<commit_before>"""

preprocess.py

This directory contains functions and classes for preprocessing raw data into
matrices and vectors we can run our EM algorithm on. A big part of this is to
reduce the overall size of our input matrixes to make our algorithm run quickly
and without using too much memory.

Sam Vohr (svohr@soe.ucsc.edu)
Wed Apr  6 10:48:21 PDT 2016

"""

import sys
import numpy
import collections

import phylotree

class HapVarBaseMatrix(object):
    """
    This class is designed to simplify a pretty large 3 dimensional matrix of
    indicator variables. Since our matrix is sparse and only need to look up
    single cells, we can store this information in a simplied way. The first
    part is that there are only two possible values that we store, a value if
    the haplogroup, variant site, observed base is expected (1) or unexpected
    (0). In practice, we will use fractional values (e.g. 0.991 and 0.003) to
    allow for sequencing error or de novo mutations.  Second, we keep a
    dictionary of haplogroup, variant site, base tuples that mark where a
    non-reference base occurs. If an entry is not present in the dictionary, it
    must be the case that the reference base is the "expected" base and all
    others are "unexpected". Finally, we will now make use of mutation counts
    available from Phylotree to adjust the probabilities based on how "mutable"
    a site appears to be.
    """
    def __init__(self, refseq, phylo, mut_wt=0.01, mut_max=0.5):
        """ Initialzes the HapVarBaseMatrix. """
        self.refseq = refseq
        self.phylo  = phylo
        self.mut_wt = mut_wt
        self.mut_max = mut_max
        self.markers = dict()

        self.add_hap_markers(phylo.hap_var)
        return

    def add_hap_markers(self, hap_var):
        """
        Populates the table with bases that are associated with haplogroups.
        """
        for hap in hap_var:
            self.markers[hap] = dict()
            for var in hap_var[hap]:
                pos = phylotree.pos_from_var(var)
                der = phylotree.der_allele(var)
                if der != self.refseq[pos]:
                    self.markers[hap][pos] = der
        return

    def _prob(self, hap_pos, pos, base):
        """
        Returns the "probability" of observing this base, at this position in
        this haplogroup, given the dictionary that describes the derived
        alleles of this haplogroup.
        """
        mut_prob = min(self.mut_wt * sum(self.phylo.variants[pos].values()),
                       self.mut_max)
        if pos in hap_pos:
            # Does this haplogroup carry a derived base?
            if hap_pos[pos] == base:
                # Is it the one we observed?
                return 1.0 - mut_prob
        else:
            # No derived base at this position, does our observed match ref.
            if self.refseq[pos] == base:
                return 1.0 - mut_prob
        return mut_prob / 3.0

    def prob_for_vars(self, hap, pos_obs):
        """
        Finds the probability of observing the read signature from the given
        haplotype by calculating the product of the probability of all
        hap, pos, base tuples it represents.
        """
        total = 1
        hap_pos = self.markers[hap]
        for pos, obs in pos_obs:
            total *= self._prob(hap_pos, pos, obs)
        return total

def process_reads(alns, var_pos, min_mq, min_bq):
    """
    Read alignments from an iterator/iterable and makes observations for known
    variant positions. Each read is simplified into a signature of observed
    base per variant site.

    Args:
        alns: An iterator/iterable of pysam AlignedSegments
        var_pos: A sorted list of all known variant positions from Phylotree.
        min_mq: The minimum mapping quality score required for a read to be
                considered.
        min_bq: The minimum base quality score required for a base observation.

    Returns:
        read_obs: A dictionary that maps AlignedSegment query_names to
                  dictionaries mapping reference positions to base
                  observations.
    """
    read_obs = collections.defaultdict(dict)
    var_pos = set(var_pos)
    for aln in alns:
        if aln.mapping_quality >= min_mq:
            for qpos, rpos in aln.get_aligned_pairs(matches_only=True):
                qpos = int(qpos)
                rpos = int(rpos)
                if (aln.query_qualities is None
                  or aln.query_qualities[qpos] >= min_bq):
                    if rpos in var_pos:
                        obs = aln.query_sequence[qpos].upper()
                        # Add this to the list, if this is a known var site.
                        if rpos in read_obs[aln.query_name]:
                            # check if this position has been observed before.
                            if read_obs[aln.query_name][rpos] != obs:
                                read_obs[aln.query_name][rpos] = "N"
                        else:
                            read_obs[aln.query_name][rpos] = obs
    # Finished, do one pass to remove Ns
    for aln_id in read_obs:
        read_obs[aln_id] = {pos:base for pos, base in
                            read_obs[aln_id].items() if base != 'N'}
    return read_obs

def read_signature(obs_by_pos):
    """
    Returns a string made from the positions where bases were able to be
    observed from the read.
    """
    return ','.join(["%d:%s" % (pos, obs_by_pos[pos])
                     for pos in sorted(obs_by_pos)])

def pos_obs_from_sig(read_sig):
    """
    Returns a list of position, observation pairs described in the read
    signature string.
    """
    def pos_obs(var):
        """ Splits var into a int position and a string base. """
        pos, obs = var.split(':')
        return int(pos), obs
    return [pos_obs(var) for var in read_sig.split(',')]

def reduce_reads(read_obs):
    """
    Takes a dictionary of read IDs and base observations at variable positions
    and returns a dictionary observation signatures (a string describing the
    base observations made at variant sites) that maps to a list of read IDs
    that carry that signature.
    """
    read_sigs = collections.defaultdict(list)
    for read_id in read_obs:
        sig = read_signature(read_obs[read_id])
        read_sigs[sig].append(read_id)
    return read_sigs

def build_em_matrix(refseq, phylo, reads, haplogroups, args):
    """
    Returns the matrix that describes the probabiliy of each read
    originating in each haplotype.
    """
    hvb_mat = HapVarBaseMatrix(refseq, phylo)
    read_hap_mat = numpy.empty((len(reads), len(haplogroups)))

    if args.verbose:
        sys.stderr.write('Building EM input matrix...\n')

    for i in xrange(len(reads)):
        pos_obs = pos_obs_from_sig(reads[i])
        for j in xrange(len(haplogroups)):
            read_hap_mat[i, j] = hvb_mat.prob_for_vars(haplogroups[j], pos_obs)
        if args.verbose and (i + 1) % 500 == 0:
            sys.stderr.write('  processed %d fragments...\n' % (i + 1))

    if args.verbose:
        sys.stderr.write('Done.\n\n')

    return read_hap_mat

def build_em_input(bamfile, refseq, phylo, args):
    """
    Builds the matrix that describes the "probability" that a read originated
    from a specific haplogroup.
    """
    var_pos = phylo.get_variant_pos()

    input_alns = bamfile.fetch()
    read_obs = process_reads(input_alns, var_pos, args.min_mq, args.min_bq)
    read_sigs = reduce_reads(read_obs)

    if args.verbose:
        sys.stderr.write('Using %d aligned fragments (MQ>=%d) '
                         '(%d distinct sub-haplotypes)\n\n'
                           % (len(read_obs), args.min_mq, len(read_sigs)))

    # This is now the order we will be using for the matrix.
    haplogroups = sorted(phylo.hap_var)
    reads = sorted(read_sigs)
    weights = numpy.array([len(read_sigs[r]) for r in reads])

    em_matrix = build_em_matrix(refseq, phylo, reads, haplogroups, args)

    # make a list mapping matrix indexes to read IDs from bam.
    read_ids = [read_sigs[reads[i]] for i in xrange(len(reads))]

    return em_matrix, weights, haplogroups, read_ids

def main():
    """ Main function for simple testing. """
#   args = {'min_mq':30, 'min_bq':30, 'verbose':True}
#   if len(sys.argv) > 2:
#       phy_fn = sys.argv[1]
#       bam_fn = sys.argv[2]
#       ref_in = pysam.FastaFile('../ref/RSRS.mtDNA.fa')
#       refseq = ref_in.fetch(ref_in.references[0])
#       with open(phy_fn, 'r') as phy_in:
#           var_pos, hap_var = phylotree.read_phylotree(phy_in,
#                                                       False, False, False)
#       with pysam.AlignmentFile(bam_fn, 'rb') as samfile:
#           build_em_input(samfile, refseq, var_pos, hap_var, args)
#   else:
        # Entirely fake data.
#       ref = "GAAAAAAAA"
#       var_pos = range(1, 9)
#       hap_var = dict({'A':['A1T', 'A3T'],
#                       'B':['A2T', 'A4T', 'A5T', 'A7T'],
#                       'C':['A2T', 'A5T'],
#                       'D':['A2T', 'A4T', 'A6T', 'A8T'],
#                       'E':['A2T', 'A3T', 'A4T', 'A6T']})
#       reads = list(["1:A,2:T,3:A", "2:T,3:A", "3:A,4:T,5:T", "5:T,6:A",
#                     "6:A,7:T", "6:A,7:T,8:A", "7:T,8:A", "4:T,5:T",
#                     "1:A,2:T,3:T,4:T", "5:A,6:T,7:A,8:A"])
#       haps = list('ABCDE')
#       print build_em_matrix(ref, hap_var, reads, haps)
    return 0

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>HapVarBaseMatrix now pre-computes site instability score. Big win.<commit_after>"""

preprocess.py

This directory contains functions and classes for preprocessing raw data into
matrices and vectors we can run our EM algorithm on. A big part of this is to
reduce the overall size of our input matrixes to make our algorithm run quickly
and without using too much memory.

Sam Vohr (svohr@soe.ucsc.edu)
Wed Apr  6 10:48:21 PDT 2016

"""

import sys
import numpy
import collections

import phylotree

class HapVarBaseMatrix(object):
    """
    This class is designed to simplify a pretty large 3 dimensional matrix of
    indicator variables. Since our matrix is sparse and only need to look up
    single cells, we can store this information in a simplied way. The first
    part is that there are only two possible values that we store, a value if
    the haplogroup, variant site, observed base is expected (1) or unexpected
    (0). In practice, we will use fractional values (e.g. 0.991 and 0.003) to
    allow for sequencing error or de novo mutations.  Second, we keep a
    dictionary of haplogroup, variant site, base tuples that mark where a
    non-reference base occurs. If an entry is not present in the dictionary, it
    must be the case that the reference base is the "expected" base and all
    others are "unexpected". Finally, we will now make use of mutation counts
    available from Phylotree to adjust the probabilities based on how "mutable"
    a site appears to be.
    """
    def __init__(self, refseq, phylo, mut_wt=0.01, mut_max=0.5):
        """ Initialzes the HapVarBaseMatrix. """
        self.refseq = refseq
        self.phylo  = phylo
        self.mut_wt = mut_wt
        self.mut_max = mut_max
        self.markers = dict()

        self.mut_prob = dict()
        for pos in self.phylo.variants:
            self.mut_prob[pos] = min(self.mut_wt *
                                       sum(self.phylo.variants[pos].values()),
                                     self.mut_max)

        self.add_hap_markers(phylo.hap_var)
        return

    def add_hap_markers(self, hap_var):
        """
        Populates the table with bases that are associated with haplogroups.
        """
        for hap in hap_var:
            self.markers[hap] = dict()
            for var in hap_var[hap]:
                pos = phylotree.pos_from_var(var)
                der = phylotree.der_allele(var)
                if der != self.refseq[pos]:
                    self.markers[hap][pos] = der
        return

    def _prob(self, hap_pos, pos, base):
        """
        Returns the "probability" of observing this base, at this position in
        this haplogroup, given the dictionary that describes the derived
        alleles of this haplogroup.
        """
        if pos in hap_pos:
            # Does this haplogroup carry a derived base?
            if hap_pos[pos] == base:
                # Is it the one we observed?
                return 1.0 - self.mut_prob[pos]
        else:
            # No derived base at this position, does our observed match ref.
            if self.refseq[pos] == base:
                return 1.0 - self.mut_prob[pos]
        return self.mut_prob[pos] / 3.0

    def prob_for_vars(self, hap, pos_obs):
        """
        Finds the probability of observing the read signature from the given
        haplotype by calculating the product of the probability of all
        hap, pos, base tuples it represents.
        """
        total = 1
        hap_pos = self.markers[hap]
        for pos, obs in pos_obs:
            total *= self._prob(hap_pos, pos, obs)
        return total

def process_reads(alns, var_pos, min_mq, min_bq):
    """
    Read alignments from an iterator/iterable and makes observations for known
    variant positions. Each read is simplified into a signature of observed
    base per variant site.

    Args:
        alns: An iterator/iterable of pysam AlignedSegments
        var_pos: A sorted list of all known variant positions from Phylotree.
        min_mq: The minimum mapping quality score required for a read to be
                considered.
        min_bq: The minimum base quality score required for a base observation.

    Returns:
        read_obs: A dictionary that maps AlignedSegment query_names to
                  dictionaries mapping reference positions to base
                  observations.
    """
    read_obs = collections.defaultdict(dict)
    var_pos = set(var_pos)
    for aln in alns:
        if aln.mapping_quality >= min_mq:
            for qpos, rpos in aln.get_aligned_pairs(matches_only=True):
                qpos = int(qpos)
                rpos = int(rpos)
                if (aln.query_qualities is None
                  or aln.query_qualities[qpos] >= min_bq):
                    if rpos in var_pos:
                        obs = aln.query_sequence[qpos].upper()
                        # Add this to the list, if this is a known var site.
                        if rpos in read_obs[aln.query_name]:
                            # check if this position has been observed before.
                            if read_obs[aln.query_name][rpos] != obs:
                                read_obs[aln.query_name][rpos] = "N"
                        else:
                            read_obs[aln.query_name][rpos] = obs
    # Finished, do one pass to remove Ns
    for aln_id in read_obs:
        read_obs[aln_id] = {pos:base for pos, base in
                            read_obs[aln_id].items() if base != 'N'}
    return read_obs

def read_signature(obs_by_pos):
    """
    Returns a string made from the positions where bases were able to be
    observed from the read.
    """
    return ','.join(["%d:%s" % (pos, obs_by_pos[pos])
                     for pos in sorted(obs_by_pos)])

def pos_obs_from_sig(read_sig):
    """
    Returns a list of position, observation pairs described in the read
    signature string.
    """
    def pos_obs(var):
        """ Splits var into a int position and a string base. """
        pos, obs = var.split(':')
        return int(pos), obs
    return [pos_obs(var) for var in read_sig.split(',')]

def reduce_reads(read_obs):
    """
    Takes a dictionary of read IDs and base observations at variable positions
    and returns a dictionary observation signatures (a string describing the
    base observations made at variant sites) that maps to a list of read IDs
    that carry that signature.
    """
    read_sigs = collections.defaultdict(list)
    for read_id in read_obs:
        sig = read_signature(read_obs[read_id])
        read_sigs[sig].append(read_id)
    return read_sigs

def build_em_matrix(refseq, phylo, reads, haplogroups, args):
    """
    Returns the matrix that describes the probabiliy of each read
    originating in each haplotype.
    """
    hvb_mat = HapVarBaseMatrix(refseq, phylo)
    read_hap_mat = numpy.empty((len(reads), len(haplogroups)))

    if args.verbose:
        sys.stderr.write('Building EM input matrix...\n')

    for i in xrange(len(reads)):
        pos_obs = pos_obs_from_sig(reads[i])
        for j in xrange(len(haplogroups)):
            read_hap_mat[i, j] = hvb_mat.prob_for_vars(haplogroups[j], pos_obs)
        if args.verbose and (i + 1) % 500 == 0:
            sys.stderr.write('  processed %d fragments...\n' % (i + 1))

    if args.verbose:
        sys.stderr.write('Done.\n\n')

    return read_hap_mat

def build_em_input(bamfile, refseq, phylo, args):
    """
    Builds the matrix that describes the "probability" that a read originated
    from a specific haplogroup.
    """
    var_pos = phylo.get_variant_pos()

    input_alns = bamfile.fetch()
    read_obs = process_reads(input_alns, var_pos, args.min_mq, args.min_bq)
    read_sigs = reduce_reads(read_obs)

    if args.verbose:
        sys.stderr.write('Using %d aligned fragments (MQ>=%d) '
                         '(%d distinct sub-haplotypes)\n\n'
                           % (len(read_obs), args.min_mq, len(read_sigs)))

    # This is now the order we will be using for the matrix.
    haplogroups = sorted(phylo.hap_var)
    reads = sorted(read_sigs)
    weights = numpy.array([len(read_sigs[r]) for r in reads])

    em_matrix = build_em_matrix(refseq, phylo, reads, haplogroups, args)

    # make a list mapping matrix indexes to read IDs from bam.
    read_ids = [read_sigs[reads[i]] for i in xrange(len(reads))]

    return em_matrix, weights, haplogroups, read_ids

def main():
    """ Main function for simple testing. """
#   args = {'min_mq':30, 'min_bq':30, 'verbose':True}
#   if len(sys.argv) > 2:
#       phy_fn = sys.argv[1]
#       bam_fn = sys.argv[2]
#       ref_in = pysam.FastaFile('../ref/RSRS.mtDNA.fa')
#       refseq = ref_in.fetch(ref_in.references[0])
#       with open(phy_fn, 'r') as phy_in:
#           var_pos, hap_var = phylotree.read_phylotree(phy_in,
#                                                       False, False, False)
#       with pysam.AlignmentFile(bam_fn, 'rb') as samfile:
#           build_em_input(samfile, refseq, var_pos, hap_var, args)
#   else:
        # Entirely fake data.
#       ref = "GAAAAAAAA"
#       var_pos = range(1, 9)
#       hap_var = dict({'A':['A1T', 'A3T'],
#                       'B':['A2T', 'A4T', 'A5T', 'A7T'],
#                       'C':['A2T', 'A5T'],
#                       'D':['A2T', 'A4T', 'A6T', 'A8T'],
#                       'E':['A2T', 'A3T', 'A4T', 'A6T']})
#       reads = list(["1:A,2:T,3:A", "2:T,3:A", "3:A,4:T,5:T", "5:T,6:A",
#                     "6:A,7:T", "6:A,7:T,8:A", "7:T,8:A", "4:T,5:T",
#                     "1:A,2:T,3:T,4:T", "5:A,6:T,7:A,8:A"])
#       haps = list('ABCDE')
#       print build_em_matrix(ref, hap_var, reads, haps)
    return 0

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>import logging
import json
import tarfile
import urllib2
import os
import gzip
import requests

resources_endpoint = "http://localhost:5002/resources"
resource_stacks_endpoint = "http://localhost:5002/resource_stacks"
environments_endpoint = "http://localhost:5002/environments"
apps_endpoint = "http://localhost:5002/apps"

SERVER_ERROR = "Something caused error in cloudark. Please submit bug report on cloudark github repo. "
SERVER_ERROR = SERVER_ERROR + "Attach logs from cld.log which is available in cloudark directory."

class TakeAction(object):

    def __init__(self):
        pass

    def _make_tarfile(self, output_filename, source_dir):
        with tarfile.open(output_filename, "w:gz") as tar:
            tar.add(source_dir, arcname=os.path.basename(source_dir))

    def _read_tarfile(self, tarfile_name):
        with gzip.open(tarfile_name, "rb") as f:
            contents = f.read()
            return contents

    def _delete_tarfile(self, tarfile_name, source_dir):
        cwd = os.getcwd()
        os.chdir(source_dir)
        if os.path.exists(tarfile_name):
            os.remove(tarfile_name)
        os.chdir(cwd)

    def deploy_app(self, app_path, app_info):
        source_dir = app_path
        k = source_dir.rfind("/")
        app_name = app_info['app_name']
        tarfile_name = app_name + ".tar"

        self._make_tarfile(tarfile_name, source_dir)
        tarfile_content = self._read_tarfile(tarfile_name)

        app_info['app_name'] = app_name
        app_info['app_tar_name'] = tarfile_name
        app_info['app_content'] = tarfile_content

        data = {'app_info': app_info}

        req = urllib2.Request(apps_endpoint)
        req.add_header('Content-Type', 'application/octet-stream')

        app_url = ''
        try:
            response = urllib2.urlopen(req, json.dumps(data, ensure_ascii=True, encoding='ISO-8859-1'))
            app_url = response.headers.get('location')
            print("Request to deploy application accepted.")
        except Exception as e:
            if e.code == 412:
                print("App cannot be deployed as Environment is not ready yet.")
            if e.code == 404:
                print("Environment with id %s not found" % app_info['env_id'])
            if e.code == 503 or e.code == 500:
                print(SERVER_ERROR)
        self._delete_tarfile(tarfile_name, source_dir)

        return app_url

    def get_app(self, app_id):
        app_url = apps_endpoint + "/" + app_id
        req = urllib2.Request(app_url)
        app_data = ''
        try:
            response = urllib2.urlopen(req)
            app_data = response.fp.read()
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                print("App with app-id %s not found." % app_id)
        return app_data

    def delete_app(self, app_id):
        app_url = apps_endpoint + "/" + app_id
        response = requests.delete(app_url)
        if response.status_code == 404:
            print("App with app-id %s not found." % app_id)
        if response.status_code == 202:
            print("Request to delete app with id %s accepted." % app_id)
        return response

    def redeploy_app(self, app_path, app_info, app_id):
        app_id_url = apps_endpoint + "/" + app_id
        source_dir = app_path
        k = source_dir.rfind("/")
        app_name = "app-redeploy-id-" + app_id
        tarfile_name = app_name + ".tar"

        self._make_tarfile(tarfile_name, source_dir)
        tarfile_content = self._read_tarfile(tarfile_name)

        app_info['app_tar_name'] = tarfile_name
        app_info['app_content'] = tarfile_content

        data = {'app_info': app_info}

        app_url = ''
        req = urllib2.Request(app_id_url)
        req.add_header('Content-Type', 'application/octet-stream')
        req.get_method = lambda: 'PUT'
        try:
            response = urllib2.urlopen(req, json.dumps(data, ensure_ascii=True, encoding='ISO-8859-1'))
            if response.code == 202:
                print("Request to redeploy app with id %s accepted." % app_id)
            app_url = response.headers.get('location')
        except Exception as e:
            if e.msg == 'NOT FOUND':
                print("App with app-id %s not found." % app_id)
            if e.msg == 'INTERNAL SERVER ERROR':
                print(SERVER_ERROR)
                return

        self._delete_tarfile(tarfile_name, source_dir)
        return app_url

    def get_app_list(self):
        req = urllib2.Request(apps_endpoint)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % apps_endpoint)
        return data

    # Functions for environment
    def create_environment(self, env_name, environment_def):
        req = urllib2.Request(environments_endpoint)
        req.add_header('Content-Type', 'application/octet-stream')

        data = {'environment_def': environment_def,
                'environment_name': env_name}

        response = urllib2.urlopen(req, json.dumps(data, ensure_ascii=True, encoding='ISO-8859-1'))

        environment_url = response.headers.get('location')
        return environment_url

    def get_environment(self, env_id):
        env_url = environments_endpoint + "/" + env_id
        req = urllib2.Request(env_url)
        env_data = ''
        try:
            response = urllib2.urlopen(req)
            env_data = response.fp.read()
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                print("Environment with env-id %s not found." % env_id)
        return env_data

    def delete_environment(self, env_id):
        env_url = environments_endpoint + "/" + env_id
        response = requests.delete(env_url)
        if response.status_code == 404:
            print("Environment with env-id %s not found." % env_id)
        if response.status_code == 202:
            print("Request to delete env with id %s accepted." % env_id)
        if response.status_code == 412:
            print("Environment cannot be deleted as there are applications still running on it.")
        return response

    def get_environment_list(self):
        req = urllib2.Request(environments_endpoint)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % environments_endpoint)
        return data

    # Functions for Individual resource
    def get_resources(self):
        req = urllib2.Request(resources_endpoint)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % resources_endpoint)
        return data

    def get_resources_for_environment(self, env_id):
        req = urllib2.Request(resources_endpoint + "?env_id=%s" % env_id)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % resources_endpoint)
        return data

    def create_resource(self, resource_obj):
        req = urllib2.Request(resources_endpoint)
        req.add_header('Content-Type', 'application/octet-stream')

        request_data = {'resource_info': resource_obj}

        response = urllib2.urlopen(req, json.dumps(request_data,
                                                   ensure_ascii=True,
                                                   encoding='ISO-8859-1'))

        resource_endpoint = response.headers.get('location')
        print("Resource URL:%s" % resource_endpoint)
        return resource_endpoint

    def get_resource(self, resource_id):
        resource_endpoint = resources_endpoint + "/" + resource_id
        req = urllib2.Request(resource_endpoint)
        resource_data = ''
        try:
            response = urllib2.urlopen(req)
            resource_data = response.fp.read()
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                print("Resource with resource-id %s not found." % resource_id)
        return resource_data

    def delete_resource(self, resource_id):
        resource_endpoint = resources_endpoint + "/" + resource_id
        response = requests.delete(resource_endpoint)
        if response.status_code == 404:
            print("Resource with resource-id %s not found." % resource_id)
        if response.status_code == 202:
            print("Request to delete resource with resource-id %s accepted." % resource_id)
        return response<commit_msg>Added error checks for environment create<commit_after>import logging
import json
import tarfile
import urllib2
import os
import gzip
import requests

resources_endpoint = "http://localhost:5002/resources"
resource_stacks_endpoint = "http://localhost:5002/resource_stacks"
environments_endpoint = "http://localhost:5002/environments"
apps_endpoint = "http://localhost:5002/apps"

SERVER_ERROR = "Something caused error in cloudark. Please submit bug report on cloudark github repo. "
SERVER_ERROR = SERVER_ERROR + "Attach logs from cld.log which is available in cloudark directory."

class TakeAction(object):

    def __init__(self):
        pass

    def _make_tarfile(self, output_filename, source_dir):
        with tarfile.open(output_filename, "w:gz") as tar:
            tar.add(source_dir, arcname=os.path.basename(source_dir))

    def _read_tarfile(self, tarfile_name):
        with gzip.open(tarfile_name, "rb") as f:
            contents = f.read()
            return contents

    def _delete_tarfile(self, tarfile_name, source_dir):
        cwd = os.getcwd()
        os.chdir(source_dir)
        if os.path.exists(tarfile_name):
            os.remove(tarfile_name)
        os.chdir(cwd)

    def deploy_app(self, app_path, app_info):
        source_dir = app_path
        k = source_dir.rfind("/")
        app_name = app_info['app_name']
        tarfile_name = app_name + ".tar"

        self._make_tarfile(tarfile_name, source_dir)
        tarfile_content = self._read_tarfile(tarfile_name)

        app_info['app_name'] = app_name
        app_info['app_tar_name'] = tarfile_name
        app_info['app_content'] = tarfile_content

        data = {'app_info': app_info}

        req = urllib2.Request(apps_endpoint)
        req.add_header('Content-Type', 'application/octet-stream')

        app_url = ''
        try:
            response = urllib2.urlopen(req, json.dumps(data, ensure_ascii=True, encoding='ISO-8859-1'))
            app_url = response.headers.get('location')
            print("Request to deploy application accepted.")
        except Exception as e:
            if e.code == 412:
                print("App cannot be deployed as Environment is not ready yet.")
            if e.code == 404:
                print("Environment with id %s not found" % app_info['env_id'])
            if e.code == 503 or e.code == 500:
                print(SERVER_ERROR)
        self._delete_tarfile(tarfile_name, source_dir)

        return app_url

    def get_app(self, app_id):
        app_url = apps_endpoint + "/" + app_id
        req = urllib2.Request(app_url)
        app_data = ''
        try:
            response = urllib2.urlopen(req)
            app_data = response.fp.read()
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                print("App with app-id %s not found." % app_id)
        return app_data

    def delete_app(self, app_id):
        app_url = apps_endpoint + "/" + app_id
        response = requests.delete(app_url)
        if response.status_code == 404:
            print("App with app-id %s not found." % app_id)
        if response.status_code == 202:
            print("Request to delete app with id %s accepted." % app_id)
        return response

    def redeploy_app(self, app_path, app_info, app_id):
        app_id_url = apps_endpoint + "/" + app_id
        source_dir = app_path
        k = source_dir.rfind("/")
        app_name = "app-redeploy-id-" + app_id
        tarfile_name = app_name + ".tar"

        self._make_tarfile(tarfile_name, source_dir)
        tarfile_content = self._read_tarfile(tarfile_name)

        app_info['app_tar_name'] = tarfile_name
        app_info['app_content'] = tarfile_content

        data = {'app_info': app_info}

        app_url = ''
        req = urllib2.Request(app_id_url)
        req.add_header('Content-Type', 'application/octet-stream')
        req.get_method = lambda: 'PUT'
        try:
            response = urllib2.urlopen(req, json.dumps(data, ensure_ascii=True, encoding='ISO-8859-1'))
            if response.code == 202:
                print("Request to redeploy app with id %s accepted." % app_id)
            app_url = response.headers.get('location')
        except Exception as e:
            if e.msg == 'NOT FOUND':
                print("App with app-id %s not found." % app_id)
            if e.msg == 'INTERNAL SERVER ERROR':
                print(SERVER_ERROR)
                return

        self._delete_tarfile(tarfile_name, source_dir)
        return app_url

    def get_app_list(self):
        req = urllib2.Request(apps_endpoint)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % apps_endpoint)
        return data

    # Functions for environment
    def create_environment(self, env_name, environment_def):
        req = urllib2.Request(environments_endpoint)
        req.add_header('Content-Type', 'application/octet-stream')

        data = {'environment_def': environment_def,
                'environment_name': env_name}
        try:
            response = urllib2.urlopen(req, json.dumps(data, ensure_ascii=True, encoding='ISO-8859-1'))
            print("Request to create environment accepted.")
        except Exception as e:
            if e.code == 503 or e.code == 500:
                print(SERVER_ERROR)
        environment_url = response.headers.get('location')
        return environment_url

    def get_environment(self, env_id):
        env_url = environments_endpoint + "/" + env_id
        req = urllib2.Request(env_url)
        env_data = ''
        try:
            response = urllib2.urlopen(req)
            env_data = response.fp.read()
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                print("Environment with env-id %s not found." % env_id)
        return env_data

    def delete_environment(self, env_id):
        env_url = environments_endpoint + "/" + env_id
        response = requests.delete(env_url)
        if response.status_code == 404:
            print("Environment with env-id %s not found." % env_id)
        if response.status_code == 202:
            print("Request to delete env with id %s accepted." % env_id)
        if response.status_code == 412:
            print("Environment cannot be deleted as there are applications still running on it.")
        return response

    def get_environment_list(self):
        req = urllib2.Request(environments_endpoint)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % environments_endpoint)
        return data

    # Functions for Individual resource
    def get_resources(self):
        req = urllib2.Request(resources_endpoint)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % resources_endpoint)
        return data

    def get_resources_for_environment(self, env_id):
        req = urllib2.Request(resources_endpoint + "?env_id=%s" % env_id)
        data = ''
        try:
            response = urllib2.urlopen(req)
            data = response.fp.read()
        except urllib2.HTTPError as e:
            print("Error occurred in querying endpoint %s" % resources_endpoint)
        return data

    def create_resource(self, resource_obj):
        req = urllib2.Request(resources_endpoint)
        req.add_header('Content-Type', 'application/octet-stream')

        request_data = {'resource_info': resource_obj}

        response = urllib2.urlopen(req, json.dumps(request_data,
                                                   ensure_ascii=True,
                                                   encoding='ISO-8859-1'))

        resource_endpoint = response.headers.get('location')
        print("Resource URL:%s" % resource_endpoint)
        return resource_endpoint

    def get_resource(self, resource_id):
        resource_endpoint = resources_endpoint + "/" + resource_id
        req = urllib2.Request(resource_endpoint)
        resource_data = ''
        try:
            response = urllib2.urlopen(req)
            resource_data = response.fp.read()
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                print("Resource with resource-id %s not found." % resource_id)
        return resource_data

    def delete_resource(self, resource_id):
        resource_endpoint = resources_endpoint + "/" + resource_id
        response = requests.delete(resource_endpoint)
        if response.status_code == 404:
            print("Resource with resource-id %s not found." % resource_id)
        if response.status_code == 202:
            print("Request to delete resource with resource-id %s accepted." % resource_id)
        return response<|endoftext|>


<commit_before>import functions.helper_functions as f
import functions.soaf as soaf
import os
import pandas as pd
import numpy as np

data_file = 'National_Generation_Capacities.xlsx'
filepath = os.path.join('input', data_file)

# Read data into pandas
data_raw = pd.read_excel(filepath,
                         sheet_name='Summary',
                         header=None,
                         na_values=['-'],
                         skiprows=0)

# Deal with merged cells from Excel: fill first three rows with information
data_raw.iloc[0:2] = data_raw.iloc[0:2].fillna(method='ffill', axis=1)

# Set index for rows
data_raw = data_raw.set_index([0])
data_raw.index.name = 'technology'

# Extract energylevels from raw data
energylevels_raw = data_raw.iloc[:, 0:5]

# Delete definition of energy levels from raw data
data_raw = data_raw.drop(data_raw.columns[[0, 1, 2, 3, 4]], axis=1)

# Set multiindex column names
data_raw.columns = pd.MultiIndex.from_arrays(data_raw[:6].values,
                                             names=['country', 'type', 'year',
                                                    'source', 'source_type',
                                                    'capacity_definition'])

# Remove 3 rows which are already used as column names
data_raw = data_raw[pd.notnull(data_raw.index)]

# Extract the ordering of technologies
technology_order = data_raw.index.str.replace('- ', '').values.tolist()

# Reshape dataframe to list
data_opsd = pd.DataFrame(data_raw.stack(level=['source', 'source_type', 'year',
                                               'type', 'country',
                                               'capacity_definition']))

# Reset index for dataframe
data_opsd = data_opsd.reset_index()
data_opsd['technology'] = data_opsd['technology'].str.replace('- ', '')

# Delete entries with missing source
data_opsd = data_opsd[data_opsd['source'].isnull() == False]
data_opsd = data_opsd[data_opsd['source'] != 0]

# Delete entries from EUROSTAT and entsoe as they will be directly used from original sources
data_opsd = data_opsd[data_opsd['source'] != 'EUROSTAT']
data_opsd = data_opsd[data_opsd['source'] != 'entsoe']

data_opsd = data_opsd.rename(columns={0: 'capacity'})

data_opsd['capacity'] = pd.to_numeric(data_opsd['capacity'], errors='coerce')

# For some source, permission to publish data
data_opsd.loc[(data_opsd['source'] == 'ELIA'),
              'comment'] = 'data available, but cannot be provided'
data_opsd.loc[(data_opsd['source'] == 'BMWi'),
              'comment'] = 'data available, but cannot be provided'
data_opsd.loc[(data_opsd['source'] == 'Mavir'),
              'comment'] = 'data available, but cannot be provided'

### Eurostat ##################################################################
url_eurostat = ('http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/'
                'BulkDownloadListing?sort=1&downfile=data%2Fnrg_113a.tsv.gz')

filepath_eurostat = f.downloadandcache(url_eurostat, 'Eurostat.tsv.gz', 'Eurostat')

data_eurostat = pd.read_csv(filepath_eurostat,
                               compression='gzip',
                               sep='\t|,',
                               engine='python'
                               )

id_vars = ['unit', 'product','indic_nrg', 'geo\\time']
data_eurostat = pd.melt(data_eurostat, id_vars=id_vars,
                        var_name='year', value_name='value')

data_definition = pd.read_csv(os.path.join('input', 'definition_EUROSTAT_indic.txt'),
                              header=None,
                              names=['indic', 'description',
                                     'energy source'],
                              sep='\t')

data_eurostat = data_eurostat.merge(data_definition,
                                    how='left',
                                    left_on='indic_nrg',
                                    right_on='indic')

data_eurostat = data_eurostat[data_eurostat['energy source'].isnull() == False]

values_as_string = data_eurostat['value'].astype(str)
string_values = values_as_string.str.split(' ', 1).str[0]
string_values.replace(':', np.nan, inplace=True)
subset_nan = string_values.isnull()

data_eurostat['value'] = string_values
#data_eurostat['comment'] = ''
#data_eurostat.loc[subset_nan, 'comment'] = 'not available'

data_eurostat['year'] = data_eurostat['year'].astype(int)
data_eurostat['value'] = data_eurostat['value'].astype(float)

data_eurostat = data_eurostat.drop(['unit', 'product', 'indic_nrg',
                                    'indic', 'description'], axis=1)

data_eurostat = data_eurostat.rename(columns={'geo\\time': 'country',
                                              'energy source': 'technology',
                                              'value': 'capacity'})

data_eurostat['country'].replace({'UK': 'GB', 'EL': 'GR'}, inplace=True)

drop_list = data_eurostat[data_eurostat['country'].isin(['EU28','EA19'])].index
data_eurostat.drop(drop_list, inplace=True)

by_columns = ['technology', 'year', 'country']
data_eurostat = pd.DataFrame(data_eurostat.groupby(by_columns)['capacity'].sum())
data_eurostat_isnull = data_eurostat['capacity'].isnull() == True
#data_eurostat.loc[data_eurostat_isnull, 'comment'] = 'not available'
#data_eurostat['comment'] = data_eurostat['comment'].fillna('').astype(str)
data_eurostat.reset_index(inplace=True)

#########

test = data_eurostat.pivot_table(values='capacity',
                                 index=['country','year'],
                                 columns='technology')

test['Differently categorized solar'] = 0
test['Solar'] = test[['Photovoltaics', 'Concentrated solar power']].sum(axis=1)
test['Differently categorized wind'] = test['Wind']
bio_arr = ['Biomass and biogas', 'Other bioenergy and renewable waste']
test['Bioenergy and renewable waste'] = test[bio_arr].sum(axis=1)
res_arr = ['Hydro', 'Wind', 'Solar', 'Geothermal', 'Marine',
           'Bioenergy and renewable waste']
test['Renewable energy sources'] = test[res_arr].sum(axis=1)

test['Fossil fuels'] = test['Fossil fuels'] - test['Bioenergy and renewable waste']
test['Differently categorized fossil fuels'] = test['Fossil fuels']\
                                             - test['Non-renewable waste']

total_arr = ['Fossil fuels','Nuclear','Renewable energy sources']
test['Total'] = test[total_arr].sum(axis=1)

data_eurostat = test.stack().reset_index().rename(columns={0: 'capacity'})

idx = data_eurostat[data_eurostat['technology'] == 'Fossil fuels'].index
data_eurostat.drop(idx, inplace=True)

data_eurostat['source'] = 'EUROSTAT'
data_eurostat['source_type'] = 'Statistical Office'
data_eurostat['capacity_definition'] = 'Unknown'
data_eurostat['type'] = 'Installed capacity in MW'

############ENTSOE SOAF###############

soafs = [soaf.SoafDataRaw('https://www.entsoe.eu/fileadmin/user_upload/_library/SDC/SOAF/SO_AF_2011_-_2025_.zip',
                          'SO_AF_2011_-_2025_.zip',
                          'SO&AF 2011 - 2025 Scenario B.xls',
                          2011),

        soaf.SoafDataRaw('https://www.entsoe.eu/fileadmin/user_upload/_library/SDC/SOAF/120705_SOAF_2012_Dataset.zip',
                         '120705_SOAF_2012_Dataset.zip',
                         'SOAF 2012 Scenario B.xls',
                         2012),

        soaf.SoafDataRaw('https://www.entsoe.eu/fileadmin/user_upload/_library/publications/entsoe/So_AF_2013-2030/130403_SOAF_2013-2030_dataset.zip',
                         '130403_SOAF_2013-2030_dataset.zip',
                         'ScB.xls',
                         2013),

        soaf.SoafDataRaw('https://www.entsoe.eu/Documents/SDC%20documents/SOAF/140602_SOAF%202014_dataset.zip',
                         '140602_SOAF%202014_dataset.zip',
                         'ScB.xlsx',
                         2014),

        soaf.SoafDataRaw('https://www.entsoe.eu/Documents/Publications/SDC/data/SO_AF_2015_dataset.zip',
                         'SO_AF_2015_dataset.zip',
                         os.path.join('SO&AF 2015 dataset', 'ScB_publication.xlsx'),
                         2016)]

soaf_data = pd.concat([s.transformed_df for s in soafs])

# Correct that in the Soaf2015 datatset the year column is 2016 instead of 2015
soaf_data['year'].replace({2016 : 2015}, inplace=True)

soaf_unstacked = f.unstackData(soaf_data)

soaf_unstacked['Differently categorized solar'] = soaf_unstacked['Solar']

soaf_unstacked['Differently categorized wind'] = soaf_unstacked['Wind']\
                                                - soaf_unstacked['Offshore']\
                                                - soaf_unstacked['Onshore']

soaf_unstacked['Differently categorized hydro'] = soaf_unstacked['Hydro']\
                                                - soaf_unstacked['Run-of-river']\
                                                - soaf_unstacked['Reservoir including pumped storage']

soaf_unstacked['Bioenergy and renewable waste'] = soaf_unstacked['Biomass and biogas']

soaf_unstacked['Differently categorized renewable energy sources'] = (
                                          soaf_unstacked['renewable']
                                        - soaf_unstacked['Wind']
                                        - soaf_unstacked['Solar']
                                        - soaf_unstacked['Biomass and biogas'])

subtract_fossils_arr = ['Lignite','Hard coal','Oil','Natural gas','Mixed fossil fuels']

soaf_unstacked['Differently categorized fossil fuels'] = soaf_unstacked['Fossil fuels']\
                                                        - soaf_unstacked[subtract_fossils_arr].sum(axis=1)

res_arr = ['Solar','Wind','Bioenergy and renewable waste','Hydro',
           'Differently categorized renewable energy sources']

soaf_unstacked['Renewable energy sources'] = soaf_unstacked[res_arr].sum(axis=1)

total_arr = ['Renewable energy sources','Fossil fuels','Nuclear',
             'Other or unspecified energy sources']

soaf_unstacked['Total'] = soaf_unstacked[total_arr].sum(axis=1)

soaf_data = f.restackData(soaf_unstacked)

soaf_data.loc[soaf_data['capacity'] < 0, 'capacity'] = 0

soaf_data['source'] = 'entsoe SOAF'
soaf_data['type'] = 'Installed capacity in MW'
soaf_data['capacity_definition'] = 'Net capacity'
soaf_data['source_type'] = 'Other association'

############ENTSOE statistical data ###############

url_entsoe = 'https://docstore.entsoe.eu/Documents/Publications/Statistics/NGC_2010-2015.xlsx'

filepath_entsoe = f.downloadandcache(url_entsoe, 'Statistics.xls',
                                     os.path.join('ENTSO-E','Statistical data 2010-2015')
                                     )

data_entsoe_raw = pd.read_excel(filepath_entsoe)
<commit_msg>entsoe statistics done<commit_after>import functions.helper_functions as f
import functions.soaf as soaf
import os
import pandas as pd
import numpy as np

data_file = 'National_Generation_Capacities.xlsx'
filepath = os.path.join('input', data_file)

# Read data into pandas
data_raw = pd.read_excel(filepath,
                         sheet_name='Summary',
                         header=None,
                         na_values=['-'],
                         skiprows=0)

# Deal with merged cells from Excel: fill first three rows with information
data_raw.iloc[0:2] = data_raw.iloc[0:2].fillna(method='ffill', axis=1)

# Set index for rows
data_raw = data_raw.set_index([0])
data_raw.index.name = 'technology'

# Extract energylevels from raw data
energylevels_raw = data_raw.iloc[:, 0:5]

# Delete definition of energy levels from raw data
data_raw = data_raw.drop(data_raw.columns[[0, 1, 2, 3, 4]], axis=1)

# Set multiindex column names
data_raw.columns = pd.MultiIndex.from_arrays(data_raw[:6].values,
                                             names=['country', 'type', 'year',
                                                    'source', 'source_type',
                                                    'capacity_definition'])

# Remove 3 rows which are already used as column names
data_raw = data_raw[pd.notnull(data_raw.index)]

# Extract the ordering of technologies
technology_order = data_raw.index.str.replace('- ', '').values.tolist()

# Reshape dataframe to list
data_opsd = pd.DataFrame(data_raw.stack(level=['source', 'source_type', 'year',
                                               'type', 'country',
                                               'capacity_definition']))

# Reset index for dataframe
data_opsd = data_opsd.reset_index()
data_opsd['technology'] = data_opsd['technology'].str.replace('- ', '')

# Delete entries with missing source
data_opsd = data_opsd[data_opsd['source'].isnull() == False]
data_opsd = data_opsd[data_opsd['source'] != 0]

# Delete entries from EUROSTAT and entsoe as they will be directly used from original sources
data_opsd = data_opsd[data_opsd['source'] != 'EUROSTAT']
data_opsd = data_opsd[data_opsd['source'] != 'entsoe']

data_opsd = data_opsd.rename(columns={0: 'capacity'})

data_opsd['capacity'] = pd.to_numeric(data_opsd['capacity'], errors='coerce')

# For some source, permission to publish data
data_opsd.loc[(data_opsd['source'] == 'ELIA'),
              'comment'] = 'data available, but cannot be provided'
data_opsd.loc[(data_opsd['source'] == 'BMWi'),
              'comment'] = 'data available, but cannot be provided'
data_opsd.loc[(data_opsd['source'] == 'Mavir'),
              'comment'] = 'data available, but cannot be provided'

### Eurostat ##################################################################
url_eurostat = ('http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/'
                'BulkDownloadListing?sort=1&downfile=data%2Fnrg_113a.tsv.gz')

filepath_eurostat = f.downloadandcache(url_eurostat, 'Eurostat.tsv.gz', 'Eurostat')

data_eurostat = pd.read_csv(filepath_eurostat,
                               compression='gzip',
                               sep='\t|,',
                               engine='python'
                               )

id_vars = ['unit', 'product','indic_nrg', 'geo\\time']
data_eurostat = pd.melt(data_eurostat, id_vars=id_vars,
                        var_name='year', value_name='value')

data_definition = pd.read_csv(os.path.join('input', 'definition_EUROSTAT_indic.txt'),
                              header=None,
                              names=['indic', 'description',
                                     'energy source'],
                              sep='\t')

data_eurostat = data_eurostat.merge(data_definition,
                                    how='left',
                                    left_on='indic_nrg',
                                    right_on='indic')

data_eurostat = data_eurostat[data_eurostat['energy source'].isnull() == False]

values_as_string = data_eurostat['value'].astype(str)
string_values = values_as_string.str.split(' ', 1).str[0]
string_values.replace(':', np.nan, inplace=True)
subset_nan = string_values.isnull()

data_eurostat['value'] = string_values
#data_eurostat['comment'] = ''
#data_eurostat.loc[subset_nan, 'comment'] = 'not available'

data_eurostat['year'] = data_eurostat['year'].astype(int)
data_eurostat['value'] = data_eurostat['value'].astype(float)

data_eurostat = data_eurostat.drop(['unit', 'product', 'indic_nrg',
                                    'indic', 'description'], axis=1)

data_eurostat = data_eurostat.rename(columns={'geo\\time': 'country',
                                              'energy source': 'technology',
                                              'value': 'capacity'})

data_eurostat['country'].replace({'UK': 'GB', 'EL': 'GR'}, inplace=True)

drop_list = data_eurostat[data_eurostat['country'].isin(['EU28','EA19'])].index
data_eurostat.drop(drop_list, inplace=True)

by_columns = ['technology', 'year', 'country']
data_eurostat = pd.DataFrame(data_eurostat.groupby(by_columns)['capacity'].sum())
data_eurostat_isnull = data_eurostat['capacity'].isnull() == True
#data_eurostat.loc[data_eurostat_isnull, 'comment'] = 'not available'
#data_eurostat['comment'] = data_eurostat['comment'].fillna('').astype(str)
data_eurostat.reset_index(inplace=True)

#########

test = data_eurostat.pivot_table(values='capacity',
                                 index=['country','year'],
                                 columns='technology')

test['Differently categorized solar'] = 0
test['Solar'] = test[['Photovoltaics', 'Concentrated solar power']].sum(axis=1)
test['Differently categorized wind'] = test['Wind']
bio_arr = ['Biomass and biogas', 'Other bioenergy and renewable waste']
test['Bioenergy and renewable waste'] = test[bio_arr].sum(axis=1)
res_arr = ['Hydro', 'Wind', 'Solar', 'Geothermal', 'Marine',
           'Bioenergy and renewable waste']
test['Renewable energy sources'] = test[res_arr].sum(axis=1)

test['Fossil fuels'] = test['Fossil fuels'] - test['Bioenergy and renewable waste']
test['Differently categorized fossil fuels'] = test['Fossil fuels']\
                                             - test['Non-renewable waste']

total_arr = ['Fossil fuels','Nuclear','Renewable energy sources']
test['Total'] = test[total_arr].sum(axis=1)

data_eurostat = test.stack().reset_index().rename(columns={0: 'capacity'})

idx = data_eurostat[data_eurostat['technology'] == 'Fossil fuels'].index
data_eurostat.drop(idx, inplace=True)

data_eurostat['source'] = 'EUROSTAT'
data_eurostat['source_type'] = 'Statistical Office'
data_eurostat['capacity_definition'] = 'Unknown'
data_eurostat['type'] = 'Installed capacity in MW'

############ENTSOE SOAF###############

soafs = [soaf.SoafDataRaw('https://www.entsoe.eu/fileadmin/user_upload/_library/SDC/SOAF/SO_AF_2011_-_2025_.zip',
                          'SO_AF_2011_-_2025_.zip',
                          'SO&AF 2011 - 2025 Scenario B.xls',
                          2011),

        soaf.SoafDataRaw('https://www.entsoe.eu/fileadmin/user_upload/_library/SDC/SOAF/120705_SOAF_2012_Dataset.zip',
                         '120705_SOAF_2012_Dataset.zip',
                         'SOAF 2012 Scenario B.xls',
                         2012),

        soaf.SoafDataRaw('https://www.entsoe.eu/fileadmin/user_upload/_library/publications/entsoe/So_AF_2013-2030/130403_SOAF_2013-2030_dataset.zip',
                         '130403_SOAF_2013-2030_dataset.zip',
                         'ScB.xls',
                         2013),

        soaf.SoafDataRaw('https://www.entsoe.eu/Documents/SDC%20documents/SOAF/140602_SOAF%202014_dataset.zip',
                         '140602_SOAF%202014_dataset.zip',
                         'ScB.xlsx',
                         2014),

        soaf.SoafDataRaw('https://www.entsoe.eu/Documents/Publications/SDC/data/SO_AF_2015_dataset.zip',
                         'SO_AF_2015_dataset.zip',
                         os.path.join('SO&AF 2015 dataset', 'ScB_publication.xlsx'),
                         2016)]

soaf_data = pd.concat([s.transformed_df for s in soafs])

# Correct that in the Soaf2015 datatset the year column is 2016 instead of 2015
soaf_data['year'].replace({2016 : 2015}, inplace=True)

soaf_unstacked = f.unstackData(soaf_data)

soaf_unstacked['Differently categorized solar'] = soaf_unstacked['Solar']

soaf_unstacked['Differently categorized wind'] = soaf_unstacked['Wind']\
                                                - soaf_unstacked['Offshore']\
                                                - soaf_unstacked['Onshore']

soaf_unstacked['Differently categorized hydro'] = soaf_unstacked['Hydro']\
                                                - soaf_unstacked['Run-of-river']\
                                                - soaf_unstacked['Reservoir including pumped storage']

soaf_unstacked['Bioenergy and renewable waste'] = soaf_unstacked['Biomass and biogas']

soaf_unstacked['Differently categorized renewable energy sources'] = (
                                          soaf_unstacked['renewable']
                                        - soaf_unstacked['Wind']
                                        - soaf_unstacked['Solar']
                                        - soaf_unstacked['Biomass and biogas'])

subtract_fossils_arr = ['Lignite','Hard coal','Oil','Natural gas','Mixed fossil fuels']

soaf_unstacked['Differently categorized fossil fuels'] = soaf_unstacked['Fossil fuels']\
                                                        - soaf_unstacked[subtract_fossils_arr].sum(axis=1)

res_arr = ['Solar','Wind','Bioenergy and renewable waste','Hydro',
           'Differently categorized renewable energy sources']

soaf_unstacked['Renewable energy sources'] = soaf_unstacked[res_arr].sum(axis=1)

total_arr = ['Renewable energy sources','Fossil fuels','Nuclear',
             'Other or unspecified energy sources']

soaf_unstacked['Total'] = soaf_unstacked[total_arr].sum(axis=1)

soaf_data = f.restackData(soaf_unstacked)

soaf_data.loc[soaf_data['capacity'] < 0, 'capacity'] = 0

soaf_data['source'] = 'entsoe SOAF'
soaf_data['type'] = 'Installed capacity in MW'
soaf_data['capacity_definition'] = 'Net capacity'
soaf_data['source_type'] = 'Other association'

############ENTSOE statistical data ###############

url_entsoe = 'https://docstore.entsoe.eu/Documents/Publications/Statistics/NGC_2010-2015.xlsx'

filepath_entsoe = f.downloadandcache(url_entsoe, 'Statistics.xls',
                                     os.path.join('ENTSO-E','Statistical data 2010-2015')
                                     )

data_entsoe_raw = pd.read_excel(filepath_entsoe)

dict_energy_source = {'hydro': 'Hydro',
                      'of which storage': 'Reservoir',
                      'of which run of river': 'Run-of-river',
                      'of which pumped storage': 'Pumped storage',
                      'nuclear': 'Nuclear',
                      'of which wind': 'Wind',
                      'of which solar': 'Solar',
                      'of which biomass': 'Biomass and biogas',
                      'fossil_fuels': 'Fossil fuels',
                      'other': 'Other or unspecified energy sources',
                      "Country": "country",
                      'fossil_fueals': 'Fossil fuels'}

data_entsoe_raw.rename(columns=dict_energy_source,
                       inplace=True)

data_entsoe_raw.drop(columns='representativity', inplace=True)

data_entsoe_raw['Differently categorized solar'] = data_entsoe_raw['Solar']
data_entsoe_raw['Differently categorized wind'] = data_entsoe_raw['Wind']
data_entsoe_raw['Bioenergy and renewable waste'] = data_entsoe_raw['Biomass and biogas']
data_entsoe_raw['Differently categorized fossil fuels'] = data_entsoe_raw['Fossil fuels']

data_entsoe_raw['Differently categorized hydro'] = (
        data_entsoe_raw['Hydro']
        - data_entsoe_raw['Run-of-river']
        - data_entsoe_raw['Reservoir']
        - data_entsoe_raw['Pumped storage'])

data_entsoe_raw['Differently categorized renewable energy sources'] = (
        data_entsoe_raw['renewable']
        - data_entsoe_raw['Wind']
        - data_entsoe_raw['Solar']
        - data_entsoe_raw['Biomass and biogas'])

data_entsoe_raw['Renewable energy sources'] = (
        data_entsoe_raw['Hydro']
        + data_entsoe_raw['Wind']
        + data_entsoe_raw['Solar']
        + data_entsoe_raw['Bioenergy and renewable waste']
        + data_entsoe_raw['Differently categorized renewable energy sources'])

data_entsoe_raw['Total'] = (
        data_entsoe_raw['Renewable energy sources']
        + data_entsoe_raw['Nuclear']
        + data_entsoe_raw['Fossil fuels']
        + data_entsoe_raw['Other or unspecified energy sources'])

data_entsoe = pd.melt(data_entsoe_raw,
                      id_vars=['country', 'year'],
                      var_name='energy_source',
                      value_name='capacity')

data_entsoe['country'].replace('NI', 'GB', inplace=True)
data_entsoe.loc[data_entsoe['capacity'] < 0, 'capacity'] = 0

data_entsoe['source'] = 'entsoe Statistics'
data_entsoe['source_type'] = 'Other association'
data_entsoe['capacity_definition'] = 'Net capacity'
data_entsoe['type'] = 'Installed capacity in MW'

##############################################################################<|endoftext|>


<commit_before># Copyright (C) 2011 John Keyes
# http://jkeyes.mit-license.org/

"""
Simple wrapper for Doc Raptor API.
"""
import json
import os
import re
import requests
from requests.auth import HTTPBasicAuth
from datetime import datetime

ENV = os.environ
API_KEY = "DOCRAPTOR_API_KEY"
URL = "DOCRAPTOR_URL"
DEFAULT_TIMEOUT = 10 #seconds
TIMEOUT = "DOCRAPTOR_TIMEOUT"

# endpoint URLs
HTTPS_URL = 'https://docraptor.com/'

class NoApiKeyProvidedError(RuntimeError):
    pass

class NoContentError(KeyError):
    pass

class DocRaptorRequestException(Exception):
    def __init__(self, message, status_code):
        super(DocRaptorRequestException, self).__init__(message)
        self.status_code = status_code

    def __str__(self):
        return "%s\nHTTP Status: %s\nReturned: %s" % (self.__class__.__name__, self.status_code, self.message)

class DocumentCreationFailure(DocRaptorRequestException):
    pass

class DocumentListingFailure(DocRaptorRequestException):
    pass

class DocumentStatusFailure(DocRaptorRequestException):
    pass

class DocumentDownloadFailure(DocRaptorRequestException):
    pass

class DocRaptor(object):

    def __init__(self, api_key=None):
        self.api_key = ENV.get(API_KEY) if api_key is None else api_key
        if not self.api_key:
            raise NoApiKeyProvidedError("No API key provided")
        self._url = ENV.get(URL, HTTPS_URL)
        self._timeout = float(ENV.get(TIMEOUT, DEFAULT_TIMEOUT))

    def create(self, options=None):
        if options is None:
            raise ValueError("Please pass in an options dict")

        if not _has_content(options):
            raise NoContentError("must supply 'document_content' or 'document_url'")

        default_options = {
            'name': 'default',
            'document_type': 'pdf',
            'test': False,
            'async': False,
            'raise_exception_on_failure': False
        }
        options = dict(list(default_options.items()) + list(options.items()))
        raise_exception_on_failure = options.pop('raise_exception_on_failure')
        query = { 'user_credentials': self.api_key }
        if options['async']:
            query['output'] = 'json'
        doc_options = _format_keys({'doc': options})

        resp = requests.post('%sdocs' % (self._url), data=doc_options, params=query, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentCreationFailure(resp.content, resp.status_code)

        if options['async']:
            return json.loads(resp.content)
        else:
            return resp

    def list_docs(self, options=None):
        if options is None:
            raise ValueError("Please pass in an options dict")

        default_options = {
            'page': 1,
            'per_page': 100,
            'raise_exception_on_failure': False,
            'user_credentials': self.api_key
        }
        options = dict(list(default_options.items()) + list(options.items()))
        raise_exception_on_failure = options.pop('raise_exception_on_failure')

        resp = requests.get('%sdocs' % (self._url), params=options, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentListingFailure(resp.content, resp.status_code)
        return resp

    def status(self, status_id, raise_exception_on_failure=False):
        query = {
            'output': 'json',
            'user_credentials': self.api_key
        }

        resp = requests.get('%sstatus/%s' % (self._url, status_id), params=query, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentStatusFailure(resp.content, resp.status_code)

        if resp.status_code == 200:
            as_json = json.loads(resp.content)
            if as_json['status'] == 'completed':
                as_json['download_key'] = re.match('.*?\/download\/(.+)', as_json['download_url']).groups()[0]
            return as_json
        return resp

    def download(self, download_key, raise_exception_on_failure=False):
        query = {
            'output': 'json',
            'user_credentials': self.api_key
        }
        resp = requests.get('%sdownload/%s' % (self._url, download_key), params=query, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentDownloadFailure(resp.content, resp.status_code)
        return resp

def _has_content(options):
    content = options.get('document_content') or options.get('document_url')
    return bool(content)

def _format_keys(options, result=None, parent_key=None):
    if result is None:
        result = {}
    for k, v in list(options.items()):
        if parent_key:
            key = '%s[%s]' % (parent_key, k)
        else:
            key= k
        if isinstance(v, dict):
            _format_keys(v, result, key)
        elif v is False: result[key] = 'false'
        elif v is True: result[key] = 'true'
        else:
            result[key] = v
    return result
<commit_msg>Decode UTF-8 JSON responses.<commit_after># Copyright (C) 2011 John Keyes
# http://jkeyes.mit-license.org/

"""
Simple wrapper for Doc Raptor API.
"""
import json
import os
import re
import requests
from requests.auth import HTTPBasicAuth
from datetime import datetime

ENV = os.environ
API_KEY = "DOCRAPTOR_API_KEY"
URL = "DOCRAPTOR_URL"
DEFAULT_TIMEOUT = 10 #seconds
TIMEOUT = "DOCRAPTOR_TIMEOUT"

# endpoint URLs
HTTPS_URL = 'https://docraptor.com/'

class NoApiKeyProvidedError(RuntimeError):
    pass

class NoContentError(KeyError):
    pass

class DocRaptorRequestException(Exception):
    def __init__(self, message, status_code):
        super(DocRaptorRequestException, self).__init__(message)
        self.status_code = status_code

    def __str__(self):
        return "%s\nHTTP Status: %s\nReturned: %s" % (self.__class__.__name__, self.status_code, self.message)

class DocumentCreationFailure(DocRaptorRequestException):
    pass

class DocumentListingFailure(DocRaptorRequestException):
    pass

class DocumentStatusFailure(DocRaptorRequestException):
    pass

class DocumentDownloadFailure(DocRaptorRequestException):
    pass

class DocRaptor(object):

    def __init__(self, api_key=None):
        self.api_key = ENV.get(API_KEY) if api_key is None else api_key
        if not self.api_key:
            raise NoApiKeyProvidedError("No API key provided")
        self._url = ENV.get(URL, HTTPS_URL)
        self._timeout = float(ENV.get(TIMEOUT, DEFAULT_TIMEOUT))

    def create(self, options=None):
        if options is None:
            raise ValueError("Please pass in an options dict")

        if not _has_content(options):
            raise NoContentError("must supply 'document_content' or 'document_url'")

        default_options = {
            'name': 'default',
            'document_type': 'pdf',
            'test': False,
            'async': False,
            'raise_exception_on_failure': False
        }
        options = dict(list(default_options.items()) + list(options.items()))
        raise_exception_on_failure = options.pop('raise_exception_on_failure')
        query = { 'user_credentials': self.api_key }
        if options['async']:
            query['output'] = 'json'
        doc_options = _format_keys({'doc': options})

        resp = requests.post('%sdocs' % (self._url), data=doc_options, params=query, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentCreationFailure(resp.content, resp.status_code)

        if options['async']:
            return json.loads(resp.content.decode("utf-8"))
        else:
            return resp

    def list_docs(self, options=None):
        if options is None:
            raise ValueError("Please pass in an options dict")

        default_options = {
            'page': 1,
            'per_page': 100,
            'raise_exception_on_failure': False,
            'user_credentials': self.api_key
        }
        options = dict(list(default_options.items()) + list(options.items()))
        raise_exception_on_failure = options.pop('raise_exception_on_failure')

        resp = requests.get('%sdocs' % (self._url), params=options, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentListingFailure(resp.content, resp.status_code)
        return resp

    def status(self, status_id, raise_exception_on_failure=False):
        query = {
            'output': 'json',
            'user_credentials': self.api_key
        }

        resp = requests.get('%sstatus/%s' % (self._url, status_id), params=query, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentStatusFailure(resp.content, resp.status_code)

        if resp.status_code == 200:
            as_json = json.loads(resp.content.decode("utf-8"))
            if as_json['status'] == 'completed':
                as_json['download_key'] = re.match('.*?\/download\/(.+)', as_json['download_url']).groups()[0]
            return as_json
        return resp

    def download(self, download_key, raise_exception_on_failure=False):
        query = {
            'output': 'json',
            'user_credentials': self.api_key
        }
        resp = requests.get('%sdownload/%s' % (self._url, download_key), params=query, timeout=self._timeout)

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentDownloadFailure(resp.content, resp.status_code)
        return resp

def _has_content(options):
    content = options.get('document_content') or options.get('document_url')
    return bool(content)

def _format_keys(options, result=None, parent_key=None):
    if result is None:
        result = {}
    for k, v in list(options.items()):
        if parent_key:
            key = '%s[%s]' % (parent_key, k)
        else:
            key= k
        if isinstance(v, dict):
            _format_keys(v, result, key)
        elif v is False: result[key] = 'false'
        elif v is True: result[key] = 'true'
        else:
            result[key] = v
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# dicom2nifti documentation build configuration file, created by
# sphinx-quickstart on Thu Jul 28 09:37:31 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import os
import sys
on_rtd = os.environ.get('READTHEDOCS') == 'True'

sys.path.append(os.path.abspath('../../dicom2nifti'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
    'sphinx.ext.inheritance_diagram'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'dicom2nifti'
copyright = '2016, icometrix'
author = 'icometrix'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = ''
# The full version, including alpha/beta/rc tags.
release = ''

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#
# today = ''
#
# Else, today_fmt is used as the format for a strftime call.
#
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
if on_rtd:
    html_theme = 'default'
else:
    html_theme = 'classic'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#
# html_title = 'dicom2nifti v'

# A shorter title for the navigation bar.  Default is the same as html_title.
#
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#
html_logo = '_static/logo.png'

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#
# html_extra_path = []

# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
#
# html_last_updated_fmt = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#
# html_additional_pages = {}

# If false, no module index is generated.
#
# html_domain_indices = True

# If false, no index is generated.
#
# html_use_index = True

# If true, the index is split into individual pages for each letter.
#
# html_split_index = False

# If true, links to the reST sources are added to the pages.
#
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
#
html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
#
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'dicom2niftidoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',

     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',

     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'dicom2nifti.tex', 'dicom2nifti Documentation',
     'icometrix', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#
# latex_use_parts = False

# If true, show page references after internal links.
#
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
#
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
#
# latex_appendices = []

# It false, will not define \strong, \code, 	itleref, \crossref ... but only
# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
# packages.
#
# latex_keep_old_macro_names = True

# If false, no module index is generated.
#
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'dicom2nifti', 'dicom2nifti Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'dicom2nifti', 'dicom2nifti Documentation',
     author, 'dicom2nifti', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#
# texinfo_appendices = []

# If false, no module index is generated.
#
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#
# texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
# epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#
# epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#
# epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
# epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#
# epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#
# epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#
# epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#
# epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#
# epub_tocdepth = 3

# Allow duplicate toc entries.
#
# epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#
# epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#
# epub_fix_images = False

# Scale large images.
#
# epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# epub_show_urls = 'inline'

# If false, no index is generated.
#
# epub_use_index = True
<commit_msg>DCMTONII-31: Add debugging info for read the docs<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# dicom2nifti documentation build configuration file, created by
# sphinx-quickstart on Thu Jul 28 09:37:31 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import os
import sys
on_rtd = os.environ.get('READTHEDOCS') == 'True'

sys.path.append(os.path.abspath('../../dicom2nifti'))
print(os.path.abspath('../../dicom2nifti'))
# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
    'sphinx.ext.inheritance_diagram'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'dicom2nifti'
copyright = '2016, icometrix'
author = 'icometrix'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = ''
# The full version, including alpha/beta/rc tags.
release = ''

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'en'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#
# today = ''
#
# Else, today_fmt is used as the format for a strftime call.
#
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
if on_rtd:
    html_theme = 'default'
else:
    html_theme = 'classic'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#
# html_title = 'dicom2nifti v'

# A shorter title for the navigation bar.  Default is the same as html_title.
#
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#
html_logo = '_static/logo.png'

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#
# html_extra_path = []

# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
#
# html_last_updated_fmt = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#
# html_additional_pages = {}

# If false, no module index is generated.
#
# html_domain_indices = True

# If false, no index is generated.
#
# html_use_index = True

# If true, the index is split into individual pages for each letter.
#
# html_split_index = False

# If true, links to the reST sources are added to the pages.
#
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
#
html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
#
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'dicom2niftidoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',

     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',

     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'dicom2nifti.tex', 'dicom2nifti Documentation',
     'icometrix', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#
# latex_use_parts = False

# If true, show page references after internal links.
#
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
#
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
#
# latex_appendices = []

# It false, will not define \strong, \code, 	itleref, \crossref ... but only
# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
# packages.
#
# latex_keep_old_macro_names = True

# If false, no module index is generated.
#
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'dicom2nifti', 'dicom2nifti Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'dicom2nifti', 'dicom2nifti Documentation',
     author, 'dicom2nifti', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#
# texinfo_appendices = []

# If false, no module index is generated.
#
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#
# texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
# epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#
# epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#
# epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
# epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#
# epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#
# epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#
# epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#
# epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#
# epub_tocdepth = 3

# Allow duplicate toc entries.
#
# epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#
# epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#
# epub_fix_images = False

# Scale large images.
#
# epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# epub_show_urls = 'inline'

# If false, no index is generated.
#
# epub_use_index = True
<|endoftext|>