<commit_before>#!/bin/bash
# Authors: Yan Kai, Jeffery Hsu
# GRCh38
import sys,re,os
from optparse import OptionParser
import pandas as pd
import HTSeq

def inner_outer_pref(line, suffix):
    '''
    Preferences outer + suffix -> suffix -> inner + suffix
    '''
    prefix = ['inner_', '', 'outer_']
    test = [i + suffix for i in prefix]
    for i in test:
        if(not pd.isnull(line[i])):
            out = int(line[i])
    return(out)

def VariantCallTabReader(filepath, chrom_size):
    """
    This function aims to read the variant calls from the merged studies which
    contain various types of variant types.

    """
    infile = pd.read_csv(filepath, sep="\t")

    # var_types is a dic keyed by var_type and valued by a list of genomic intervals
    var_types_ga = {}
    for _, line in infile.iterrows():
        var_type = str(line['var_type'])
        var_type = var_type.replace(" ","_")

        if var_type not in var_types_ga.keys():
            var_types_ga[var_type] = []

        chrom = 'chr'+ str(line['chr'])
        if (chrom in chrom_size.keys()):
            start = inner_outer_pref(line, 'start')
            end = inner_outer_pref(line, 'stop')

            # Create a 'Genomic interval' from this variant call
            iv = HTSeq.GenomicInterval(chrom, start, end, ".")
            var_types_ga[var_type].append(iv)
    return(var_types_ga)

def write_to_gvf(ga,var_type, outfile):
    """
    ga is the genomic array.
    """
    outfile = open(outfile, 'w')
    ivs = list(ga.steps())
    num_iv = len(ivs)
    outline = '##gvf-version 1.00\n'
    outfile.write(outline)

    source = 'dbVar'
    score = '.'
    strand = '.'
    phase = '.'
    for i in xrange(num_iv):
        chrom = ivs[i][0].chrom
        start = ivs[i][0].start+1
        end = ivs[i][0].end
        ID = str(i+1)
        count = ivs[i][1]
        attributes = 'ID='+ID+';'+'count='+str(count)
        outline =chrom+'\t'+source+'\t'+var_type+'\t'+str(start)+'\t'+str(end)+'\t'+score+'\t'+strand+'\t'+phase+'\t'+attributes+'\n'
        outfile.write(outline)
    outfile.close()

def main(argv):
    parser = OptionParser()
    parser.add_option("-r", "--chromsize", action="store", type="string", dest="chromsize", help="GRCh38 chromosome size file", metavar="<str>")
    parser.add_option("-v", "--variantfile", action="store", type="string", dest="variantfile", metavar="<file>", help="the variant calls files in a specific format")
    parser.add_option("-o", "--outdir", action="store", type="string", dest="outdir", metavar="<file>", help="the directory to store the output files")

    (opt, args) = parser.parse_args(argv)
    if len(argv) < 6:
        parser.print_help()
        sys.exit(1)

    chrom_size_file = open(opt.chromsize,'r')
    # Read chrom size information from the chrom_size_file.
    chrom_size = {}
    for line in chrom_size_file:
        pline = line.strip()
        sline = pline.split('\t')
        chrom_size[sline[1]] = int(sline[0])
    chrom_size_file.close()

    var_types_ga = VariantCallTabReader(opt.variantfile,chrom_size)
    for var_type in var_types_ga.keys():
        # Creat a 'Genomic Array' using HTSeq package
        ga = HTSeq.GenomicArray( chrom_size, stranded=False, typecode="i" )

        variant_interval = var_types_ga[var_type]

        # Get the count of variant calls in each region
        variant_num = len(variant_interval)
        print "For "+var_type+", there are "+str(variant_num)+" variant calls from the clustersed studies..."
        for i in xrange(variant_num):
            iv = variant_interval[i]
            try:
                ga[iv] += 1
            except:
                iv.length == 0

        bedgraph = opt.outdir+'/'+var_type+'_dbVar.bedgraph'
        ga.write_bedgraph_file(bedgraph, strand=".", track_options="")

        gvf = opt.outdir+'/VarType='+var_type+'_dbVar.gvf'
        write_to_gvf(ga,var_type,gvf)

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>variant id added<commit_after>#!/bin/bash
# Authors: Yan Kai, Jeffery Hsu
# GRCh38
import sys,re,os
from optparse import OptionParser
import pandas as pd
import HTSeq

def inner_outer_pref(line, suffix):
    '''
    Preferences outer + suffix -> suffix -> inner + suffix
    '''
    prefix = ['inner_', '', 'outer_']
    test = [i + suffix for i in prefix]
    for i in test:
        if(not pd.isnull(line[i])):
            out = int(line[i])
    return(out)

def VariantCallTabReader(filepath, chrom_size):
    """
    This function aims to read the variant calls from the merged studies which
    contain various types of variant types.

    """
    infile = pd.read_csv(filepath, sep="\t")

    # var_types is a dic keyed by var_type and valued by a list of genomic intervals
    var_types_ga = {}
    var_types_id = {}

    for _, line in infile.iterrows():
        var_type = str(line['var_type'])
        var_type = var_type.replace(" ","_")

        if var_type not in var_types_ga.keys():
            var_types_ga[var_type] = []
            var_types_id[var_type] = []

        chrom = 'chr'+ str(line['chr'])
        accesion = line['accession_num']
        if (chrom in chrom_size.keys()):
            start = inner_outer_pref(line, 'start')
            end = inner_outer_pref(line, 'stop')

            # Create a 'Genomic interval' from this variant call
            iv = HTSeq.GenomicInterval(chrom, start, end, ".")
            var_types_ga[var_type].append(iv)
            var_types_id[var_type].append(accesion)
    return((var_types_ga,var_types_id))

def write_to_gvf(ga,nssd, var_type, outfile):
    """
    ga is the genomic array.
    """
    outfile = open(outfile, 'w')
    ivs = list(ga.steps())
    ids = list(nssd.steps())
    num_iv = len(ivs)
    outline = '##gvf-version 1.00\n'
    outfile.write(outline)

    source = 'dbVar'
    score = '.'
    strand = '.'
    phase = '.'
    for i in xrange(num_iv):
        chrom = ivs[i][0].chrom
        start = ivs[i][0].start+1
        end = ivs[i][0].end
        ID = str(i+1)
        count = ivs[i][1]
        SSV = ids[i][1]
        attributes = 'ID='+ID+';'+'count='+str(count)+';'+'SSV='+str(SSV)

        outline =chrom+'\t'+source+'\t'+var_type+'\t'+str(start)+'\t'+str(end)+'\t'+score+'\t'+strand+'\t'+phase+'\t'+attributes+'\n'
        outfile.write(outline)
    outfile.close()

def main(argv):
    parser = OptionParser()
    parser.add_option("-r", "--chromsize", action="store", type="string", dest="chromsize", help="GRCh38 chromosome size file", metavar="<str>")
    parser.add_option("-v", "--variantfile", action="store", type="string", dest="variantfile", metavar="<file>", help="the variant calls files in a specific format")
    parser.add_option("-o", "--outdir", action="store", type="string", dest="outdir", metavar="<file>", help="the directory to store the output files")

    (opt, args) = parser.parse_args(argv)
    if len(argv) < 6:
        parser.print_help()
        sys.exit(1)

    chrom_size_file = open(opt.chromsize,'r')
    # Read chrom size information from the chrom_size_file.
    chrom_size = {}
    for line in chrom_size_file:
        pline = line.strip()
        sline = pline.split('\t')
        chrom_size[sline[1]] = int(sline[0])
    chrom_size_file.close()

    var_types_ga = VariantCallTabReader(opt.variantfile,chrom_size)[0]
    var_types_id = VariantCallTabReader(opt.variantfile,chrom_size)[1]
    for var_type in var_types_ga.keys():
        # Creat a 'Genomic Array' using HTSeq package
        ga = HTSeq.GenomicArray( chrom_size, stranded=False, typecode="i" )
        nssd = HTSeq.GenomicArrayOfSets(chrom_size, stranded=False)

        variant_interval = var_types_ga[var_type]
        variant_id = var_types_id[var_type]

        # Get the count of variant calls in each region
        variant_num = len(variant_interval)
        print "For "+var_type+", there are "+str(variant_num)+" variant calls from the clustersed studies..."
        for i in xrange(variant_num):
            iv = variant_interval[i]
            try:
                ga[iv] += 1
                nssd[iv] += variant_id[i]
            except:
                iv.length == 0

        bedgraph = opt.outdir+'/'+var_type+'_dbVar.bedgraph'
        ga.write_bedgraph_file(bedgraph, strand=".", track_options="")

        gvf = opt.outdir+'/VarType='+var_type+'_dbVar.gvf'
        write_to_gvf(ga,nssd,var_type,gvf)

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from __future__ import print_function
import logging
import os
import pynmrstar

# Set the log level to INFO
logging.getLogger().setLevel(logging.INFO)

three_letter_code = {'I': 'ILE', 'Q': 'GLN', 'G': 'GLY', 'E': 'GLU', 'C': 'CYS',
                     'D': 'ASP', 'S': 'SER', 'K': 'LYS', 'P': 'PRO', 'N': 'ASN',
                     'V': 'VAL', 'T': 'THR', 'H': 'HIS', 'W': 'TRP', 'F': 'PHE',
                     'A': 'ALA', 'M': 'MET', 'L': 'LEU', 'R': 'ARG', 'Y': 'TYR'}
one_letter_code = dict([(value, key) for key, value in three_letter_code.items()])

class ChemicalShift(object):
    '''
    Generates chemical shift dictionary from BMRB entry or NMR-STAR file
    '''

    @staticmethod
    def from_entry(entry_data, data_set_id, auth_tag=False, ):
        '''
        Extracts chemical shift data as dictionary from PyNMRSTAR entry object

        :param entry_data: PyNMRSTAR entry object
        :param data_set_id: Data set identifier (bmrb id or filename or user defined id)
        :param auth_tag: Use author sequence numbering True/False default: False
        :return: Chemical shift dictionary {data_set_id:{chain_id:{seq_id:{atom_id:cs_value}},'seq_ids':[1,2,3,4..]}}
        '''
        cs_data = {}
        # entry_data = pynmrstar.Entry.from_database(bmrb_id)
        cs_loops = entry_data.get_loops_by_category('Atom_chem_shift')
        logging.debug('Chemical shift loop read')
        cs_list_id = 1
        for cs_loop in cs_loops:
            cs_id = '{}-{}'.format(data_set_id, cs_list_id)
            cs_list_id += 1
            cs_data[cs_id] = {}
            column_name = cs_loop.get_tag_names()
            data = cs_loop.data
            if auth_tag:
                # chain = column_name.index('_Atom_chem_shift.Auth_asym_ID')
                seq_no = column_name.index('_Atom_chem_shift.Auth_seq_ID')
                # residue = column_name.index('_Atom_chem_shift.Auth_comp_ID')
                # atom = column_name.index('_Atom_chem_shift.Auth_atom_ID')
            else:
                # chain = column_name.index('_Atom_chem_shift.Entity_assembly_ID')
                seq_no = column_name.index('_Atom_chem_shift.Comp_index_ID')
                # residue = column_name.index('_Atom_chem_shift.Comp_ID')
                # atom = column_name.index('_Atom_chem_shift.Atom_ID')
            residue = column_name.index('_Atom_chem_shift.Comp_ID')
            atom = column_name.index('_Atom_chem_shift.Atom_ID')
            chain = column_name.index('_Atom_chem_shift.Entity_assembly_ID')
            cs_value = column_name.index('_Atom_chem_shift.Val')
            for cs_row in data:
                c = cs_row[chain]
                s = int(cs_row[seq_no])
                r = cs_row[residue]
                a = cs_row[atom]
                v = float(cs_row[cs_value])
                if c not in cs_data[cs_id].keys():
                    cs_data[cs_id][c] = {}
                if s not in cs_data[cs_id][c].keys():
                    cs_data[cs_id][c][s] = {}
                if a not in cs_data[cs_id][c][s].keys():
                    cs_data[cs_id][c][s][a] = []
                cs_data[cs_id][c][s][a].append(r)
                try:
                    cs_data[cs_id][c][s][a].append(one_letter_code[r])
                except KeyError:
                    cs_data[cs_id][c][s][a].append(r)
                # cs_data[cs_id][c][s][a].append(a)
                cs_data[cs_id][c][s][a].append(v)
        logging.debug('Getting sequence information from chemical shift loop')
        for cs_id in cs_data.keys():
            for chain in cs_data[cs_id].keys():
                seq_ids = sorted(set([i for i in cs_data[cs_id][chain].keys()]))
                cs_data[cs_id][chain]['seq_ids'] = seq_ids
        return cs_data

    @classmethod
    def from_file(cls, file_names, auth_tag=False, data_set_id=None):
        '''
        Extracts chemical shift information one or more NMR-STAR files

        :param file_name: list of NMR-STAR file names with full path or single file name with full path
        :param auth_tag: Use author sequence numbering True/False default: False
        :param data_set_id: User defined data set id default: filename
        :return: Chemical shift dictionary {data_set_id:{chain_id:{seq_id:{atom_id:cs_value}},'seq_ids':[1,2,3,4..]}}
        '''
        # handle the data_set_id list for more than one files : todo
        if type(file_names) is list:
            all_cs_data = {}
            for file_name in file_names:
                logging.debug('Reading file {}'.format(file_name))
                if os.path.exists(file_name):
                    entry_data = pynmrstar.Entry.from_file(file_name)
                    data_set_id = os.path.splitext(os.path.basename(file_name))[0]
                    cs_data = cls.from_entry(entry_data, data_set_id, auth_tag)
                else:
                    logging.error('File not found {}'.format(file_name))
                    raise IOError('File not found : {}'.format(file_name))
                all_cs_data.update(cs_data)
        else:
            logging.debug('Reading file {}'.format(file_names))
            if os.path.exists(file_names):
                entry_data = pynmrstar.Entry.from_file(file_names)
                if data_set_id is None:
                    data_set_id = os.path.splitext(os.path.basename(file_names))[0]
                all_cs_data = cls.from_entry(entry_data, data_set_id, auth_tag)
            else:
                logging.error('File not found {}'.format(file_names))
                raise IOError('File not found : {}'.format(file_names))
        return all_cs_data

    @classmethod
    def from_bmrb(cls, bmrb_ids, auth_tag=False):
        '''
        Extracts chemical shift information directly from BMRB database for a given BMRB entry or list of entries

        :param bmrb_ids: List of BMRB entries ids or single BMRB ID
        :param auth_tag: Use author sequence numbering True/False default: False
        :return: Chemical shift dictionary {data_set_id:{chain_id:{seq_id:{atom_id:cs_value}},'seq_ids':[1,2,3,4..]}}
        '''
        if type(bmrb_ids) is list:
            all_cs_data={}
            for bmrb_id in bmrb_ids:
                try:
                    logging.debug('Getting entry {} from BMRB'.format(bmrb_id))
                    entry_data = pynmrstar.Entry.from_database(bmrb_id)
                except OSError:
                    entry_data = None
                except KeyError:
                    entry_data = None
                except IOError:
                    entry_data = None
                if entry_data is not None:
                    cs_data = cls.from_entry(entry_data, bmrb_id, auth_tag)
                else:
                    logging.error('Entry {} not found in public database'.format(bmrb_id))
                    raise IOError('Entry not found in public database: {}'.format(bmrb_id))
                all_cs_data.update(cs_data)
        else:

            try:
                logging.debug('Getting entry {} from BMRB'.format(bmrb_ids))
                entry_data = pynmrstar.Entry.from_database(bmrb_ids)
            except OSError:
                entry_data = None
            except KeyError:
                entry_data = None
            except IOError:
                entry_data = None
            if entry_data is not None:
                all_cs_data = cls.from_entry(entry_data, bmrb_ids, auth_tag)
            else:
                logging.error('Entry {} not found in public database'.format(bmrb_ids))
                raise IOError('Entry not found in public database: {}'.format(bmrb_ids))
        return all_cs_data
#
# if __name__ == "__main__":
#     p = ChemicalShift.from_bmrb(15060)
#     # p.from_file('/Users/kumaran/MyData.str',data_set_id='test')
#     # p.
#
<commit_msg>debugging rate limit<commit_after>#!/usr/bin/env python3

from __future__ import print_function
import logging
import os
import pynmrstar

# Set the log level to INFO
logging.getLogger().setLevel(logging.INFO)

three_letter_code = {'I': 'ILE', 'Q': 'GLN', 'G': 'GLY', 'E': 'GLU', 'C': 'CYS',
                     'D': 'ASP', 'S': 'SER', 'K': 'LYS', 'P': 'PRO', 'N': 'ASN',
                     'V': 'VAL', 'T': 'THR', 'H': 'HIS', 'W': 'TRP', 'F': 'PHE',
                     'A': 'ALA', 'M': 'MET', 'L': 'LEU', 'R': 'ARG', 'Y': 'TYR'}
one_letter_code = dict([(value, key) for key, value in three_letter_code.items()])

class ChemicalShift(object):
    '''
    Generates chemical shift dictionary from BMRB entry or NMR-STAR file
    '''

    @staticmethod
    def from_entry(entry_data, data_set_id, auth_tag=False, ):
        '''
        Extracts chemical shift data as dictionary from PyNMRSTAR entry object

        :param entry_data: PyNMRSTAR entry object
        :param data_set_id: Data set identifier (bmrb id or filename or user defined id)
        :param auth_tag: Use author sequence numbering True/False default: False
        :return: Chemical shift dictionary {data_set_id:{chain_id:{seq_id:{atom_id:cs_value}},'seq_ids':[1,2,3,4..]}}
        '''
        cs_data = {}
        # entry_data = pynmrstar.Entry.from_database(bmrb_id)
        cs_loops = entry_data.get_loops_by_category('Atom_chem_shift')
        logging.debug('Chemical shift loop read')
        cs_list_id = 1
        for cs_loop in cs_loops:
            cs_id = '{}-{}'.format(data_set_id, cs_list_id)
            cs_list_id += 1
            cs_data[cs_id] = {}
            column_name = cs_loop.get_tag_names()
            data = cs_loop.data
            if auth_tag:
                # chain = column_name.index('_Atom_chem_shift.Auth_asym_ID')
                seq_no = column_name.index('_Atom_chem_shift.Auth_seq_ID')
                # residue = column_name.index('_Atom_chem_shift.Auth_comp_ID')
                # atom = column_name.index('_Atom_chem_shift.Auth_atom_ID')
            else:
                # chain = column_name.index('_Atom_chem_shift.Entity_assembly_ID')
                seq_no = column_name.index('_Atom_chem_shift.Comp_index_ID')
                # residue = column_name.index('_Atom_chem_shift.Comp_ID')
                # atom = column_name.index('_Atom_chem_shift.Atom_ID')
            residue = column_name.index('_Atom_chem_shift.Comp_ID')
            atom = column_name.index('_Atom_chem_shift.Atom_ID')
            chain = column_name.index('_Atom_chem_shift.Entity_assembly_ID')
            cs_value = column_name.index('_Atom_chem_shift.Val')
            for cs_row in data:
                c = cs_row[chain]
                s = int(cs_row[seq_no])
                r = cs_row[residue]
                a = cs_row[atom]
                v = float(cs_row[cs_value])
                if c not in cs_data[cs_id].keys():
                    cs_data[cs_id][c] = {}
                if s not in cs_data[cs_id][c].keys():
                    cs_data[cs_id][c][s] = {}
                if a not in cs_data[cs_id][c][s].keys():
                    cs_data[cs_id][c][s][a] = []
                cs_data[cs_id][c][s][a].append(r)
                try:
                    cs_data[cs_id][c][s][a].append(one_letter_code[r])
                except KeyError:
                    cs_data[cs_id][c][s][a].append(r)
                # cs_data[cs_id][c][s][a].append(a)
                cs_data[cs_id][c][s][a].append(v)
        logging.debug('Getting sequence information from chemical shift loop')
        for cs_id in cs_data.keys():
            for chain in cs_data[cs_id].keys():
                seq_ids = sorted(set([i for i in cs_data[cs_id][chain].keys()]))
                cs_data[cs_id][chain]['seq_ids'] = seq_ids
        return cs_data

    @classmethod
    def from_file(cls, file_names, auth_tag=False, data_set_id=None):
        '''
        Extracts chemical shift information one or more NMR-STAR files

        :param file_name: list of NMR-STAR file names with full path or single file name with full path
        :param auth_tag: Use author sequence numbering True/False default: False
        :param data_set_id: User defined data set id default: filename
        :return: Chemical shift dictionary {data_set_id:{chain_id:{seq_id:{atom_id:cs_value}},'seq_ids':[1,2,3,4..]}}
        '''
        # handle the data_set_id list for more than one files : todo
        if type(file_names) is list:
            all_cs_data = {}
            for file_name in file_names:
                logging.debug('Reading file {}'.format(file_name))
                if os.path.exists(file_name):
                    entry_data = pynmrstar.Entry.from_file(file_name)
                    data_set_id = os.path.splitext(os.path.basename(file_name))[0]
                    cs_data = cls.from_entry(entry_data, data_set_id, auth_tag)
                else:
                    logging.error('File not found {}'.format(file_name))
                    raise IOError('File not found : {}'.format(file_name))
                all_cs_data.update(cs_data)
        else:
            logging.debug('Reading file {}'.format(file_names))
            if os.path.exists(file_names):
                entry_data = pynmrstar.Entry.from_file(file_names)
                if data_set_id is None:
                    data_set_id = os.path.splitext(os.path.basename(file_names))[0]
                all_cs_data = cls.from_entry(entry_data, data_set_id, auth_tag)
            else:
                logging.error('File not found {}'.format(file_names))
                raise IOError('File not found : {}'.format(file_names))
        return all_cs_data

    @classmethod
    def from_bmrb(cls, bmrb_ids, auth_tag=False):
        '''
        Extracts chemical shift information directly from BMRB database for a given BMRB entry or list of entries

        :param bmrb_ids: List of BMRB entries ids or single BMRB ID
        :param auth_tag: Use author sequence numbering True/False default: False
        :return: Chemical shift dictionary {data_set_id:{chain_id:{seq_id:{atom_id:cs_value}},'seq_ids':[1,2,3,4..]}}
        '''
        if type(bmrb_ids) is list:
            all_cs_data={}
            for bmrb_id in bmrb_ids:
                try:
                    logging.debug('Getting entry {} from BMRB'.format(bmrb_id))
                    entry_data = pynmrstar.Entry.from_database(bmrb_id)
                except OSError as e:
                    entry_data = None
                except KeyError as e:
                    entry_data = None
                except IOError as e:
                    entry_data = None
                if entry_data is not None:
                    cs_data = cls.from_entry(entry_data, bmrb_id, auth_tag)
                else:

                    logging.error('Entry {} not found in public database{}'.format(bmrb_id,e))
                    raise IOError('Entry not found in public database: {}{}'.format(bmrb_id,e))
                all_cs_data.update(cs_data)
        else:

            try:
                logging.debug('Getting entry {} from BMRB'.format(bmrb_ids))
                entry_data = pynmrstar.Entry.from_database(bmrb_ids)
            except OSError:
                entry_data = None
            except KeyError:
                entry_data = None
            except IOError:
                entry_data = None
            if entry_data is not None:
                all_cs_data = cls.from_entry(entry_data, bmrb_ids, auth_tag)
            else:
                logging.error('Entry {} not found in public database'.format(bmrb_ids))
                raise IOError('Entry not found in public database: {}'.format(bmrb_ids))
        return all_cs_data
#
# if __name__ == "__main__":
#     p = ChemicalShift.from_bmrb(15060)
#     # p.from_file('/Users/kumaran/MyData.str',data_set_id='test')
#     # p.
#
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
"""Analysis module for pybossa-analyst."""

import time
import enki
import numpy
from pybossa_analyst import object_loader

def _drop_excluded_keys(df, excluded_keys):
    """Drop excluded keys from task run info fields."""
    keyset = set()
    for i in range(len(df)):
        for k in df.iloc[i]['info'].keys():
            keyset.add(k)
    keys = [k for k in keyset if k not in excluded_keys]
    return df[keys]

def _drop_empty_rows(df):
    """Drop rows that contain no data."""
    df = df.replace('', numpy.nan)
    df = df.dropna(how='all')
    return df

def _check_for_n_percent_of_matches(df, n_task_runs, match_percentage):
    """Check if n percent of answers match for each key."""
    required_matches = int(round(n_task_runs*(match_percentage / 100.0)))
    info = {}
    df = df.replace(numpy.nan, '')  # Replace Nan with the empty string
    for k in df.keys():
        if df[k].value_counts().max() < required_matches:
            info = 'Unanalysed'
            break
        info[k] = df[k].value_counts().idxmax()
    return info

def _get_task_run_df(api_key, endpoint, project_short_name, task_id):
    """Return a dataframe containing all task run info for a task."""
    e = enki.Enki(api_key, endpoint, project_short_name, all=1)
    e.get_tasks(task_id=task_id)
    e.get_task_runs()
    t = e.tasks[0]
    return e.task_runs_df[t.id]

def _filter_results(results, info_filter):
    """Filter results based on their info field."""
    return filter(lambda x: (info_filter == "All") or
                            (not x.info and info_filter == "New") or
                            (x.info == info_filter), results)

def analyse(api_key, endpoint, project_id, result_id, project_short_name,
            match_percentage, excluded_keys=[], **kwargs):
    """Analyser for all projects.

    Check that the info fields of each task run match n percent of the time
    or above, disregarding task runs where all of the info fields are blank.
    For tasks where we have a sufficent number of matches the result will be
    set to a dictionary containing the task run info keys and the matched
    values. For tasks where all info fields of all task runs are empty the
    result will be set to a dictionary containing the task run info keys and
    empty values. For all other tasks the result will be set to the string
    'Unanalysed' so that the different answers can be checked manually later.
    """
    enki.pbclient.set('api_key', api_key)
    enki.pbclient.set('endpoint', endpoint)
    r = enki.pbclient.find_results(project_id, id=result_id, limit=1, all=1)[0]
    df = _get_task_run_df(api_key, endpoint, project_short_name, r.task_id)
    n_task_runs = len(df.index)

    df = _drop_excluded_keys(df, excluded_keys)
    df = _drop_empty_rows(df)

    if df.empty:
        r.info = {k: "" for k in df.keys()}
        enki.pbclient.update_result(r)
        return

    r.info = _check_for_n_percent_of_matches(df, n_task_runs, match_percentage)
    enki.pbclient.update_result(r)

def analyse_multiple(api_key, endpoint, project_id, project_short_name,
                     info_filter, match_percentage, excluded_keys=[]):
    """Analyse multiple results."""
    enki.pbclient.set('api_key', api_key)
    enki.pbclient.set('endpoint', endpoint)
    time.sleep(2)  # To handle API rate limit when analysing many results
    results = object_loader.load(enki.pbclient.find_results,
                                 project_id=project_id, all=1)
    results = _filter_results(results, info_filter)
    for result in results:
        kwargs = {
            'api_key': api_key,
            'endpoint': endpoint,
            'project_id': project_id,
            'result_id': result.id,
            'project_short_name': project_short_name,
            'match_percentage': match_percentage,
            'excluded_keys': excluded_keys
        }
        analyse(**kwargs)
<commit_msg>Update comment<commit_after># -*- coding: utf8 -*-
"""Analysis module for pybossa-analyst."""

import time
import enki
import numpy
from pybossa_analyst import object_loader

def _drop_excluded_keys(df, excluded_keys):
    """Drop excluded keys from task run info fields."""
    keyset = set()
    for i in range(len(df)):
        for k in df.iloc[i]['info'].keys():
            keyset.add(k)
    keys = [k for k in keyset if k not in excluded_keys]
    return df[keys]

def _drop_empty_rows(df):
    """Drop rows that contain no data."""
    df = df.replace('', numpy.nan)
    df = df.dropna(how='all')
    return df

def _check_for_n_percent_of_matches(df, n_task_runs, match_percentage):
    """Check if n percent of answers match for each key."""
    required_matches = int(round(n_task_runs*(match_percentage / 100.0)))
    info = {}
    df = df.replace(numpy.nan, '')  # Replace Nan with the empty string
    for k in df.keys():
        if df[k].value_counts().max() < required_matches:
            info = 'Unanalysed'
            break
        info[k] = df[k].value_counts().idxmax()
    return info

def _get_task_run_df(api_key, endpoint, project_short_name, task_id):
    """Return a dataframe containing all task run info for a task."""
    e = enki.Enki(api_key, endpoint, project_short_name, all=1)
    e.get_tasks(task_id=task_id)
    e.get_task_runs()
    t = e.tasks[0]
    return e.task_runs_df[t.id]

def _filter_results(results, info_filter):
    """Filter results based on their info field."""
    return filter(lambda x: (info_filter == "All") or
                            (not x.info and info_filter == "New") or
                            (x.info == info_filter), results)

def analyse(api_key, endpoint, project_id, result_id, project_short_name,
            match_percentage, excluded_keys=[], **kwargs):
    """Analyser for all projects.

    Check that the info fields of each task run match n percent of the time
    or above, disregarding task runs where all of the info fields are blank.
    For tasks where we have a sufficent number of matches the result will be
    set to a dictionary containing the task run info keys and the matched
    values. For tasks where all info fields of all task runs are empty the
    result will be set to a dictionary containing the task run info keys and
    empty values. For all other tasks the result will be set to the string
    'Unanalysed' so that the different answers can be checked manually later.
    """
    enki.pbclient.set('api_key', api_key)
    enki.pbclient.set('endpoint', endpoint)
    r = enki.pbclient.find_results(project_id, id=result_id, limit=1, all=1)[0]
    df = _get_task_run_df(api_key, endpoint, project_short_name, r.task_id)
    n_task_runs = len(df.index)

    df = _drop_excluded_keys(df, excluded_keys)
    df = _drop_empty_rows(df)

    if df.empty:
        r.info = {k: "" for k in df.keys()}
        enki.pbclient.update_result(r)
        return

    r.info = _check_for_n_percent_of_matches(df, n_task_runs, match_percentage)
    enki.pbclient.update_result(r)

def analyse_multiple(api_key, endpoint, project_id, project_short_name,
                     info_filter, match_percentage, excluded_keys=[]):
    """Analyse multiple results."""
    enki.pbclient.set('api_key', api_key)
    enki.pbclient.set('endpoint', endpoint)
    time.sleep(2)  # To handle API rate limit
    results = object_loader.load(enki.pbclient.find_results,
                                 project_id=project_id, all=1)
    results = _filter_results(results, info_filter)
    for result in results:
        kwargs = {
            'api_key': api_key,
            'endpoint': endpoint,
            'project_id': project_id,
            'result_id': result.id,
            'project_short_name': project_short_name,
            'match_percentage': match_percentage,
            'excluded_keys': excluded_keys
        }
        analyse(**kwargs)
<|endoftext|>


<commit_before>"""
The energy_surf module contains a routine for calculating the
energy surface of a system.
"""

from __future__ import division
from pycalphad import Model
from pycalphad.model import DofError
from pycalphad.eq.utils import make_callable, point_sample, generate_dof
from pycalphad.eq.utils import endmember_matrix, unpack_kwarg
from pycalphad.log import logger
import pycalphad.variables as v
from sympy import Symbol
import pandas as pd
import numpy as np
import itertools
import collections
import functools

try:
    set
except NameError:
    from sets import Set as set #pylint: disable=W0622

def _listify(val):
    "Return val if val is iterable; otherwise return list(val)."
    if isinstance(val, collections.Iterable):
        return val
    else:
        return [val]

def energy_surf(dbf, comps, phases, mode=None, output='GM', **kwargs):
    """
    Sample the property surface of 'output' containing the specified
    components and phases. Model parameters are taken from 'dbf' and any
    state variables (T, P, etc.) can be specified as keyword arguments.

    Parameters
    ----------
    dbf : Database
        Thermodynamic database containing the relevant parameters.
    comps : list
        Names of components to consider in the calculation.
    phases : list
        Names of phases to consider in the calculation.
    mode : string, optional
        See 'make_callable' docstring for details.
    output : string, optional
        Model attribute to sample. Default is 'energy'.
    pdens : int, a dict of phase names to int, or a list of both, optional
        Number of points to sample per degree of freedom.
    model : Model, a dict of phase names to Model, or a list of both, optional
        Model class to use for each phase.

    Returns
    -------
    DataFrame of the output as a function of composition, temperature, etc.

    Examples
    --------
    None yet.
    """
    # Here we check for any keyword arguments that are special, i.e.,
    # there may be keyword arguments that aren't state variables
    pdens_dict = unpack_kwarg(kwargs.pop('pdens', 2000), default_arg=2000)
    model_dict = unpack_kwarg(kwargs.pop('model', Model), default_arg=Model)

    # Convert keyword strings to proper state variable objects
    # If we don't do this, sympy will get confused during substitution
    statevar_dict = \
        dict((v.StateVariable(key), value) \
             for (key, value) in kwargs.items())

    # Generate all combinations of state variables for 'map' calculation
    # Wrap single values of state variables in lists
    # Use 'kwargs' because we want state variable names to be stringified
    statevar_values = [_listify(val) for val in kwargs.values()]
    statevars_to_map = [dict(zip(kwargs.keys(), prod)) \
        for prod in itertools.product(*statevar_values)]

    # Consider only the active phases
    active_phases = dict((name.upper(), dbf.phases[name.upper()]) \
        for name in phases)
    comp_sets = {}
    # Construct a list to hold all the data
    all_phase_data = []
    for phase_name, phase_obj in sorted(active_phases.items()):
        # Build the symbolic representation of the energy
        mod = model_dict[phase_name]
        # if this is an object type, we need to construct it
        if isinstance(mod, type):
            try:
                mod = mod(dbf, comps, phase_name)
            except DofError:
                # we can't build the specified phase because the
                # specified components aren't found in every sublattice
                # we'll just skip it
                logger.warning("""Suspending specified phase %s due to
                some sublattices containing only unspecified components""",
                               phase_name)
                continue
        try:
            out = getattr(mod, output)
        except AttributeError:
            raise AttributeError('Missing Model attribute {0} specified for {1}'
                                 .format(output, mod.__class__))
        # As a last resort, treat undefined symbols as zero
        # But warn the user when we do this
        # This is consistent with TC's behavior
        undefs = list(out.atoms(Symbol) - out.atoms(v.StateVariable))
        for undef in undefs:
            out = out.xreplace({undef: float(0)})
            logger.warning('Setting undefined symbol %s for phase %s to zero',
                           undef, phase_name)
        # Construct an ordered list of the variables
        variables, sublattice_dof = generate_dof(phase_obj, mod.components)

        # Build the "fast" representation of that model
        comp_sets[phase_name] = make_callable(out, \
            list(statevar_dict.keys()) + variables, mode=mode)

        # Get the site ratios in each sublattice
        site_ratios = list(phase_obj.sublattices)

        # Eliminate pure vacancy endmembers from the calculation
        vacancy_indices = list()
        for idx, sublattice in enumerate(phase_obj.constituents):
            if 'VA' in sorted(sublattice) and 'VA' in sorted(comps):
                vacancy_indices.append(sorted(sublattice).index('VA'))
        if len(vacancy_indices) != len(phase_obj.constituents):
            vacancy_indices = None
        logger.debug('vacancy_indices: %s', vacancy_indices)
        # Add all endmembers to guarantee their presence
        points = endmember_matrix(sublattice_dof,
                                  vacancy_indices=vacancy_indices)

        # Sample composition space for more points
        if sum(sublattice_dof) > len(sublattice_dof):
            points = np.concatenate((points,
                                     point_sample(sublattice_dof,
                                                  pdof=pdens_dict[phase_name])
                                    ))

        # If there are nontrivial sublattices with vacancies in them,
        # generate a set of points where their fraction is zero and renormalize
        for idx, sublattice in enumerate(phase_obj.constituents):
            if 'VA' in set(sublattice) and len(sublattice) > 1:
                var_idx = variables.index(v.SiteFraction(phase_name, idx, 'VA'))
                addtl_pts = np.copy(points)
                # set vacancy fraction to log-spaced between 1e-10 and 1e-6
                addtl_pts[:, var_idx] = np.power(10.0, -10.0*(1.0 - addtl_pts[:, var_idx]))
                # renormalize site fractions
                cur_idx = 0
                for ctx in sublattice_dof:
                    end_idx = cur_idx + ctx
                    addtl_pts[:, cur_idx:end_idx] /= \
                        addtl_pts[:, cur_idx:end_idx].sum(axis=1)[:, None]
                    cur_idx = end_idx
                # add to points matrix
                points = np.concatenate((points, addtl_pts), axis=0)

        data_dict = {'Phase': phase_name}
        # Generate input d.o.f matrix for all state variable combinations
        for statevars in statevars_to_map:
            # Prefill the state variable arguments to the energy function
            energy_func = functools.partial(comp_sets[phase_name],
                                            *list(statevars.values()))
            energies = energy_func(*points.T)
            try:
                data_dict[output].extend(energies)
                for statevar in kwargs.keys():
                    data_dict[statevar].extend(
                        list(np.repeat(list(statevars.values()),
                                       len(points))))
            except KeyError:
                data_dict[output] = list(energies)
                for statevar in kwargs.keys():
                    data_dict[statevar] = \
                        list(np.repeat(list(statevars.values()),
                                       len(points)))

            # Map the internal degrees of freedom to global coordinates

            # Normalize site ratios
            # Normalize by the sum of site ratios times a factor
            # related to the site fraction of vacancies
            site_ratio_normalization = np.zeros(len(points))
            for idx, sublattice in enumerate(phase_obj.constituents):
                vacancy_column = np.ones(len(points))
                if 'VA' in set(sublattice):
                    var_idx = variables.index(v.SiteFraction(phase_name, idx, 'VA'))
                    vacancy_column -= points[:, var_idx]
                site_ratio_normalization += site_ratios[idx] * vacancy_column

            for comp in sorted(comps):
                if comp == 'VA':
                    continue
                avector = [float(cur_var.species == comp) * \
                    site_ratios[cur_var.sublattice_index] for cur_var in variables]
                try:
                    data_dict['X('+comp+')'].extend(list(np.divide(np.dot(
                        points[:, :], avector), site_ratio_normalization)))
                except KeyError:
                    data_dict['X('+comp+')'] = list(np.divide(np.dot(
                        points[:, :], avector), site_ratio_normalization))

            # Copy coordinate information into data_dict
            # TODO: Is there a more memory-efficient way to deal with this?
            # Perhaps with hierarchical indexing...
            try:
                for column_idx, data in enumerate(points.T):
                    data_dict[str(variables[column_idx])].extend(list(data))
            except KeyError:
                for column_idx, data in enumerate(points.T):
                    data_dict[str(variables[column_idx])] = list(data)

        all_phase_data.append(pd.DataFrame(data_dict))

    # all_phases_data now contains energy surface information for the system
    return pd.concat(all_phase_data, axis=0, join='outer', \
                            ignore_index=True, verify_integrity=False)
<commit_msg>ENH: energy_surf: 'callables' keyword argument support<commit_after>"""
The energy_surf module contains a routine for calculating the
energy surface of a system.
"""

from __future__ import division
from pycalphad import Model
from pycalphad.model import DofError
from pycalphad.eq.utils import make_callable, point_sample, generate_dof
from pycalphad.eq.utils import endmember_matrix, unpack_kwarg
from pycalphad.log import logger
import pycalphad.variables as v
from sympy import Symbol
import pandas as pd
import numpy as np
import itertools
import collections
import functools

try:
    set
except NameError:
    from sets import Set as set #pylint: disable=W0622

def _listify(val):
    "Return val if val is iterable; otherwise return list(val)."
    if isinstance(val, collections.Iterable):
        return val
    else:
        return [val]

def energy_surf(dbf, comps, phases, mode=None, output='GM', **kwargs):
    """
    Sample the property surface of 'output' containing the specified
    components and phases. Model parameters are taken from 'dbf' and any
    state variables (T, P, etc.) can be specified as keyword arguments.

    Parameters
    ----------
    dbf : Database
        Thermodynamic database containing the relevant parameters.
    comps : list
        Names of components to consider in the calculation.
    phases : list
        Names of phases to consider in the calculation.
    mode : string, optional
        See 'make_callable' docstring for details.
    output : string, optional
        Model attribute to sample. Default is 'energy'.
    pdens : int, a dict of phase names to int, or a list of both, optional
        Number of points to sample per degree of freedom.
    model : Model, a dict of phase names to Model, or a list of both, optional
        Model class to use for each phase.

    Returns
    -------
    DataFrame of the output as a function of composition, temperature, etc.

    Examples
    --------
    None yet.
    """
    # Here we check for any keyword arguments that are special, i.e.,
    # there may be keyword arguments that aren't state variables
    pdens_dict = unpack_kwarg(kwargs.pop('pdens', 2000), default_arg=2000)
    model_dict = unpack_kwarg(kwargs.pop('model', Model), default_arg=Model)
    callable_dict = unpack_kwarg(kwargs.pop('callables', None), default_arg=None)

    # Convert keyword strings to proper state variable objects
    # If we don't do this, sympy will get confused during substitution
    statevar_dict = \
        dict((v.StateVariable(key), value) \
             for (key, value) in kwargs.items())

    # Generate all combinations of state variables for 'map' calculation
    # Wrap single values of state variables in lists
    # Use 'kwargs' because we want state variable names to be stringified
    statevar_values = [_listify(val) for val in kwargs.values()]
    statevars_to_map = [dict(zip(kwargs.keys(), prod)) \
        for prod in itertools.product(*statevar_values)]

    # Consider only the active phases
    active_phases = dict((name.upper(), dbf.phases[name.upper()]) \
        for name in phases)
    comp_sets = {}
    # Construct a list to hold all the data
    all_phase_data = []
    for phase_name, phase_obj in sorted(active_phases.items()):
        # Build the symbolic representation of the energy
        mod = model_dict[phase_name]
        # if this is an object type, we need to construct it
        if isinstance(mod, type):
            try:
                mod = mod(dbf, comps, phase_name)
            except DofError:
                # we can't build the specified phase because the
                # specified components aren't found in every sublattice
                # we'll just skip it
                logger.warning("""Suspending specified phase %s due to
                some sublattices containing only unspecified components""",
                               phase_name)
                continue
        try:
            out = getattr(mod, output)
        except AttributeError:
            raise AttributeError('Missing Model attribute {0} specified for {1}'
                                 .format(output, mod.__class__))

        # Construct an ordered list of the variables
        variables, sublattice_dof = generate_dof(phase_obj, mod.components)

        site_ratios = list(phase_obj.sublattices)
        # Build the "fast" representation of that model
        if callable_dict[phase_name] is None:
            # As a last resort, treat undefined symbols as zero
            # But warn the user when we do this
            # This is consistent with TC's behavior
            undefs = list(out.atoms(Symbol) - out.atoms(v.StateVariable))
            for undef in undefs:
                out = out.xreplace({undef: float(0)})
                logger.warning('Setting undefined symbol %s for phase %s to zero',
                               undef, phase_name)
            comp_sets[phase_name] = make_callable(out, \
                list(statevar_dict.keys()) + variables, mode=mode)
        else:
            comp_sets[phase_name] = callable_dict[phase_name]

        # Eliminate pure vacancy endmembers from the calculation
        vacancy_indices = list()
        for idx, sublattice in enumerate(phase_obj.constituents):
            if 'VA' in sorted(sublattice) and 'VA' in sorted(comps):
                vacancy_indices.append(sorted(sublattice).index('VA'))
        if len(vacancy_indices) != len(phase_obj.constituents):
            vacancy_indices = None
        logger.debug('vacancy_indices: %s', vacancy_indices)
        # Add all endmembers to guarantee their presence
        points = endmember_matrix(sublattice_dof,
                                  vacancy_indices=vacancy_indices)

        # Sample composition space for more points
        if sum(sublattice_dof) > len(sublattice_dof):
            points = np.concatenate((points,
                                     point_sample(sublattice_dof,
                                                  pdof=pdens_dict[phase_name])
                                    ))

        # If there are nontrivial sublattices with vacancies in them,
        # generate a set of points where their fraction is zero and renormalize
        for idx, sublattice in enumerate(phase_obj.constituents):
            if 'VA' in set(sublattice) and len(sublattice) > 1:
                var_idx = variables.index(v.SiteFraction(phase_name, idx, 'VA'))
                addtl_pts = np.copy(points)
                # set vacancy fraction to log-spaced between 1e-10 and 1e-6
                addtl_pts[:, var_idx] = np.power(10.0, -10.0*(1.0 - addtl_pts[:, var_idx]))
                # renormalize site fractions
                cur_idx = 0
                for ctx in sublattice_dof:
                    end_idx = cur_idx + ctx
                    addtl_pts[:, cur_idx:end_idx] /= \
                        addtl_pts[:, cur_idx:end_idx].sum(axis=1)[:, None]
                    cur_idx = end_idx
                # add to points matrix
                points = np.concatenate((points, addtl_pts), axis=0)

        data_dict = {'Phase': phase_name}
        # Generate input d.o.f matrix for all state variable combinations
        for statevars in statevars_to_map:
            # Prefill the state variable arguments to the energy function
            energy_func = functools.partial(comp_sets[phase_name],
                                            *list(statevars.values()))
            energies = energy_func(*points.T)
            try:
                data_dict[output].extend(energies)
                for statevar in kwargs.keys():
                    data_dict[statevar].extend(
                        list(np.repeat(list(statevars.values()),
                                       len(points))))
            except KeyError:
                data_dict[output] = list(energies)
                for statevar in kwargs.keys():
                    data_dict[statevar] = \
                        list(np.repeat(list(statevars.values()),
                                       len(points)))

            # Map the internal degrees of freedom to global coordinates

            # Normalize site ratios
            # Normalize by the sum of site ratios times a factor
            # related to the site fraction of vacancies
            site_ratio_normalization = np.zeros(len(points))
            for idx, sublattice in enumerate(phase_obj.constituents):
                vacancy_column = np.ones(len(points))
                if 'VA' in set(sublattice):
                    var_idx = variables.index(v.SiteFraction(phase_name, idx, 'VA'))
                    vacancy_column -= points[:, var_idx]
                site_ratio_normalization += site_ratios[idx] * vacancy_column

            for comp in sorted(comps):
                if comp == 'VA':
                    continue
                avector = [float(cur_var.species == comp) * \
                    site_ratios[cur_var.sublattice_index] for cur_var in variables]
                try:
                    data_dict['X('+comp+')'].extend(list(np.divide(np.dot(
                        points[:, :], avector), site_ratio_normalization)))
                except KeyError:
                    data_dict['X('+comp+')'] = list(np.divide(np.dot(
                        points[:, :], avector), site_ratio_normalization))

            # Copy coordinate information into data_dict
            # TODO: Is there a more memory-efficient way to deal with this?
            # Perhaps with hierarchical indexing...
            try:
                for column_idx, data in enumerate(points.T):
                    data_dict[str(variables[column_idx])].extend(list(data))
            except KeyError:
                for column_idx, data in enumerate(points.T):
                    data_dict[str(variables[column_idx])] = list(data)

        all_phase_data.append(pd.DataFrame(data_dict))

    # all_phases_data now contains energy surface information for the system
    return pd.concat(all_phase_data, axis=0, join='outer', \
                            ignore_index=True, verify_integrity=False)
<|endoftext|>


<commit_before>from substanced.schema import (
    NameSchemaNode
)
from zope.interface import (
    Interface,
    taggedValue,
)
from zope.interface.interfaces import IInterface
from adhocracy.schema import ReferenceSetSchemaNode

import colander

class IIProperty(IInterface):

    """Mark IProperty class to allow registering a MultiAdapter.

    See adhocracy.properties.includme for examples.

    """

class IProperty(Interface):

    """Marker interface with tagged values to define resource data."""

    taggedValue("schema", "substanced.schema.Schema")
    """Reference colander data schema.

    This schema has to define the facade to work with this resource.
    To set/get the data you can adapt to IPropertySheet objects.

    Subtype has to override.

    """
    taggedValue("key", "")
    """Key to store the schema data, defaults to IProperyXY.__identifier__"""
    taggedValue("permission_view", "view")
    """Permission to view or index this data. Subtype should override."""
    taggedValue("permission_edit", "edit")
    """Permission to edit this data. Subtype should override."""

class IName(IProperty):

    """Set human readable name of this resource and create identifier."""

    taggedValue("schema", "adhocracy.properties.interfaces.NameSchema")

class NameSchema(colander.Schema):

    name = NameSchemaNode(default="", missing=colander.drop)

class INameReadOnly(IName):

    """ Same as IName but name is autogenerated."""

    taggedValue("schema", "adhocracy.properties.interfaces.NameReadOnlySchema")

class NameReadOnlySchema(colander.Schema):

    name = NameSchemaNode(readonly=True, default="", missing=colander.drop)

class IPool(IProperty):

    """Get listing with child objects of this resource."""

    taggedValue("schema", "adhocracy.properties.interfaces.PoolSchema")

class PoolSchema(colander.Schema):

    elements = ReferenceSetSchemaNode(default=[],
                                      missing=colander.drop,
                                      interfaces=[Interface]
                                      )

class IVersionable(IProperty):

    """Make this Fubel a versionable."""

    taggedValue("schema", "adhocracy.properties.interfaces.VersionableSchema")

# class IForkable(IVersionable):
# #     """Marker interface representing a forkable node with version data"""
#
class VersionableSchema(colander.Schema):

    follows = ReferenceSetSchemaNode(default=[],
                                     missing=colander.drop,
                                     interfaces=[IVersionable]
                                     )
    """follows blas"""

# followed_by = ReferenceSetSchemaNode(
#         default=[],
#         missing=colander.drop,
#         interface=IVersionable,
#         readonly=True,
#     )

class IVersions(IProperty):

    """Dag for collecting all versions of one Fubel."""
    taggedValue("schema", "adhocracy.properties.interfaces.VersionsSchema")

class VersionsSchema(colander.Schema):

    elements = ReferenceSetSchemaNode(default=[],
                                      missing=colander.drop,
                                      interface=[IVersionable],
                                      )

class ITags(IProperty):

    """List all tags for this FubelVersionsPool."""

    taggedValue("schema", "adhocracy.properties.interfaces.TagsSchema")

class TagsSchema(colander.Schema):

    elements = ReferenceSetSchemaNode(default=[],
                                      missing=colander.drop,
                                      interface=IVersionable,
                                      )

#
# # Document Data
#
#
# class IDocument(IProperty):
#
#     """Marker interface representing a Fubel with document data """
#
#     taggedValue("schema", "adhocracy.properties.interfaces.DocumentSchema")
#
#
# class DocumentSchema(colander.Schema):
#
#     elements = ReferenceSetSchemaNode(essence_refs=True,
#                                       default=[],
#                                       missing=colander.drop,
#                                       interface=
#                                    "adhocracy.resources.interfaces.ISection")
#
#
# class ISection(IProperty):
#
#     """Marker interface representing a document section """
#
#     taggedValue("schema", "adhocracy.properties.interfaces.SectionSchema")
#
#
# class SectionSchema(colander.Schema):
#     title = colander.SchemaNode(colander.String(), default="")
#     elements = ReferenceSetSchemaNode(essence_refs=True,
#                                       default=[],
#                                       missing=colander.drop,
#                                       interface=

#                                    "adhocracy.resources.interfaces.ISection")
<commit_msg>make IPool readonly<commit_after>from substanced.schema import (
    NameSchemaNode
)
from zope.interface import (
    Interface,
    taggedValue,
)
from zope.interface.interfaces import IInterface
from adhocracy.schema import ReferenceSetSchemaNode

import colander

class IIProperty(IInterface):

    """Mark IProperty class to allow registering a MultiAdapter.

    See adhocracy.properties.includme for examples.

    """

class IProperty(Interface):

    """Marker interface with tagged values to define resource data."""

    taggedValue("schema", "substanced.schema.Schema")
    """Reference colander data schema.

    This schema has to define the facade to work with this resource.
    To set/get the data you can adapt to IPropertySheet objects.

    Subtype has to override.

    """
    taggedValue("key", "")
    """Key to store the schema data, defaults to IProperyXY.__identifier__"""
    taggedValue("permission_view", "view")
    """Permission to view or index this data. Subtype should override."""
    taggedValue("permission_edit", "edit")
    """Permission to edit this data. Subtype should override."""

class IName(IProperty):

    """Set human readable name of this resource and create identifier."""

    taggedValue("schema", "adhocracy.properties.interfaces.NameSchema")

class NameSchema(colander.Schema):

    name = NameSchemaNode(default="", missing=colander.drop)

class INameReadOnly(IName):

    """ Same as IName but name is autogenerated."""

    taggedValue("schema", "adhocracy.properties.interfaces.NameReadOnlySchema")

class NameReadOnlySchema(colander.Schema):

    name = NameSchemaNode(readonly=True, default="", missing=colander.drop)

class IPool(IProperty):

    """Get listing with child objects of this resource."""

    taggedValue("schema", "adhocracy.properties.interfaces.PoolSchema")

class PoolSchema(colander.Schema):

    elements = ReferenceSetSchemaNode(default=[],
                                      missing=colander.drop,
                                      interfaces=[Interface],
                                      readonly=True,
                                      )

class IVersionable(IProperty):

    """Make this Fubel a versionable."""

    taggedValue("schema", "adhocracy.properties.interfaces.VersionableSchema")

# class IForkable(IVersionable):
# #     """Marker interface representing a forkable node with version data"""
#
class VersionableSchema(colander.Schema):

    follows = ReferenceSetSchemaNode(default=[],
                                     missing=colander.drop,
                                     interfaces=[IVersionable]
                                     )
    """follows blas"""

# followed_by = ReferenceSetSchemaNode(
#         default=[],
#         missing=colander.drop,
#         interface=IVersionable,
#         readonly=True,
#     )

class IVersions(IProperty):

    """Dag for collecting all versions of one Fubel."""
    taggedValue("schema", "adhocracy.properties.interfaces.VersionsSchema")

class VersionsSchema(colander.Schema):

    elements = ReferenceSetSchemaNode(default=[],
                                      missing=colander.drop,
                                      interface=[IVersionable],
                                      )

class ITags(IProperty):

    """List all tags for this FubelVersionsPool."""

    taggedValue("schema", "adhocracy.properties.interfaces.TagsSchema")

class TagsSchema(colander.Schema):

    elements = ReferenceSetSchemaNode(default=[],
                                      missing=colander.drop,
                                      interface=IVersionable,
                                      )

#
# # Document Data
#
#
# class IDocument(IProperty):
#
#     """Marker interface representing a Fubel with document data """
#
#     taggedValue("schema", "adhocracy.properties.interfaces.DocumentSchema")
#
#
# class DocumentSchema(colander.Schema):
#
#     elements = ReferenceSetSchemaNode(essence_refs=True,
#                                       default=[],
#                                       missing=colander.drop,
#                                       interface=
#                                    "adhocracy.resources.interfaces.ISection")
#
#
# class ISection(IProperty):
#
#     """Marker interface representing a document section """
#
#     taggedValue("schema", "adhocracy.properties.interfaces.SectionSchema")
#
#
# class SectionSchema(colander.Schema):
#     title = colander.SchemaNode(colander.String(), default="")
#     elements = ReferenceSetSchemaNode(essence_refs=True,
#                                       default=[],
#                                       missing=colander.drop,
#                                       interface=

#                                    "adhocracy.resources.interfaces.ISection")
<|endoftext|>


<commit_before># coding: utf-8

"""
組み込み関数についてのサンプルです。
"""
from trypython.common.commoncls import SampleBase
from trypython.common.commonfunc import pr

class Sample(SampleBase):
    def exec(self):
        pass

def go():
    obj = Sample()
    obj.exec()

if __name__ == '__main__':
    go()
<commit_msg>コード追加<commit_after># coding: utf-8

"""
組み込み関数についてのサンプルです。
"""
from trypython.common.commoncls import SampleBase
from trypython.common.commonfunc import pr

class Sample(SampleBase):
    def exec(self):
        #
        # abs()
        # 絶対値を返す
        #
        int01 = -1024
        pr('abs()', abs(int01))

        #
        # all(iterable)
        # 指定した iterable の全要素が True の場合に True を返す
        #
        list01 = list(range(1, 10))  # 要素に 0 がいると False となる
        pr('all()', all(list01))

        #
        # any(iterable)
        # 指定した iterable のどれかの要素が True の場合に True を返す
        #
        pr('any()', any(list01))

        #
        # ascii()
        # 印字可能な文字列を返す。非asciiな文字はエスケープされる
        #
        str01 = 'こんにちわ世界'
        pr('ascii()', ascii(str01))

        #
        # bin()
        # 指定された整数を2進数文字列にして返す
        #
        int02 = 1024
        pr('bin()', bin(int02))

        #
        # bool()
        # 指定したオブジェクトを判定して bool値 を返す
        #
        pr('bool()', bool(int02))

        #
        # bytearray()
        # bytearrayオブジェクトを生成する
        # bytearrayは、ミュータブルなバイト配列を表す
        # (bytesはイミュータブル)
        #
        ba01 = bytearray(5)  # 初期容量が 5バイト のbytearray生成
        ba01[0] = 0xff
        pr('bytearray()', ba01)

        #
        # bytes()
        # bytesオブジェクトを生成する
        # 使い方は bytearray() と同様だが、bytesはイミュータブル
        #
        b01 = bytes([1, 11, 16])
        pr('bytes()', b01)

        try:
            b01[0] = b'10'
        except TypeError as e:
            pr('bytesはイミュータブル', e)

        #
        # callable()
        # 呼び出し可能オブジェクトかどうかを返す
        # クラスは呼び出し可能
        # インスタンスは __call__() を持っている場合は呼び出し可能
        #
        pr('callable()', callable(Sample))
        pr('callable()', callable(self))  # __call__を定義していないのでFalse

def go():
    obj = Sample()
    obj.exec()

if __name__ == '__main__':
    go()
<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Doxygen(CMakePackage):
    """Doxygen is the de facto standard tool for generating documentation
    from annotated C++ sources, but it also supports other popular programming
    languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba,
    Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some
    extent D.."""

    homepage = "http://www.doxygen.nl/"
    url      = "https://sourceforge.net/projects/doxygen/files/rel-1.8.14/doxygen-1.8.14.src.tar.gz/download"

    version('1.8.14', '41d8821133e8d8104280030553e2b42b')
    version('1.8.12', '08e0f7850c4d22cb5188da226b209a96')
    version('1.8.11', 'f4697a444feaed739cfa2f0644abc19b')
    version('1.8.10', '79767ccd986f12a0f949015efb5f058f')

    # graphviz appears to be a run-time optional dependency
    variant('graphviz', default=False,
            description='Build with dot command support from Graphviz.')

    depends_on("cmake@2.8.12:", type='build')
    depends_on("flex", type='build')
    # code.l just checks subminor version <=2.5.4 or >=2.5.33
    # but does not recognize 2.6.x as newer...could be patched if needed
    depends_on("flex@2.5.39", type='build', when='@1.8.10')
    depends_on("bison", type='build')

    # optional dependencies
    depends_on("graphviz", when="+graphviz", type='run')

    # Support C++14's std::shared_ptr. For details about this patch, see
    # https://github.com/Sleepyowl/doxygen/commit/6c380ba91ae41c6d5c409a5163119318932ae2a3?diff=unified
    # Also - https://github.com/doxygen/doxygen/pull/6588
    patch('shared_ptr.patch', when='@1.8.14')
<commit_msg>doxygen: URL and version hashes updated to project's GitHub home (#10250)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Doxygen(CMakePackage):
    """Doxygen is the de facto standard tool for generating documentation
    from annotated C++ sources, but it also supports other popular programming
    languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba,
    Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some
    extent D.."""

    homepage  = "https://github.com/doxygen/doxygen/"
    git       = "https://github.com/doxygen/doxygen.git"

    # Doxygen versions on GitHub
    version('1.8.15', commit='dc89ac01407c24142698c1374610f2cee1fbf200')
    version('1.8.14', commit='2f4139de014bf03898320a45fe52c92872c1e0f4')
    version('1.8.12', commit='4951df8d0d0acf843b4147136f945504b96536e7')
    version('1.8.11', commit='a6d4f4df45febe588c38de37641513fd576b998f')
    version('1.8.10', commit='fdae7519a2e29f94e65c0e718513343f07302ddb')

    # graphviz appears to be a run-time optional dependency
    variant('graphviz', default=False,
            description='Build with dot command support from Graphviz.')

    depends_on("cmake@2.8.12:", type='build')
    depends_on("flex", type='build')
    # code.l just checks subminor version <=2.5.4 or >=2.5.33
    # but does not recognize 2.6.x as newer...could be patched if needed
    depends_on("flex@2.5.39", type='build', when='@1.8.10')
    depends_on("bison", type='build')

    # optional dependencies
    depends_on("graphviz", when="+graphviz", type='run')

    # Support C++14's std::shared_ptr. For details about this patch, see
    # https://github.com/Sleepyowl/doxygen/commit/6c380ba91ae41c6d5c409a5163119318932ae2a3?diff=unified
    # Also - https://github.com/doxygen/doxygen/pull/6588
    patch('shared_ptr.patch', when='@1.8.14')
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import glob
from spack import *

class Leveldb(MakefilePackage):
    """LevelDB is a fast key-value storage library written at Google
    that provides an ordered mapping from string keys to string values."""

    homepage = "https://github.com/google/leveldb"
    url      = "https://github.com/google/leveldb/archive/v1.20.tar.gz"

    version('1.20', '298b5bddf12c675d6345784261302252')
    version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0')

    depends_on("snappy")

    def install(self, spec, prefix):
        mkdirp(prefix.lib.pkgconfig)

        libraries  = glob.glob('out-shared/libleveldb.*')
        libraries += glob.glob('out-static/libleveldb.*')
        for library in libraries:
            install(library, prefix.lib)

        install_tree('include/leveldb', prefix.include)

        with open(join_path(prefix.lib, 'pkgconfig', 'leveldb.pc'), 'w') as f:
            f.write('prefix={0}\n'.format(prefix))
            f.write('exec_prefix=${prefix}\n')
            f.write('libdir={0}\n'.format(prefix.lib))
            f.write('includedir={0}\n'.format(prefix.include))
            f.write('\n')
            f.write('Name: leveldb\n')
            f.write('Description: LevelDB is a fast key-value storage library'
                    ' written at Google that provides an ordered mapping from'
                    ' string keys to string values.\n')
            f.write('Version: {0}\n'.format(spec.version))
            f.write('Cflags: -I${includedir}\n')
            f.write('Libs: -L${libdir} -lleveldb\n')
<commit_msg>leveldb: Fix header installation (#9069)<commit_after>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import glob
from spack import *

class Leveldb(MakefilePackage):
    """LevelDB is a fast key-value storage library written at Google
    that provides an ordered mapping from string keys to string values."""

    homepage = "https://github.com/google/leveldb"
    url      = "https://github.com/google/leveldb/archive/v1.20.tar.gz"

    version('1.20', '298b5bddf12c675d6345784261302252')
    version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0')

    depends_on("snappy")

    def install(self, spec, prefix):
        mkdirp(prefix.lib.pkgconfig)

        libraries  = glob.glob('out-shared/libleveldb.*')
        libraries += glob.glob('out-static/libleveldb.*')
        for library in libraries:
            install(library, prefix.lib)

        install_tree('include', prefix.include)

        with open(join_path(prefix.lib, 'pkgconfig', 'leveldb.pc'), 'w') as f:
            f.write('prefix={0}\n'.format(prefix))
            f.write('exec_prefix=${prefix}\n')
            f.write('libdir={0}\n'.format(prefix.lib))
            f.write('includedir={0}\n'.format(prefix.include))
            f.write('\n')
            f.write('Name: leveldb\n')
            f.write('Description: LevelDB is a fast key-value storage library'
                    ' written at Google that provides an ordered mapping from'
                    ' string keys to string values.\n')
            f.write('Version: {0}\n'.format(spec.version))
            f.write('Cflags: -I${includedir}\n')
            f.write('Libs: -L${libdir} -lleveldb\n')
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Libcint(CMakePackage):
    """Library for analytical Gaussian integrals for quantum chemistry."""

    homepage    = "https://github.com/sunqm/libcint"
    url         = "https://github.com/sunqm/libcint/archive/v3.0.4.tar.gz"
    maintainers = ['mfherbst']

    #
    # Versions
    #
    version('3.0.10', 'b368e257dba99febf1cdd1391f2e58a3')
    version('3.0.8',  '61f415ad9c7854963136c6bba7a661eb')
    version('3.0.7',  'f0f3c971c0d5bb5565ac9ab5e271aa79')
    version('3.0.6',  '1aafd91107e2418f794ce02b9d88ea8b')
    version('3.0.5',  'facb1ddabd9497e99d22176a9b9a895f')
    version('3.0.4',  '55607a61313225ef4434d3e96624a008')

    #
    # Variants
    #
    variant('f12', default=True,
            description="Enable explicitly correlated f12 integrals.")
    variant('coulomb_erf', default=True,
            description="Enable attenuated coulomb operator integrals.")
    variant('test', default=False, description="Build test programs")
    variant('shared', default=True,
            description="Build the shared library")

    #
    # Dependencies and conflicts
    #
    depends_on('cmake@2.6:', type="build")
    depends_on('blas')
    depends_on('python', type=("build", "test"), when="+test")
    depends_on('py-numpy', type=("build", "test"), when="+test")

    # Libcint tests only work with a shared libcint library
    conflicts('+test~shared')

    #
    # Settings and cmake cache
    #
    def cmake_args(self):
        spec = self.spec
        args = [
            "-DWITH_COULOMB_ERF=" + str("+coulomb_erf" in spec),
            "-DWITH_F12=" + str("+f12" in spec),
            "-DBUILD_SHARED_LIBS=" + str("+shared" in spec),
            "-DENABLE_TEST=" + str("+test" in spec),
            "-DENABLE_EXAMPLE=OFF",  # Requires fortran compiler
        ]
        return args
<commit_msg>Add libcint version 3.0.11 (#7764)<commit_after>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Libcint(CMakePackage):
    """Library for analytical Gaussian integrals for quantum chemistry."""

    homepage    = "https://github.com/sunqm/libcint"
    url         = "https://github.com/sunqm/libcint/archive/v3.0.4.tar.gz"
    maintainers = ['mfherbst']

    #
    # Versions
    #
    version('3.0.11', '1fb1db5a426280d38cddbb0098f6c678')
    version('3.0.10', 'b368e257dba99febf1cdd1391f2e58a3')
    version('3.0.8',  '61f415ad9c7854963136c6bba7a661eb')
    version('3.0.7',  'f0f3c971c0d5bb5565ac9ab5e271aa79')
    version('3.0.6',  '1aafd91107e2418f794ce02b9d88ea8b')
    version('3.0.5',  'facb1ddabd9497e99d22176a9b9a895f')
    version('3.0.4',  '55607a61313225ef4434d3e96624a008')

    #
    # Variants
    #
    variant('f12', default=True,
            description="Enable explicitly correlated f12 integrals.")
    variant('coulomb_erf', default=True,
            description="Enable attenuated coulomb operator integrals.")
    variant('test', default=False, description="Build test programs")
    variant('shared', default=True,
            description="Build the shared library")

    #
    # Dependencies and conflicts
    #
    depends_on('cmake@2.6:', type="build")
    depends_on('blas')
    depends_on('python', type=("build", "test"), when="+test")
    depends_on('py-numpy', type=("build", "test"), when="+test")

    # Libcint tests only work with a shared libcint library
    conflicts('+test~shared')

    #
    # Settings and cmake cache
    #
    def cmake_args(self):
        spec = self.spec
        args = [
            "-DWITH_COULOMB_ERF=" + str("+coulomb_erf" in spec),
            "-DWITH_F12=" + str("+f12" in spec),
            "-DBUILD_SHARED_LIBS=" + str("+shared" in spec),
            "-DENABLE_TEST=" + str("+test" in spec),
            "-DENABLE_EXAMPLE=OFF",  # Requires fortran compiler
        ]
        return args
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Libgit2(CMakePackage):
    """libgit2 is a portable, pure C implementation of the Git core
    methods provided as a re-entrant linkable library with a solid
    API, allowing you to write native speed custom Git applications in
    any language which supports C bindings.
    """

    homepage = "https://libgit2.github.com/"
    url      = "https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz"

    version('0.99.0',  sha256='174024310c1563097a6613a0d3f7539d11a9a86517cd67ce533849065de08a11')
    version('0.28.4',  sha256='30f3877469d09f2e4a21be933b4e2800560d16646028dd800744dc5f7fb0c749')
    version('0.28.3',  sha256='ee5344730fe11ce7c86646e19c2d257757be293f5a567548d398fb3af8b8e53b')
    version('0.28.2',  sha256='42b5f1e9b9159d66d86fff0394215c5733b6ef8f9b9d054cdd8c73ad47177fc3')
    version('0.28.1',  sha256='0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b')
    version('0.28.0',  sha256='9d60d64dc77085e8e530e5c66314057eafe0c06e4a7a61149a70ff3e0688f284')
    version('0.27.10', sha256='f6fd26378ff71bd7a4b17b576c82c774a2e9c2d6b74b24718a8fb29551e1c4a5')
    version('0.27.9',  sha256='adf17310b590e6e7618f070c742b5ee028aeeed2c60099bc4190c386b5060de1')
    version('0.27.8',  sha256='8313873d49dc01e8b880ec334d7430ae67496a89aaa8c6e7bbd3affb47a00c76')
    version('0.27.7',  sha256='1a5435a483759b1cd96feb12b11abb5231b0688016db506ce5947178f6ba2531')
    version('0.27.6',  sha256='d98db2ed11ec82fee94dce4819b466524613852c2c9c3426d351c57729ec49da')
    version('0.27.5',  sha256='15f2775f4f325951d9139ed906502b6c71fee6787cada9b045f5994072ccbd33')
    version('0.27.4',  sha256='0b7ca31cb959ff1b22afa0da8621782afe61f99242bf716c403802ffbdb21d51')
    version('0.27.3',  sha256='50a57bd91f57aa310fb7d5e2a340b3779dc17e67b4e7e66111feac5c2432f1a5')
    version('0.27.2',  sha256='ffacdbd5588aeb03e98e3866a7e2ceace468723a439bdc9bb01362fe140fa9e5')
    version('0.27.1',  sha256='837b11927bc5f64e7f9ab0376f57cfe3ca5aa52ffd2007ac41184b21124fb086')
    version('0.27.0',  sha256='545b0458292c786aba334f1bf1c8f73600ae73dd7205a7bb791a187ee48ab8d2')
    version('0.26.8',  sha256='0f20d7e239be030db33d7350bab38ada2830b3bffab5539730074e71b0267796')
    version('0.26.7',  sha256='65584ac1f4de2c3ab8491351c8629eb68bad2d65e67f6411bf0333b8976dc4ef')
    version('0.26.6',  sha256='7669dd47ebdab86ced8888816c552596ec923b6e126704a3445b2081cb0e5662')
    version('0.26.5',  sha256='52e28a5166564bc4365a2e4112f5e5c6e334708dbf13596241b2fd34efc1b0a9')
    version('0.26.4',  sha256='292fa2600bbb4e52641793cfcc1c19ffc0bf97b5fd8378d422a6bfe7afffcb97')
    version('0.26.3',  sha256='0da4e211dfb63c22e5f43f2a4a5373e86a140afa88a25ca6ba3cc2cae58263d2')
    version('0.26.2',  sha256='747b47d5b02a2387ff81301c694763785181b895690b6eb91ed1ae4b7904307b')
    version('0.26.1',  sha256='68cd0f8ee9e0ca84dcf0f0267d0a8297471d3365622d22d3da67c57165bb0722')
    version('0.26.0',  sha256='6a62393e0ceb37d02fe0d5707713f504e7acac9006ef33da1e88960bd78b6eac')

    # Backends
    variant(
        'https', default='system', description='HTTPS support',
        values=('system', 'openssl', 'none'), multi=False)
    variant('ssh', default=True, description='Enable SSH support')
    variant('curl', default=False, description='Enable libcurl support (only supported through v0.27)')

    # Build Dependencies
    depends_on('cmake@2.8:', type='build', when="@:0.28")
    depends_on('cmake@3.5:', type='build', when="@0.99:")

    # Runtime Dependencies
    depends_on('libssh2', when='+ssh')
    depends_on('openssl', when='https=system platform=linux')
    depends_on('openssl', when='https=system platform=cray')
    depends_on('openssl', when='https=openssl')
    depends_on('curl', when='+curl')

    conflicts('+curl', when='@0.28:')

    def cmake_args(self):
        args = []
        if 'https=system' in self.spec:
            if 'platform=linux' in self.spec or 'platform=cray' in self.spec:
                args.append('-DUSE_HTTPS=OpenSSL')
            elif 'platform=darwin' in self.spec:
                args.append('-DUSE_HTTPS=SecureTransport')
            else:
                # Let CMake try to find an HTTPS implementation. Mileage on
                # your platform may vary
                args.append('-DUSE_HTTPS=ON')
        elif 'https=openssl' in self.spec:
            args.append('-DUSE_HTTPS=OpenSSL')
        else:
            args.append('-DUSE_HTTPS=OFF')

        args.append(
            '-DUSE_SSH={0}'.format('ON' if '+ssh' in self.spec else 'OFF'))

        # The curl backed is not supported after 0.27.x
        if '@:0.27 +curl' in self.spec:
            args.append(
                '-DCURL={0}'.format('ON' if '+curl' in self.spec else 'OFF'))

        # Control tests
        args.append(
            '-DBUILD_CLAR={0}'.format('ON' if self.run_tests else 'OFF'))

        return args
<commit_msg>Add libgit2 1.0.0 and 0.28.5 (#15801)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Libgit2(CMakePackage):
    """libgit2 is a portable, pure C implementation of the Git core
    methods provided as a re-entrant linkable library with a solid
    API, allowing you to write native speed custom Git applications in
    any language which supports C bindings.
    """

    homepage = "https://libgit2.github.com/"
    url      = "https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz"

    maintainers = ["AndrewGaspar"]

    version('1.0.0',   sha256='6a1fa16a7f6335ce8b2630fbdbb5e57c4027929ebc56fcd1ac55edb141b409b4')
    version('0.99.0',  sha256='174024310c1563097a6613a0d3f7539d11a9a86517cd67ce533849065de08a11')
    version('0.28.5',  sha256='2b7b68aee6f123bc84cc502a9c12738435b8054e7d628962e091cd2a25be4f42')
    version('0.28.4',  sha256='30f3877469d09f2e4a21be933b4e2800560d16646028dd800744dc5f7fb0c749')
    version('0.28.3',  sha256='ee5344730fe11ce7c86646e19c2d257757be293f5a567548d398fb3af8b8e53b')
    version('0.28.2',  sha256='42b5f1e9b9159d66d86fff0394215c5733b6ef8f9b9d054cdd8c73ad47177fc3')
    version('0.28.1',  sha256='0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b')
    version('0.28.0',  sha256='9d60d64dc77085e8e530e5c66314057eafe0c06e4a7a61149a70ff3e0688f284')
    version('0.27.10', sha256='f6fd26378ff71bd7a4b17b576c82c774a2e9c2d6b74b24718a8fb29551e1c4a5')
    version('0.27.9',  sha256='adf17310b590e6e7618f070c742b5ee028aeeed2c60099bc4190c386b5060de1')
    version('0.27.8',  sha256='8313873d49dc01e8b880ec334d7430ae67496a89aaa8c6e7bbd3affb47a00c76')
    version('0.27.7',  sha256='1a5435a483759b1cd96feb12b11abb5231b0688016db506ce5947178f6ba2531')
    version('0.27.6',  sha256='d98db2ed11ec82fee94dce4819b466524613852c2c9c3426d351c57729ec49da')
    version('0.27.5',  sha256='15f2775f4f325951d9139ed906502b6c71fee6787cada9b045f5994072ccbd33')
    version('0.27.4',  sha256='0b7ca31cb959ff1b22afa0da8621782afe61f99242bf716c403802ffbdb21d51')
    version('0.27.3',  sha256='50a57bd91f57aa310fb7d5e2a340b3779dc17e67b4e7e66111feac5c2432f1a5')
    version('0.27.2',  sha256='ffacdbd5588aeb03e98e3866a7e2ceace468723a439bdc9bb01362fe140fa9e5')
    version('0.27.1',  sha256='837b11927bc5f64e7f9ab0376f57cfe3ca5aa52ffd2007ac41184b21124fb086')
    version('0.27.0',  sha256='545b0458292c786aba334f1bf1c8f73600ae73dd7205a7bb791a187ee48ab8d2')
    version('0.26.8',  sha256='0f20d7e239be030db33d7350bab38ada2830b3bffab5539730074e71b0267796')
    version('0.26.7',  sha256='65584ac1f4de2c3ab8491351c8629eb68bad2d65e67f6411bf0333b8976dc4ef')
    version('0.26.6',  sha256='7669dd47ebdab86ced8888816c552596ec923b6e126704a3445b2081cb0e5662')
    version('0.26.5',  sha256='52e28a5166564bc4365a2e4112f5e5c6e334708dbf13596241b2fd34efc1b0a9')
    version('0.26.4',  sha256='292fa2600bbb4e52641793cfcc1c19ffc0bf97b5fd8378d422a6bfe7afffcb97')
    version('0.26.3',  sha256='0da4e211dfb63c22e5f43f2a4a5373e86a140afa88a25ca6ba3cc2cae58263d2')
    version('0.26.2',  sha256='747b47d5b02a2387ff81301c694763785181b895690b6eb91ed1ae4b7904307b')
    version('0.26.1',  sha256='68cd0f8ee9e0ca84dcf0f0267d0a8297471d3365622d22d3da67c57165bb0722')
    version('0.26.0',  sha256='6a62393e0ceb37d02fe0d5707713f504e7acac9006ef33da1e88960bd78b6eac')

    # Backends
    variant(
        'https', default='system', description='HTTPS support',
        values=('system', 'openssl', 'none'), multi=False)
    variant('ssh', default=True, description='Enable SSH support')
    variant('curl', default=False, description='Enable libcurl support (only supported through v0.27)')

    # Build Dependencies
    depends_on('cmake@2.8:', type='build', when="@:0.28")
    depends_on('cmake@3.5:', type='build', when="@0.99:")

    # Runtime Dependencies
    depends_on('libssh2', when='+ssh')
    depends_on('openssl', when='https=system platform=linux')
    depends_on('openssl', when='https=system platform=cray')
    depends_on('openssl', when='https=openssl')
    depends_on('curl', when='+curl')

    conflicts('+curl', when='@0.28:')

    def cmake_args(self):
        args = []
        if 'https=system' in self.spec:
            if 'platform=linux' in self.spec or 'platform=cray' in self.spec:
                args.append('-DUSE_HTTPS=OpenSSL')
            elif 'platform=darwin' in self.spec:
                args.append('-DUSE_HTTPS=SecureTransport')
            else:
                # Let CMake try to find an HTTPS implementation. Mileage on
                # your platform may vary
                args.append('-DUSE_HTTPS=ON')
        elif 'https=openssl' in self.spec:
            args.append('-DUSE_HTTPS=OpenSSL')
        else:
            args.append('-DUSE_HTTPS=OFF')

        args.append(
            '-DUSE_SSH={0}'.format('ON' if '+ssh' in self.spec else 'OFF'))

        # The curl backed is not supported after 0.27.x
        if '@:0.27 +curl' in self.spec:
            args.append(
                '-DCURL={0}'.format('ON' if '+curl' in self.spec else 'OFF'))

        # Control tests
        args.append(
            '-DBUILD_CLAR={0}'.format('ON' if self.run_tests else 'OFF'))

        return args
<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Libtiff(AutotoolsPackage):
    """LibTIFF - Tag Image File Format (TIFF) Library and Utilities."""

    homepage = "http://www.simplesystems.org/libtiff/"
    url      = "http://download.osgeo.org/libtiff/tiff-4.0.9.tar.gz"

    version('4.0.9', '54bad211279cc93eb4fca31ba9bfdc79')
    version('4.0.8', '2a7d1c1318416ddf36d5f6fa4600069b')
    version('4.0.7', '77ae928d2c6b7fb46a21c3a29325157b')
    version('4.0.6', 'd1d2e940dea0b5ad435f21f03d96dd72')
    version('4.0.3', '051c1068e6a0627f461948c365290410')
    version('3.9.7', '626102f448ba441d42e3212538ad67d2')

    depends_on('jpeg')
    depends_on('zlib')
    depends_on('xz')
<commit_msg>Add libtiff 4.0.10 (#11732)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Libtiff(AutotoolsPackage):
    """LibTIFF - Tag Image File Format (TIFF) Library and Utilities."""

    homepage = "http://www.simplesystems.org/libtiff/"
    url      = "http://download.osgeo.org/libtiff/tiff-4.0.10.tar.gz"

    version('4.0.10', sha256='2c52d11ccaf767457db0c46795d9c7d1a8d8f76f68b0b800a3dfe45786b996e4')
    version('4.0.9', '54bad211279cc93eb4fca31ba9bfdc79')
    version('4.0.8', '2a7d1c1318416ddf36d5f6fa4600069b')
    version('4.0.7', '77ae928d2c6b7fb46a21c3a29325157b')
    version('4.0.6', 'd1d2e940dea0b5ad435f21f03d96dd72')
    version('4.0.3', '051c1068e6a0627f461948c365290410')
    version('3.9.7', '626102f448ba441d42e3212538ad67d2')

    depends_on('jpeg')
    depends_on('zlib')
    depends_on('xz')
<|endoftext|>


<commit_before>from __future__ import print_function
import logging
import os
from argparse import ArgumentDefaultsHelpFormatter

from django.core.management.base import BaseCommand
from django.conf import settings
from django.db import transaction

from container.models import Container

logger = logging.getLogger(__name__)

# This assumes we are using the default FileSystemStorage class.
class Command(BaseCommand):
    help = "Consolidates Container file locations into their default directory."

    def add_arguments(self, parser):
        parser.formatter_class = ArgumentDefaultsHelpFormatter

        parser.add_argument('--directory',
                            help='Directory to consolidate Container files in',
                            default=os.path.join(settings.MEDIA_ROOT, Container.UPLOAD_DIR))

    def handle(self, directory, **options):
        self.consolidate_containers(directory=directory)

    def consolidate_containers(self, directory):
        """
        Consolidate container files in the specified directory.

        :param directory: an absolute path
        :return:
        """
        new_container_directory = os.path.normpath(directory)

        # Scan through all Containers and examine the filenames.
        for container in Container.objects.all():
            current_absolute_path = os.path.join(settings.MEDIA_ROOT, container.file.name)
            directory_in_storage = os.path.normpath(os.path.dirname(current_absolute_path))
            if directory_in_storage == new_container_directory:
                continue

            # This incantation is copied from
            # https://docs.djangoproject.com/en/1.11/topics/files/#using-files-in-models
            with transaction.atomic():
                current_basename = os.path.basename(current_absolute_path)
                new_absolute_path = os.path.join(new_container_directory, current_basename)
                print("Moving {} to {}....".format(current_absolute_path, new_absolute_path))
                container.file.name = new_absolute_path
                os.rename(current_absolute_path, new_absolute_path)
                container.save()
<commit_msg>Added a confirmation message and prompt to the consolidate_containers command.<commit_after>from __future__ import print_function
import logging
import os
import sys
from argparse import ArgumentDefaultsHelpFormatter

from django.core.management.base import BaseCommand
from django.conf import settings
from django.db import transaction

from container.models import Container

logger = logging.getLogger(__name__)

# This assumes we are using the default FileSystemStorage class.
class Command(BaseCommand):
    help = "Consolidates Container file locations into their default directory."

    def add_arguments(self, parser):
        parser.formatter_class = ArgumentDefaultsHelpFormatter

        parser.add_argument('--directory',
                            help='Directory to consolidate Container files in',
                            default=os.path.join(settings.MEDIA_ROOT, Container.UPLOAD_DIR))

    def handle(self, directory, **options):
        move_instructions = self.identify_containers_to_move(directory=directory)
        prompt_message = "The following Container files will be moved:\n"
        for container, curr_loc, new_loc in move_instructions:
            prompt_message += "{} ({} -> {})\n".format(container, curr_loc, new_loc)
        prompt_message += "Proceed?"
        proceed = self.confirm(prompt_message)
        if not proceed:
            return
        self.move_container_files(move_instructions)

    def identify_containers_to_move(self, directory):
        """
        Identify Containers whose files must move.
        :param directory:
        :return:
        """
        new_container_directory = os.path.normpath(directory)

        containers_to_consolidate = []  # this will be a list of 3-tuples containing Container, old path, and new path
        # Scan through all Containers and examine the filenames.
        for container in Container.objects.all():
            current_absolute_path = os.path.join(settings.MEDIA_ROOT, container.file.name)
            directory_in_storage = os.path.normpath(os.path.dirname(current_absolute_path))
            if directory_in_storage == new_container_directory:
                continue

            current_basename = os.path.basename(current_absolute_path)
            new_absolute_path = os.path.join(new_container_directory, current_basename)
            containers_to_consolidate.append((container, current_absolute_path, new_absolute_path))

        return containers_to_consolidate

    def move_container_files(self, move_instructions):
        """
        Move Container files around as specified.

        :param move_instructions: a list of 3-tuples (container, current location, new location)
        :return:
        """
        for container, curr_loc, new_loc in move_instructions:
            # This incantation is copied from
            # https://docs.djangoproject.com/en/1.11/topics/files/#using-files-in-models
            with transaction.atomic():
                print("Moving {} to {}....".format(curr_loc, new_loc))
                container.file.name = new_loc
                os.rename(curr_loc, new_loc)
                container.save()

    # This is copied from the "purge.py" management command.
    def confirm(self, prompt):
        print(prompt, end=' ')
        confirmation = sys.stdin.readline().strip()
        is_confirmed = confirmation.lower() == 'y'
        return is_confirmed
<|endoftext|>