<commit_before># -*- coding: utf-8 -*-

# Banco de dados
db = DAL('sqlite://storage.sqlite', pool_size=1, check_reserved=['all'])

# by default give a view/generic.extension to all actions from localhost
# none otherwise. a pattern can be 'controller/function.extension'
response.generic_patterns = ['*'] if request.is_local else []

from gluon.tools import Auth
# autenticação do usuário
auth = Auth(db)

# campos adicionais do usuário
auth.settings.extra_fields['auth_user'] = [
    Field('instituicao', label="Instituição de Ensino", length=120),
    Field('matricula', label='Matrícula', length=32),
    Field('telefone')
]

# create all tables needed by auth if not custom tables
auth.define_tables(username=False, signature=False)

# configure email
mail = auth.settings.mailer
mail.settings.server = 'logging' if request.is_local else 'smtp.gmail.com:587'
mail.settings.sender = 'you@gmail.com'
mail.settings.login = 'username:password'

# configure auth policy
auth.settings.registration_requires_verification = False
auth.settings.registration_requires_approval = False
auth.settings.reset_password_requires_verification = True

db.define_table(
    'evento',
    Field('nome', length=120, notnull=True),
    Field(
        'data_inicio',
        'date',
        label="Data Inicial",
        notnull=True
    ),
    Field(
        'data_final',
        'date',
        label="Data Final",
        notnull=True
    ),
    Field('localizacao', notnull=True, length=540, label="Localização"),
    Field('descricao', 'text', label="Descrição", notnull=True),
    Field(
        'url_youtube',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Youtube (https://youtube.com/seu-canal) "
    ),
    Field(
        'url_facebook',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Facebook (https://facebook.com/seu-nome)",
    ),
    Field(
        'url_twitter',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Twitter (https://twitter.com/seu-nome)",
    ),
    Field(
        'url_gplus',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Google Plus (https://plus.google.com/u/0/+Seu-nome)"
    ),
	Field('logotipo', 'upload', requires=IS_EMPTY_OR(IS_IMAGE())),
)

db.define_table(
    'organizador',
    Field('usuario', 'reference auth_user'),
    Field('foto', 'upload', requires=IS_EMPTY_OR(IS_IMAGE())),
    Field(
        'url_facebook',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Facebook (https://facebook.com/seu-nome)",
    ),
    Field(
        'url_twitter',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Twitter (https://twitter.com/seu-nome)",
    ),
    Field(
        'url_gplus',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Google Plus (https://plus.google.com/u/0/+Seu-nome)"
    ),
	Field(
        'url_Github',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Github (https://github.com/Seu-Git)"
    ),
	Field(
        'url_LinkedIn',
        requires=IS_EMPTY_OR(IS_URL()),
        label="LinkedIn "
    ),
)

db.define_table(
	'palestrante',
	Field('nome', length=120, notnull=True),
	Field('foto', 'upload', requires=IS_EMPTY_OR(IS_IMAGE()) ),
	Field(
        'url_facebook',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Facebook (https://facebook.com/seu-nome)",
    ),
    Field(
        'url_twitter',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Twitter (https://twitter.com/seu-nome)",
    ),
    Field(
        'url_gplus',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Google Plus (https://plus.google.com/u/0/+Seu-nome)"
    ),
	Field(
        'url_Github',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Github (https://github.com/Seu-Git)"
    ),
	Field(
        'url_LinkedIn',
        requires=IS_EMPTY_OR(IS_URL()),
        label="LinkedIn "
    ),
	Field('bio', 'text'),
)

db.define_table(
	'atividade',
	Field('titulo', length=120, notnull=True),
	Field('tipo_atividade',
	length=12,
	requires=IS_IN_SET(['palestra','minicurso','workshop'],
	zero='Escolha um tipo de atividade')
	),
	Field(
        'data_hora_inicio',
        'datetime',
        label="Data/Horário Inicial",
        notnull=True
    ),
    Field(
        'data_hora_final',
        'datetime',
        label="Data/Horário Final",
        notnull=True
    ),
    Field('palestrante'),
    Field('evento_relacionado')
)

db.atividade.palestrante.requires = IS_IN_DB(db, 'palestrante.id', '%(nome)s')
db.atividade.evento_relacionado.requires = IS_IN_DB(db, 'evento.id', '%(nome)s')

db.define_table(
    'patrocinador',
    Field('nome', length=120, notnull=True),
	Field(
        'url_empresa',
        requires=IS_EMPTY_OR(IS_URL() ),
        label="Link da empresa"
    ),
	Field('foto', 'upload', requires=IS_IMAGE()),
	Field('plano', label='Tipo de patrocínio',
	requires=IS_IN_SET(['Bronze','Prata','Ouro','Platina'],
	zero='Escolha o tipo de patrocínio')
	),
)

db.define_table(
    'vinculo_organizador_evento',
    Field('organizador', 'reference auth_user'),
    Field('evento', 'reference evento'),
)

db.vinculo_organizador_evento.organizador.requires = IS_IN_DB(
    db, 'auth_user.id', '%(first_name)s')
db.vinculo_organizador_evento.evento.requires = IS_IN_DB(
    db, 'evento.id', '%(nome)s')

db.define_table(
    'vinculo_patrocinador_evento',
    Field('patrocinador', 'reference patrocinador'),
    Field('evento', 'reference evento'),
)

db.vinculo_patrocinador_evento.patrocinador.requires = IS_IN_DB(
    db, 'patrocinador.id', '%(nome)s')
db.vinculo_patrocinador_evento.evento.requires = IS_IN_DB(
    db, 'evento.id', '%(nome)s')

db.define_table(
	'vinculo_usuario_atividade',
	Field('usuario', 'reference auth_user'),
	Field('atividade', 'reference atividade'),
)

db.vinculo_usuario_atividade.usuario.requires = IS_IN_DB(
	db, 'auth_user.id', '%(first_name)s')
db.vinculo_usuario_atividade.atividade.requires = IS_IN_DB(
	db, 'atividade.id', '%(titulo)s')<commit_msg>fix #9, resolve #11<commit_after># -*- coding: utf-8 -*-

# Banco de dados
db = DAL('sqlite://storage.sqlite', pool_size=1, check_reserved=['all'])

# by default give a view/generic.extension to all actions from localhost
# none otherwise. a pattern can be 'controller/function.extension'
response.generic_patterns = ['*'] if request.is_local else []

from gluon.tools import Auth
# autenticação do usuário
auth = Auth(db)

# campos adicionais do usuário
auth.settings.extra_fields['auth_user'] = [
    Field('instituicao', label="Instituição de Ensino", length=120),
    Field('matricula', label='Matrícula', length=32),
    Field('telefone')
]

# create all tables needed by auth if not custom tables
auth.define_tables(username=False, signature=False)

# configure email
mail = auth.settings.mailer
mail.settings.server = 'logging' if request.is_local else 'smtp.gmail.com:587'
mail.settings.sender = 'you@gmail.com'
mail.settings.login = 'username:password'

# configure auth policy
auth.settings.registration_requires_verification = False
auth.settings.registration_requires_approval = False
auth.settings.reset_password_requires_verification = True

db.define_table(
    'evento',
    Field('nome', length=120, notnull=True),
    Field(
        'data_inicio',
        'date',
        label="Data Inicial",
        notnull=True
    ),
    Field(
        'data_final',
        'date',
        label="Data Final",
        notnull=True
    ),
    Field('localizacao', notnull=True, length=540, label="Localização"),
    Field('descricao', 'text', label="Descrição", notnull=True),
    Field(
        'url_youtube',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Youtube (https://youtube.com/seu-canal) "
    ),
    Field(
        'url_facebook',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Facebook (https://facebook.com/seu-nome)",
    ),
    Field(
        'url_twitter',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Twitter (https://twitter.com/seu-nome)",
    ),
    Field(
        'url_gplus',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Google Plus (https://plus.google.com/u/0/+Seu-nome)"
    ),
	Field('logotipo', 'upload', requires=IS_EMPTY_OR(IS_IMAGE())),
)

db.define_table(
    'organizador',
    Field('usuario', 'reference auth_user'),
    Field('foto', 'upload', requires=IS_EMPTY_OR(IS_IMAGE())),
    Field(
        'url_facebook',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Facebook (https://facebook.com/seu-nome)",
    ),
    Field(
        'url_twitter',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Twitter (https://twitter.com/seu-nome)",
    ),
    Field(
        'url_gplus',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Google Plus (https://plus.google.com/u/0/+Seu-nome)"
    ),
	Field(
        'url_Github',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Github (https://github.com/Seu-Git)"
    ),
	Field(
        'url_LinkedIn',
        requires=IS_EMPTY_OR(IS_URL()),
        label="LinkedIn "
    ),
)

db.define_table(
	'palestrante',
	Field('nome', length=120, notnull=True),
	Field('foto', 'upload', requires=IS_EMPTY_OR(IS_IMAGE()) ),
	Field(
        'url_facebook',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Facebook (https://facebook.com/seu-nome)",
    ),
    Field(
        'url_twitter',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Twitter (https://twitter.com/seu-nome)",
    ),
    Field(
        'url_gplus',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Google Plus (https://plus.google.com/u/0/+Seu-nome)"
    ),
	Field(
        'url_Github',
        requires=IS_EMPTY_OR(IS_URL()),
        label="Github (https://github.com/Seu-Git)"
    ),
	Field(
        'url_LinkedIn',
        requires=IS_EMPTY_OR(IS_URL()),
        label="LinkedIn "
    ),
	Field('bio', 'text'),
)

db.define_table(
	'atividade',
	Field('titulo', length=120, notnull=True),
	Field('tipo_atividade',
	length=12,
	requires=IS_IN_SET(['palestra','minicurso','workshop'],
	zero='Escolha um tipo de atividade')
	),
	Field(
        'data_hora_inicio',
        'datetime',
        label="Data/Horário Inicial",
        notnull=True
    ),
    Field(
        'data_hora_final',
        'datetime',
        label="Data/Horário Final",
        notnull=True
    ),
    Field('palestrante'),
    Field('evento_relacionado')
)

db.atividade.palestrante.requires = IS_IN_DB(db, 'palestrante.id', '%(nome)s')
db.atividade.evento_relacionado.requires = IS_IN_DB(db, 'evento.id', '%(nome)s')

db.define_table(
    'patrocinador',
    Field('nome', length=120, notnull=True),
	Field(
        'url_empresa',
        requires=IS_EMPTY_OR(IS_URL() ),
        label="Link da empresa"
    ),
	Field('foto', 'upload', requires=IS_IMAGE()),
	Field('plano', label='Tipo de patrocínio',
	requires=IS_IN_SET(['Bronze','Prata','Ouro','Platina'],
	zero='Escolha o tipo de patrocínio')
	),
)

db.define_table(
    'vinculo_organizador_evento',
    Field('organizador', 'reference auth_user'),
    Field('evento', 'reference evento'),
)

db.vinculo_organizador_evento.organizador.requires = IS_IN_DB(
    db, 'auth_user.id', '%(first_name)s')
db.vinculo_organizador_evento.evento.requires = IS_IN_DB(
    db, 'evento.id', '%(nome)s')

db.define_table(
    'vinculo_patrocinador_evento',
    Field('patrocinador', 'reference patrocinador'),
    Field('evento', 'reference evento'),
)

db.vinculo_patrocinador_evento.patrocinador.requires = IS_IN_DB(
    db, 'patrocinador.id', '%(nome)s')
db.vinculo_patrocinador_evento.evento.requires = IS_IN_DB(
    db, 'evento.id', '%(nome)s')

db.define_table(
	'vinculo_usuario_atividade',
	Field('usuario', 'reference auth_user'),
	Field('atividade','reference atividade'),
)

db.vinculo_usuario_atividade.usuario.requires = IS_IN_DB(
	db, 'auth_user.id', '%(first_name)s')
db.vinculo_usuario_atividade.atividade.requires = IS_IN_DB(
	db, 'atividade.id', '%(titulo)s')<|endoftext|>


<commit_before>"""High level parallel SNP and indel calling using multiple variant callers.
"""
import os
import collections
import copy

from bcbio import utils
from bcbio.distributed.split import grouped_parallel_split_combine
from bcbio.pipeline import region
from bcbio.variation import gatk, gatkfilter, multi, phasing, ploidy, vfilter

# ## Variant filtration -- shared functionality

def variant_filtration(call_file, ref_file, vrn_files, data):
    """Filter variant calls using Variant Quality Score Recalibration.

    Newer GATK with Haplotype calling has combined SNP/indel filtering.
    """
    caller = data["config"]["algorithm"].get("variantcaller")
    call_file = ploidy.filter_vcf_by_sex(call_file, data)
    if caller in ["freebayes"]:
        return vfilter.freebayes(call_file, ref_file, vrn_files, data)
    elif caller in ["gatk", "gatk-haplotype"]:
        return gatkfilter.run(call_file, ref_file, vrn_files, data)
    # no additional filtration for callers that filter as part of call process
    else:
        return call_file

# ## High level functionality to run genotyping in parallel

def get_variantcaller(data):
    if data.get("align_bam"):
        return data["config"]["algorithm"].get("variantcaller", "gatk")

def combine_multiple_callers(samples):
    """Collapse together variant calls from multiple approaches into single data item with `variants`.
    """
    by_bam = collections.OrderedDict()
    for data in (x[0] for x in samples):
        work_bam = utils.get_in(data, ("combine", "work_bam", "out"), data.get("align_bam"))
        variantcaller = get_variantcaller(data)
        key = (data["description"], work_bam)
        try:
            by_bam[key][variantcaller] = data
        except KeyError:
            by_bam[key] = {variantcaller: data}
    out = []
    for grouped_calls in [d.values() for d in by_bam.values()]:
        ready_calls = [{"variantcaller": get_variantcaller(x),
                        "vrn_file": x.get("vrn_file"),
                        "vrn_file_batch": x.get("vrn_file_batch"),
                        "validate": x.get("validate")}
                       for x in grouped_calls]
        final = grouped_calls[0]
        def orig_variantcaller_order(x):
            return final["config"]["algorithm"]["orig_variantcaller"].index(x["variantcaller"])
        if len(ready_calls) > 1 and "orig_variantcaller" in final["config"]["algorithm"]:
            final["variants"] = sorted(ready_calls, key=orig_variantcaller_order)
            final["config"]["algorithm"]["variantcaller"] = final["config"]["algorithm"].pop("orig_variantcaller")
        else:
            final["variants"] = ready_calls
        final.pop("vrn_file_batch", None)
        out.append([final])
    return out

def _split_by_ready_regions(ext, file_key, dir_ext_fn):
    """Organize splits based on regions generated by parallel_prep_region.
    """
    def _do_work(data):
        if "region" in data:
            name = data["group"][0] if "group" in data else data["description"]
            out_dir = os.path.join(data["dirs"]["work"], dir_ext_fn(data))
            out_file = os.path.join(out_dir, "%s%s" % (name, ext))
            assert isinstance(data["region"], (list, tuple))
            out_parts = []
            for i, r in enumerate(data["region"]):
                out_region_dir = os.path.join(out_dir, r[0])
                out_region_file = os.path.join(out_region_dir,
                                               "%s-%s%s" % (name, region.to_safestr(r), ext))
                work_bams = []
                for xs in data["region_bams"]:
                    if len(xs) == 1:
                        work_bams.append(xs[0])
                    else:
                        work_bams.append(xs[i])
                for work_bam in work_bams:
                    assert os.path.exists(work_bam), work_bam
                out_parts.append((r, work_bams, out_region_file))
            return out_file, out_parts
        else:
            return None, []
    return _do_work

def _collapse_by_bam_variantcaller(samples):
    """Collapse regions to a single representative by BAM input and variant caller.
    """
    by_bam = collections.OrderedDict()
    for data in (x[0] for x in samples):
        work_bam = utils.get_in(data, ("combine", "work_bam", "out"), data.get("align_bam"))
        variantcaller = get_variantcaller(data)
        if isinstance(work_bam, list):
            work_bam = tuple(work_bam)
        key = (data["description"], work_bam, variantcaller)
        try:
            by_bam[key].append(data)
        except KeyError:
            by_bam[key] = [data]
    out = []
    for grouped_data in by_bam.values():
        cur = grouped_data[0]
        cur.pop("region", None)
        region_bams = cur.pop("region_bams", None)
        if region_bams and len(region_bams[0]) > 1:
            cur.pop("work_bam", None)
        out.append([cur])
    return out

def parallel_variantcall_region(samples, run_parallel):
    """Perform variant calling and post-analysis on samples by region.
    """
    to_process = []
    extras = []
    for x in samples:
        added = False
        for add in handle_multiple_variantcallers(x):
            added = True
            to_process.append(add)
        if not added:
            extras.append(x)
    split_fn = _split_by_ready_regions(".vcf.gz", "work_bam", get_variantcaller)
    samples = _collapse_by_bam_variantcaller(
        grouped_parallel_split_combine(to_process, split_fn,
                                       multi.group_batches, run_parallel,
                                       "variantcall_sample", "concat_variant_files",
                                       "vrn_file", ["region", "sam_ref", "config"]))
    return extras + samples

def handle_multiple_variantcallers(data):
    """Split samples that potentially require multiple variant calling approaches.
    """
    assert len(data) == 1
    callers = get_variantcaller(data[0])
    if isinstance(callers, basestring):
        return [data]
    elif not callers:
        return []
    else:
        out = []
        for caller in callers:
            base = copy.deepcopy(data[0])
            base["config"]["algorithm"]["orig_variantcaller"] = \
              base["config"]["algorithm"]["variantcaller"]
            base["config"]["algorithm"]["variantcaller"] = caller
            out.append([base])
        return out

def get_variantcallers():
    from bcbio.variation import freebayes, cortex, samtools, varscan, mutect
    return {"gatk": gatk.unified_genotyper,
            "gatk-haplotype": gatk.haplotype_caller,
            "freebayes": freebayes.run_freebayes,
            "cortex": cortex.run_cortex,
            "samtools": samtools.run_samtools,
            "varscan": varscan.run_varscan,
            "mutect": mutect.mutect_caller}

def variantcall_sample(data, region=None, align_bams=None, out_file=None):
    """Parallel entry point for doing genotyping of a region of a sample.
    """
    if out_file is None or not os.path.exists(out_file) or not os.path.lexists(out_file):
        utils.safe_makedir(os.path.dirname(out_file))
        sam_ref = data["sam_ref"]
        config = data["config"]
        caller_fns = get_variantcallers()
        caller_fn = caller_fns[config["algorithm"].get("variantcaller", "gatk")]
        if len(align_bams) == 1:
            items = [data]
        else:
            items = multi.get_orig_items(data)
            assert len(items) == len(align_bams)
        call_file = "%s-raw%s" % utils.splitext_plus(out_file)
        call_file = caller_fn(align_bams, items, sam_ref,
                              data["genome_resources"]["variation"],
                              region, call_file)
        if data["config"]["algorithm"].get("phasing", False) == "gatk":
            call_file = phasing.read_backed_phasing(call_file, align_bams, sam_ref, region, config)
        utils.symlink_plus(call_file, out_file)
    if region:
        data["region"] = region
    data["vrn_file"] = out_file
    return [data]
<commit_msg>Added standalone support for Scalpel calling<commit_after>"""High level parallel SNP and indel calling using multiple variant callers.
"""
import os
import collections
import copy

from bcbio import utils
from bcbio.distributed.split import grouped_parallel_split_combine
from bcbio.pipeline import region
from bcbio.variation import gatk, gatkfilter, multi, phasing, ploidy, vfilter

# ## Variant filtration -- shared functionality

def variant_filtration(call_file, ref_file, vrn_files, data):
    """Filter variant calls using Variant Quality Score Recalibration.

    Newer GATK with Haplotype calling has combined SNP/indel filtering.
    """
    caller = data["config"]["algorithm"].get("variantcaller")
    call_file = ploidy.filter_vcf_by_sex(call_file, data)
    if caller in ["freebayes"]:
        return vfilter.freebayes(call_file, ref_file, vrn_files, data)
    elif caller in ["gatk", "gatk-haplotype"]:
        return gatkfilter.run(call_file, ref_file, vrn_files, data)
    # no additional filtration for callers that filter as part of call process
    else:
        return call_file

# ## High level functionality to run genotyping in parallel

def get_variantcaller(data):
    if data.get("align_bam"):
        return data["config"]["algorithm"].get("variantcaller", "gatk")

def combine_multiple_callers(samples):
    """Collapse together variant calls from multiple approaches into single data item with `variants`.
    """
    by_bam = collections.OrderedDict()
    for data in (x[0] for x in samples):
        work_bam = utils.get_in(data, ("combine", "work_bam", "out"), data.get("align_bam"))
        variantcaller = get_variantcaller(data)
        key = (data["description"], work_bam)
        try:
            by_bam[key][variantcaller] = data
        except KeyError:
            by_bam[key] = {variantcaller: data}
    out = []
    for grouped_calls in [d.values() for d in by_bam.values()]:
        ready_calls = [{"variantcaller": get_variantcaller(x),
                        "vrn_file": x.get("vrn_file"),
                        "vrn_file_batch": x.get("vrn_file_batch"),
                        "validate": x.get("validate")}
                       for x in grouped_calls]
        final = grouped_calls[0]
        def orig_variantcaller_order(x):
            return final["config"]["algorithm"]["orig_variantcaller"].index(x["variantcaller"])
        if len(ready_calls) > 1 and "orig_variantcaller" in final["config"]["algorithm"]:
            final["variants"] = sorted(ready_calls, key=orig_variantcaller_order)
            final["config"]["algorithm"]["variantcaller"] = final["config"]["algorithm"].pop("orig_variantcaller")
        else:
            final["variants"] = ready_calls
        final.pop("vrn_file_batch", None)
        out.append([final])
    return out

def _split_by_ready_regions(ext, file_key, dir_ext_fn):
    """Organize splits based on regions generated by parallel_prep_region.
    """
    def _do_work(data):
        if "region" in data:
            name = data["group"][0] if "group" in data else data["description"]
            out_dir = os.path.join(data["dirs"]["work"], dir_ext_fn(data))
            out_file = os.path.join(out_dir, "%s%s" % (name, ext))
            assert isinstance(data["region"], (list, tuple))
            out_parts = []
            for i, r in enumerate(data["region"]):
                out_region_dir = os.path.join(out_dir, r[0])
                out_region_file = os.path.join(out_region_dir,
                                               "%s-%s%s" % (name, region.to_safestr(r), ext))
                work_bams = []
                for xs in data["region_bams"]:
                    if len(xs) == 1:
                        work_bams.append(xs[0])
                    else:
                        work_bams.append(xs[i])
                for work_bam in work_bams:
                    assert os.path.exists(work_bam), work_bam
                out_parts.append((r, work_bams, out_region_file))
            return out_file, out_parts
        else:
            return None, []
    return _do_work

def _collapse_by_bam_variantcaller(samples):
    """Collapse regions to a single representative by BAM input and variant caller.
    """
    by_bam = collections.OrderedDict()
    for data in (x[0] for x in samples):
        work_bam = utils.get_in(data, ("combine", "work_bam", "out"), data.get("align_bam"))
        variantcaller = get_variantcaller(data)
        if isinstance(work_bam, list):
            work_bam = tuple(work_bam)
        key = (data["description"], work_bam, variantcaller)
        try:
            by_bam[key].append(data)
        except KeyError:
            by_bam[key] = [data]
    out = []
    for grouped_data in by_bam.values():
        cur = grouped_data[0]
        cur.pop("region", None)
        region_bams = cur.pop("region_bams", None)
        if region_bams and len(region_bams[0]) > 1:
            cur.pop("work_bam", None)
        out.append([cur])
    return out

def parallel_variantcall_region(samples, run_parallel):
    """Perform variant calling and post-analysis on samples by region.
    """
    to_process = []
    extras = []
    for x in samples:
        added = False
        for add in handle_multiple_variantcallers(x):
            added = True
            to_process.append(add)
        if not added:
            extras.append(x)
    split_fn = _split_by_ready_regions(".vcf.gz", "work_bam", get_variantcaller)
    samples = _collapse_by_bam_variantcaller(
        grouped_parallel_split_combine(to_process, split_fn,
                                       multi.group_batches, run_parallel,
                                       "variantcall_sample", "concat_variant_files",
                                       "vrn_file", ["region", "sam_ref", "config"]))
    return extras + samples

def handle_multiple_variantcallers(data):
    """Split samples that potentially require multiple variant calling approaches.
    """
    assert len(data) == 1
    callers = get_variantcaller(data[0])
    if isinstance(callers, basestring):
        return [data]
    elif not callers:
        return []
    else:
        out = []
        for caller in callers:
            base = copy.deepcopy(data[0])
            base["config"]["algorithm"]["orig_variantcaller"] = \
              base["config"]["algorithm"]["variantcaller"]
            base["config"]["algorithm"]["variantcaller"] = caller
            out.append([base])
        return out

def get_variantcallers():
    from bcbio.variation import freebayes, cortex, samtools, varscan, mutect, scalpel
    return {"gatk": gatk.unified_genotyper,
            "gatk-haplotype": gatk.haplotype_caller,
            "freebayes": freebayes.run_freebayes,
            "cortex": cortex.run_cortex,
            "samtools": samtools.run_samtools,
            "varscan": varscan.run_varscan,
            "mutect": mutect.mutect_caller,
            "scalpel": scalpel.run_scalpel}

def variantcall_sample(data, region=None, align_bams=None, out_file=None):
    """Parallel entry point for doing genotyping of a region of a sample.
    """
    if out_file is None or not os.path.exists(out_file) or not os.path.lexists(out_file):
        utils.safe_makedir(os.path.dirname(out_file))
        sam_ref = data["sam_ref"]
        config = data["config"]
        caller_fns = get_variantcallers()
        caller_fn = caller_fns[config["algorithm"].get("variantcaller", "gatk")]
        if len(align_bams) == 1:
            items = [data]
        else:
            items = multi.get_orig_items(data)
            assert len(items) == len(align_bams)
        call_file = "%s-raw%s" % utils.splitext_plus(out_file)
        call_file = caller_fn(align_bams, items, sam_ref,
                              data["genome_resources"]["variation"],
                              region, call_file)
        if data["config"]["algorithm"].get("phasing", False) == "gatk":
            call_file = phasing.read_backed_phasing(call_file, align_bams, sam_ref, region, config)
        utils.symlink_plus(call_file, out_file)
    if region:
        data["region"] = region
    data["vrn_file"] = out_file
    return [data]
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division

import os as _os

import numpy as np
import pandas as pd

from mousestyles import data_dir

def load_all_features():
    """
    Returns a (21131, 13) size pandas.DataFrame object corresponding to
    9 features over each mouse's 2-hour time bin. The first four columns
    index each mouses's 2-hour bin:

    Column 0: the strain of the mouse (0-15)
    Column 1: the mouse number (number depends on strain)
    Column 2: the day number (5-16)
    Column 3: the 2-hour time bin (e.g., value 4 corresponds to hours 4 to 6)

    The remaining 9 columns are the computed features.

    Returns
    -------
    features_data_frame : pandas.DataFrame
        A dataframe of computed features.
    """
    features = [
        'ASProbability',
        'ASNumbers',
        'ASDurations',
        'Food',
        'Water',
        'Distance',
        'ASFoodIntensity',
        'ASWaterIntensity',
        'MoveASIntensity']

    # 9 x 1921 x (3 labels + 11 feature time bins)
    all_features = np.load(
        _os.path.join(
            data_dir,
            'all_features_mousedays_11bins.npy'))

    # Here we begin reshaping the 3-d numpy array into a pandas 2-d dataframe
    columns = ['strain', 'mouse', 'day']
    # Append 2-hour time bin values
    columns += list(range(0, 22, 2))

    # For each feature, unpivot its dataframe so that the 2-hour
    # time bins become a value column, rather than a dimension
    data_frames = []
    for (i, feature) in enumerate(features):
        df = pd.DataFrame(data=all_features[i, :, :], columns=columns)
        melted_df = pd.melt(df, id_vars=columns[:3], value_vars=columns[3:],
                            var_name='hour', value_name=feature)
        data_frames.append(melted_df)

    # Connect each of the feature dataframes to make one dataframe
    all_features_df = data_frames.pop(0)
    other_features = [x.iloc[:, -1] for x in data_frames]
    other_features.insert(0, all_features_df)
    return pd.concat(other_features, axis=1)

def load_intervals(feature):
    """
    Parameters
    ----------
    feature: {"AS", "F", "IS", "M_AS", "M_IS", "W"}

    Returns
    -------
    intervals : pandas.DataFrame
        All the data from the feature directory into one data frame
        columns are strain, mouse, day, start, stop

    Examples
    --------
    >>> AS = load_intervals('AS')
    >>> IS = load_intervals('IS')
    """
    # check input is one of the provided choices
    if feature not in {"AS", "F", "IS", "M_AS", "M_IS", "W"}:
        raise ValueError(
            'Input value must be one of {"AS", "F", "IS", "M_AS", "M_IS", "W"}'
            )
    # get all file names
    file_names = _os.listdir(_os.path.join(data_dir, "intervals", feature))
    # check if directory is empty
    if len(file_names) == 0:
        raise ValueError('Directory is empty; no file found.')
    # initialized data frame
    # use for loop to load every file and concat to overall data frame
    dt = pd.DataFrame()
    for item in file_names:
        strain = int(item.split("strain")[1].split("_mouse")[0])
        mouse = int(item.split("mouse")[1].split("_day")[0])
        day = int(item.split("day")[1].split(".npy")[0])
        path = _os.path.join(data_dir, "intervals", feature, item)
        sub = np.load(path)
        dt_sub = pd.DataFrame()
        dt_sub["strain"] = [strain] * sub.shape[0]
        dt_sub["mouse"] = [mouse] * sub.shape[0]
        dt_sub["day"] = [day] * sub.shape[0]
        dt_sub["start"] = sub[:, 0]
        dt_sub["stop"] = sub[:, 1]
        dt = pd.concat([dt, dt_sub])
    # sort based on strain, mouse and day
    dt = dt.sort(["strain", "mouse", "day"])
    dt.index = range(dt.shape[0])
    return dt

def load_movement(strain, mouse, day):
    """
    Parameters
    ----------
    strain: int
        nonnegative integer indicating the strain number
    mouse: int
        nonnegative integer indicating the mouse number
    day: int
        nonnegative integer indicating the day number

    Returns
    -------
    movement : pandas.DataFrame
        CT, CX, CY coordinates of the specified strain-mouse-day
        columns are t, x, y and isHB
        isHB indicates whether the point is in the home base or not

    Examples
    --------
    >>> movement = load_movement(0, 0, 0)
    >>> movement = load_movement(1, 2, 1)
    """
    # check if all inputs are nonnegative integers
    conditions_value = [strain < 0, mouse < 0, day < 0]
    conditions_type = [type(strain) != int, type(mouse) != int,
                       type(day) != int]
    if any(conditions_value):
        raise ValueError("Input values need to be nonnegative")
    if any(conditions_type):
        raise TypeError("Input values need to be integer")
    # load all four files of HB, CT, CX and CY data
    HB_path = "txy_coords/C_idx_HB/C_idx_HB_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    CT_path = "txy_coords/CT/CT_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    CX_path = "txy_coords/CX/CX_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    CY_path = "txy_coords/CY/CY_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    HB = np.load(_os.path.join(data_dir, HB_path))
    CT = np.load(_os.path.join(data_dir, CT_path))
    CX = np.load(_os.path.join(data_dir, CX_path))
    CY = np.load(_os.path.join(data_dir, CY_path))
    # make data frame
    dt = pd.DataFrame()
    dt["t"] = CT
    dt["x"] = CX
    dt["y"] = CY
    dt["isHB"] = HB
    return dt
<commit_msg>add descriptions to functions<commit_after>from __future__ import print_function, absolute_import, division

import os as _os

import numpy as np
import pandas as pd

from mousestyles import data_dir

def load_all_features():
    """
    Returns a (21131, 13) size pandas.DataFrame object corresponding to
    9 features over each mouse's 2-hour time bin. The first four columns
    index each mouses's 2-hour bin:

    Column 0: the strain of the mouse (0-15)
    Column 1: the mouse number (number depends on strain)
    Column 2: the day number (5-16)
    Column 3: the 2-hour time bin (e.g., value 4 corresponds to hours 4 to 6)

    The remaining 9 columns are the computed features.

    Returns
    -------
    features_data_frame : pandas.DataFrame
        A dataframe of computed features.
    """
    features = [
        'ASProbability',
        'ASNumbers',
        'ASDurations',
        'Food',
        'Water',
        'Distance',
        'ASFoodIntensity',
        'ASWaterIntensity',
        'MoveASIntensity']

    # 9 x 1921 x (3 labels + 11 feature time bins)
    all_features = np.load(
        _os.path.join(
            data_dir,
            'all_features_mousedays_11bins.npy'))

    # Here we begin reshaping the 3-d numpy array into a pandas 2-d dataframe
    columns = ['strain', 'mouse', 'day']
    # Append 2-hour time bin values
    columns += list(range(0, 22, 2))

    # For each feature, unpivot its dataframe so that the 2-hour
    # time bins become a value column, rather than a dimension
    data_frames = []
    for (i, feature) in enumerate(features):
        df = pd.DataFrame(data=all_features[i, :, :], columns=columns)
        melted_df = pd.melt(df, id_vars=columns[:3], value_vars=columns[3:],
                            var_name='hour', value_name=feature)
        data_frames.append(melted_df)

    # Connect each of the feature dataframes to make one dataframe
    all_features_df = data_frames.pop(0)
    other_features = [x.iloc[:, -1] for x in data_frames]
    other_features.insert(0, all_features_df)
    return pd.concat(other_features, axis=1)

def load_intervals(feature):
    """
    Return a pandas.DataFrame object of project interval data
    for the specified feature.

    There are 5 columns in the dataframe:
    strain: the strain number of the mouse
    mouse: the mouse number in its strain
    day: the day number
    start: the start time
    stop: the stop time

    Parameters
    ----------
    feature: {"AS", "F", "IS", "M_AS", "M_IS", "W"}

    Returns
    -------
    intervals : pandas.DataFrame
        All data of the specified feature as a dataframe

    Examples
    --------
    >>> AS = load_intervals('AS')
    >>> IS = load_intervals('IS')
    """
    # check input is one of the provided choices
    if feature not in {"AS", "F", "IS", "M_AS", "M_IS", "W"}:
        raise ValueError(
            'Input value must be one of {"AS", "F", "IS", "M_AS", "M_IS", "W"}'
            )
    # get all file names
    file_names = _os.listdir(_os.path.join(data_dir, "intervals", feature))
    # check if directory is empty
    if len(file_names) == 0:
        raise ValueError('Directory is empty; no file found.')
    # initialized data frame
    # use for loop to load every file and concat to overall data frame
    dt = pd.DataFrame()
    for item in file_names:
        strain = int(item.split("strain")[1].split("_mouse")[0])
        mouse = int(item.split("mouse")[1].split("_day")[0])
        day = int(item.split("day")[1].split(".npy")[0])
        path = _os.path.join(data_dir, "intervals", feature, item)
        sub = np.load(path)
        dt_sub = pd.DataFrame()
        dt_sub["strain"] = [strain] * sub.shape[0]
        dt_sub["mouse"] = [mouse] * sub.shape[0]
        dt_sub["day"] = [day] * sub.shape[0]
        dt_sub["start"] = sub[:, 0]
        dt_sub["stop"] = sub[:, 1]
        dt = pd.concat([dt, dt_sub])
    # sort based on strain, mouse and day
    dt = dt.sort(["strain", "mouse", "day"])
    dt.index = range(dt.shape[0])
    return dt

def load_movement(strain, mouse, day):
    """
    Return a pandas.DataFrame object of project movement data
    for the specified combination of strain, mouse and day.

    There are 4 columns in the dataframe:
    t: Time coordinates (in seconds)
    x: X coordinates indicating the left-right position of the cage
    y: Y coordinates indicating the front-back position of the cage
    isHB: Boolean indicating whether the point is in the home base or not

    Parameters
    ----------
    strain: int
        nonnegative integer indicating the strain number
    mouse: int
        nonnegative integer indicating the mouse number
    day: int
        nonnegative integer indicating the day number

    Returns
    -------
    movement : pandas.DataFrame
        CT, CX, CY coordinates and home base status
        of the combination of strain, mouse and day

    Examples
    --------
    >>> movement = load_movement(0, 0, 0)
    >>> movement = load_movement(1, 2, 1)
    """
    # check if all inputs are nonnegative integers
    conditions_value = [strain < 0, mouse < 0, day < 0]
    conditions_type = [type(strain) != int, type(mouse) != int,
                       type(day) != int]
    if any(conditions_value):
        raise ValueError("Input values need to be nonnegative")
    if any(conditions_type):
        raise TypeError("Input values need to be integer")
    # load all four files of HB, CT, CX and CY data
    HB_path = "txy_coords/C_idx_HB/C_idx_HB_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    CT_path = "txy_coords/CT/CT_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    CX_path = "txy_coords/CX/CX_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    CY_path = "txy_coords/CY/CY_strain{}_mouse{}_day{}.npy".\
        format(strain, mouse, day)
    HB = np.load(_os.path.join(data_dir, HB_path))
    CT = np.load(_os.path.join(data_dir, CT_path))
    CX = np.load(_os.path.join(data_dir, CX_path))
    CY = np.load(_os.path.join(data_dir, CY_path))
    # make data frame
    dt = pd.DataFrame()
    dt["t"] = CT
    dt["x"] = CX
    dt["y"] = CY
    dt["isHB"] = HB
    return dt
<|endoftext|>


<commit_before># These are versions of the functions in django.utils.translation.trans_real
# that don't actually do anything. This is purely for performance, so that
# settings.USE_I18N = False can use this module rather than trans_real.py.

from django.conf import settings

def ngettext(singular, plural, number):
    if number == 1: return singular
    return plural
ngettext_lazy = ngettext

gettext = gettext_noop = gettext_lazy = _ = lambda x: x
string_concat = lambda *strings: ''.join([str(el) for el in strings])
activate = lambda x: None
deactivate = install = lambda: None
get_language = lambda: 'en'
get_date_formats = lambda: settings.DATE_FORMAT, settings.DATETIME_FORMAT, settings.TIME_FORMAT
get_partial_date_formats = lambda: settings.YEAR_MONTH_FORMAT, settings.MONTH_DAY_FORMAT
<commit_msg>Fixed #2310 -- Changed trans_null.py to use settings.LANGUAGE_CODE instead of hard-coded 'en'<commit_after># These are versions of the functions in django.utils.translation.trans_real
# that don't actually do anything. This is purely for performance, so that
# settings.USE_I18N = False can use this module rather than trans_real.py.

from django.conf import settings

def ngettext(singular, plural, number):
    if number == 1: return singular
    return plural
ngettext_lazy = ngettext

gettext = gettext_noop = gettext_lazy = _ = lambda x: x
string_concat = lambda *strings: ''.join([str(el) for el in strings])
activate = lambda x: None
deactivate = install = lambda: None
get_language = lambda: settings.LANGUAGE_CODE
get_date_formats = lambda: settings.DATE_FORMAT, settings.DATETIME_FORMAT, settings.TIME_FORMAT
get_partial_date_formats = lambda: settings.YEAR_MONTH_FORMAT, settings.MONTH_DAY_FORMAT
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
import sys
import os
import json
import codecs
#from threading import Timer
import time
import requests
import urllib

if sys.stdout.encoding != 'UTF-8':
	if sys.version_info.major == 3:
		sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
	else:
		sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict')
if sys.stderr.encoding != 'UTF-8':
	if sys.version_info.major == 3:
		sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
	else:
		sys.stderr = codecs.getwriter('utf-8')(sys.stderr, 'strict')

# load artifact
artifact = json.load(sys.stdin)

def error(message):
	print('{{"errorMessage":"{}"}}'.format(message))
	sys.exit(1)

def get_param(name, default=None, message=None, current=artifact):
	if isinstance(name, str):
		name = name.split('.')
	if len(name) == 0:
		return current
	else:
		value = current.get(name[0])
		if value == None:
			if message != None:
				error(message)
			else:
				return default
		else:
			return get_param(name[1:], default, message, value)

def debug(msg):
	#print >> sys.stderr, msg
	pass

http_proxy     = get_param('config.proxy.http')
https_proxy    = get_param('config.proxy.https')
max_tlp        = get_param('config.max_tlp', 1)
tlp            = get_param('tlp', 2) # amber by default
data_type      = get_param('dataType', None, 'Missing dataType field')
service        = get_param('config.service', None, 'Service parameter is missing')
otx_key        = get_param('config.key', None, 'Missing OTX API key')

def OTX_Query_IP(data):
	debug('>> OTX_Query_IP ' + str(data))
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/IPv4/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','reputation','geo','malware','url_list','passive_dns']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	json.dump({
		'pulse_count': IP_['general']['pulse_info']['count'],
		'pulses': IP_['general']['pulse_info']['pulses'],
		'whois': IP_['general']['whois'],
		'continent_code': IP_['geo']['continent_code'],
		'country_code': IP_['geo']['country_code'],
		'country_name': IP_['geo']['country_name'],
		'city': IP_['geo']['city'],
		'longitude': IP_['general']['longitude'],
		'latitude': IP_['general']['latitude'],
		'asn': IP_['geo']['asn'],
		'malware_samples': IP_['malware']['result'],
		'url_list': IP_['url_list']['url_list'],
		'passive_dns': IP_['passive_dns']['passive_dns']
	}, sys.stdout, ensure_ascii=False)

def OTX_Query_Domain(data):
	debug('>> OTX_Query_Domain ' + str(data))
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/domain/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','geo','malware','url_list','passive_dns']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	json.dump({
		'pulse_count': IP_['general']['pulse_info']['count'],
		'pulses': IP_['general']['pulse_info']['pulses'],
		'whois': IP_['general']['whois'],
		'continent_code': IP_['geo']['continent_code'],
		'country_code': IP_['geo']['country_code'],
		'country_name': IP_['geo']['country_name'],
		'city': IP_['geo']['city'],
		'asn': IP_['geo']['asn'],
		'malware_samples': IP_['malware']['result'],
		'url_list': IP_['url_list']['url_list'],
		'passive_dns': IP_['passive_dns']['passive_dns']
	}, sys.stdout, ensure_ascii=False)

def OTX_Query_File(data):
	debug('>> OTX_Query_File ' + str(data))
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/file/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','analysis']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	if IP_['analysis']['analysis']: # file has been analyzed before
		json.dump({
			'pulse_count': IP_['general']['pulse_info']['count'],
			'pulses': IP_['general']['pulse_info']['pulses'],
			'malware': IP_['analysis']['malware'],
			'page_type': IP_['analysis']['page_type'],
			'sha1': IP_['analysis']['analysis']['info']['results']['sha1'],
			'sha256': IP_['analysis']['analysis']['info']['results']['sha256'],
			'md5': IP_['analysis']['analysis']['info']['results']['md5'],
			'file_class': IP_['analysis']['analysis']['info']['results']['file_class'],
			'file_type': IP_['analysis']['analysis']['info']['results']['file_type'],
			'filesize': IP_['analysis']['analysis']['info']['results']['filesize'],
			'ssdeep': IP_['analysis']['analysis']['info']['results']['ssdeep']
		}, sys.stdout, ensure_ascii=False)
	else: # file has not been analyzed before
		json.dump({
			'errortext': 'File has not previously been analyzed by OTX!',
			'pulse_count': IP_['general']['pulse_info']['count'],
			'pulses': IP_['general']['pulse_info']['pulses']
		}, sys.stdout, ensure_ascii=False)

def OTX_Query_URL(data):
	debug('>> OTX_Query_URL ' + str(data))
	data = urllib.quote_plus(data) # urlencode the URL that we are searching for
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/url/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','url_list']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	json.dump({
		'pulse_count': IP_['general']['pulse_info']['count'],
		'pulses': IP_['general']['pulse_info']['pulses'],
		'alexa': IP_['general']['alexa'],
		'whois': IP_['general']['whois'],
		'url_list': IP_['url_list']['url_list']
	}, sys.stdout, ensure_ascii=False)

# run  only if TLP condition is met
if tlp > max_tlp:
	error('Error with TLP value ; see max_tlp in config or tlp value in input data')

# setup proxy
if http_proxy != None:
	os.environ['http_proxy'] = http_proxy
if https_proxy != None:
	os.environ['https_proxy'] = https_proxy

if service == 'query':
	if data_type == 'file':
		hashes = get_param('attachment.hashes', None, 'Hash is missing')
		# find SHA256 hash
		hash = next(h for h in hashes if len(h) == 64)
		OTX_Query_File(hash)
	elif data_type == 'url':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_URL(data)
	elif data_type == 'domain':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_Domain(data)
	elif data_type == 'ip':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_IP(data)
	elif data_type == 'hash':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_File(data)
	else:
		error('Invalid data type')
else:
	error('Invalid service')
<commit_msg>#32 Manage exception for a domain that does not exist : geo query returns {} and the program raises an error<commit_after>#!/usr/bin/env python
# encoding: utf-8
import sys
import os
import json
import codecs
#from threading import Timer
import time
import requests
import urllib

if sys.stdout.encoding != 'UTF-8':
	if sys.version_info.major == 3:
		sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
	else:
		sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict')
if sys.stderr.encoding != 'UTF-8':
	if sys.version_info.major == 3:
		sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
	else:
		sys.stderr = codecs.getwriter('utf-8')(sys.stderr, 'strict')

# load artifact
artifact = json.load(sys.stdin)

def error(message):
	print('{{"errorMessage":"{}"}}'.format(message))
	sys.exit(1)

def get_param(name, default=None, message=None, current=artifact):
	if isinstance(name, str):
		name = name.split('.')
	if len(name) == 0:
		return current
	else:
		value = current.get(name[0])
		if value == None:
			if message != None:
				error(message)
			else:
				return default
		else:
			return get_param(name[1:], default, message, value)

def debug(msg):
	#print >> sys.stderr, msg
	pass

http_proxy     = get_param('config.proxy.http')
https_proxy    = get_param('config.proxy.https')
max_tlp        = get_param('config.max_tlp', 1)
tlp            = get_param('tlp', 2) # amber by default
data_type      = get_param('dataType', None, 'Missing dataType field')
service        = get_param('config.service', None, 'Service parameter is missing')
otx_key        = get_param('config.key', None, 'Missing OTX API key')

def OTX_Query_IP(data):
	debug('>> OTX_Query_IP ' + str(data))
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/IPv4/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','reputation','geo','malware','url_list','passive_dns']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	json.dump({
		'pulse_count': IP_['general']['pulse_info']['count'],
		'pulses': IP_['general']['pulse_info']['pulses'],
		'whois': IP_['general']['whois'],
		'continent_code': IP_['geo']['continent_code'],
		'country_code': IP_['geo']['country_code'],
		'country_name': IP_['geo']['country_name'],
		'city': IP_['geo']['city'],
		'longitude': IP_['general']['longitude'],
		'latitude': IP_['general']['latitude'],
		'asn': IP_['geo']['asn'],
		'malware_samples': IP_['malware']['result'],
		'url_list': IP_['url_list']['url_list'],
		'passive_dns': IP_['passive_dns']['passive_dns']
	}, sys.stdout, ensure_ascii=False)

def OTX_Query_Domain(data):
	debug('>> OTX_Query_Domain ' + str(data))
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/domain/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','geo','malware','url_list','passive_dns']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	result = {
		'pulse_count': IP_['general']['pulse_info']['count'],
		'pulses': IP_['general']['pulse_info']['pulses'],
		'whois': IP_['general']['whois'],
		'malware_samples': IP_['malware']['result'],
		'url_list': IP_['url_list']['url_list'],
		'passive_dns': IP_['passive_dns']['passive_dns']}

	try:
		result.update({
			'continent_code': IP_['geo']['continent_code'],
			'country_code': IP_['geo']['country_code'],
			'country_name': IP_['geo']['country_name'],
			'city': IP_['geo']['city'],
			'asn': IP_['geo']['asn']})
	except Exception:
		pass

	json.dump(result,sys.stdout, ensure_ascii=False)

def OTX_Query_File(data):
	debug('>> OTX_Query_File ' + str(data))
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/file/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','analysis']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	if IP_['analysis']['analysis']: # file has been analyzed before
		json.dump({
			'pulse_count': IP_['general']['pulse_info']['count'],
			'pulses': IP_['general']['pulse_info']['pulses'],
			'malware': IP_['analysis']['malware'],
			'page_type': IP_['analysis']['page_type'],
			'sha1': IP_['analysis']['analysis']['info']['results']['sha1'],
			'sha256': IP_['analysis']['analysis']['info']['results']['sha256'],
			'md5': IP_['analysis']['analysis']['info']['results']['md5'],
			'file_class': IP_['analysis']['analysis']['info']['results']['file_class'],
			'file_type': IP_['analysis']['analysis']['info']['results']['file_type'],
			'filesize': IP_['analysis']['analysis']['info']['results']['filesize'],
			'ssdeep': IP_['analysis']['analysis']['info']['results']['ssdeep']
		}, sys.stdout, ensure_ascii=False)
	else: # file has not been analyzed before
		json.dump({
			'errortext': 'File has not previously been analyzed by OTX!',
			'pulse_count': IP_['general']['pulse_info']['count'],
			'pulses': IP_['general']['pulse_info']['pulses']
		}, sys.stdout, ensure_ascii=False)

def OTX_Query_URL(data):
	debug('>> OTX_Query_URL ' + str(data))
	data = urllib.quote_plus(data) # urlencode the URL that we are searching for
	baseurl = "https://otx.alienvault.com:443/api/v1/indicators/url/%s/" % data
	headers = {'X-OTX-API-KEY': otx_key, 'Accept':'application/json'}
	sections = ['general','url_list']
	IP_={}
	try:
		for section in sections:
			queryurl = baseurl + section
			IP_[section] = json.loads(requests.get(queryurl, headers=headers).content)
	except:
		error('API Error! Please verify data type is correct.')

	json.dump({
		'pulse_count': IP_['general']['pulse_info']['count'],
		'pulses': IP_['general']['pulse_info']['pulses'],
		'alexa': IP_['general']['alexa'],
		'whois': IP_['general']['whois'],
		'url_list': IP_['url_list']['url_list']
	}, sys.stdout, ensure_ascii=False)

# run  only if TLP condition is met
if tlp > max_tlp:
	error('Error with TLP value ; see max_tlp in config or tlp value in input data')

# setup proxy
if http_proxy != None:
	os.environ['http_proxy'] = http_proxy
if https_proxy != None:
	os.environ['https_proxy'] = https_proxy

if service == 'query':
	if data_type == 'file':
		hashes = get_param('attachment.hashes', None, 'Hash is missing')
		# find SHA256 hash
		hash = next(h for h in hashes if len(h) == 64)
		OTX_Query_File(hash)
	elif data_type == 'url':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_URL(data)
	elif data_type == 'domain':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_Domain(data)
	elif data_type == 'ip':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_IP(data)
	elif data_type == 'hash':
		data = get_param('data', None, 'Data is missing')
		OTX_Query_File(data)
	else:
		error('Invalid data type')
else:
	error('Invalid service')
<|endoftext|>


<commit_before># Copyright (C) 2015 UCSC Computational Genomics Lab
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os
import random
import socket
import sys
import threading
import pickle
import logging
import subprocess
import traceback
from time import sleep, time

import psutil
import mesos.interface
from bd2k.util.expando import Expando
from mesos.interface import mesos_pb2
import mesos.native
from struct import pack
from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem, WorkerCleanupInfo
from toil.resource import Resource

log = logging.getLogger(__name__)

class MesosExecutor(mesos.interface.Executor):
    """
    Part of mesos framework, runs on mesos slave. A toil job is passed to it via the task.data field, and launched
    via call(toil.command). Uses the ExecutorDriver to communicate.
    """

    def __init__(self):
        super(MesosExecutor, self).__init__()
        self.popenLock = threading.Lock()
        self.runningTasks = {}
        self.workerCleanupInfo = None
        Resource.prepareSystem()
        self.address = socket.gethostbyname(socket.gethostname())

    def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
        """
        Invoked once the executor driver has been able to successfully connect with Mesos.
        """
        log.info("Registered with framework")
        nodeInfoThread = threading.Thread(target=self._sendFrameworkMessage, args=[driver])
        nodeInfoThread.daemon = True
        nodeInfoThread.start()

    def reregistered(self, driver, slaveInfo):
        """
        Invoked when the executor re-registers with a restarted slave.
        """
        log.info("Re-registered")

    def disconnected(self, driver):
        """
        Invoked when the executor becomes "disconnected" from the slave (e.g., the slave is being
        restarted due to an upgrade).
        """
        log.critical("Disconnected from slave")

    def killTask(self, driver, taskId):
        try:
            pid = self.runningTasks[taskId]
        except KeyError:
            pass
        else:
            os.kill(pid, 9)

    def shutdown(self, driver):
        log.critical('Shutting down executor ...')
        for taskId in self.runningTasks.keys():
            self.killTask(driver, taskId)
        Resource.cleanSystem()
        AbstractBatchSystem.workerCleanup(self.workerCleanupInfo)
        log.critical('... executor shut down.')

    def error(self, driver, message):
        """
        Invoked when a fatal error has occurred with the executor and/or executor driver.
        """
        log.critical("FATAL ERROR: " + message)

    def _sendFrameworkMessage(self, driver):
        message = None
        while True:
            # The psutil documentation recommends that we ignore the value returned by the first
            # invocation of cpu_percent(). However, we do want to send a sign of life early after
            # starting (e.g. to unblock the provisioner waiting for an instance to come up) so
            # the first message we send omits the load info.
            if message is None:
                message = Expando(address=self.address)
                psutil.cpu_percent()
            else:
                message.nodeInfo = dict(cores=float(psutil.cpu_percent()) * .01,
                                        memory=float(psutil.virtual_memory().percent) * .01,
                                        workers=len(self.runningTasks))
            driver.sendFrameworkMessage(repr(message))
            # Prevent workers launched together from hitting the leader at the same time,
            # all the time.
            sleep(random.randint(45, 75))

    def launchTask(self, driver, task):
        """
        Invoked by SchedulerDriver when a Mesos task should be launched by this executor
        """

        def runTask():
            log.debug("Running task %s", task.task_id.value)
            sendUpdate(mesos_pb2.TASK_RUNNING)
            # This is where task.data is first invoked. Using this position to setup cleanupInfo
            taskData = pickle.loads(task.data)
            if self.workerCleanupInfo is not None:
                assert self.workerCleanupInfo == taskData.workerCleanupInfo
            else:
                self.workerCleanupInfo = taskData.workerCleanupInfo
            try:
                popen = runJob(taskData)
                startTime = time()
                self.runningTasks[task.task_id.value] = popen.pid
                try:
                    exitStatus = popen.wait()
                    wallTime = time() - startTime
                    if 0 == exitStatus:
                        sendUpdate(mesos_pb2.TASK_FINISHED, wallTime)
                    elif -9 == exitStatus:
                        sendUpdate(mesos_pb2.TASK_KILLED, wallTime)
                    else:
                        sendUpdate(mesos_pb2.TASK_FAILED, wallTime, message=str(exitStatus))
                finally:
                    del self.runningTasks[task.task_id.value]
            except:
                wallTime = time() - startTime
                exc_info = sys.exc_info()
                log.error('Exception while running task:', exc_info=exc_info)
                exc_type, exc_value, exc_trace = exc_info
                sendUpdate(mesos_pb2.TASK_FAILED, wallTime,
                           message=''.join(traceback.format_exception_only(exc_type, exc_value)))

        def runJob(job):
            """
            :type job: toil.batchSystems.mesos.ToilJob

            :rtype: subprocess.Popen
            """
            if job.userScript:
                job.userScript.register()
            log.debug("Invoking command: '%s'", job.command)
            with self.popenLock:
                return subprocess.Popen(job.command,
                                        shell=True, env=dict(os.environ, **job.environment))

        def sendUpdate(taskState, wallTime, message=''):
            log.debug('Sending task status update ...')
            status = mesos_pb2.TaskStatus()
            status.task_id.value = task.task_id.value
            status.message = message
            status.state = taskState
            status.data = pack('d', wallTime)
            driver.sendStatusUpdate(status)
            log.debug('... done sending task status update.')

        thread = threading.Thread(target=runTask)
        thread.start()

    def frameworkMessage(self, driver, message):
        """
        Invoked when a framework message has arrived for this executor.
        """
        log.debug("Received message from framework: {}".format(message))

def main(executorClass=MesosExecutor):
    logging.basicConfig(level=logging.DEBUG)
    log.debug("Starting executor")
    executor = executorClass()
    driver = mesos.native.MesosExecutorDriver(executor)
    exit_value = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
    assert len(executor.runningTasks) == 0
    sys.exit(exit_value)
<commit_msg>Fix attribute error in Mesos executor shutdown<commit_after># Copyright (C) 2015 UCSC Computational Genomics Lab
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os
import random
import socket
import sys
import threading
import pickle
import logging
import subprocess
import traceback
from time import sleep, time

import psutil
import mesos.interface
from bd2k.util.expando import Expando
from mesos.interface import mesos_pb2
import mesos.native
from struct import pack
from toil.batchSystems.abstractBatchSystem import BatchSystemSupport
from toil.resource import Resource

log = logging.getLogger(__name__)

class MesosExecutor(mesos.interface.Executor):
    """
    Part of mesos framework, runs on mesos slave. A toil job is passed to it via the task.data field, and launched
    via call(toil.command). Uses the ExecutorDriver to communicate.
    """

    def __init__(self):
        super(MesosExecutor, self).__init__()
        self.popenLock = threading.Lock()
        self.runningTasks = {}
        self.workerCleanupInfo = None
        Resource.prepareSystem()
        self.address = socket.gethostbyname(socket.gethostname())

    def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
        """
        Invoked once the executor driver has been able to successfully connect with Mesos.
        """
        log.info("Registered with framework")
        nodeInfoThread = threading.Thread(target=self._sendFrameworkMessage, args=[driver])
        nodeInfoThread.daemon = True
        nodeInfoThread.start()

    def reregistered(self, driver, slaveInfo):
        """
        Invoked when the executor re-registers with a restarted slave.
        """
        log.info("Re-registered")

    def disconnected(self, driver):
        """
        Invoked when the executor becomes "disconnected" from the slave (e.g., the slave is being
        restarted due to an upgrade).
        """
        log.critical("Disconnected from slave")

    def killTask(self, driver, taskId):
        try:
            pid = self.runningTasks[taskId]
        except KeyError:
            pass
        else:
            os.kill(pid, 9)

    def shutdown(self, driver):
        log.critical('Shutting down executor ...')
        for taskId in self.runningTasks.keys():
            self.killTask(driver, taskId)
        Resource.cleanSystem()
        BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
        log.critical('... executor shut down.')

    def error(self, driver, message):
        """
        Invoked when a fatal error has occurred with the executor and/or executor driver.
        """
        log.critical("FATAL ERROR: " + message)

    def _sendFrameworkMessage(self, driver):
        message = None
        while True:
            # The psutil documentation recommends that we ignore the value returned by the first
            # invocation of cpu_percent(). However, we do want to send a sign of life early after
            # starting (e.g. to unblock the provisioner waiting for an instance to come up) so
            # the first message we send omits the load info.
            if message is None:
                message = Expando(address=self.address)
                psutil.cpu_percent()
            else:
                message.nodeInfo = dict(cores=float(psutil.cpu_percent()) * .01,
                                        memory=float(psutil.virtual_memory().percent) * .01,
                                        workers=len(self.runningTasks))
            driver.sendFrameworkMessage(repr(message))
            # Prevent workers launched together from hitting the leader at the same time,
            # all the time.
            sleep(random.randint(45, 75))

    def launchTask(self, driver, task):
        """
        Invoked by SchedulerDriver when a Mesos task should be launched by this executor
        """

        def runTask():
            log.debug("Running task %s", task.task_id.value)
            sendUpdate(mesos_pb2.TASK_RUNNING)
            # This is where task.data is first invoked. Using this position to setup cleanupInfo
            taskData = pickle.loads(task.data)
            if self.workerCleanupInfo is not None:
                assert self.workerCleanupInfo == taskData.workerCleanupInfo
            else:
                self.workerCleanupInfo = taskData.workerCleanupInfo
            try:
                popen = runJob(taskData)
                startTime = time()
                self.runningTasks[task.task_id.value] = popen.pid
                try:
                    exitStatus = popen.wait()
                    wallTime = time() - startTime
                    if 0 == exitStatus:
                        sendUpdate(mesos_pb2.TASK_FINISHED, wallTime)
                    elif -9 == exitStatus:
                        sendUpdate(mesos_pb2.TASK_KILLED, wallTime)
                    else:
                        sendUpdate(mesos_pb2.TASK_FAILED, wallTime, message=str(exitStatus))
                finally:
                    del self.runningTasks[task.task_id.value]
            except:
                wallTime = time() - startTime
                exc_info = sys.exc_info()
                log.error('Exception while running task:', exc_info=exc_info)
                exc_type, exc_value, exc_trace = exc_info
                sendUpdate(mesos_pb2.TASK_FAILED, wallTime,
                           message=''.join(traceback.format_exception_only(exc_type, exc_value)))

        def runJob(job):
            """
            :type job: toil.batchSystems.mesos.ToilJob

            :rtype: subprocess.Popen
            """
            if job.userScript:
                job.userScript.register()
            log.debug("Invoking command: '%s'", job.command)
            with self.popenLock:
                return subprocess.Popen(job.command,
                                        shell=True, env=dict(os.environ, **job.environment))

        def sendUpdate(taskState, wallTime, message=''):
            log.debug('Sending task status update ...')
            status = mesos_pb2.TaskStatus()
            status.task_id.value = task.task_id.value
            status.message = message
            status.state = taskState
            status.data = pack('d', wallTime)
            driver.sendStatusUpdate(status)
            log.debug('... done sending task status update.')

        thread = threading.Thread(target=runTask)
        thread.start()

    def frameworkMessage(self, driver, message):
        """
        Invoked when a framework message has arrived for this executor.
        """
        log.debug("Received message from framework: {}".format(message))

def main(executorClass=MesosExecutor):
    logging.basicConfig(level=logging.DEBUG)
    log.debug("Starting executor")
    executor = executorClass()
    driver = mesos.native.MesosExecutorDriver(executor)
    exit_value = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
    assert len(executor.runningTasks) == 0
    sys.exit(exit_value)
<|endoftext|>


<commit_before>import sys
import pandas as pd
from edisgo.flex_opt import check_tech_constraints as checks
from .reinforce_measures import reinforce_branches_current, \
    reinforce_branches_voltage, extend_distribution_substation
import logging

logger = logging.getLogger('edisgo')

def reinforce_grid(network, while_counter_max=10):
    """ Evaluates grid reinforcement needs and performs measures. This function
        is the parent function for all grid reinforcements.

    Parameters
    ----------
    network : :class:`~.grid.network.Network`
    while_counter_max : int
        Maximum number of iterations when solving overvoltage problems, to
        prevent infinite grid expansion.

    Notes
    -----
    Vorgehen laut BW-Studie:
    * getrennte oder kombinierte Betrachtung von NS und MS muss noch entschieden
      werden, BW-Studie führt getrennte Betrachtung durch
    * Reihenfolge der Behebung von Grenzwertverletzungen:
    ** Trafo
    ** Spannung
    ** Leitungen
    * Thermische Belastung:
    ** vorhandene BM werden maximal durch ein weiteres gleiches BM verstärkt
    ** ist das nicht ausreichend, wird das BM durch beliebig viele Standard-
       BM ersetzt
    * Spannungsbandverletztung
    ** Strangauftrennung nach 2/3 der Distanz
    ** danach eventuell weitere Strangauftrennung wenn sinnvoll, sonst parallele
       BM

    Sonstiges:
    * nur Rückspeisefall
    ** NS: 20% Last, 85-100% Einspeisung, BM-Belastung 100%
    ** MS: 30% Last, 85-100% Einspeisung, BM-Belastung 100%
    * Spannungsbandaufteilung wie in Wuppertal Studie
    * bei Spannungsproblemen am Trafo wird nicht Trafo ausgebaut, sondern
      Leistung in der MS

    References
    ----------
    .. [1] dena VNS
    .. [2] Ackermann et al. (RP VNS)

    """

    def _add_lines_changes_to_equipment_changes():
        equipment, index, quantity = [], [], []
        for line, number_of_lines in lines_changes.items():
            equipment.append(line.type.name)
            index.append(line)
            quantity.append(number_of_lines)
        network.results.equipment_changes = \
            network.results.equipment_changes.append(
                pd.DataFrame(
                    {'iteration_step': [iteration_step] * len(
                        lines_changes),
                     'change': ['changed'] * len(lines_changes),
                     'equipment': equipment,
                     'quantity': quantity},
                    index=index))

    def _add_transformer_changes_to_equipment_changes(mode):
        for station, transformer_list in transformer_changes[mode].items():
            network.results.equipment_changes = \
                network.results.equipment_changes.append(
                    pd.DataFrame(
                        {'iteration_step': [iteration_step] * len(
                            transformer_list),
                         'change': [mode] * len(transformer_list),
                         'equipment': transformer_list,
                         'quantity': [1] * len(transformer_list)},
                        index=[station] * len(transformer_list)))

    # STEP 1: reinforce overloaded transformers
    iteration_step = 1

    logger.info('==> Check LV stations')

    # ToDo: check overloading of HV/MV Trafo?
    overloaded_stations = checks.mv_lv_station_load(network)

    if overloaded_stations:
        # reinforce substations
        transformer_changes = extend_distribution_substation(
            network, overloaded_stations)
        # write added and removed transformers to results.equipment_changes
        _add_transformer_changes_to_equipment_changes('added')
        _add_transformer_changes_to_equipment_changes('removed')

    # STEP 2: reinforce branches due to overloading

    crit_lines_lv = checks.lv_line_load(network)
    crit_lines_mv = checks.mv_line_load(network)
    crit_lines = {**crit_lines_lv, **crit_lines_mv}

    # do reinforcement
    if crit_lines:
        lines_changes = reinforce_branches_current(network, crit_lines)
        # write changed lines to results.equipment_changes
        _add_lines_changes_to_equipment_changes()

    # run power flow analysis again and check if all overloading
    # problems were solved
    network.analyze()
    overloaded_stations = checks.mv_lv_station_load(network)
    if overloaded_stations:
        logger.error("==> Overloading issues of LV stations were not "
                     "solved in the first iteration step.")
        sys.exit()

    crit_lines_lv = checks.lv_line_load(network)
    crit_lines_mv = checks.mv_line_load(network)
    crit_lines = {**crit_lines_lv, **crit_lines_mv}
    if crit_lines:
        logger.error("==> Overloading issues of lines were not "
                     "solved in the first iteration step.")
        sys.exit()

    # STEP 3: reinforce branches due to voltage problems

    iteration_step += 1

    # solve voltage problems in MV grid
    crit_nodes = checks.mv_voltage_deviation(network)

    # as long as there are voltage issues, do reinforcement
    while_counter = 0
    while crit_nodes and while_counter < while_counter_max:
        # ToDo: get crit_nodes as objects instead of string
        # for now iterate through grid to find node for repr
        crit_nodes_objects = pd.Series()
        for node in network.mv_grid.graph.nodes():
            if repr(node) in crit_nodes[network.mv_grid].index:
                crit_nodes_objects = pd.concat(
                    [crit_nodes_objects,
                     pd.Series(crit_nodes[network.mv_grid].loc[repr(node)],
                               index=[node])])
                break
        lines_changes = reinforce_branches_voltage(
            network, network.mv_grid, crit_nodes_objects)
        # write changed lines to results.equipment_changes
        _add_lines_changes_to_equipment_changes()
        network.analyze()
        iteration_step += 1
        crit_nodes = checks.mv_voltage_deviation(network)
        while_counter += 1

    logger.info('==> All voltage issues in MV grid are solved.')

    # solve voltage problems in LV grid
    crit_nodes = checks.lv_voltage_deviation(network)

    # as long as there are voltage issues, do reinforcement
    while_counter = 0
    while crit_nodes and while_counter < while_counter_max:
        # for every grid in crit_nodes do reinforcement
        for grid in crit_nodes:
            # for now iterate through grid to find node for repr
            crit_nodes_objects = pd.Series()
            for node in grid.graph.nodes():
                if repr(node) in crit_nodes[grid].index:
                    crit_nodes_objects = pd.concat(
                        [crit_nodes_objects,
                         pd.Series(crit_nodes[grid].loc[repr(node)],
                                   index=[node])])
                    break
            lines_changes = reinforce_branches_voltage(
                network, grid, crit_nodes_objects)
            # write changed lines to results.equipment_changes
            _add_lines_changes_to_equipment_changes()
        network.analyze()
        iteration_step += 1
        crit_nodes = checks.lv_voltage_deviation(network)
        while_counter += 1

    logger.info('==> All voltage issues in LV grids are solved.')

    # ToDo: recheck overloading problems of lines

<commit_msg>Check if all voltage problems were solved after maximum number of iterations allowed<commit_after>import sys
import pandas as pd
from edisgo.flex_opt import check_tech_constraints as checks
from .reinforce_measures import reinforce_branches_current, \
    reinforce_branches_voltage, extend_distribution_substation
import logging

logger = logging.getLogger('edisgo')

def reinforce_grid(network, while_counter_max=10):
    """ Evaluates grid reinforcement needs and performs measures. This function
        is the parent function for all grid reinforcements.

    Parameters
    ----------
    network : :class:`~.grid.network.Network`
    while_counter_max : int
        Maximum number of iterations when solving overvoltage problems, to
        prevent infinite grid expansion.

    Notes
    -----
    Vorgehen laut BW-Studie:
    * getrennte oder kombinierte Betrachtung von NS und MS muss noch entschieden
      werden, BW-Studie führt getrennte Betrachtung durch
    * Reihenfolge der Behebung von Grenzwertverletzungen:
    ** Trafo
    ** Spannung
    ** Leitungen
    * Thermische Belastung:
    ** vorhandene BM werden maximal durch ein weiteres gleiches BM verstärkt
    ** ist das nicht ausreichend, wird das BM durch beliebig viele Standard-
       BM ersetzt
    * Spannungsbandverletztung
    ** Strangauftrennung nach 2/3 der Distanz
    ** danach eventuell weitere Strangauftrennung wenn sinnvoll, sonst parallele
       BM

    Sonstiges:
    * nur Rückspeisefall
    ** NS: 20% Last, 85-100% Einspeisung, BM-Belastung 100%
    ** MS: 30% Last, 85-100% Einspeisung, BM-Belastung 100%
    * Spannungsbandaufteilung wie in Wuppertal Studie
    * bei Spannungsproblemen am Trafo wird nicht Trafo ausgebaut, sondern
      Leistung in der MS

    References
    ----------
    .. [1] dena VNS
    .. [2] Ackermann et al. (RP VNS)

    """

    def _add_lines_changes_to_equipment_changes():
        equipment, index, quantity = [], [], []
        for line, number_of_lines in lines_changes.items():
            equipment.append(line.type.name)
            index.append(line)
            quantity.append(number_of_lines)
        network.results.equipment_changes = \
            network.results.equipment_changes.append(
                pd.DataFrame(
                    {'iteration_step': [iteration_step] * len(
                        lines_changes),
                     'change': ['changed'] * len(lines_changes),
                     'equipment': equipment,
                     'quantity': quantity},
                    index=index))

    def _add_transformer_changes_to_equipment_changes(mode):
        for station, transformer_list in transformer_changes[mode].items():
            network.results.equipment_changes = \
                network.results.equipment_changes.append(
                    pd.DataFrame(
                        {'iteration_step': [iteration_step] * len(
                            transformer_list),
                         'change': [mode] * len(transformer_list),
                         'equipment': transformer_list,
                         'quantity': [1] * len(transformer_list)},
                        index=[station] * len(transformer_list)))

    # STEP 1: reinforce overloaded transformers
    iteration_step = 1

    logger.info('==> Check LV stations')

    # ToDo: check overloading of HV/MV Trafo?
    overloaded_stations = checks.mv_lv_station_load(network)

    if overloaded_stations:
        # reinforce substations
        transformer_changes = extend_distribution_substation(
            network, overloaded_stations)
        # write added and removed transformers to results.equipment_changes
        _add_transformer_changes_to_equipment_changes('added')
        _add_transformer_changes_to_equipment_changes('removed')

    # STEP 2: reinforce branches due to overloading

    crit_lines_lv = checks.lv_line_load(network)
    crit_lines_mv = checks.mv_line_load(network)
    crit_lines = {**crit_lines_lv, **crit_lines_mv}

    # do reinforcement
    if crit_lines:
        lines_changes = reinforce_branches_current(network, crit_lines)
        # write changed lines to results.equipment_changes
        _add_lines_changes_to_equipment_changes()

    # run power flow analysis again and check if all overloading
    # problems were solved
    network.analyze()
    overloaded_stations = checks.mv_lv_station_load(network)
    if overloaded_stations:
        logger.error("==> Overloading issues of LV stations were not "
                     "solved in the first iteration step.")
        sys.exit()

    crit_lines_lv = checks.lv_line_load(network)
    crit_lines_mv = checks.mv_line_load(network)
    crit_lines = {**crit_lines_lv, **crit_lines_mv}
    if crit_lines:
        logger.error("==> Overloading issues of lines were not "
                     "solved in the first iteration step.")
        sys.exit()

    # STEP 3: reinforce branches due to voltage problems

    iteration_step += 1

    # solve voltage problems in MV grid
    crit_nodes = checks.mv_voltage_deviation(network)

    # as long as there are voltage issues, do reinforcement
    while_counter = 0
    while crit_nodes and while_counter < while_counter_max:
        # ToDo: get crit_nodes as objects instead of string
        # for now iterate through grid to find node for repr
        crit_nodes_objects = pd.Series()
        for node in network.mv_grid.graph.nodes():
            if repr(node) in crit_nodes[network.mv_grid].index:
                crit_nodes_objects = pd.concat(
                    [crit_nodes_objects,
                     pd.Series(crit_nodes[network.mv_grid].loc[repr(node)],
                               index=[node])])
                break
        lines_changes = reinforce_branches_voltage(
            network, network.mv_grid, crit_nodes_objects)
        # write changed lines to results.equipment_changes
        _add_lines_changes_to_equipment_changes()
        network.analyze()
        iteration_step += 1
        crit_nodes = checks.mv_voltage_deviation(network)
        while_counter += 1

    # check if all voltage problems were solved after maximum number of
    # iterations allowed
    if while_counter == while_counter_max and crit_nodes:
        logger.error("==> Voltage issues in MV grid were not solved.")
        sys.exit()

    logger.info('==> All voltage issues in MV grid are solved.')

    # solve voltage problems in LV grid
    crit_nodes = checks.lv_voltage_deviation(network)

    # as long as there are voltage issues, do reinforcement
    while_counter = 0
    while crit_nodes and while_counter < while_counter_max:
        # for every grid in crit_nodes do reinforcement
        for grid in crit_nodes:
            # for now iterate through grid to find node for repr
            crit_nodes_objects = pd.Series()
            for node in grid.graph.nodes():
                if repr(node) in crit_nodes[grid].index:
                    crit_nodes_objects = pd.concat(
                        [crit_nodes_objects,
                         pd.Series(crit_nodes[grid].loc[repr(node)],
                                   index=[node])])
                    break
            lines_changes = reinforce_branches_voltage(
                network, grid, crit_nodes_objects)
            # write changed lines to results.equipment_changes
            _add_lines_changes_to_equipment_changes()
        network.analyze()
        iteration_step += 1
        crit_nodes = checks.lv_voltage_deviation(network)
        while_counter += 1

    # check if all voltage problems were solved after maximum number of
    # iterations allowed
    if while_counter == while_counter_max and crit_nodes:
        logger.error("==> Voltage issues in LV grids were not solved.")
        sys.exit()

    logger.info('==> All voltage issues in LV grids are solved.')

    # ToDo: recheck overloading problems of lines

<|endoftext|>


<commit_before>VERSION = '1.5.2-dev'
<commit_msg>bump waiter-cli version to 1.5.2 to release<commit_after>VERSION = '1.5.2'
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
CKAN Gallery Item class (formally known as "related").
This class fetches all relevant information about
a gallery items / related items from a specified
CKAN instance.

'''
import ckanapi

from app.classes.ckan import CKAN

class GalleryItem:
  '''
  Complete CKAN gallery item / related item class.

  '''
  def __init__(self, id):
    self.id = id
    self.hdx = CKAN().init()

  def info(self):
    self.object = self.hdx.action.related_show(id=self.id)
    return self.object
<commit_msg>removing unnecessary items<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
CKAN Gallery Item class (formally known as "related").
This class fetches all relevant information about
a gallery items / related items from a specified
CKAN instance.

'''
import ckanapi

from app.classes.ckan import CKAN

class GalleryItem:
  '''
  Complete CKAN gallery item / related item class.

  '''
  def __init__(self, id):
    self.id = id
    self.hdx = CKAN().init()

  def info(self):
    self.object = self.hdx.action.related_show(id=self.id)
    self.object['view_count'] = self.object['__extras']['view_count']
    self.object.pop('__extras')
    return self.object
<|endoftext|>