<commit_before>from pprint import pprint

import django.utils.timezone
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import IntegrityError, models
from django.utils.text import slugify

from utils.classes import LoggedInUser
from utils.functions.cadastro import CNPJ

class PosicaoCarga(models.Model):
    nome = models.CharField(
        max_length=20,
        db_index=True, unique=True)

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_pos_carga"
        verbose_name = "Posição de carga(NF)"
        verbose_name_plural = "Posições de carga(NF)"

class NotaFiscal(models.Model):
    # campos importados
    numero = models.IntegerField(
        db_index=True, unique=True, verbose_name='número')
    ativa = models.BooleanField(
        db_index=True, default=True)
    faturamento = models.DateTimeField(
        db_index=True, null=True, blank=True)
    cod_status = models.IntegerField(
        null=True, blank=True,
        verbose_name='status')
    msg_status = models.CharField(
        max_length=200, null=True, blank=True,
        verbose_name='status (descr.)')
    dest_cnpj = models.CharField(
        max_length=20, null=True, blank=True,
        verbose_name='CNPJ')
    dest_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='Destinatário')
    natu_venda = models.BooleanField(
        db_index=True, default=False, verbose_name='venda')
    uf = models.CharField(
        max_length=2, null=True, blank=True,
        verbose_name='UF')
    natu_descr = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='natureza')
    transp_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='transportadora')
    valor = models.DecimalField(
        decimal_places=2, max_digits=10,
        null=True, blank=True)
    volumes = models.IntegerField(
        null=True, blank=True)
    pedido = models.IntegerField(
        null=True, blank=True,
        verbose_name='pedido')
    ped_cliente = models.CharField(
        max_length=30, null=True, blank=True,
        verbose_name='pedido cliente')
    nf_devolucao = models.IntegerField(
        null=True, blank=True, verbose_name='Devolução')
    trail = models.CharField(
        db_index=True, max_length=32, null=True, blank=True, default='')
    posicao = models.ForeignKey(
        PosicaoCarga, default=1,
        verbose_name='Posição',
        on_delete=models.PROTECT)

    # campos editáveis
    saida = models.DateField(null=True, blank=True, verbose_name='saída')
    entrega = models.DateField(
        null=True, blank=True,
        verbose_name='agendamento')
    confirmada = models.BooleanField(
        default=False,
        verbose_name='entregue')
    observacao = models.TextField(
        null=True, blank=True,
        verbose_name='observação')

    def save(self, *args, **kwargs):
        if self.id:
            if self.saida:
                if self.posicao.id != 3:
                    self.posicao = PosicaoCarga.objects.get(id=3)
            else:
                if self.posicao.id == 3:
                    self.posicao = PosicaoCarga.objects.get(id=1)
        if self.id:
            if self.confirmada:
                if self.posicao.id != 5:
                    self.posicao = PosicaoCarga.objects.get(id=5)
            else:
                if self.posicao.id == 5:
                    if self.saida:
                        self.posicao = PosicaoCarga.objects.get(id=3)
                    else:
                        self.posicao = PosicaoCarga.objects.get(id=1)
        super(NotaFiscal, self).save(*args, **kwargs)

    class Meta:
        db_table = "fo2_fat_nf"
        verbose_name = "Nota Fiscal"
        verbose_name_plural = "Notas Fiscais"
        permissions = (("can_beep_shipment", "Can beep shipment"),
                       )

class RotinaLogistica(models.Model):
    nome = models.CharField(
        max_length=30,
        db_index=True, unique=True)
    slug = models.SlugField()

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_logist_rotina"
        verbose_name = "Rotina ligada à app Logistica"
        verbose_name_plural = "Rotinas ligadas à app Logistica"

    def save(self, *args, **kwargs):
        self.slug = slugify(self.nome)
        super(RotinaLogistica, self).save(*args, **kwargs)

class PosicaoCargaAlteracao(models.Model):
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='posicao_inicial_set',
        on_delete=models.PROTECT)
    ordem = models.IntegerField(default=0)
    descricao = models.CharField(
        'descrição', max_length=200)
    efeito = models.ForeignKey(
        RotinaLogistica, default=1,
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='posicao_final_set',
        on_delete=models.PROTECT)
    so_nfs_ativas = models.BooleanField(
        verbose_name='Só NFs ativas',
        default=True)

    class Meta:
        db_table = "fo2_pos_carga_alt"
        verbose_name = "Alteração de posição de carga(NF)"
        verbose_name_plural = "Alterações de Posição de carga(NF)"

class PosicaoCargaAlteracaoLog(models.Model):
    numero = models.IntegerField(
        db_index=True, verbose_name='número')
    time = models.DateTimeField(
        db_index=True,
        auto_now_add=True, verbose_name='Hora')
    user = models.CharField(
        db_index=True,
        max_length=64, verbose_name='Usuário')
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='log_posicao_inicial_set',
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='log_posicao_final_set',
        on_delete=models.PROTECT)
    saida = models.DateField(null=True, blank=True, verbose_name='saída')

    class Meta:
        db_table = "fo2_pos_carga_alt_log"
        verbose_name = "Log de alteração de posição de carga(NF)"

class NfEntrada(models.Model):
    cadastro = models.CharField(
        'CNPJ', max_length=20, default='')
    emissor = models.CharField(
        max_length=200)
    numero = models.IntegerField(
        'número')
    descricao = models.CharField(
        'descrição', max_length=200)
    qtd = models.DecimalField(
        'quantidade', max_digits=13, decimal_places=4)
    hora_entrada = models.TimeField(
        'hora de entrada', default=django.utils.timezone.now)
    transportadora = models.CharField(
        max_length=100)
    motorista = models.CharField(
        max_length=100)
    placa = models.CharField(
        max_length=10)
    responsavel = models.CharField(
        'responsável', max_length=100)
    usuario = models.ForeignKey(
        User, models.PROTECT,
        verbose_name='usuário')
    quando = models.DateTimeField(
        null=True, auto_now_add=True)

    def __str__(self):
        cnpj = CNPJ()
        return f'{cnpj.mask(self.cadastro)} NF {self.numero}'

    def save(self, *args, **kwargs):
        val_cnpj = CNPJ()
        if val_cnpj.validate(self.cadastro):
            self.cadastro = val_cnpj.cnpj
        else:
            raise ValidationError(f"Cadastro nacional inválido.")
        logged_in = LoggedInUser()
        self.usuario = logged_in.user
        try:
            super(NfEntrada, self).save(*args, **kwargs)
        except IntegrityError as e:
            if 'Key (cadastro, numero)' in str(e):
                raise IntegrityError(
                    f"CNPJ '{self.cadastro}' com número de NF '{self.numero}' já gravado"
                )
            else:
                raise e

    class Meta:
        db_table = "fo2_nf_entrada"
        verbose_name = "Nota fiscal de entrada"
        verbose_name_plural = "Notas fiscais de entrada"
        unique_together = [["cadastro", "numero"]]
<commit_msg>model NfEntrada melhorando teste de erro de unique_together cadastro + numero<commit_after>from pprint import pprint

import django.utils.timezone
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import IntegrityError, models
from django.utils.text import slugify

from utils.classes import LoggedInUser
from utils.functions.cadastro import CNPJ

class PosicaoCarga(models.Model):
    nome = models.CharField(
        max_length=20,
        db_index=True, unique=True)

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_pos_carga"
        verbose_name = "Posição de carga(NF)"
        verbose_name_plural = "Posições de carga(NF)"

class NotaFiscal(models.Model):
    # campos importados
    numero = models.IntegerField(
        db_index=True, unique=True, verbose_name='número')
    ativa = models.BooleanField(
        db_index=True, default=True)
    faturamento = models.DateTimeField(
        db_index=True, null=True, blank=True)
    cod_status = models.IntegerField(
        null=True, blank=True,
        verbose_name='status')
    msg_status = models.CharField(
        max_length=200, null=True, blank=True,
        verbose_name='status (descr.)')
    dest_cnpj = models.CharField(
        max_length=20, null=True, blank=True,
        verbose_name='CNPJ')
    dest_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='Destinatário')
    natu_venda = models.BooleanField(
        db_index=True, default=False, verbose_name='venda')
    uf = models.CharField(
        max_length=2, null=True, blank=True,
        verbose_name='UF')
    natu_descr = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='natureza')
    transp_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='transportadora')
    valor = models.DecimalField(
        decimal_places=2, max_digits=10,
        null=True, blank=True)
    volumes = models.IntegerField(
        null=True, blank=True)
    pedido = models.IntegerField(
        null=True, blank=True,
        verbose_name='pedido')
    ped_cliente = models.CharField(
        max_length=30, null=True, blank=True,
        verbose_name='pedido cliente')
    nf_devolucao = models.IntegerField(
        null=True, blank=True, verbose_name='Devolução')
    trail = models.CharField(
        db_index=True, max_length=32, null=True, blank=True, default='')
    posicao = models.ForeignKey(
        PosicaoCarga, default=1,
        verbose_name='Posição',
        on_delete=models.PROTECT)

    # campos editáveis
    saida = models.DateField(null=True, blank=True, verbose_name='saída')
    entrega = models.DateField(
        null=True, blank=True,
        verbose_name='agendamento')
    confirmada = models.BooleanField(
        default=False,
        verbose_name='entregue')
    observacao = models.TextField(
        null=True, blank=True,
        verbose_name='observação')

    def save(self, *args, **kwargs):
        if self.id:
            if self.saida:
                if self.posicao.id != 3:
                    self.posicao = PosicaoCarga.objects.get(id=3)
            else:
                if self.posicao.id == 3:
                    self.posicao = PosicaoCarga.objects.get(id=1)
        if self.id:
            if self.confirmada:
                if self.posicao.id != 5:
                    self.posicao = PosicaoCarga.objects.get(id=5)
            else:
                if self.posicao.id == 5:
                    if self.saida:
                        self.posicao = PosicaoCarga.objects.get(id=3)
                    else:
                        self.posicao = PosicaoCarga.objects.get(id=1)
        super(NotaFiscal, self).save(*args, **kwargs)

    class Meta:
        db_table = "fo2_fat_nf"
        verbose_name = "Nota Fiscal"
        verbose_name_plural = "Notas Fiscais"
        permissions = (("can_beep_shipment", "Can beep shipment"),
                       )

class RotinaLogistica(models.Model):
    nome = models.CharField(
        max_length=30,
        db_index=True, unique=True)
    slug = models.SlugField()

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_logist_rotina"
        verbose_name = "Rotina ligada à app Logistica"
        verbose_name_plural = "Rotinas ligadas à app Logistica"

    def save(self, *args, **kwargs):
        self.slug = slugify(self.nome)
        super(RotinaLogistica, self).save(*args, **kwargs)

class PosicaoCargaAlteracao(models.Model):
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='posicao_inicial_set',
        on_delete=models.PROTECT)
    ordem = models.IntegerField(default=0)
    descricao = models.CharField(
        'descrição', max_length=200)
    efeito = models.ForeignKey(
        RotinaLogistica, default=1,
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='posicao_final_set',
        on_delete=models.PROTECT)
    so_nfs_ativas = models.BooleanField(
        verbose_name='Só NFs ativas',
        default=True)

    class Meta:
        db_table = "fo2_pos_carga_alt"
        verbose_name = "Alteração de posição de carga(NF)"
        verbose_name_plural = "Alterações de Posição de carga(NF)"

class PosicaoCargaAlteracaoLog(models.Model):
    numero = models.IntegerField(
        db_index=True, verbose_name='número')
    time = models.DateTimeField(
        db_index=True,
        auto_now_add=True, verbose_name='Hora')
    user = models.CharField(
        db_index=True,
        max_length=64, verbose_name='Usuário')
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='log_posicao_inicial_set',
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='log_posicao_final_set',
        on_delete=models.PROTECT)
    saida = models.DateField(null=True, blank=True, verbose_name='saída')

    class Meta:
        db_table = "fo2_pos_carga_alt_log"
        verbose_name = "Log de alteração de posição de carga(NF)"

class NfEntrada(models.Model):
    cadastro = models.CharField(
        'CNPJ', max_length=20, default='')
    emissor = models.CharField(
        max_length=200)
    numero = models.IntegerField(
        'número')
    descricao = models.CharField(
        'descrição', max_length=200)
    qtd = models.DecimalField(
        'quantidade', max_digits=13, decimal_places=4)
    hora_entrada = models.TimeField(
        'hora de entrada', default=django.utils.timezone.now)
    transportadora = models.CharField(
        max_length=100)
    motorista = models.CharField(
        max_length=100)
    placa = models.CharField(
        max_length=10)
    responsavel = models.CharField(
        'responsável', max_length=100)
    usuario = models.ForeignKey(
        User, models.PROTECT,
        verbose_name='usuário')
    quando = models.DateTimeField(
        null=True, auto_now_add=True)

    def __str__(self):
        cnpj = CNPJ()
        return f'{cnpj.mask(self.cadastro)} NF {self.numero}'

    def save(self, *args, **kwargs):
        val_cnpj = CNPJ()
        if val_cnpj.validate(self.cadastro):
            self.cadastro = val_cnpj.cnpj
        else:
            raise ValidationError(f"Cadastro nacional inválido.")
        logged_in = LoggedInUser()
        self.usuario = logged_in.user
        try:
            super(NfEntrada, self).save(*args, **kwargs)
        except IntegrityError as e:
            if '(cadastro, numero)' in str(e):
                raise IntegrityError(
                    f"CNPJ '{self.cadastro}' com número de NF '{self.numero}' já gravado"
                )
            else:
                raise e

    class Meta:
        db_table = "fo2_nf_entrada"
        verbose_name = "Nota fiscal de entrada"
        verbose_name_plural = "Notas fiscais de entrada"
        unique_together = [["cadastro", "numero"]]
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging

import numpy as np
from builtins import object

from rasa_core.domain import check_domain_sanity
from rasa_core.interpreter import RegexInterpreter

logger = logging.getLogger(__name__)

class PolicyTrainer(object):
    def __init__(self, ensemble, domain, featurizer):
        self.domain = domain
        self.ensemble = ensemble
        self.featurizer = featurizer

    def train(self, filename=None, max_history=3,
              augmentation_factor=20, max_training_samples=None,
              max_number_of_trackers=2000, **kwargs):
        """Trains a policy on a domain using training data from a file.

        :param augmentation_factor: how many stories should be created by
                                    randomly concatenating stories
        :param filename: story file containing the training conversations
        :param max_history: number of past actions to consider for the
                            prediction of the next action
        :param max_training_samples: specifies how many training samples to
                                     train on - `None` to use all examples
        :param max_number_of_trackers: limits the tracker generation during
                                       story file parsing - `None` for unlimited
        :param kwargs: additional arguments passed to the underlying ML trainer
                       (e.g. keras parameters)
        :return: trained policy
        """

        logger.debug("Policy trainer got kwargs: {}".format(kwargs))
        check_domain_sanity(self.domain)

        X, y = self._prepare_training_data(filename, max_history,
                                           augmentation_factor,
                                           max_training_samples,
                                           max_number_of_trackers)

        self.ensemble.train(X, y, self.domain, self.featurizer, **kwargs)

    def _prepare_training_data(self, filename, max_history, augmentation_factor,
                               max_training_samples=None,
                               should_remove_duplicates=True,
                               max_number_of_trackers=2000):
        """Reads training data from file and prepares it for the training."""

        from rasa_core.training_utils import extract_training_data_from_file

        if filename:
            X, y = extract_training_data_from_file(
                    filename,
                    augmentation_factor=augmentation_factor,
                    max_history=max_history,
                    remove_duplicates=should_remove_duplicates,
                    domain=self.domain,
                    featurizer=self.featurizer,
                    interpreter=RegexInterpreter(),
                    max_number_of_trackers=max_number_of_trackers)
            if max_training_samples is not None:
                X = X[:max_training_samples, :]
                y = y[:max_training_samples]
        else:
            X = np.zeros((0, self.domain.num_features))
            y = np.zeros(self.domain.num_actions)
        return X, y
<commit_msg>fixed max tracker parameter passing<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging

import numpy as np
from builtins import object

from rasa_core.domain import check_domain_sanity
from rasa_core.interpreter import RegexInterpreter

logger = logging.getLogger(__name__)

class PolicyTrainer(object):
    def __init__(self, ensemble, domain, featurizer):
        self.domain = domain
        self.ensemble = ensemble
        self.featurizer = featurizer

    def train(self, filename=None, max_history=3,
              augmentation_factor=20, max_training_samples=None,
              max_number_of_trackers=2000, **kwargs):
        """Trains a policy on a domain using training data from a file.

        :param augmentation_factor: how many stories should be created by
                                    randomly concatenating stories
        :param filename: story file containing the training conversations
        :param max_history: number of past actions to consider for the
                            prediction of the next action
        :param max_training_samples: specifies how many training samples to
                                     train on - `None` to use all examples
        :param max_number_of_trackers: limits the tracker generation during
                                       story file parsing - `None` for unlimited
        :param kwargs: additional arguments passed to the underlying ML trainer
                       (e.g. keras parameters)
        :return: trained policy
        """

        logger.debug("Policy trainer got kwargs: {}".format(kwargs))
        check_domain_sanity(self.domain)

        X, y = self._prepare_training_data(filename, max_history,
                                           augmentation_factor,
                                           max_training_samples,
                                           max_number_of_trackers)

        self.ensemble.train(X, y, self.domain, self.featurizer, **kwargs)

    def _prepare_training_data(self, filename, max_history, augmentation_factor,
                               max_training_samples=None,
                               max_number_of_trackers=2000):
        """Reads training data from file and prepares it for the training."""

        from rasa_core.training_utils import extract_training_data_from_file

        if filename:
            X, y = extract_training_data_from_file(
                    filename,
                    augmentation_factor=augmentation_factor,
                    max_history=max_history,
                    remove_duplicates=True,
                    domain=self.domain,
                    featurizer=self.featurizer,
                    interpreter=RegexInterpreter(),
                    max_number_of_trackers=max_number_of_trackers)
            if max_training_samples is not None:
                X = X[:max_training_samples, :]
                y = y[:max_training_samples]
        else:
            X = np.zeros((0, self.domain.num_features))
            y = np.zeros(self.domain.num_actions)
        return X, y
<|endoftext|>


<commit_before>#System import subprocess
import subprocess
import getopt
import sys

# PID imports
from b2handle.clientcredentials import PIDClientCredentials
from b2handle.handleclient import EUDATHandleClient

import uuid
import hashlib
import os, shutil

RED     = "\033[31m"
GREEN   = "\033[92m"
BLUE    = "\033[34m"
DEFAULT = "\033[0m"

#Upload dataset to gridFTP server
def gridftp_upload(dataset, server, protocol, destination):
    exit_code = subprocess.call(["grid-proxy-init"])
    if exit_code == 0:
        print GREEN, "DEBUG", DEFAULT, \
            "Uplopading", dataset, "to", protocol+"://"+server+destination
        exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
                    dataset,
                    protocol+"://"+server+destination])
        print GREEN, "DEBUG"
        exit_code = subprocess.call(["globus-url-copy", "-list",
                    protocol+"://"+server+destination])
        print DEFAULT
    return exit_code

#Connect to handle server --> return client instance
def conn_handle(credentials='cred_21.T12995.json'):
    cred = PIDClientCredentials.load_from_JSON(credentials)
    print GREEN, "DEBUG"
    print('PID prefix ' + cred.get_prefix())
    print('Server ' + cred.get_server_URL())
    print DEFAULT

    ec = EUDATHandleClient.instantiate_with_credentials(cred)

    return ec, cred

#Create a PID for the root collection
def register_dataset(ec, cred, dataset, protocol, server):
    #reverse lookup
    rev_args = dict([('URL', dataset)])
    if ec.search_handle(**rev_args)==[]:
        #Register dataset (folder) on gridftp server
        uid = uuid.uuid1()
        Handle  = ec.register_handle(cred.get_prefix() + '/' + str(uid), dataset)
        print GREEN, "DEBUG", DEFAULT, \
            "Creating handle", Handle, "for", dataset
        #Add information types
        args        = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol),
                    ('SITE', server)])
        ec.modify_handle_value(Handle, ttl=None,
                    add_if_not_exist=True, **args)
    else:
        print RED, "WARNING", DEFAULT, dataset, \
                        "already has handles", ec.search_handle(**rev_args)
        Handle = ec.search_handle(**rev_args)[0]

    return Handle

# Returns a list of children or empty list
def get_children(pid, ec):
    entry = ec.get_value_from_handle(pid, 'CHILDREN')
    if entry == None:
        return []
    else:
        return entry.replace("u'", "").replace("'", "").strip("]").strip("[").split(', ')

#Given the root collection, registers folders and files
#Data needs to reside on a gridFTP server
def register_files(ec, cred, dataset, protocol, server):
    #Create PID for each file and subcollection in the dataset
    args       = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol), ('SITE', server)])
    parent_args = dict()
    collection = [dataset] # root collection

    while len(collection) > 0:
        children = []
        coll = collection[0]
        coll_pid = ec.search_handle(**dict([('URL', coll)]))[0]
        p = subprocess.Popen(["globus-url-copy -list "+ protocol+"://"+server+coll],
            shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in p.stdout.readlines()[1:]: #get files and folders
            if line.strip() == "":
                continue
            else:
                #Check if folder or file
                if line.strip().endswith('/'):
                    args["TYPE"]    = "Folder"
                    collection.append(coll+line.strip())
                else:
                    args["TYPE"]    = "File"

                args["PARENT"]  = coll_pid
                #reverse lookup for field URL, do not regenerate PIDs for same paths
                rev_args = dict([('URL', coll+line.strip())])
                if ec.search_handle(**rev_args)==[]:
                    uid = uuid.uuid1()
                    h   = ec.register_handle(cred.get_prefix() + '/' + str(uid),
                        coll+line.strip())
                    children.append(h)
                    ec.modify_handle_value(h, ttl=None,
                        add_if_not_exist=True, **args)
                    print GREEN, "DEBUG", DEFAULT, "Created handle", h, \
                        "for", coll+line.strip()
                else:
                    children.extend(ec.search_handle(**rev_args))
                    print RED, "WARNING", DEFAULT, coll+line.strip(), \
                        "already has handles", ec.search_handle(**rev_args)
        # Update collection with all PIDs to children
        parent_args['CHILDREN'] = ', '.join(children)
        print GREEN, "DEBUG", DEFAULT, "Update ", coll_pid
        #print GREEN, "DEBUG", DEFAULT, "CHILDREN ", children
        ec.modify_handle_value(coll_pid, ttl=None,
            add_if_not_exist=True, **parent_args)
        collection.remove(coll)

def sync_dataset(pid, local_data, ec, cred):
# Synchronises a local dataset to an existing registered dataset on a grdiFTP server.
    # Get location of dataset
    assert ec.get_value_from_handle(pid, 'TYPE') == 'Folder'

    server      = ec.get_value_from_handle(pid, 'SITE').strip('/')
    protocol    = ec.get_value_from_handle(pid, 'PROTOCOL')
    dest_coll   = ec.get_value_from_handle(pid, 'URL')

    # By default only files that are not present will be transfered
    # -sync -sync-level 0

    exit_code = subprocess.call(["grid-proxy-init"])
    if exit_code == 0:
        print GREEN, "DEBUG", DEFAULT, \
            "Uplopading", local_data, "to", protocol+"://"+server+dest_coll
        exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
            "-sync", "-sync-level", "0", local_data, protocol+"://"+server+dest_coll])
        print GREEN, "DEBUG", exit_code
        exit_code = subprocess.call(["globus-url-copy", "-list",
            protocol+"://"+server+dest_coll])
        print GREEN, "DEBUG", exit_code
        print DEFAULT

    # Update PID entries for a whole dataset, e.g. after adding more data
    # Add children and link to parent
    # Assumes that data cannot be deleted from dataset.

    register_files(ec, cred, dest_coll, protocol, server)

def download_dataset(pid, destination):
    #Instantiate client for reading --> credentials necessary
    ec      = EUDATHandleClient.instantiate_for_read_access('https://hdl.handle.net')
    record  = ec.retrieve_handle_record(pid)

    assert record != None

    assert 'URL'        in record
    assert 'PROTOCOL'   in record
    assert 'SITE'       in record

    protocol    = record['PROTOCOL']
    site        = record['SITE']
    source      = record['URL']

    print GREEN, "DEBUG", DEFAULT, \
            "PID", pid, "resolves to", protocol+"://"+site+source

    exit_code = subprocess.call(["grid-proxy-init"])

    print GREEN, "DEBUG downloading:"
    exit_code = subprocess.call(["globus-url-copy", "-list",
                    protocol+"://"+site+source])
    print "Destination", destination
    print DEFAULT

    exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
        protocol+"://"+site+source, destination])

    return exit_code

def update_url(pid, new_location, ec):
    # After moving a dataset to another location all the URL fields need to be updated.
    # new_location needs a trailing '/'!!

    # Check whether we have the dataset pid:
    # No PARENT entry
    record  = ec.retrieve_handle_record(pid)
    assert record != None
    assert 'PARENT' not in record

    url = record['URL'] # root of the dataset
    # Check if new location is really new
    assert url  != new_location

    print GREEN, "DEBUG", DEFAULT, "Update URL of collection", pid
    print GREEN, "DEBUG", DEFAULT, "Old location", url
    print GREEN, "DEBUG", DEFAULT, "New location", new_location
    print

    # Update all URL fields in the directory tree
    collection = [pid] # root collection
    args = {}
    while len(collection) > 0:
        # get all children
        collection.extend(get_children(collection[0], ec))
        #print GREEN, "DEBUG", DEFAULT, collection
        r           = ec.retrieve_handle_record(collection[0])
        cur_url     = r['URL']
        args['URL'] = cur_url.replace(url, new_location)
        ec.modify_handle_value(collection[0], ttl=None, **args)
        collection.remove(collection[0])
    print GREEN, "DEBUG", DEFAULT, "URL Update DONE."

def main():
    """
    Usage:
    Upload to gridFTP server
        python gridftp.py -u <upload collection> -g <destination>
    Synchronise existing dataset
        python gridftp.py -u <upload collection> -p <PID>
    Update URL of a dataset
        python gridftp.py -p <PID> -n <new URL to dataset>
    Download with PID
        pythoe gridftp.py -d <local download destination> -p <PID>
    """
    # parse command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hu:d:p:g:e:n:s", ["help"])
    except getopt.error, msg:
        print msg
        print "for help use --help"
        sys.exit(2)

    if args != []:
        print "for help use --help"
        sys.exit(2)
    # process options
    #for upload
    dataset_up  = ""
    protocol    = "gsiftp"
    server      = "nlnode.elixirgridftp-sara.surf-hosted.nl/"
    destination_ftp = ""

    #for download
    pid         = "21.T12995/A866A7A8-E947-11E6-A26B-040091643BEA"
    destination = ""

    #for update
    new_loc     = ""

    for o, a in opts:
        print o, a
        if o in ("-h", "--help"):
            print "Help"
            print __doc__
            sys.exit(0)
        elif o == "--sync":
            sync = True
        elif o == "-u":
            dataset_up = a
        elif o == "-d":
            destination = a
        elif o == "-p":
            pid = a
        elif o == "-g":
            destination_ftp = a
        elif o == "-e":
            protocol = a
        elif o == "-s":
            server = a
        elif o == "-n":
            new_loc = a
        else:
            print "option unknown"
            sys.exit(2)

    if not (protocol and server):
        print "%sDefine server and protocol. For help use --help%s"  %(RED, DEFAULT)
        return 0

    if dataset_up and destination_ftp:
        print "Uploading data to gridFTP server"
        gridftp_upload(dataset_up, server, protocol, destination_ftp)
        print "Registering PIDs"
        ec, cred = conn_handle(credentials='cred_21.T12995.json')
        pid = register_dataset(ec, cred, destination_ftp, protocol, server)
        print "Dataset PID:", pid
        register_files(ec, cred, destination_ftp, protocol, server)
    elif dataset_up and pid:
        ec, cred = conn_handle(credentials='cred_21.T12995.json')
        sync_dataset(pid, dataset_up, ec, cred)
    elif pid and destination:
        print "Downloading data fom gridFTP server"
        download_dataset(pid, destination)
    elif pid and new_loc:
        ec, cred = conn_handle(credentials='cred_21.T12995.json')
        update_url(pid, new_loc, ec)
    else:
        print "%sNot a valid option. For help use --help%s"  %(RED, DEFAULT)
        return 0

if __name__ == "__main__":
    main()
<commit_msg>Option -s needed a ":"<commit_after>#System import subprocess
import subprocess
import getopt
import sys

# PID imports
from b2handle.clientcredentials import PIDClientCredentials
from b2handle.handleclient import EUDATHandleClient

import uuid
import hashlib
import os, shutil

RED     = "\033[31m"
GREEN   = "\033[92m"
BLUE    = "\033[34m"
DEFAULT = "\033[0m"

#Upload dataset to gridFTP server
def gridftp_upload(dataset, server, protocol, destination):
    exit_code = subprocess.call(["grid-proxy-init"])
    if exit_code == 0:
        print GREEN, "DEBUG", DEFAULT, \
            "Uplopading", dataset, "to", protocol+"://"+server+destination
        exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
                    dataset,
                    protocol+"://"+server+destination])
        print GREEN, "DEBUG"
        exit_code = subprocess.call(["globus-url-copy", "-list",
                    protocol+"://"+server+destination])
        print DEFAULT
    return exit_code

#Connect to handle server --> return client instance
def conn_handle(credentials='cred_21.T12995.json'):
    cred = PIDClientCredentials.load_from_JSON(credentials)
    print GREEN, "DEBUG"
    print('PID prefix ' + cred.get_prefix())
    print('Server ' + cred.get_server_URL())
    print DEFAULT

    ec = EUDATHandleClient.instantiate_with_credentials(cred)

    return ec, cred

#Create a PID for the root collection
def register_dataset(ec, cred, dataset, protocol, server):
    #reverse lookup
    rev_args = dict([('URL', dataset)])
    if ec.search_handle(**rev_args)==[]:
        #Register dataset (folder) on gridftp server
        uid = uuid.uuid1()
        Handle  = ec.register_handle(cred.get_prefix() + '/' + str(uid), dataset)
        print GREEN, "DEBUG", DEFAULT, \
            "Creating handle", Handle, "for", dataset
        #Add information types
        args        = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol),
                    ('SITE', server)])
        ec.modify_handle_value(Handle, ttl=None,
                    add_if_not_exist=True, **args)
    else:
        print RED, "WARNING", DEFAULT, dataset, \
                        "already has handles", ec.search_handle(**rev_args)
        Handle = ec.search_handle(**rev_args)[0]

    return Handle

# Returns a list of children or empty list
def get_children(pid, ec):
    entry = ec.get_value_from_handle(pid, 'CHILDREN')
    if entry == None:
        return []
    else:
        return entry.replace("u'", "").replace("'", "").strip("]").strip("[").split(', ')

#Given the root collection, registers folders and files
#Data needs to reside on a gridFTP server
def register_files(ec, cred, dataset, protocol, server):
    #Create PID for each file and subcollection in the dataset
    args       = dict([('TYPE', 'Folder'), ('PROTOCOL', protocol), ('SITE', server)])
    parent_args = dict()
    collection = [dataset] # root collection

    while len(collection) > 0:
        children = []
        coll = collection[0]
        coll_pid = ec.search_handle(**dict([('URL', coll)]))[0]
        p = subprocess.Popen(["globus-url-copy -list "+ protocol+"://"+server+coll],
            shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in p.stdout.readlines()[1:]: #get files and folders
            if line.strip() == "":
                continue
            else:
                #Check if folder or file
                if line.strip().endswith('/'):
                    args["TYPE"]    = "Folder"
                    collection.append(coll+line.strip())
                else:
                    args["TYPE"]    = "File"

                args["PARENT"]  = coll_pid
                #reverse lookup for field URL, do not regenerate PIDs for same paths
                rev_args = dict([('URL', coll+line.strip())])
                if ec.search_handle(**rev_args)==[]:
                    uid = uuid.uuid1()
                    h   = ec.register_handle(cred.get_prefix() + '/' + str(uid),
                        coll+line.strip())
                    children.append(h)
                    ec.modify_handle_value(h, ttl=None,
                        add_if_not_exist=True, **args)
                    print GREEN, "DEBUG", DEFAULT, "Created handle", h, \
                        "for", coll+line.strip()
                else:
                    children.extend(ec.search_handle(**rev_args))
                    print RED, "WARNING", DEFAULT, coll+line.strip(), \
                        "already has handles", ec.search_handle(**rev_args)
        # Update collection with all PIDs to children
        parent_args['CHILDREN'] = ', '.join(children)
        print GREEN, "DEBUG", DEFAULT, "Update ", coll_pid
        #print GREEN, "DEBUG", DEFAULT, "CHILDREN ", children
        ec.modify_handle_value(coll_pid, ttl=None,
            add_if_not_exist=True, **parent_args)
        collection.remove(coll)

def sync_dataset(pid, local_data, ec, cred):
# Synchronises a local dataset to an existing registered dataset on a grdiFTP server.
    # Get location of dataset
    assert ec.get_value_from_handle(pid, 'TYPE') == 'Folder'

    server      = ec.get_value_from_handle(pid, 'SITE').strip('/')
    protocol    = ec.get_value_from_handle(pid, 'PROTOCOL')
    dest_coll   = ec.get_value_from_handle(pid, 'URL')

    # By default only files that are not present will be transfered
    # -sync -sync-level 0

    exit_code = subprocess.call(["grid-proxy-init"])
    if exit_code == 0:
        print GREEN, "DEBUG", DEFAULT, \
            "Uplopading", local_data, "to", protocol+"://"+server+dest_coll
        exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
            "-sync", "-sync-level", "0", local_data, protocol+"://"+server+dest_coll])
        print GREEN, "DEBUG", exit_code
        exit_code = subprocess.call(["globus-url-copy", "-list",
            protocol+"://"+server+dest_coll])
        print GREEN, "DEBUG", exit_code
        print DEFAULT

    # Update PID entries for a whole dataset, e.g. after adding more data
    # Add children and link to parent
    # Assumes that data cannot be deleted from dataset.

    register_files(ec, cred, dest_coll, protocol, server)

def download_dataset(pid, destination):
    #Instantiate client for reading --> credentials necessary
    ec      = EUDATHandleClient.instantiate_for_read_access('https://hdl.handle.net')
    record  = ec.retrieve_handle_record(pid)

    assert record != None

    assert 'URL'        in record
    assert 'PROTOCOL'   in record
    assert 'SITE'       in record

    protocol    = record['PROTOCOL']
    site        = record['SITE']
    source      = record['URL']

    print GREEN, "DEBUG", DEFAULT, \
            "PID", pid, "resolves to", protocol+"://"+site+source

    exit_code = subprocess.call(["grid-proxy-init"])

    print GREEN, "DEBUG downloading:"
    exit_code = subprocess.call(["globus-url-copy", "-list",
                    protocol+"://"+site+source])
    print "Destination", destination
    print DEFAULT

    exit_code = subprocess.call(["globus-url-copy", "-cd", "-r",
        protocol+"://"+site+source, destination])

    return exit_code

def update_url(pid, new_location, ec):
    # After moving a dataset to another location all the URL fields need to be updated.
    # new_location needs a trailing '/'!!

    # Check whether we have the dataset pid:
    # No PARENT entry
    record  = ec.retrieve_handle_record(pid)
    assert record != None
    assert 'PARENT' not in record

    url = record['URL'] # root of the dataset
    # Check if new location is really new
    assert url  != new_location

    print GREEN, "DEBUG", DEFAULT, "Update URL of collection", pid
    print GREEN, "DEBUG", DEFAULT, "Old location", url
    print GREEN, "DEBUG", DEFAULT, "New location", new_location
    print

    # Update all URL fields in the directory tree
    collection = [pid] # root collection
    args = {}
    while len(collection) > 0:
        # get all children
        collection.extend(get_children(collection[0], ec))
        #print GREEN, "DEBUG", DEFAULT, collection
        r           = ec.retrieve_handle_record(collection[0])
        cur_url     = r['URL']
        args['URL'] = cur_url.replace(url, new_location)
        ec.modify_handle_value(collection[0], ttl=None, **args)
        collection.remove(collection[0])
    print GREEN, "DEBUG", DEFAULT, "URL Update DONE."

def main():
    """
    Usage:
    Upload to gridFTP server
        python gridftp.py -u <upload collection> -g <destination>
    Synchronise existing dataset
        python gridftp.py -u <upload collection> -p <PID>
    Update URL of a dataset
        python gridftp.py -p <PID> -n <new URL to dataset>
    Download with PID
        pythoe gridftp.py -d <local download destination> -p <PID>
    """
    # parse command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hu:d:p:g:e:n:s:", ["help"])
    except getopt.error, msg:
        print msg
        print "for help use --help"
        sys.exit(2)

    if args != []:
        print "for help use --help"
        sys.exit(2)
    # process options
    #for upload
    dataset_up  = ""
    protocol    = "gsiftp"
    server      = "nlnode.elixirgridftp-sara.surf-hosted.nl/"
    destination_ftp = ""

    #for download
    pid         = "21.T12995/A866A7A8-E947-11E6-A26B-040091643BEA"
    destination = ""

    #for update
    new_loc     = ""

    for o, a in opts:
        print o, a
        if o in ("-h", "--help"):
            print "Help"
            print __doc__
            sys.exit(0)
        elif o == "--sync":
            sync = True
        elif o == "-u":
            dataset_up = a
        elif o == "-d":
            destination = a
        elif o == "-p":
            pid = a
        elif o == "-g":
            destination_ftp = a
        elif o == "-e":
            protocol = a
        elif o == "-s":
            server = a
        elif o == "-n":
            new_loc = a
        else:
            print "option unknown"
            sys.exit(2)

    if not (protocol and server):
        print "%sDefine server and protocol. For help use --help%s"  %(RED, DEFAULT)
        return 0

    if dataset_up and destination_ftp:
        print "Uploading data to gridFTP server"
        gridftp_upload(dataset_up, server, protocol, destination_ftp)
        print "Registering PIDs"
        ec, cred = conn_handle(credentials='cred_21.T12995.json')
        pid = register_dataset(ec, cred, destination_ftp, protocol, server)
        print "Dataset PID:", pid
        register_files(ec, cred, destination_ftp, protocol, server)
    elif dataset_up and pid:
        ec, cred = conn_handle(credentials='cred_21.T12995.json')
        sync_dataset(pid, dataset_up, ec, cred)
    elif pid and destination:
        print "Downloading data fom gridFTP server"
        download_dataset(pid, destination)
    elif pid and new_loc:
        ec, cred = conn_handle(credentials='cred_21.T12995.json')
        update_url(pid, new_loc, ec)
    else:
        print "%sNot a valid option. For help use --help%s"  %(RED, DEFAULT)
        return 0

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import collections
import os
import os.path
import sys
import time
import traceback

import yaml

from behave import decorators, model, parser, runner
from behave.configuration import Configuration
from behave.formatter.ansi_escapes import escapes
from behave.formatter.pretty_formatter import PrettyFormatter

TAG_HELP = """
Scenarios inherit tags declared on the Feature level. The simplest
TAG_EXPRESSION is simply a tag:

--tags @dev

When a tag in a tag expression starts with a ~, this represents boolean NOT:

--tags ~@dev

A tag expression can have several tags separated by a comma, which represents
logical OR:

--tags @dev,@wip

The --tags option can be specified several times, and this represents logical
AND, for instance this represents the boolean expression
(@foo || !@bar) && @zap:

--tags @foo,~@bar --tags @zap.

Beware that if you want to use several negative tags to exclude several tags
you have to use logical AND:

--tags ~@fixme --tags ~@buggy.

Positive tags can be given a threshold to limit the number of occurrences. Which
can be practical if you are practicing Kanban or CONWIP. This will fail if there
are more than 3 occurrences of the @qa tag:

--tags @qa:3
""".strip()

class Language(object):
    @classmethod
    def load(cls, language_name, default_language_name="en"):
        directory, _f = os.path.split(os.path.abspath(__file__))
        language_path = os.path.join(directory, 'languages.yml')
        languages = yaml.load(open(language_path))
        if language_name not in languages:
            return None
        return cls(languages[language_name], languages[default_language_name])

    def __init__(self, mappings, default_mappings=None):
        self.mappings = mappings
        self.default_mappings = default_mappings

    def words(self, key):
        """
        Give all the synonymns of a word in the requested language
        (or the default language if no word is available).
        """
        if self.default_mappings is not None and key not in self.mappings:
            return self.default_mappings[key].encode('utf').split("|")
        else:
            return self.mappings[key].encode('utf').split("|")

def feature_files(paths):
    files = []
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if filename.endswith('.feature'):
                        files.append(os.path.join(dirpath, filename))
        elif path.startswith('@'):
            files.extend([filename.strip() for filename in open(path)])
        elif os.path.exists(path):
            files.append(path)
        else:
            raise Exception("Can't find path: " + path)
    return files

def main():
    config = Configuration()

    if config.tags and config.tags == 'help':
        print TAG_HELP
        sys.exit(0)

    if not config.paths:
        config.paths = ['features']

    passed = []
    failed = []
    undefined = []
    skipped = []

    stream = sys.stdout

    base_dir = os.path.abspath(config.paths[0])
    if not os.path.isdir(base_dir):
        base_dir = os.path.dirname(base_dir)
    while not os.path.isdir(os.path.join(base_dir, 'steps')):
        base_dir = os.path.dirname(base_dir)
        if base_dir == os.getcwd():
            base_dir = None
            break

    hooks = {}

    if base_dir:
        sys.path.insert(0, base_dir)

        steps_dir = os.path.join(base_dir, 'steps')
        for name in os.listdir(steps_dir):
            step_globals = {
                'Given': decorators.Given,
                'When': decorators.When,
                'Then': decorators.Then,
            }
            if name.endswith('.py'):
                execfile(os.path.join(steps_dir, name), step_globals)

        hooks_path = os.path.join(base_dir, 'hooks.py')
        if os.path.exists(hooks_path):
            execfile(hooks_path, hooks)

    def run_hook(name, *args):
        if name in hooks:
            hooks[name](*args)

    context = runner.Context()
    features = []

    run_hook('before_all', context)

    for filename in feature_files(config.paths):
        context._push()

        feature = parser.parse_file(os.path.abspath(filename), Language.load('en'))
        features.append(feature)

        feature_match = config.tags.check(feature.tags)

        formatter = PrettyFormatter(stream, False, True)
        formatter.uri(filename)
        formatter.feature(feature)

        run_hook('before_feature', context, feature)

        if feature.has_background():
            formatter.background(feature.background)

        for scenario in feature:
            run_scenario = feature_match or config.tags.check(scenario.tags)
            run_steps = run_scenario

            formatter.scenario(scenario)

            context._push()

            if run_scenario:
                run_hook('before_scenario', context, scenario)

            for step in scenario:
                formatter.step(step)

            for step in scenario:
                if run_steps:
                    match = runner.steps.find_match(step)
                    if match is None:
                        undefined.append(step)
                        formatter.match(model.NoMatch())
                        step.status = 'undefined'
                        formatter.result(step)
                        run_steps = False
                    else:
                        formatter.match(match)
                        run_hook('before_step', context, step)
                        try:
                            start = time.time()
                            match.run(context)
                            elapsed = time.time() - start
                            step.status = 'passed'
                            step.duration = elapsed
                        except:
                            elapsed = time.time() - start
                            error = traceback.format_exc()
                            step.status = 'failed'
                            step.duration = elapsed
                            step.error_message = error
                            run_steps = False

                        formatter.result(step)
                        run_hook('after_step', context, step)
                else:
                    step.status = 'skipped'
                    if scenario.status is None:
                        scenario.status = 'skipped'

            if run_scenario:
                run_hook('after_scenario', context, scenario)

            context._pop()

        formatter.eof()

        run_hook('after_feature', context, feature)

        context._pop()

        print ''

    run_hook('after_all', context)

    feature_summary = {'passed': 0, 'failed': 0, 'skipped': 0}
    scenario_summary = {'passed': 0, 'failed': 0, 'skipped': 0}
    step_summary = {'passed': 0, 'failed': 0, 'skipped': 0, 'undefined': 0}
    duration = 0.0

    for feature in features:
        feature_summary[feature.status or 'skipped'] += 1
        for scenario in feature:
            scenario_summary[scenario.status or 'skipped'] += 1
            for step in scenario:
                step_summary[step.status or 'skipped'] += 1
                duration += step.duration or 0.0

    def format_summary(statement_type, summary):
        first = True
        parts = []
        for status in ('passed', 'failed', 'skipped', 'undefined'):
            if status not in summary:
                continue
            if first:
                label = statement_type
                if summary[status] != 1:
                    label += 's'
                part = '{:d} {:s} {:s}'.format(summary[status], label, status)
                first = False
            else:
                part = '{:d} {:s}'.format(summary[status], status)
            parts.append(part)
        return ', '.join(parts) + '\n'

    stream.write(format_summary('feature', feature_summary))
    stream.write(format_summary('scenario', scenario_summary))
    stream.write(format_summary('step', step_summary))

    if undefined:
        msg =  "\nYou can implement step definitions for undefined steps with "
        msg += "these snippets:\n\n"
        printed = set()
        for step in undefined:
            if step in printed:
                continue
            printed.add(step)

            func_name = step.name.lower().replace(' ', '_')
            msg += "@" + step.keyword + "('" + step.name + "')\n"
            msg += "def " + func_name + "(context):\n"
            msg += "    assert False\n\n"

        stream.write(escapes['undefined'] + msg + escapes['reset'])
        stream.flush()
<commit_msg>Better behaviour around how feature tags are passed down to scenarios.<commit_after>import collections
import os
import os.path
import sys
import time
import traceback

import yaml

from behave import decorators, model, parser, runner
from behave.configuration import Configuration
from behave.formatter.ansi_escapes import escapes
from behave.formatter.pretty_formatter import PrettyFormatter

TAG_HELP = """
Scenarios inherit tags declared on the Feature level. The simplest
TAG_EXPRESSION is simply a tag:

--tags @dev

When a tag in a tag expression starts with a ~, this represents boolean NOT:

--tags ~@dev

A tag expression can have several tags separated by a comma, which represents
logical OR:

--tags @dev,@wip

The --tags option can be specified several times, and this represents logical
AND, for instance this represents the boolean expression
(@foo || !@bar) && @zap:

--tags @foo,~@bar --tags @zap.

Beware that if you want to use several negative tags to exclude several tags
you have to use logical AND:

--tags ~@fixme --tags ~@buggy.

Positive tags can be given a threshold to limit the number of occurrences. Which
can be practical if you are practicing Kanban or CONWIP. This will fail if there
are more than 3 occurrences of the @qa tag:

--tags @qa:3
""".strip()

class Language(object):
    @classmethod
    def load(cls, language_name, default_language_name="en"):
        directory, _f = os.path.split(os.path.abspath(__file__))
        language_path = os.path.join(directory, 'languages.yml')
        languages = yaml.load(open(language_path))
        if language_name not in languages:
            return None
        return cls(languages[language_name], languages[default_language_name])

    def __init__(self, mappings, default_mappings=None):
        self.mappings = mappings
        self.default_mappings = default_mappings

    def words(self, key):
        """
        Give all the synonymns of a word in the requested language
        (or the default language if no word is available).
        """
        if self.default_mappings is not None and key not in self.mappings:
            return self.default_mappings[key].encode('utf').split("|")
        else:
            return self.mappings[key].encode('utf').split("|")

def feature_files(paths):
    files = []
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if filename.endswith('.feature'):
                        files.append(os.path.join(dirpath, filename))
        elif path.startswith('@'):
            files.extend([filename.strip() for filename in open(path)])
        elif os.path.exists(path):
            files.append(path)
        else:
            raise Exception("Can't find path: " + path)
    return files

def main():
    config = Configuration()

    if config.tags and config.tags == 'help':
        print TAG_HELP
        sys.exit(0)

    if not config.paths:
        config.paths = ['features']

    passed = []
    failed = []
    undefined = []
    skipped = []

    stream = sys.stdout

    base_dir = os.path.abspath(config.paths[0])
    if not os.path.isdir(base_dir):
        base_dir = os.path.dirname(base_dir)
    while not os.path.isdir(os.path.join(base_dir, 'steps')):
        base_dir = os.path.dirname(base_dir)
        if base_dir == os.getcwd():
            base_dir = None
            break

    hooks = {}

    if base_dir:
        sys.path.insert(0, base_dir)

        steps_dir = os.path.join(base_dir, 'steps')
        for name in os.listdir(steps_dir):
            step_globals = {
                'Given': decorators.Given,
                'When': decorators.When,
                'Then': decorators.Then,
            }
            if name.endswith('.py'):
                execfile(os.path.join(steps_dir, name), step_globals)

        hooks_path = os.path.join(base_dir, 'hooks.py')
        if os.path.exists(hooks_path):
            execfile(hooks_path, hooks)

    def run_hook(name, *args):
        if name in hooks:
            hooks[name](*args)

    context = runner.Context()
    features = []

    run_hook('before_all', context)

    for filename in feature_files(config.paths):
        context._push()

        feature = parser.parse_file(os.path.abspath(filename), Language.load('en'))
        features.append(feature)

        formatter = PrettyFormatter(stream, False, True)
        formatter.uri(filename)
        formatter.feature(feature)

        run_hook('before_feature', context, feature)

        if feature.has_background():
            formatter.background(feature.background)

        for scenario in feature:
            run_scenario = config.tags.check(feature.tags + scenario.tags)
            run_steps = run_scenario

            formatter.scenario(scenario)

            context._push()

            if run_scenario:
                run_hook('before_scenario', context, scenario)

            for step in scenario:
                formatter.step(step)

            for step in scenario:
                if run_steps:
                    match = runner.steps.find_match(step)
                    if match is None:
                        undefined.append(step)
                        formatter.match(model.NoMatch())
                        step.status = 'undefined'
                        formatter.result(step)
                        run_steps = False
                    else:
                        formatter.match(match)
                        run_hook('before_step', context, step)
                        try:
                            start = time.time()
                            match.run(context)
                            elapsed = time.time() - start
                            step.status = 'passed'
                            step.duration = elapsed
                        except:
                            elapsed = time.time() - start
                            error = traceback.format_exc()
                            step.status = 'failed'
                            step.duration = elapsed
                            step.error_message = error
                            run_steps = False

                        formatter.result(step)
                        run_hook('after_step', context, step)
                else:
                    step.status = 'skipped'
                    if scenario.status is None:
                        scenario.status = 'skipped'

            if run_scenario:
                run_hook('after_scenario', context, scenario)

            context._pop()

        formatter.eof()

        run_hook('after_feature', context, feature)

        context._pop()

        print ''

    run_hook('after_all', context)

    feature_summary = {'passed': 0, 'failed': 0, 'skipped': 0}
    scenario_summary = {'passed': 0, 'failed': 0, 'skipped': 0}
    step_summary = {'passed': 0, 'failed': 0, 'skipped': 0, 'undefined': 0}
    duration = 0.0

    for feature in features:
        feature_summary[feature.status or 'skipped'] += 1
        for scenario in feature:
            scenario_summary[scenario.status or 'skipped'] += 1
            for step in scenario:
                step_summary[step.status or 'skipped'] += 1
                duration += step.duration or 0.0

    def format_summary(statement_type, summary):
        first = True
        parts = []
        for status in ('passed', 'failed', 'skipped', 'undefined'):
            if status not in summary:
                continue
            if first:
                label = statement_type
                if summary[status] != 1:
                    label += 's'
                part = '{:d} {:s} {:s}'.format(summary[status], label, status)
                first = False
            else:
                part = '{:d} {:s}'.format(summary[status], status)
            parts.append(part)
        return ', '.join(parts) + '\n'

    stream.write(format_summary('feature', feature_summary))
    stream.write(format_summary('scenario', scenario_summary))
    stream.write(format_summary('step', step_summary))

    if undefined:
        msg =  "\nYou can implement step definitions for undefined steps with "
        msg += "these snippets:\n\n"
        printed = set()
        for step in undefined:
            if step in printed:
                continue
            printed.add(step)

            func_name = step.name.lower().replace(' ', '_')
            msg += "@" + step.keyword + "('" + step.name + "')\n"
            msg += "def " + func_name + "(context):\n"
            msg += "    assert False\n\n"

        stream.write(escapes['undefined'] + msg + escapes['reset'])
        stream.flush()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import logging
import re
import subprocess
import os
import yaml

import messagebuilder
import configfetcher
import rqueue
from wblogging import LoggingSetupParser

parser = LoggingSetupParser(
    description='Sends events from Gerrit to IRC'
)
args = parser.parse_args()

logger = logging.getLogger('wikibugs.wb2-grrrrit')

def trim_repo(repo: str) -> str:
    if repo.startswith('mediawiki/') or repo.startswith('operations/'):
        repo = repo.split('/', 1)[-1]
    return repo

def extract_bug(commit_msg: str):
    search = re.search('Bug: T(\d+)', commit_msg)
    if search:
        return 'T' + search.group(1)

def process_event(event: dict):
    ret = None
    if event['type'] == 'patchset-created':
        ret = {
            'type': 'PS' + event['patchSet']['number'],
            'user': event['uploader']['name'],
            'message': event['change']['subject'],
            'repo': event['change']['project'],
            'branch': event['change']['branch'],
            'url': event['change']['url'],
            'task': extract_bug(event['change']['commitMessage']),
        }
        owner = event['change']['owner']['name']
        if ret['user'] != owner:
            ret['owner'] = owner
    elif event['type'] == 'draft-published':
        ret = {
            'type': 'Draft' + event['patchSet']['number'],
            'user': event['uploader']['name'],
            'message': event['change']['subject'],
            'repo': event['change']['project'],
            'branch': event['change']['branch'],
            'url': event['change']['url'],
            'task': extract_bug(event['change']['commitMessage']),
        }
        owner = event['change']['owner']['name']
        if ret['user'] != owner:
            ret['owner'] = owner
    elif event['type'] == 'comment-added':
        ret = {
            'type': 'CR',
            'user': event['author']['name'],
            'repo': event['change']['project'],
            'branch': event['change']['branch'],
            'url': event['change']['url'],
            'owner': event['change']['owner']['name'],
            'task': extract_bug(event['change']['commitMessage']),
        }
        comment = ''
        original_comment = event.get('comment')
        inline = 0
        if original_comment:
            inline_match = re.search('\((\d+) comments?\)', original_comment)
            if inline_match:
                try:
                    inline = int(inline_match.group(1))
                except ValueError:
                    pass
            comment = "\n".join(original_comment.split('\n')[1:]).strip().split('\n')[0].strip()
        if comment:
            comment = '"' + comment[:138] + '"'
        else:
            comment = event['change']['subject'][:140]
        ret['message'] = comment
        if ret['user'] == 'jenkins-bot':
            ret['message'] = event['change']['subject']
        ret['inline'] = inline
        ret['approvals'] = {}
        if event.get('approvals'):
            for approval in event['approvals']:
                value = int(approval['value'])
                # First, if it's jenkins-bot, skip if the value is not negative
                if ret['user'] == 'jenkins-bot' and value >= 0:
                    return None
                old_value = int(approval.get('oldValue', 0))
                # If the value didn't change, don't mention a score.
                if value == old_value:
                    continue
                if approval['type'] == 'Verified' and value != 0:
                    ret['approvals']['V'] = value
                    if ret['user'] == 'jenkins-bot' and value == -1:
                        ret['user'] = 'jerkins-bot'  # For MaxSem
                elif approval['type'] == 'Code-Review' and value != 0:
                    ret['approvals']['C'] = value
    elif event['type'] == 'change-merged':
        ret = process_simple(event, 'Merged', 'submitter')
        if ret['user'] == 'jenkins-bot' and ret['owner'] == 'L10n-bot':
            return None
        elif ret['user'] != 'jenkins-bot':
            return None
    elif event['type'] == 'change-restored':
        ret = process_simple(event, 'Restored', 'restorer')
    elif event['type'] == 'change-abandoned':
        ret = process_simple(event, 'Abandoned', 'abandoner')

    return ret

def process_simple(event: dict, type_: str, user_property: str) -> dict:
    return {
        'type': type_,
        'user': event[user_property]['name'],
        'message': event['change']['subject'],
        'repo': event['change']['project'],
        'branch': event['change']['branch'],
        'url': event['change']['url'],
        'owner': event['change']['owner']['name'],
        'task': extract_bug(event['change']['commitMessage']),
    }

def build_message(processed: dict) -> str:
    helper = messagebuilder.IRCMessageBuilder().ircformat
    text = '({})'.format(helper(processed['type'], foreground='green'))
    text += ' {}:'.format(helper(processed['user'], foreground='teal', style='bold'))
    if 'approvals' in processed and processed['approvals']:
        def format_approval(value: int) -> str:
            if value == 1:
                return helper(str(value), foreground='green')
            elif value == 2:
                return helper(str(value), foreground='green', style='bold')
            elif value == -1:
                return helper(str(value), foreground='red')
            else:  # -2
                return helper(str(value), foreground='red', style='bold')

        text += ' ['
        has_c = 'C' in processed['approvals']
        has_v = 'V' in processed['approvals']
        if has_v:
            text += 'V: {}'.format(format_approval(processed['approvals']['V']))
        if has_v and has_c:
            text += ' '
        if has_c:
            text += 'C: {}'.format(format_approval(processed['approvals']['C']))
        text += ']'
    text += ' {}'.format(processed['message'])
    if 'inline' in processed and processed['inline']:
        text += ' ({} {})'.format(
            helper(str(processed['inline']), foreground='green', style='bold'),
            'comments' if processed['inline'] > 1 else 'comment'
        )
    text += ' [{}]'.format(trim_repo(processed['repo']))
    if processed['branch'] not in ('master', 'production'):
        text += ' ({})'.format(processed['branch'])
    text += ' {}'.format(helper(processed['url'], foreground='teal'))
    if processed['task']:
        text += ' (https://phabricator.wikimedia.org/{})'.format(processed['task'])
    if 'owner' in processed:
        text += ' (owner: {})'.format(helper(processed['owner'], foreground='teal', style='bold'))

    return text

def channel_filter(repo: str, branch: str) -> set:
    # TODO use the channelfilter module
    with open(os.path.join(os.path.dirname(__file__), 'gerrit-channels.yaml')) as f:
        data = yaml.safe_load(f)
    channels = set()
    for channel in data['channels']:
        repos = data['channels'][channel]
        for repo_re, filters in repos.items():
            if re.match(repo_re, repo):
                if filters:
                    if not re.match(filters['branch'], branch):
                        continue
                channels.add(channel)

    if not channels:
        channels.add(data['default-channel'])

    channels.add(data['firehose-channel'])
    return channels

def main():
    conf = configfetcher.ConfigFetcher()
    queue = rqueue.RedisQueue(
        conf.get('REDIS_QUEUE_NAME'),
        conf.get('REDIS_HOST')
    )
    ssh = subprocess.Popen(
        ['ssh', 'suchabot@gerrit.wikimedia.org',
         '-i', 'id_rsa',
         '-p', '29418',
         'gerrit', 'stream-events'
         ],
        shell=False,
        bufsize=1,  # line buffered
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT
    )
    logger.info('Opened SSH connection')

    for line in ssh.stdout:
        logger.info(line.decode())
        parsed = json.loads(line.decode())
        processed = process_event(parsed)
        if processed:
            logger.info(json.dumps(processed))
            try:
                msg = build_message(processed)
                channels = channel_filter(processed['repo'], processed['branch'])
                queue.put({'raw': True, 'msg': ', '.join(channels) + ': ' + msg, 'channels': list(channels)})
            except:
                logger.exception('Error queuing message')
        ssh.stdout.flush()

if __name__ == '__main__':
    while True:
        try:
            main()
        except json.decoder.JSONDecodeError:
            logger.exception('JSONDecodeError, probably SSH connection dropped.')
<commit_msg>grrrrit: Use absolute path to ssh key<commit_after>#!/usr/bin/env python3

import json
import logging
import re
import subprocess
import os
import yaml

import messagebuilder
import configfetcher
import rqueue
from wblogging import LoggingSetupParser

parser = LoggingSetupParser(
    description='Sends events from Gerrit to IRC'
)
args = parser.parse_args()

logger = logging.getLogger('wikibugs.wb2-grrrrit')

def trim_repo(repo: str) -> str:
    if repo.startswith('mediawiki/') or repo.startswith('operations/'):
        repo = repo.split('/', 1)[-1]
    return repo

def extract_bug(commit_msg: str):
    search = re.search('Bug: T(\d+)', commit_msg)
    if search:
        return 'T' + search.group(1)

def process_event(event: dict):
    ret = None
    if event['type'] == 'patchset-created':
        ret = {
            'type': 'PS' + event['patchSet']['number'],
            'user': event['uploader']['name'],
            'message': event['change']['subject'],
            'repo': event['change']['project'],
            'branch': event['change']['branch'],
            'url': event['change']['url'],
            'task': extract_bug(event['change']['commitMessage']),
        }
        owner = event['change']['owner']['name']
        if ret['user'] != owner:
            ret['owner'] = owner
    elif event['type'] == 'draft-published':
        ret = {
            'type': 'Draft' + event['patchSet']['number'],
            'user': event['uploader']['name'],
            'message': event['change']['subject'],
            'repo': event['change']['project'],
            'branch': event['change']['branch'],
            'url': event['change']['url'],
            'task': extract_bug(event['change']['commitMessage']),
        }
        owner = event['change']['owner']['name']
        if ret['user'] != owner:
            ret['owner'] = owner
    elif event['type'] == 'comment-added':
        ret = {
            'type': 'CR',
            'user': event['author']['name'],
            'repo': event['change']['project'],
            'branch': event['change']['branch'],
            'url': event['change']['url'],
            'owner': event['change']['owner']['name'],
            'task': extract_bug(event['change']['commitMessage']),
        }
        comment = ''
        original_comment = event.get('comment')
        inline = 0
        if original_comment:
            inline_match = re.search('\((\d+) comments?\)', original_comment)
            if inline_match:
                try:
                    inline = int(inline_match.group(1))
                except ValueError:
                    pass
            comment = "\n".join(original_comment.split('\n')[1:]).strip().split('\n')[0].strip()
        if comment:
            comment = '"' + comment[:138] + '"'
        else:
            comment = event['change']['subject'][:140]
        ret['message'] = comment
        if ret['user'] == 'jenkins-bot':
            ret['message'] = event['change']['subject']
        ret['inline'] = inline
        ret['approvals'] = {}
        if event.get('approvals'):
            for approval in event['approvals']:
                value = int(approval['value'])
                # First, if it's jenkins-bot, skip if the value is not negative
                if ret['user'] == 'jenkins-bot' and value >= 0:
                    return None
                old_value = int(approval.get('oldValue', 0))
                # If the value didn't change, don't mention a score.
                if value == old_value:
                    continue
                if approval['type'] == 'Verified' and value != 0:
                    ret['approvals']['V'] = value
                    if ret['user'] == 'jenkins-bot' and value == -1:
                        ret['user'] = 'jerkins-bot'  # For MaxSem
                elif approval['type'] == 'Code-Review' and value != 0:
                    ret['approvals']['C'] = value
    elif event['type'] == 'change-merged':
        ret = process_simple(event, 'Merged', 'submitter')
        if ret['user'] == 'jenkins-bot' and ret['owner'] == 'L10n-bot':
            return None
        elif ret['user'] != 'jenkins-bot':
            return None
    elif event['type'] == 'change-restored':
        ret = process_simple(event, 'Restored', 'restorer')
    elif event['type'] == 'change-abandoned':
        ret = process_simple(event, 'Abandoned', 'abandoner')

    return ret

def process_simple(event: dict, type_: str, user_property: str) -> dict:
    return {
        'type': type_,
        'user': event[user_property]['name'],
        'message': event['change']['subject'],
        'repo': event['change']['project'],
        'branch': event['change']['branch'],
        'url': event['change']['url'],
        'owner': event['change']['owner']['name'],
        'task': extract_bug(event['change']['commitMessage']),
    }

def build_message(processed: dict) -> str:
    helper = messagebuilder.IRCMessageBuilder().ircformat
    text = '({})'.format(helper(processed['type'], foreground='green'))
    text += ' {}:'.format(helper(processed['user'], foreground='teal', style='bold'))
    if 'approvals' in processed and processed['approvals']:
        def format_approval(value: int) -> str:
            if value == 1:
                return helper(str(value), foreground='green')
            elif value == 2:
                return helper(str(value), foreground='green', style='bold')
            elif value == -1:
                return helper(str(value), foreground='red')
            else:  # -2
                return helper(str(value), foreground='red', style='bold')

        text += ' ['
        has_c = 'C' in processed['approvals']
        has_v = 'V' in processed['approvals']
        if has_v:
            text += 'V: {}'.format(format_approval(processed['approvals']['V']))
        if has_v and has_c:
            text += ' '
        if has_c:
            text += 'C: {}'.format(format_approval(processed['approvals']['C']))
        text += ']'
    text += ' {}'.format(processed['message'])
    if 'inline' in processed and processed['inline']:
        text += ' ({} {})'.format(
            helper(str(processed['inline']), foreground='green', style='bold'),
            'comments' if processed['inline'] > 1 else 'comment'
        )
    text += ' [{}]'.format(trim_repo(processed['repo']))
    if processed['branch'] not in ('master', 'production'):
        text += ' ({})'.format(processed['branch'])
    text += ' {}'.format(helper(processed['url'], foreground='teal'))
    if processed['task']:
        text += ' (https://phabricator.wikimedia.org/{})'.format(processed['task'])
    if 'owner' in processed:
        text += ' (owner: {})'.format(helper(processed['owner'], foreground='teal', style='bold'))

    return text

def channel_filter(repo: str, branch: str) -> set:
    # TODO use the channelfilter module
    with open(os.path.join(os.path.dirname(__file__), 'gerrit-channels.yaml')) as f:
        data = yaml.safe_load(f)
    channels = set()
    for channel in data['channels']:
        repos = data['channels'][channel]
        for repo_re, filters in repos.items():
            if re.match(repo_re, repo):
                if filters:
                    if not re.match(filters['branch'], branch):
                        continue
                channels.add(channel)

    if not channels:
        channels.add(data['default-channel'])

    channels.add(data['firehose-channel'])
    return channels

def main():
    conf = configfetcher.ConfigFetcher()
    queue = rqueue.RedisQueue(
        conf.get('REDIS_QUEUE_NAME'),
        conf.get('REDIS_HOST')
    )
    ssh = subprocess.Popen(
        ['ssh', 'suchabot@gerrit.wikimedia.org',
         '-i', os.path.join(os.path.dirname(__file__), 'id_rsa'),
         '-p', '29418',
         'gerrit', 'stream-events'
         ],
        shell=False,
        bufsize=1,  # line buffered
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT
    )
    logger.info('Opened SSH connection')

    for line in ssh.stdout:
        logger.info(line.decode())
        parsed = json.loads(line.decode())
        processed = process_event(parsed)
        if processed:
            logger.info(json.dumps(processed))
            try:
                msg = build_message(processed)
                channels = channel_filter(processed['repo'], processed['branch'])
                queue.put({'raw': True, 'msg': ', '.join(channels) + ': ' + msg, 'channels': list(channels)})
            except:
                logger.exception('Error queuing message')
        ssh.stdout.flush()

if __name__ == '__main__':
    while True:
        try:
            main()
        except json.decoder.JSONDecodeError:
            logger.exception('JSONDecodeError, probably SSH connection dropped.')
<|endoftext|>


<commit_before>#!/usr/bin/python3

"""
Script for automatically testing the REV7 hardware.
"""
import os
import serial
import time
#import traceback
try:
    import RPi.GPIO as GPIO
    MOTOR_PINS = {'LEFT': 3, 'RIGHT': 5}  # dict to store motor test pins.
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(MOTOR_PINS['LEFT'], GPIO.IN)
    GPIO.setup(MOTOR_PINS['RIGHT'], GPIO.IN)
except ImportError:
    print("RPi.GPIO not found. Using null test for motor.")
    GPIO = None

# Constants
OUTPUT_FILE = os.path.realpath('/home/pi/dev/hw_test/output.txt')
SERIAL_DEV  = '/dev/ttyUSB0'
SERIAL_BAUD = 4800

class AvailableTests:
    """
    A class containing the implementations of the tests.
    """
    def __init__(self):
        self.voltage = 300  # Set supply voltage here.

    def pass_fail(self, result):
        """
        Check if the string returned is PASS or FAIL
        :param result: String with the values PASS or FAIL
        :return: 1 on 'PASS', 0 on 'FAIL' and None for any invalid input.
        """
        if result == 'PASS':
            return 1
        if result == 'FAIL':
            return 0
        else:
            return None

    def supply(self, voltage):
        """
        Check the supply voltage is within bounds.
        The value is in the init routine for now.
        :param voltage: Supply voltage in cV
        :return: 1 if within bounds, else 0.
        """
        if (voltage > self.voltage * 0.9) and (voltage < self.voltage * 1.1):
            return 1
        else:
            return 0

    def serial(self, dev):
        """
        Loop back test to make sure serial Rx and Tx both work.
        :param dev: The serial device to use.
        :return: 1 on pass, 0 on fail.
        """
        test_string = b'PING!'  # This is the buffer that is sent.
        dev.write(test_string)
        response = dev.readline()
        if response[:5] == test_string:
            return 1
        else:
            return 0

    def led_flashing(self):
        """
        Accepts user input to determine if LED works.
        :return: 1 on pass, 0 on fail.
        """
        if input("Press Enter if LED flashing. Otherwise, press any key...") == '':
            return 1
        else:
            return 0

    def motor(self, direction):
        """
        Checks motor is working.
        NOT YET IMPLEMENTED!
        :param direction:
        :return:
        """
        # return None
        try:
            # Read pins
            for i in range(10):
                if not GPIO.input(MOTOR_PINS['LEFT']):
                    return 1
                if not GPIO.input(MOTOR_PINS['RIGHT']):
                    return 1
                time.sleep(0.100)
            else:
                return 0
        except: # AttributeError:
        #    # Not on the RPi.
            print("Skipping Motor Test")
            return None

    def dummy(self):
        """
        Dummy routine.
        :return: None to indicate test is disabled.
        """
        return None

# create test class and make a dispatch table.
available_tests = AvailableTests()
TESTS = {"UILED":           available_tests.dummy,  # This will be checked by the user
         "Vcc":             available_tests.supply,  # Ignored or set to 3.3 V?
         "Xtal":            available_tests.pass_fail,
         "LightSensor":     available_tests.pass_fail,
         "SHT21":           available_tests.pass_fail,
         "RFM23B":          available_tests.pass_fail,
         "Buttons":         available_tests.pass_fail,
         "Potentiometer":   available_tests.pass_fail,
         "MotorLeft":       available_tests.motor,  # This is checked by the optoisolator
         "MotorRight":      available_tests.motor,
         "Serial":          available_tests.serial}  # This will need some thought.

class DoTest:
    """
    Appends valid tests and results to a dictionary.
    The dictionary is returned if the test is successfully completed.
    """
    def __init__(self, tests, dev):
        """
        Init DoTest.
        :param test: A dictionary containing the expected test names and pass results.
        """
        self.tests = tests
        self.dev = dev

        self.first_token = 'Testing'
        self.results = {}
        self.n_devices_tested = 0
        self.n_devices_passed = 0
        self.n_devices_failed = 0

    def do(self, input_string):
        result_list = self.parse(input_string)
        ret_val = self.test(result_list)
        if ret_val is True:
            return False
        if ret_val is False:
            self.log_result(self.results)
            return self.results
        if ret_val is None:
            pass

    def parse(self, input_string):
        try:
            input_string = input_string.split()  # split at spaces
            if input_string[0] == "Testing":  # Make sure it is a test.
                return input_string[1:]
        except:
            print(traceback.format_exc())
            return None

    def test(self, var):
        try:
            if var[0] in TESTS:
                # This is the case where the REV7 cannot test itself.
                if var[0] == 'Serial':
                    self.results[var[0]] = TESTS[var[0]](self.dev)
                elif var[0] == 'Vcc':  # len(var) == 3:
                    self.results[var[0]] = TESTS[var[0]](int(var[2]))
                elif var[0] == 'UILED':
                    self.results[var[0]] = TESTS[var[0]]()
                elif var[0] == 'MotorLeft':
                    self.results[var[0]] = TESTS[var[0]]('LEFT')
                elif var[0] == 'MotorRight':
                    self.results[var[0]] = TESTS[var[0]]('RIGHT')
                else:
                    self.results[var[0]] = TESTS[var[0]](var[1])
                return True
            elif var[0] == 'START':
                print("Device found. Running tests...")
                self.results = {}
                self.n_devices_tested += 1
            elif var[0] == 'DONE':
                failed_tests = [x for x in self.results if self.results[x] == 0]
                if len(failed_tests) == 0:
                    self.n_devices_passed += 1
                    print("\033[92m" + "ALL TESTS PASSED!" + '\033[0m')
                else:
                    self.n_devices_failed += 1
                    print('\033[91m' + str(len(failed_tests)) + " TESTS FAILED!")
                    [print('\t' + x) for x in failed_tests]
                    print('\033[0m')
                print("Tested: " + str(self.n_devices_tested) + ',\t', end=' ')
                print("Passes: " + str(self.n_devices_passed) + ',\t', end=' ')
                print("Fails: " + str(self.n_devices_failed) + '.')
                return False
            else:
                print('\033[91m' + "Unsupported Test: " + var[0] + '\033[0m')
        except:
            #print(traceback.format_exc())
            return None

    def log_result(self, result):
        """
        Writes out the results as a JSON string.
        :param result:
        :return: None
        """
        try:
            with open(OUTPUT_FILE, 'a') as outfile:
                result = repr(result) + '\n'
                outfile.write(result)
        except:
            #print(traceback.format_exc())
            print("Could not log!")

# The actual program.
try:
    print("\n\nInitialising serial device " + SERIAL_DEV + " to " + str(SERIAL_BAUD) + ".")
    with serial.Serial(SERIAL_DEV, SERIAL_BAUD, timeout=1) as ser:  # Open serial port.
        do_test = DoTest(TESTS, ser)  # Make an instance of DoTest
        print("Waiting for device...")
        while True:
            string = ser.readline().decode()
            if len(string) > 3:
                do_test.do(string)
except UnicodeDecodeError:
    print("Serial error. Please try again.")
finally:
    try:
        print("\nCleaning up GPIO")
        GPIO.cleanup()
    except:
        pass
    print("Exiting")
<commit_msg>Added version number and updated output<commit_after>#!/usr/bin/python3

"""
Script for automatically testing the REV7 hardware.
"""
import sys
import os
import serial
import time
#import traceback

print("\nREV7 Hardware Testing Script v0.1")
print("Press CTRL-C to exit.")

try:
    import RPi.GPIO as GPIO
    MOTOR_PINS = {'LEFT': 3, 'RIGHT': 5}  # dict to store motor test pins.
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(MOTOR_PINS['LEFT'], GPIO.IN)
    GPIO.setup(MOTOR_PINS['RIGHT'], GPIO.IN)
except ImportError:
    print("\nRPi.GPIO not found. Using null test for motor.")
    GPIO = None

# Constants
OUTPUT_FILE = os.path.realpath('/home/pi/dev/hw_test/output.txt')
SERIAL_DEV  = '/dev/ttyUSB0'
SERIAL_BAUD = 4800

class AvailableTests:
    """
    A class containing the implementations of the tests.
    """
    def __init__(self):
        self.voltage = 300  # Set supply voltage here.

    def pass_fail(self, result):
        """
        Check if the string returned is PASS or FAIL
        :param result: String with the values PASS or FAIL
        :return: 1 on 'PASS', 0 on 'FAIL' and None for any invalid input.
        """
        if result == 'PASS':
            return 1
        if result == 'FAIL':
            return 0
        else:
            return None

    def supply(self, voltage):
        """
        Check the supply voltage is within bounds.
        The value is in the init routine for now.
        :param voltage: Supply voltage in cV
        :return: 1 if within bounds, else 0.
        """
        if (voltage > self.voltage * 0.9) and (voltage < self.voltage * 1.1):
            return 1
        else:
            return 0

    def serial(self, dev):
        """
        Loop back test to make sure serial Rx and Tx both work.
        :param dev: The serial device to use.
        :return: 1 on pass, 0 on fail.
        """
        test_string = b'PING!'  # This is the buffer that is sent.
        dev.write(test_string)
        response = dev.readline()
        if response[:5] == test_string:
            return 1
        else:
            return 0

    def led_flashing(self):
        """
        Accepts user input to determine if LED works.
        :return: 1 on pass, 0 on fail.
        """
        if input("Press Enter if LED flashing. Otherwise, press any key...") == '':
            return 1
        else:
            return 0

    def motor(self, direction):
        """
        Checks motor is working.
        NOT YET IMPLEMENTED!
        :param direction:
        :return:
        """
        # return None
        try:
            # Read pins
            for i in range(10):
                if not GPIO.input(MOTOR_PINS['LEFT']):
                    return 1
                if not GPIO.input(MOTOR_PINS['RIGHT']):
                    return 1
                time.sleep(0.100)
            else:
                return 0
        except: # AttributeError:
        #    # Not on the RPi.
            print("Skipping Motor Test")
            return None

    def dummy(self):
        """
        Dummy routine.
        :return: None to indicate test is disabled.
        """
        return None

# create test class and make a dispatch table.
available_tests = AvailableTests()
TESTS = {"UILED":           available_tests.dummy,  # This will be checked by the user
         "Vcc":             available_tests.supply,  # Ignored or set to 3.3 V?
         "Xtal":            available_tests.pass_fail,
         "LightSensor":     available_tests.pass_fail,
         "SHT21":           available_tests.pass_fail,
         "RFM23B":          available_tests.pass_fail,
         "Buttons":         available_tests.pass_fail,
         "Potentiometer":   available_tests.pass_fail,
         "MotorLeft":       available_tests.motor,  # This is checked by the optoisolator
         "MotorRight":      available_tests.motor,
         "Serial":          available_tests.serial}  # This will need some thought.

class DoTest:
    """
    Appends valid tests and results to a dictionary.
    The dictionary is returned if the test is successfully completed.
    """
    def __init__(self, tests, dev):
        """
        Init DoTest.
        :param test: A dictionary containing the expected test names and pass results.
        """
        self.tests = tests
        self.dev = dev

        self.first_token = 'Testing'
        self.results = {}
        self.n_devices_tested = 0
        self.n_devices_passed = 0
        self.n_devices_failed = 0

    def do(self, input_string):
        result_list = self.parse(input_string)
        ret_val = self.test(result_list)
        if ret_val is True:
            return False
        if ret_val is False:
            self.log_result(self.results)
            print("\nConnect next device or press CTRL-C to exit.\n")
            return self.results
        if ret_val is None:
            pass

    def parse(self, input_string):
        try:
            input_string = input_string.split()  # split at spaces
            if input_string[0] == "Testing":  # Make sure it is a test.
                return input_string[1:]
        except:
            print(traceback.format_exc())
            return None

    def test(self, var):
        try:
            if var[0] in TESTS:
                # This is the case where the REV7 cannot test itself.
                if var[0] == 'Serial':
                    self.results[var[0]] = TESTS[var[0]](self.dev)
                elif var[0] == 'Vcc':  # len(var) == 3:
                    self.results[var[0]] = TESTS[var[0]](int(var[2]))
                elif var[0] == 'UILED':
                    self.results[var[0]] = TESTS[var[0]]()
                elif var[0] == 'MotorLeft':
                    self.results[var[0]] = TESTS[var[0]]('LEFT')
                elif var[0] == 'MotorRight':
                    self.results[var[0]] = TESTS[var[0]]('RIGHT')
                else:
                    self.results[var[0]] = TESTS[var[0]](var[1])
                return True
            elif var[0] == 'START':
                print("Device found. Running tests...")
                self.results = {}
                self.n_devices_tested += 1
            elif var[0] == 'DONE':
                failed_tests = [x for x in self.results if self.results[x] == 0]
                if len(failed_tests) == 0:
                    self.n_devices_passed += 1
                    print("\033[92m" + "ALL TESTS PASSED!" + '\033[0m')
                else:
                    self.n_devices_failed += 1
                    print('\033[91m' + str(len(failed_tests)) + " TESTS FAILED!")
                    [print('\t' + x) for x in failed_tests]
                    print('\033[0m')
                print("Tested: " + str(self.n_devices_tested) + ',\t', end=' ')
                print("Passes: " + str(self.n_devices_passed) + ',\t', end=' ')
                print("Fails: " + str(self.n_devices_failed) + '.')
                return False
            else:
                print('\033[91m' + "Unsupported Test: " + var[0] + '\033[0m')
        except:
            #print(traceback.format_exc())
            return None

    def log_result(self, result):
        """
        Writes out the results as a JSON string.
        :param result:
        :return: None
        """
        try:
            with open(OUTPUT_FILE, 'a') as outfile:
                result = repr(result) + '\n'
                outfile.write(result)
        except:
            #print(traceback.format_exc())
            print("Could not log!")

# The actual program.
try:
    print("Initialising serial device " + SERIAL_DEV + " to " + str(SERIAL_BAUD) + ".")
    with serial.Serial(SERIAL_DEV, SERIAL_BAUD, timeout=1) as ser:  # Open serial port.
        do_test = DoTest(TESTS, ser)  # Make an instance of DoTest
        print("Waiting for device...")
        while True:
            string = ser.readline().decode()
            if len(string) > 3:
                do_test.do(string)
except UnicodeDecodeError:
    print("Serial error. Please try again.")
except KeyboardInterrupt:
    sys.exit(0)
finally:
    try:
        print("\nCleaning up GPIO")
        GPIO.cleanup()
    except:
        pass
    print("Exiting")
<|endoftext|>