<commit_before># -*- coding: utf-8; Mode: Python -*-
#
# Copyright (C) 2010  Lincoln de Sousa <lincoln@comum.org>
# Copyright (C) 2010  Rogerio Hilbert Lima <rogerhil@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Campos:
#
# * nome
# * tipo
# * tipo_convenio
# * numero_convenio
#
# * entidade proponente
#   * nome
#   * telefones
#   * endereco
#   * contato
#
# * responsavel
#   * nome
#   * email
#   * telefones
#
# * endereco
#   * logradouro
#   * numero
#   * bairro
#   * cidade
#   * estado
#   * complemento
#   * cep
#
# * contato
#   * telefone(s)
#   * email
#   * blog

import csv
import sys
sys.path.insert(0, '..')
from datetime import datetime
from time import strptime

from elixir import session
from sqlalchemy.exc import IntegrityError

from cdpc.app.usuarios.models import Pessoa
from cdpc.app.projetos.models import Projeto, Entidade
from cdpc.app.common.models import Endereco, Telefone
from cdpc.app.common.cadastro import item_format
from cdpc.app.utils.model import get_or_create
from cdpc.main import setup_models

setup_models()

TIPOSTEL = {'8': u'Celular',
            '9': u'Celular',
            '7': u'Rádio'}
            #'2': u'Rural',
            #'2': u'Público'}

def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
    # csv.py doesn't do Unicode; encode temporarily as UTF-8:
    csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
                            dialect=dialect, **kwargs)
    for row in csv_reader:
        # decode UTF-8 back to Unicode, cell by cell:
        yield [unicode(cell, 'utf-8') for cell in row]

def utf_8_encoder(unicode_csv_data):
    for line in unicode_csv_data:
        yield line.encode('utf-8')

def get_tipo(tel):
    p = tel[0]
    tipo = TIPOSTEL.get(p, u'Fixo')
    return item_format(tipo)

def add_tel(tel_list, ddd, tel):
    ddd = ddd.strip()
    tel = tel.strip()
    if ddd and tel:
        tel = '%s%s' % (ddd, tel)
        tipo = get_tipo(tel)
        inst, _ = get_or_create(Telefone, numero=tel, tipo=tipo)
        session.commit()
        if not inst in tel_list:
            tel_list.append(inst)
        session.commit()

def clean_name(name):
    name = unicode(name.decode('utf-8')).strip()
    cap = lambda x: x.lower() if len(x) <= 2 or (len(x) == 3 and \
                    x.lower().endswith('s')) else x.capitalize()
    splited = name.split(' ')
    if splited:
        name = ' '.join(filter(None, [splited[0].capitalize()] + map(cap, splited[1:])))
    return name

def main():
    # Iterando sobre as entradas no arquivo csv gerado à partir do dump
    # do banco de dados colhido durante a teia2010.
    #
    # Todos os campos que começam com "p_" são relativos ao projeto, já
    # os que começam com "r_", são relativos ao responsável e,
    # finalmente, os que começam com "e_" são relativos à entidade
    # proponente. Meio tosco, mas é um script utilitário que vai ser
    # usado apenas uma vez, então... let the hammer fall!

    for i in list(csv.reader(file('pontos_teia2010.csv')))[1:]:
        (_, _, _, _,
         p_tipo, p_tipo_convenio, p_nome,
         p_logradouro, p_num, p_complemento, p_bairro, p_cep, p_cidade, p_uf,
         p_ddd1, p_tel1, p_ddd2, p_tel2, p_email, p_blog,
         r_nome, r_email, r_ddd, r_tel,
         e_nome,
         e_logradouro, e_num, e_complemento, e_bairro, e_cep, e_cidade, e_uf,
         e_ddd1, e_tel1, e_ddd2, e_tel2, e_email, e_blog,
         data_cadastro, hora_cadastro, _
         ) = i

        # Criando a linha para a pessoa responsável pelo projeto e suas
        # relações
        responsavel, created = get_or_create(Pessoa, email=r_email)
        if created:
            responsavel.nome = clean_name(r_nome)
            add_tel(responsavel.telefones, r_ddd, r_tel)

        # Criando a linha para a entidade responsável pelo projeto e
        # suas relações
        entidade = get_or_create(Entidade, nome=e_nome, email=e_email,
                                 website=e_blog)[0]

        entidade.endereco = get_or_create(Endereco,
            logradouro=e_logradouro,
            numero=e_num,
            complemento=e_complemento,
            bairro=e_bairro,
            cep=e_cep,
            cidade=e_cidade,
            uf=e_uf)[0]

        add_tel(entidade.telefones, e_ddd1, e_tel1)
        add_tel(entidade.telefones, e_ddd2, e_tel2)

        # Finalmente, criando o projeto
        projeto = Projeto(
            nome=clean_name(p_nome),
            tipo=p_tipo,
            tipo_convenio=p_tipo_convenio,
            email=p_email,
            site=p_blog,
            entidade=entidade)

        projeto.endereco_sede = get_or_create(Endereco,
                logradouro=clean_name(p_logradouro),
                numero=p_num,
                complemento=p_complemento,
                bairro=clean_name(p_bairro),
                cep=p_cep,
                cidade=clean_name(p_cidade),
                uf=p_uf)[0]

        add_tel(projeto.telefones, p_ddd1, p_tel1)
        add_tel(projeto.telefones, p_ddd2, p_tel2)

        data_cadastro = data_cadastro.strip()
        hora_cadastro = hora_cadastro.strip()
        data = ''
        if data_cadastro:
            data = data_cadastro
            formato = '%Y-%m-%d'
        if data_cadastro and hora_cadastro:
            data += ' %s' % hora_cadastro
            formato += ' %H:%M:%S'
        if data:
            time_st = strptime(data, formato)
            projeto.data_cadastro = datetime(*tuple(time_st)[:6])

        session.commit()

if __name__ == '__main__':
    main()
<commit_msg>Resposavel do projeton associado.<commit_after># -*- coding: utf-8; Mode: Python -*-
#
# Copyright (C) 2010  Lincoln de Sousa <lincoln@comum.org>
# Copyright (C) 2010  Rogerio Hilbert Lima <rogerhil@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Campos:
#
# * nome
# * tipo
# * tipo_convenio
# * numero_convenio
#
# * entidade proponente
#   * nome
#   * telefones
#   * endereco
#   * contato
#
# * responsavel
#   * nome
#   * email
#   * telefones
#
# * endereco
#   * logradouro
#   * numero
#   * bairro
#   * cidade
#   * estado
#   * complemento
#   * cep
#
# * contato
#   * telefone(s)
#   * email
#   * blog

import csv
import sys
sys.path.insert(0, '..')
from datetime import datetime
from time import strptime

from elixir import session
from sqlalchemy.exc import IntegrityError

from cdpc.app.usuarios.models import Pessoa
from cdpc.app.projetos.models import Projeto, Entidade
from cdpc.app.common.models import Endereco, Telefone
from cdpc.app.common.cadastro import item_format
from cdpc.app.utils.model import get_or_create
from cdpc.main import setup_models

setup_models()

TIPOSTEL = {'8': u'Celular',
            '9': u'Celular',
            '7': u'Rádio'}
            #'2': u'Rural',
            #'2': u'Público'}

def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
    # csv.py doesn't do Unicode; encode temporarily as UTF-8:
    csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
                            dialect=dialect, **kwargs)
    for row in csv_reader:
        # decode UTF-8 back to Unicode, cell by cell:
        yield [unicode(cell, 'utf-8') for cell in row]

def utf_8_encoder(unicode_csv_data):
    for line in unicode_csv_data:
        yield line.encode('utf-8')

def get_tipo(tel):
    p = tel[0]
    tipo = TIPOSTEL.get(p, u'Fixo')
    return item_format(tipo)

def add_tel(tel_list, ddd, tel):
    ddd = ddd.strip()
    tel = tel.strip()
    if ddd and tel:
        tel = '%s%s' % (ddd, tel)
        tipo = get_tipo(tel)
        inst, _ = get_or_create(Telefone, numero=tel, tipo=tipo)
        session.commit()
        if not inst in tel_list:
            tel_list.append(inst)
        session.commit()

def clean_name(name):
    name = unicode(name.decode('utf-8')).strip()
    cap = lambda x: x.lower() if len(x) <= 2 or (len(x) == 3 and \
                    x.lower().endswith('s')) else x.capitalize()
    splited = name.split(' ')
    if splited:
        name = ' '.join(filter(None, [splited[0].capitalize()] + map(cap, splited[1:])))
    return name

def main():
    # Iterando sobre as entradas no arquivo csv gerado à partir do dump
    # do banco de dados colhido durante a teia2010.
    #
    # Todos os campos que começam com "p_" são relativos ao projeto, já
    # os que começam com "r_", são relativos ao responsável e,
    # finalmente, os que começam com "e_" são relativos à entidade
    # proponente. Meio tosco, mas é um script utilitário que vai ser
    # usado apenas uma vez, então... let the hammer fall!

    for i in list(csv.reader(file('pontos_teia2010.csv')))[1:]:
        (_, _, _, _,
         p_tipo, p_tipo_convenio, p_nome,
         p_logradouro, p_num, p_complemento, p_bairro, p_cep, p_cidade, p_uf,
         p_ddd1, p_tel1, p_ddd2, p_tel2, p_email, p_blog,
         r_nome, r_email, r_ddd, r_tel,
         e_nome,
         e_logradouro, e_num, e_complemento, e_bairro, e_cep, e_cidade, e_uf,
         e_ddd1, e_tel1, e_ddd2, e_tel2, e_email, e_blog,
         data_cadastro, hora_cadastro, _
         ) = i

        # Criando a linha para a pessoa responsável pelo projeto e suas
        # relações
        responsavel, created = get_or_create(Pessoa, email=r_email)
        if created:
            responsavel.nome = clean_name(r_nome)
            add_tel(responsavel.telefones, r_ddd, r_tel)

        # Criando a linha para a entidade responsável pelo projeto e
        # suas relações
        entidade = get_or_create(Entidade, nome=e_nome, email=e_email,
                                 website=e_blog)[0]

        entidade.endereco = get_or_create(Endereco,
            logradouro=e_logradouro,
            numero=e_num,
            complemento=e_complemento,
            bairro=e_bairro,
            cep=e_cep,
            cidade=e_cidade,
            uf=e_uf)[0]

        add_tel(entidade.telefones, e_ddd1, e_tel1)
        add_tel(entidade.telefones, e_ddd2, e_tel2)

        # Finalmente, criando o projeto
        projeto = Projeto(
            nome=clean_name(p_nome),
            tipo=p_tipo,
            tipo_convenio=p_tipo_convenio,
            email=p_email,
            site=p_blog,
            entidade=entidade)

        projeto.responsavel = responsavel

        projeto.endereco_sede = get_or_create(Endereco,
                logradouro=clean_name(p_logradouro),
                numero=p_num,
                complemento=p_complemento,
                bairro=clean_name(p_bairro),
                cep=p_cep,
                cidade=clean_name(p_cidade),
                uf=p_uf)[0]

        add_tel(projeto.telefones, p_ddd1, p_tel1)
        add_tel(projeto.telefones, p_ddd2, p_tel2)

        data_cadastro = data_cadastro.strip()
        hora_cadastro = hora_cadastro.strip()
        data = ''
        if data_cadastro:
            data = data_cadastro
            formato = '%Y-%m-%d'
        if data_cadastro and hora_cadastro:
            data += ' %s' % hora_cadastro
            formato += ' %H:%M:%S'
        if data:
            time_st = strptime(data, formato)
            projeto.data_cadastro = datetime(*tuple(time_st)[:6])

        session.commit()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># monkeyrunner script to install current klaxon, and run its tests.
# Author: Marc Dougherty <muncus@gmail.com>

from com.android.monkeyrunner import MonkeyDevice, MonkeyRunner

INSTRUMENTATION_RUNNER = 'android.test.InstrumentationTestRunner'

device = MonkeyRunner.waitForConnection()

device.installPackage('bin/Klaxon-debug.apk')
device.installPackage('tests/bin/KlaxonTests-debug.apk')
package = 'org.nerdcircus.android.klaxon'
activity = package + ".KlaxonList"

test_dict = device.instrument("%s/%s" % (package+'.tests', INSTRUMENTATION_RUNNER))

assert 'OK' in test_dict['stream']
print test_dict['stream']

# try to send some smses through adb.

import socket
sock = socket.socket()
sock.connect(('localhost', 5554))
sock.recv(300) # eat the banner.

pattern = "sms send 424242 %(msg)s\n"

for m in ['foo', 'Foo', 'FOO']:
  sock.send(pattern % {'msg': m})
  assert 'OK' in sock.recv(10)
<commit_msg>leave out the sms sending, until we can test it properly<commit_after># monkeyrunner script to install current klaxon, and run its tests.
# Author: Marc Dougherty <muncus@gmail.com>

from com.android.monkeyrunner import MonkeyDevice, MonkeyRunner

INSTRUMENTATION_RUNNER = 'android.test.InstrumentationTestRunner'

device = MonkeyRunner.waitForConnection()

device.installPackage('bin/Klaxon-debug.apk')
device.installPackage('tests/bin/KlaxonTests-debug.apk')
package = 'org.nerdcircus.android.klaxon'
activity = package + ".KlaxonList"

test_dict = device.instrument("%s/%s" % (package+'.tests', INSTRUMENTATION_RUNNER))

assert 'OK' in test_dict['stream']
print test_dict['stream']

# try to send some smses through adb.

#import socket
#sock = socket.socket()
#sock.connect(('localhost', 5554))
#sock.recv(300) # eat the banner.

#pattern = "sms send 424242 %(msg)s\n"

#for m in ['foo', 'Foo', 'FOO']:
#  sock.send(pattern % {'msg': m})
#  assert 'OK' in sock.recv(10)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
from __future__ import print_function
"""
optimize_path
"""
import itertools
import json
import logging
import os
import re
import subprocess
import sys
from collections import OrderedDict, Counter

if sys.version_info.major > 2:
    iterkeys = lambda x: x.keys()
else:
    iterkeys = lambda x: s.iterkeys()

import distutils.spawn

log = logging.getLogger()

md5_bin = distutils.spawn.find_executable('md5')
if md5_bin:  # OSX
    CHECKSUM_BIN = (md5_bin,'-q')
else:
    md5sum_bin = distutils.spawn.find_executable('md5sum')
    if md5sum_bin:
        CHECKSUM_BIN = (md5sum_bin, )
if not CHECKSUM_BIN:
    raise Exception('neither md5 nor md5sum were found in $PATH')
CHECKSUM_BIN = list(CHECKSUM_BIN)

def sbin_policy(iterable):
    """
    # TODO: static optimizations
    """
    for s in sorted(
        (
            (
                (int('/sbin' not in x)*2) +
                (int('/usr/local' in x)*3) +
                (int('/home' in x)*4)
            ),
            len(x),
            x)
            for x in iterable ):
        #logging.debug(s)
        yield s[-1]

def optimize_path(iterable, strip=None, sortfunc=sbin_policy):
    """
    optimize paths
    """
    if strip:
        pattern = re.compile( strip )
        log.debug((pattern, strip, ))
        keys = (p.strip() for p in iterable if not pattern.match(p))
    else:
        keys = (p.strip() for p in iterable)

    paths = OrderedDict.fromkeys(keys)

    if sortfunc:
        return sortfunc(iterkeys(paths))
    else:
        return iterkeys(paths)

def get_checksum(path, hashbin=CHECKSUM_BIN):
    """
    Get a file checksum

    Args:
        path (str): path to get checksum of
        hashbin (list): command argument list
    Returns:
        (path, checksum or None, symlink_to or None)
    """
    logging.debug((hashbin,path))
    if os.path.isfile(path):
        if os.access(path, os.R_OK):
            cmd = hashbin + [path,]
            return (path,
                    subprocess.check_output(cmd).rstrip(),
                    (os.path.islink(path) and os.path.realpath(path) or None))
    return (path, None, None)

def detect_duplicates(iterable, do_checksums=False):
    """
    Detect duplicate paths in iterable

    Args:
        iterable (iterable): iterable of paths
    Returns:

    """

    def get_files_on_path():
        """
        Returns:
            paths (OrderedDict): (basename, [paths])
        """
        paths = OrderedDict()
        for path in iterable:
            if not os.path.exists(path):
                continue  # TODO
            for f in os.listdir(path):
                filename = os.path.join(path, f)
                fullpath = os.path.abspath(filename)
                basename = os.path.basename(f)
                paths.setdefault(basename, [])
                paths[basename].append(fullpath)
                #print(basename, basename_paths)
        return paths

    def find_name_collisions(paths):
        """
        Args:
            paths (OrderedDict): (basename, [paths])
        Returns:
            None
        """
        for key, value in paths.items():
            if len(value) > 1:
                print('# duplicate: {} {}'.format(key, value))
            print('{} {}'.format(key, value))

    def get_annotated_paths(paths):
        """
        Args:
            paths (iterable): iterable of paths
        Yields:
            tuple: (key, (path, checksum, symlink_to))
        """
        for (key, value) in paths.iteritems():
            for p in value:
                path_ = p  # TODO: os.path.join(p, key)
                if os.path.exists(path_):
                    try:
                        yield (key, get_checksum(path_))
                    except Exception as e:
                        log.error('checksum_err: {} # {}'.format(path_, e))
                        continue

    paths = get_files_on_path()
    output = paths

    find_name_collisions(paths)

    if do_checksums:
        iterable = list( get_annotated_paths(paths) )

        counterdict = OrderedDict()
        for key, value in iterable:
            counterdict.setdefault(key, [])
            counterdict[key].append(value)

        for key, value in counterdict.items():
            if len(value) > 1:
                print('duplicate: {} {}'.format(key, value))
            print('{} {}'.format(key, value))

        output = counterdict

    return output

def check_difference(iterable):
    raise NotImplementedError()
    import difflib

    iterable = list(iterable)
    regular = list( optimize_path( iterable, sortfunc=None) )
    optimized = list( optimize_path( iterable, sortfunc=sbin_policy ) )

    logging.debug(regular)
    logging.debug(optimized)

    return '\n'.join( difflib.ndiff(regular, optimized) )
    # detect colli

import unittest
import sys

if sys.version_info.major > 2:
    from io import StringIO
else:
    from StringIO import StringIO

class ShellTestCase(unittest.TestCase):
    __stdout = sys.stdout
    __stderr = sys.stderr
    def _run_command(self, _cmd):
        # TODO: not monkey-patch sys.stderr
        try:
            #process = subprocess.Popen((here,) + cmd,
            #    stdout=subprocess.PIPE,
            #    stderr=subprocess.PIPE)
            #stdout,stderr = process.communicate()
            #logging.debug(_cmd)
            logging.info("### TEST CASE: %r" % _cmd)

            stdout = sys.stdout = StringIO()
            stderr = sys.stderr = StringIO()

            sys.argv = _cmd
            main()

            _stderr = stderr.read()
            _stdout = stdout.read() # ..

            logging.debug('# >2\n%s',_stderr)
            logging.debug('# >1\n%s', _stdout)
            return _stderr, _stdout

        finally:
            sys.stdout = self.__stdout
            sys.stderr = self.__stderr

class Test_optimize_path(ShellTestCase):
    def test_optimize_path(self):
        print( '\n'.join( optimize_path(os.environ['PATH'].split(':')) ) )

    def test_script(self):
        cmds = (
         ('-p', ),
         ('-c', ),
         #('-w','hashes', ),
         #('-w','duplicates', ),
        )
        here = os.path.abspath(__file__)

        for cmd in cmds:
            _cmd = [here, ] + list(cmd)
            stderr, stdout = self._run_command(_cmd)
            self.assertEqual(len(stderr), 0)

def main(*_args):
    import optparse
    import logging
    import sys

    prs = optparse.OptionParser(usage="./%prog : args")

    prs.add_option('-v', '--verbose',
                    dest='verbose',
                    action='store_true',)
    prs.add_option('-q', '--quiet',
                    dest='quiet',
                    action='store_true',)
    prs.add_option('-t', '--test',
                    dest='run_tests',
                    action='store_true',)

    prs.add_option('-p', '--print',
                    dest='print',
                    action='store_true',)
    prs.add_option('-d', '--delim',
                    dest='delim',
                    action='store',
                    default='\n')
    prs.add_option('-s', '--strip',
                    dest='strip',
                    action='store',
                    help='Regex expression to strip from path')

    prs.add_option('-c', '--check',
                    dest='check',
                    action='store_true',
                    help='Detect lookup variance on path')
    prs.add_option('-w', '--whichpackage',
                    dest='pkg_lookup',
                    action='store',
                    default=None,
                    help='Run `wajig whichpackage $path`')

    (opts, args) = prs.parse_args(_args or sys.argv[1:])

    if not opts.quiet:
        logging.basicConfig()

        if opts.verbose:
            logging.getLogger().setLevel(logging.DEBUG)

    if opts.run_tests:
        sys.argv = [sys.argv[0]] + args
        import unittest
        exit(unittest.main())

    iterable = os.environ['PATH'].split(':')

    #iterable = list( optimize_path(iterable, opts.strip) )

    if opts.verbose:
        iterable, output = itertools.tee(iterable)
        for i in output:
            logging.info(i)

    if opts.print:
        print( opts.delim.join( iterable ) )

    duplicates = tuple()
    if opts.check:
        duplicates = detect_duplicates(iterable)
        print(json.dumps(duplicates, indent=3))

    if opts.pkg_lookup:
        duplicates = duplicates or detect_duplicates(iterable)

        # emit,yield,return
        from collections import namedtuple
        File = namedtuple('File', ('path','package', 'hash'))
        File.__str__ = lambda self: ("%-20s\t%s\t%s" %
                        (   self.package or '?',
                            self.path,
                            not self.package and self.hash or ''))

        for (k,v) in duplicates.iteritems():
            if v.get('type', None) == opts.pkg_lookup:
                for path in v.get('files'):
                    hash, path = path.split()
                    package = subprocess.check_output(
                            ('wajig','whichpackage', path)
                        ).split(':')[0].strip()
                    _file = File(path, package, hash)
                    logging.info(str(_file))

if __name__ == "__main__":
    main()
<commit_msg>BUG: scripts/optimizepath.py: typo in iterkeys lambda<commit_after>#!/usr/bin/env python
# encoding: utf-8
from __future__ import print_function
"""
optimize_path
"""
import itertools
import json
import logging
import os
import re
import subprocess
import sys
from collections import OrderedDict, Counter

if sys.version_info.major > 2:
    iterkeys = lambda x: x.keys()
else:
    iterkeys = lambda x: x.iterkeys()

import distutils.spawn

log = logging.getLogger()

md5_bin = distutils.spawn.find_executable('md5')
if md5_bin:  # OSX
    CHECKSUM_BIN = (md5_bin,'-q')
else:
    md5sum_bin = distutils.spawn.find_executable('md5sum')
    if md5sum_bin:
        CHECKSUM_BIN = (md5sum_bin, )
if not CHECKSUM_BIN:
    raise Exception('neither md5 nor md5sum were found in $PATH')
CHECKSUM_BIN = list(CHECKSUM_BIN)

def sbin_policy(iterable):
    """
    # TODO: static optimizations
    """
    for s in sorted(
        (
            (
                (int('/sbin' not in x)*2) +
                (int('/usr/local' in x)*3) +
                (int('/home' in x)*4)
            ),
            len(x),
            x)
            for x in iterable ):
        #logging.debug(s)
        yield s[-1]

def optimize_path(iterable, strip=None, sortfunc=sbin_policy):
    """
    optimize paths
    """
    if strip:
        pattern = re.compile( strip )
        log.debug((pattern, strip, ))
        keys = (p.strip() for p in iterable if not pattern.match(p))
    else:
        keys = (p.strip() for p in iterable)

    paths = OrderedDict.fromkeys(keys)

    if sortfunc:
        return sortfunc(iterkeys(paths))
    else:
        return iterkeys(paths)

def get_checksum(path, hashbin=CHECKSUM_BIN):
    """
    Get a file checksum

    Args:
        path (str): path to get checksum of
        hashbin (list): command argument list
    Returns:
        (path, checksum or None, symlink_to or None)
    """
    logging.debug((hashbin,path))
    if os.path.isfile(path):
        if os.access(path, os.R_OK):
            cmd = hashbin + [path,]
            return (path,
                    subprocess.check_output(cmd).rstrip(),
                    (os.path.islink(path) and os.path.realpath(path) or None))
    return (path, None, None)

def detect_duplicates(iterable, do_checksums=False):
    """
    Detect duplicate paths in iterable

    Args:
        iterable (iterable): iterable of paths
    Returns:

    """

    def get_files_on_path():
        """
        Returns:
            paths (OrderedDict): (basename, [paths])
        """
        paths = OrderedDict()
        for path in iterable:
            if not os.path.exists(path):
                continue  # TODO
            for f in os.listdir(path):
                filename = os.path.join(path, f)
                fullpath = os.path.abspath(filename)
                basename = os.path.basename(f)
                paths.setdefault(basename, [])
                paths[basename].append(fullpath)
                #print(basename, basename_paths)
        return paths

    def find_name_collisions(paths):
        """
        Args:
            paths (OrderedDict): (basename, [paths])
        Returns:
            None
        """
        for key, value in paths.items():
            if len(value) > 1:
                print('# duplicate: {} {}'.format(key, value))
            print('{} {}'.format(key, value))

    def get_annotated_paths(paths):
        """
        Args:
            paths (iterable): iterable of paths
        Yields:
            tuple: (key, (path, checksum, symlink_to))
        """
        for (key, value) in paths.iteritems():
            for p in value:
                path_ = p  # TODO: os.path.join(p, key)
                if os.path.exists(path_):
                    try:
                        yield (key, get_checksum(path_))
                    except Exception as e:
                        log.error('checksum_err: {} # {}'.format(path_, e))
                        continue

    paths = get_files_on_path()
    output = paths

    find_name_collisions(paths)

    if do_checksums:
        iterable = list( get_annotated_paths(paths) )

        counterdict = OrderedDict()
        for key, value in iterable:
            counterdict.setdefault(key, [])
            counterdict[key].append(value)

        for key, value in counterdict.items():
            if len(value) > 1:
                print('duplicate: {} {}'.format(key, value))
            print('{} {}'.format(key, value))

        output = counterdict

    return output

def check_difference(iterable):
    raise NotImplementedError()
    import difflib

    iterable = list(iterable)
    regular = list( optimize_path( iterable, sortfunc=None) )
    optimized = list( optimize_path( iterable, sortfunc=sbin_policy ) )

    logging.debug(regular)
    logging.debug(optimized)

    return '\n'.join( difflib.ndiff(regular, optimized) )
    # detect colli

import unittest
import sys

if sys.version_info.major > 2:
    from io import StringIO
else:
    from StringIO import StringIO

class ShellTestCase(unittest.TestCase):
    __stdout = sys.stdout
    __stderr = sys.stderr
    def _run_command(self, _cmd):
        # TODO: not monkey-patch sys.stderr
        try:
            #process = subprocess.Popen((here,) + cmd,
            #    stdout=subprocess.PIPE,
            #    stderr=subprocess.PIPE)
            #stdout,stderr = process.communicate()
            #logging.debug(_cmd)
            logging.info("### TEST CASE: %r" % _cmd)

            stdout = sys.stdout = StringIO()
            stderr = sys.stderr = StringIO()

            sys.argv = _cmd
            main()

            _stderr = stderr.read()
            _stdout = stdout.read() # ..

            logging.debug('# >2\n%s',_stderr)
            logging.debug('# >1\n%s', _stdout)
            return _stderr, _stdout

        finally:
            sys.stdout = self.__stdout
            sys.stderr = self.__stderr

class Test_optimize_path(ShellTestCase):
    def test_optimize_path(self):
        print( '\n'.join( optimize_path(os.environ['PATH'].split(':')) ) )

    def test_script(self):
        cmds = (
         ('-p', ),
         ('-c', ),
         #('-w','hashes', ),
         #('-w','duplicates', ),
        )
        here = os.path.abspath(__file__)

        for cmd in cmds:
            _cmd = [here, ] + list(cmd)
            stderr, stdout = self._run_command(_cmd)
            self.assertEqual(len(stderr), 0)

def main(*_args):
    import optparse
    import logging
    import sys

    prs = optparse.OptionParser(usage="./%prog : args")

    prs.add_option('-v', '--verbose',
                    dest='verbose',
                    action='store_true',)
    prs.add_option('-q', '--quiet',
                    dest='quiet',
                    action='store_true',)
    prs.add_option('-t', '--test',
                    dest='run_tests',
                    action='store_true',)

    prs.add_option('-p', '--print',
                    dest='print',
                    action='store_true',)
    prs.add_option('-d', '--delim',
                    dest='delim',
                    action='store',
                    default='\n')
    prs.add_option('-s', '--strip',
                    dest='strip',
                    action='store',
                    help='Regex expression to strip from path')

    prs.add_option('-c', '--check',
                    dest='check',
                    action='store_true',
                    help='Detect lookup variance on path')
    prs.add_option('-w', '--whichpackage',
                    dest='pkg_lookup',
                    action='store',
                    default=None,
                    help='Run `wajig whichpackage $path`')

    (opts, args) = prs.parse_args(_args or sys.argv[1:])

    if not opts.quiet:
        logging.basicConfig()

        if opts.verbose:
            logging.getLogger().setLevel(logging.DEBUG)

    if opts.run_tests:
        sys.argv = [sys.argv[0]] + args
        import unittest
        exit(unittest.main())

    iterable = os.environ['PATH'].split(':')

    #iterable = list( optimize_path(iterable, opts.strip) )

    if opts.verbose:
        iterable, output = itertools.tee(iterable)
        for i in output:
            logging.info(i)

    if opts.print:
        print( opts.delim.join( iterable ) )

    duplicates = tuple()
    if opts.check:
        duplicates = detect_duplicates(iterable)
        print(json.dumps(duplicates, indent=3))

    if opts.pkg_lookup:
        duplicates = duplicates or detect_duplicates(iterable)

        # emit,yield,return
        from collections import namedtuple
        File = namedtuple('File', ('path','package', 'hash'))
        File.__str__ = lambda self: ("%-20s\t%s\t%s" %
                        (   self.package or '?',
                            self.path,
                            not self.package and self.hash or ''))

        for (k,v) in duplicates.iteritems():
            if v.get('type', None) == opts.pkg_lookup:
                for path in v.get('files'):
                    hash, path = path.split()
                    package = subprocess.check_output(
                            ('wajig','whichpackage', path)
                        ).split(':')[0].strip()
                    _file = File(path, package, hash)
                    logging.info(str(_file))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import subprocess
import io
import os
import tempfile
import glob
import json
import yaml
import logging
import sys
import requests
from . import docker
from .process import get_feature, empty_subtree
from .errors import WorkflowException
import shutil
import stat
import re
import shellescape
from .docker_uid import docker_vm_uid
from .builder import Builder
from typing import Union, Iterable, Callable, Any, Mapping, IO, cast, Tuple
from .pathmapper import PathMapper
import functools

_logger = logging.getLogger("cwltool")

needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")

def deref_links(outputs):  # type: (Any) -> None
    if isinstance(outputs, dict):
        if outputs.get("class") == "File":
            st = os.lstat(outputs["path"])
            if stat.S_ISLNK(st.st_mode):
                outputs["path"] = os.readlink(outputs["path"])
        else:
            for v in outputs.values():
                deref_links(v)
    if isinstance(outputs, list):
        for v in outputs:
            deref_links(v)

class CommandLineJob(object):

    def __init__(self):  # type: () -> None
        self.builder = None  # type: Builder
        self.joborder = None  # type: Dict[str,str]
        self.stdin = None  # type: str
        self.stderr = None  # type: str
        self.stdout = None  # type: str
        self.successCodes = None  # type: Iterable[int]
        self.temporaryFailCodes = None  # type: Iterable[int]
        self.permanentFailCodes = None  # type: Iterable[int]
        self.requirements = None  # type: List[Dict[str, str]]
        self.hints = None  # type: Dict[str,str]
        self.name = None  # type: unicode
        self.command_line = None  # type: List[unicode]
        self.pathmapper = None  # type: PathMapper
        self.collect_outputs = None  # type: Union[Callable[[Any], Any],functools.partial[Any]]
        self.output_callback = None  # type: Callable[[Any, Any], Any]
        self.outdir = None  # type: str
        self.tmpdir = None  # type: str
        self.environment = None  # type: Dict[str,str]
        self.generatefiles = None  # type: Dict[str,Union[Dict[str,str],str]]

    def run(self, dry_run=False, pull_image=True, rm_container=True,
            rm_tmpdir=True, move_outputs=True, **kwargs):
        # type: (bool, bool, bool, bool, bool, **Any) -> Union[Tuple[str,Dict[None,None]],None]
        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp:
        #    json.dump(self.joborder, fp)

        runtime = []  # type: List[unicode]

        # spec currently says "HOME must be set to the designated output
        # directory." but spec might change to designated temp directory.
        # env = {"TMPDIR": self.tmpdir, "HOME": self.tmpdir}  # type: Mapping[str,str]
        env = {"TMPDIR": self.tmpdir, "HOME": self.outdir}  # type: Mapping[str,str]

        (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")

        for f in self.pathmapper.files():
            if not os.path.isfile(self.pathmapper.mapper(f)[0]):
                raise WorkflowException(u"Required input file %s not found or is not a regular file." % self.pathmapper.mapper(f)[0])

        img_id = None
        if docker_req and kwargs.get("use_container") is not False:
            env = os.environ
            img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)

        if docker_is_req and img_id is None:
            raise WorkflowException("Docker is required for running this tool.")

        if img_id:
            runtime = ["docker", "run", "-i"]
            for src in self.pathmapper.files():
                vol = self.pathmapper.mapper(src)
                runtime.append(u"--volume=%s:%s:ro" % vol)
            runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl"))
            runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp"))
            runtime.append(u"--workdir=%s" % ("/var/spool/cwl"))
            runtime.append("--read-only=true")
            if (kwargs.get("enable_net", None) is None and
                    kwargs.get("custom_net", None) is not None):
                runtime.append("--net=none")
            elif kwargs.get("custom_net", None) is not None:
                runtime.append("--net={0}".format(kwargs.get("custom_net")))

            if self.stdout:
                runtime.append("--log-driver=none")

            euid = docker_vm_uid() or os.geteuid()
            runtime.append(u"--user=%s" % (euid))

            if rm_container:
                runtime.append("--rm")

            runtime.append("--env=TMPDIR=/tmp")

            # spec currently says "HOME must be set to the designated output
            # directory." but spec might change to designated temp directory.
            # runtime.append("--env=HOME=/tmp")
            runtime.append("--env=HOME=/var/spool/cwl")

            for t,v in self.environment.items():
                runtime.append(u"--env=%s=%s" % (t, v))

            runtime.append(img_id)
        else:
            env = self.environment
            if not os.path.exists(self.tmpdir):
                os.makedirs(self.tmpdir)
            env["TMPDIR"] = self.tmpdir
            vars_to_preserve = kwargs.get("preserve_environment")
            if vars_to_preserve is not None:
                for key, value in os.environ.items():
                    if key in vars_to_preserve and key not in env:
                        env[key] = value

        stdin = None  # type: Union[IO[Any],int]
        stderr = None  # type: IO[Any]
        stdout = None  # type: IO[Any]

        scr, _ = get_feature(self, "ShellCommandRequirement")

        if scr:
            shouldquote = lambda x: False
        else:
            shouldquote = needs_shell_quoting_re.search

        _logger.info(u"[job %s] %s$ %s%s%s%s",
                     self.name,
                     self.outdir,
                     " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
                     u' < %s' % (self.stdin) if self.stdin else '',
                     u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '',
                     u' \2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '')

        if dry_run:
            return (self.outdir, {})

        outputs = {}  # type: Dict[str,str]

        try:
            for t in self.generatefiles:
                entry = self.generatefiles[t]
                if isinstance(entry, dict):
                    src = entry["path"]
                    dst = os.path.join(self.outdir, t)
                    if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir:
                        _logger.debug(u"symlinking %s to %s", dst, src)
                        os.symlink(src, dst)
                elif isinstance(entry, str):
                    with open(os.path.join(self.outdir, t), "w") as fout:
                        fout.write(entry)
                else:
                    raise Exception("Unhandled type")

            if self.stdin:
                stdin = open(self.pathmapper.mapper(self.stdin)[0], "rb")
            else:
                stdin = subprocess.PIPE

            if self.stderr:
                abserr = os.path.join(self.outdir, self.stderr)
                dnerr = os.path.dirname(abserr)
                if dnerr and not os.path.exists(dnerr):
                    os.makedirs(dnerr)
                stderr = open(absout, "wb")
            else:
                stderr = sys.stderr

            if self.stdout:
                absout = os.path.join(self.outdir, self.stdout)
                dn = os.path.dirname(absout)
                if dn and not os.path.exists(dn):
                    os.makedirs(dn)
                stdout = open(absout, "wb")
            else:
                stdout = sys.stderr

            sp = subprocess.Popen([str(x) for x in runtime + self.command_line],
                                  shell=False,
                                  close_fds=True,
                                  stdin=stdin,
                                  stderr=stderr,
                                  stdout=stdout,
                                  env=env,
                                  cwd=self.outdir)

            if sp.stdin:
                sp.stdin.close()

            rcode = sp.wait()

            if isinstance(stdin, file):
                stdin.close()

            if stderr is not sys.stderr:
                stderr.close()

            if stdout is not sys.stderr:
                stdout.close()

            if self.successCodes and rcode in self.successCodes:
                processStatus = "success"
            elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
                processStatus = "temporaryFail"
            elif self.permanentFailCodes and rcode in self.permanentFailCodes:
                processStatus = "permanentFail"
            elif rcode == 0:
                processStatus = "success"
            else:
                processStatus = "permanentFail"

            for t in self.generatefiles:
                if isinstance(self.generatefiles[t], dict):
                    src = cast(dict, self.generatefiles[t])["path"]
                    dst = os.path.join(self.outdir, t)
                    if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir:
                        os.remove(dst)
                        os.symlink(self.pathmapper.reversemap(src)[1], dst)

            outputs = self.collect_outputs(self.outdir)

        except OSError as e:
            if e.errno == 2:
                if runtime:
                    _logger.error(u"'%s' not found", runtime[0])
                else:
                    _logger.error(u"'%s' not found", self.command_line[0])
            else:
                _logger.exception("Exception while running job")
            processStatus = "permanentFail"
        except WorkflowException as e:
            _logger.error(u"Error while running job: %s" % e)
            processStatus = "permanentFail"
        except Exception as e:
            _logger.exception("Exception while running job")
            processStatus = "permanentFail"

        if processStatus != "success":
            _logger.warn(u"[job %s] completed %s", self.name, processStatus)
        else:
            _logger.debug(u"[job %s] completed %s", self.name, processStatus)
        _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4))

        self.output_callback(outputs, processStatus)

        if rm_tmpdir:
            _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir)
            shutil.rmtree(self.tmpdir, True)

        if move_outputs and empty_subtree(self.outdir):
            _logger.debug(u"[job %s] Removing empty output directory %s", self.name, self.outdir)
            shutil.rmtree(self.outdir, True)
<commit_msg>stderr err<commit_after>import subprocess
import io
import os
import tempfile
import glob
import json
import yaml
import logging
import sys
import requests
from . import docker
from .process import get_feature, empty_subtree
from .errors import WorkflowException
import shutil
import stat
import re
import shellescape
from .docker_uid import docker_vm_uid
from .builder import Builder
from typing import Union, Iterable, Callable, Any, Mapping, IO, cast, Tuple
from .pathmapper import PathMapper
import functools

_logger = logging.getLogger("cwltool")

needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")

def deref_links(outputs):  # type: (Any) -> None
    if isinstance(outputs, dict):
        if outputs.get("class") == "File":
            st = os.lstat(outputs["path"])
            if stat.S_ISLNK(st.st_mode):
                outputs["path"] = os.readlink(outputs["path"])
        else:
            for v in outputs.values():
                deref_links(v)
    if isinstance(outputs, list):
        for v in outputs:
            deref_links(v)

class CommandLineJob(object):

    def __init__(self):  # type: () -> None
        self.builder = None  # type: Builder
        self.joborder = None  # type: Dict[str,str]
        self.stdin = None  # type: str
        self.stderr = None  # type: str
        self.stdout = None  # type: str
        self.successCodes = None  # type: Iterable[int]
        self.temporaryFailCodes = None  # type: Iterable[int]
        self.permanentFailCodes = None  # type: Iterable[int]
        self.requirements = None  # type: List[Dict[str, str]]
        self.hints = None  # type: Dict[str,str]
        self.name = None  # type: unicode
        self.command_line = None  # type: List[unicode]
        self.pathmapper = None  # type: PathMapper
        self.collect_outputs = None  # type: Union[Callable[[Any], Any],functools.partial[Any]]
        self.output_callback = None  # type: Callable[[Any, Any], Any]
        self.outdir = None  # type: str
        self.tmpdir = None  # type: str
        self.environment = None  # type: Dict[str,str]
        self.generatefiles = None  # type: Dict[str,Union[Dict[str,str],str]]

    def run(self, dry_run=False, pull_image=True, rm_container=True,
            rm_tmpdir=True, move_outputs=True, **kwargs):
        # type: (bool, bool, bool, bool, bool, **Any) -> Union[Tuple[str,Dict[None,None]],None]
        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp:
        #    json.dump(self.joborder, fp)

        runtime = []  # type: List[unicode]

        # spec currently says "HOME must be set to the designated output
        # directory." but spec might change to designated temp directory.
        # env = {"TMPDIR": self.tmpdir, "HOME": self.tmpdir}  # type: Mapping[str,str]
        env = {"TMPDIR": self.tmpdir, "HOME": self.outdir}  # type: Mapping[str,str]

        (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")

        for f in self.pathmapper.files():
            if not os.path.isfile(self.pathmapper.mapper(f)[0]):
                raise WorkflowException(u"Required input file %s not found or is not a regular file." % self.pathmapper.mapper(f)[0])

        img_id = None
        if docker_req and kwargs.get("use_container") is not False:
            env = os.environ
            img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)

        if docker_is_req and img_id is None:
            raise WorkflowException("Docker is required for running this tool.")

        if img_id:
            runtime = ["docker", "run", "-i"]
            for src in self.pathmapper.files():
                vol = self.pathmapper.mapper(src)
                runtime.append(u"--volume=%s:%s:ro" % vol)
            runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl"))
            runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp"))
            runtime.append(u"--workdir=%s" % ("/var/spool/cwl"))
            runtime.append("--read-only=true")
            if (kwargs.get("enable_net", None) is None and
                    kwargs.get("custom_net", None) is not None):
                runtime.append("--net=none")
            elif kwargs.get("custom_net", None) is not None:
                runtime.append("--net={0}".format(kwargs.get("custom_net")))

            if self.stdout:
                runtime.append("--log-driver=none")

            euid = docker_vm_uid() or os.geteuid()
            runtime.append(u"--user=%s" % (euid))

            if rm_container:
                runtime.append("--rm")

            runtime.append("--env=TMPDIR=/tmp")

            # spec currently says "HOME must be set to the designated output
            # directory." but spec might change to designated temp directory.
            # runtime.append("--env=HOME=/tmp")
            runtime.append("--env=HOME=/var/spool/cwl")

            for t,v in self.environment.items():
                runtime.append(u"--env=%s=%s" % (t, v))

            runtime.append(img_id)
        else:
            env = self.environment
            if not os.path.exists(self.tmpdir):
                os.makedirs(self.tmpdir)
            env["TMPDIR"] = self.tmpdir
            vars_to_preserve = kwargs.get("preserve_environment")
            if vars_to_preserve is not None:
                for key, value in os.environ.items():
                    if key in vars_to_preserve and key not in env:
                        env[key] = value

        stdin = None  # type: Union[IO[Any],int]
        stderr = None  # type: IO[Any]
        stdout = None  # type: IO[Any]

        scr, _ = get_feature(self, "ShellCommandRequirement")

        if scr:
            shouldquote = lambda x: False
        else:
            shouldquote = needs_shell_quoting_re.search

        _logger.info(u"[job %s] %s$ %s%s%s%s",
                     self.name,
                     self.outdir,
                     " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
                     u' < %s' % (self.stdin) if self.stdin else '',
                     u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '',
                     u' \2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '')

        if dry_run:
            return (self.outdir, {})

        outputs = {}  # type: Dict[str,str]

        try:
            for t in self.generatefiles:
                entry = self.generatefiles[t]
                if isinstance(entry, dict):
                    src = entry["path"]
                    dst = os.path.join(self.outdir, t)
                    if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir:
                        _logger.debug(u"symlinking %s to %s", dst, src)
                        os.symlink(src, dst)
                elif isinstance(entry, str):
                    with open(os.path.join(self.outdir, t), "w") as fout:
                        fout.write(entry)
                else:
                    raise Exception("Unhandled type")

            if self.stdin:
                stdin = open(self.pathmapper.mapper(self.stdin)[0], "rb")
            else:
                stdin = subprocess.PIPE

            if self.stderr:
                abserr = os.path.join(self.outdir, self.stderr)
                dnerr = os.path.dirname(abserr)
                if dnerr and not os.path.exists(dnerr):
                    os.makedirs(dnerr)
                stderr = open(abserr, "wb")
            else:
                stderr = sys.stderr

            if self.stdout:
                absout = os.path.join(self.outdir, self.stdout)
                dn = os.path.dirname(absout)
                if dn and not os.path.exists(dn):
                    os.makedirs(dn)
                stdout = open(absout, "wb")
            else:
                stdout = sys.stderr

            sp = subprocess.Popen([str(x) for x in runtime + self.command_line],
                                  shell=False,
                                  close_fds=True,
                                  stdin=stdin,
                                  stderr=stderr,
                                  stdout=stdout,
                                  env=env,
                                  cwd=self.outdir)

            if sp.stdin:
                sp.stdin.close()

            rcode = sp.wait()

            if isinstance(stdin, file):
                stdin.close()

            if stderr is not sys.stderr:
                stderr.close()

            if stdout is not sys.stderr:
                stdout.close()

            if self.successCodes and rcode in self.successCodes:
                processStatus = "success"
            elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
                processStatus = "temporaryFail"
            elif self.permanentFailCodes and rcode in self.permanentFailCodes:
                processStatus = "permanentFail"
            elif rcode == 0:
                processStatus = "success"
            else:
                processStatus = "permanentFail"

            for t in self.generatefiles:
                if isinstance(self.generatefiles[t], dict):
                    src = cast(dict, self.generatefiles[t])["path"]
                    dst = os.path.join(self.outdir, t)
                    if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir:
                        os.remove(dst)
                        os.symlink(self.pathmapper.reversemap(src)[1], dst)

            outputs = self.collect_outputs(self.outdir)

        except OSError as e:
            if e.errno == 2:
                if runtime:
                    _logger.error(u"'%s' not found", runtime[0])
                else:
                    _logger.error(u"'%s' not found", self.command_line[0])
            else:
                _logger.exception("Exception while running job")
            processStatus = "permanentFail"
        except WorkflowException as e:
            _logger.error(u"Error while running job: %s" % e)
            processStatus = "permanentFail"
        except Exception as e:
            _logger.exception("Exception while running job")
            processStatus = "permanentFail"

        if processStatus != "success":
            _logger.warn(u"[job %s] completed %s", self.name, processStatus)
        else:
            _logger.debug(u"[job %s] completed %s", self.name, processStatus)
        _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4))

        self.output_callback(outputs, processStatus)

        if rm_tmpdir:
            _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir)
            shutil.rmtree(self.tmpdir, True)

        if move_outputs and empty_subtree(self.outdir):
            _logger.debug(u"[job %s] Removing empty output directory %s", self.name, self.outdir)
            shutil.rmtree(self.outdir, True)
<|endoftext|>


<commit_before>from wsgi_intercept import requests_intercept, add_wsgi_intercept
from wsgi_proxy import WSGIProxyApplication, reconstruct_url
import sys
import urlparse
from httplib import HTTPConnection
from fibpro.logsink import LogSinkClient
from fibpro.userstore import UserStoreServer, UserStoreClient

class Transaction(object):
    def __init__(self):
        self.request = {}
        self.response = {}

    def __repr__(self):
        return "Transaction(%s, %s)" % (repr(self.request), repr(self.response))

class RecordingProxy(object):

    def __init__(self, wsgi_app):
        self.wsgi_app = wsgi_app
        self.transactions = []

    def start_response_proxy(self, start_response, current_transaction):
        def proxy(status, headers):
            current_transaction.request['status'] = status
            current_transaction.request['headers'] = headers
            start_response(status, headers)
        return proxy

    def __call__(self, environ, start_response):
        current_transaction = Transaction()
        current_transaction.request['url'] = urlparse.urlparse(reconstruct_url(environ))
        response_iter = self.wsgi_app(environ, self.start_response_proxy(start_response, current_transaction))
        response_body = "".join([chunk for chunk in response_iter])
        current_transaction.response['body'] = response_body
        self.transactions.append(current_transaction)
        return iter([response_body])

def intercept(host, port, app):
    add_wsgi_intercept(host, port, lambda: app)

requests_intercept.install()
#proxy = RecordingProxy(UserStoreServer(register=False).app())
proxy = RecordingProxy(WSGIProxyApplication())
intercept('127.0.0.1', 9003, proxy)
UserStoreClient().get_user("username")
print proxy.__dict__
<commit_msg>more work on reqtool<commit_after>from wsgi_intercept import requests_intercept, add_wsgi_intercept
from wsgi_proxy import WSGIProxyApplication, reconstruct_url
import sys
import urlparse
from httplib import HTTPConnection
from fibpro.logsink import LogSinkClient
from fibpro.userstore import UserStoreServer, UserStoreClient
from fibpro.rpc import RPCBase
from StringIO import StringIO

class Transaction(object):
    def __init__(self):
        self.request = {}
        self.response = {}

    def _read_request_body(self, environ):
        try:
            body_length = int(environ['CONTENT_LENGTH'])
            request_body = environ['wsgi.input'].read(body_length)
            self.request['body_length'] = body_length
            self.request['body'] = request_body
            environ['wsgi.input'] = StringIO(request_body)
        except (KeyError, ValueError):
            pass

    def _read_request_url(self, environ):
        self.request['url'] = urlparse.urlparse(reconstruct_url(environ))

    def _read_request_call(self, environ):
        rpcbase = RPCBase()
        self.request['call'] = rpcbase.decode_arguments(environ)

    def _read_request_headers(self, environ):
        # from wsgi-proxy source
        headers = dict(
            (key, value)
            for key, value in (
                # This is a hacky way of getting the header names right
                (key[5:].lower().replace('_', '-'), value)
                for key, value in environ.items()
                # Keys that start with HTTP_ are all headers
                if key.startswith('HTTP_')
            )
        )
        # Handler headers that aren't HTTP_ in environ
        try:
            headers['content-type'] = environ['CONTENT_TYPE']
        except KeyError:
            pass
        # Add our host if one isn't defined
        if 'host' not in headers:
            headers['host'] = environ['SERVER_NAME']
        self.request['headers'] = headers

    def read_request(self, environ):
        self._read_request_url(environ)
        self._read_request_body(environ)
        self._read_request_headers(environ)
        self._read_request_call(environ)

    def __repr__(self):
        return "Transaction(%s, %s)" % (repr(self.request), repr(self.response))

class RecordingProxy(object):

    def __init__(self, wsgi_app):
        self.wsgi_app = wsgi_app
        self.transactions = []

    def start_response_proxy(self, start_response, current_transaction):
        def proxy(status, headers):
            current_transaction.response['status'] = status
            current_transaction.response['headers'] = headers
            start_response(status, headers)
        return proxy

    def __call__(self, environ, start_response):
        current_transaction = Transaction()
        current_transaction.read_request(environ)
        response_iter = self.wsgi_app(environ, self.start_response_proxy(start_response, current_transaction))
        response_body = "".join([chunk for chunk in response_iter])
        current_transaction.response['body'] = response_body
        self.transactions.append(current_transaction)
        return iter([response_body])

def intercept(host, port, app):
    add_wsgi_intercept(host, port, lambda: app)

requests_intercept.install()
#proxy = RecordingProxy(UserStoreServer(register=False).app())
proxy = RecordingProxy(WSGIProxyApplication())
intercept('127.0.0.1', 9003, proxy)
UserStoreClient().get_user("username")
print proxy.__dict__
<|endoftext|>


<commit_before>"""
django-filer - file and image management for django projects
"""
VERSION = (0,6,2,'a8')
__version__ = "0.6.2a8"
__authors__ = ["Stefan Foulis <stefan.foulis@gmail.com>", ]<commit_msg>fixed version number to 0.6.2a9<commit_after>"""
django-filer - file and image management for django projects
"""
VERSION = (0,6,2,'a9')
__version__ = "0.6.2a9"
__authors__ = ["Stefan Foulis <stefan.foulis@gmail.com>", ]<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
# version string following pep-0396 and pep-0386
__version__ = '0.9pbs.1'  # pragma: nocover
<commit_msg>Bump version.<commit_after>#-*- coding: utf-8 -*-
# version string following pep-0396 and pep-0386
__version__ = '0.9pbs.2'  # pragma: nocover
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

from datetime import datetime
import json
import logging
import os
import re
import sh
import shlex

import sys

from .exceptions import BranchError, VersionError

'''
    Uses a slightly modified version of this regex
    https://regex101.com/r/E0iVVS/2
'''
SEMVER_RE = re.compile('''
                       ^(?P<VersionTripple>
                            (?P<Major>0|[1-9][0-9]*)\.
                            (?P<Minor>0|[1-9][0-9]*)\.
                            (?P<Patch>0|[1-9][0-9]*)
                        ){1}
                        (?P<Tags>(?:\-
                            (?P<Prerelease>
                                (?:(?=[0]{1}[0-9A-Za-z-]{0})(?:[0]{1})|(?=[1-9]{1}[0-9]*[A-Za-z]{0})(?:[0-9]+)|(?=[0-9]*[A-Za-z-]+[0-9A-Za-z-]*)(?:[0-9A-Za-z-]+)){1}(?:\.(?=[0]{1}[0-9A-Za-z-]{0})(?:[0]{1})|\.(?=[1-9]{1}[0-9]*[A-Za-z]{0})(?:[0-9]+)|\.(?=[0-9]*[A-Za-z-]+[0-9A-Za-z-]*)(?:[0-9A-Za-z-]+))*){1}
                            ){0,1}(?:\+
                            (?P<Build>
                                (?:[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*
                            ))
                        ){0,1})$
                       ''', re.VERBOSE)

RC_RE = re.compile(r'.*-rc(\d+).*')

INITIAL_VERSION = '0.0.0'

MAJOR = 0
MINOR = 1
PATCH = 2

def print_error(buf):
    print(buf, file=sys.stderr)

def is_calver(calver_version, calver_format):
    # ex: 201809.25 from 201809.25.1-rc
    d = '.'.join(calver_version.split('.', 2)[:2])
    try:
        datetime.strptime(d, calver_format)
    except AttributeError:
        return False
    except ValueError:
        return False
    else:
        return True

def is_semver(semver_version):
    return SEMVER_RE.match(semver_version) is not None

class GitVersion(object):
    """
    Get and set git version tag
    """
    def __init__(self, args=None):
        self.args = args

    @property
    def logger(self):
        return logging.getLogger('{}.{}'.format(
            __name__,
            self.__class__.__name__))

    @property
    def branch(self):
        branch = os.environ.get('GIT_BRANCH')
        if branch is None:
            command = sh.git(*shlex.split('rev-parse --abbrev-ref HEAD'))
            lines = command.stdout.decode('utf8').strip().splitlines()
            branch = lines[0].strip()

            color_marker_idx = branch.find('\x1b')
            if color_marker_idx >= 0:
                self.logger.warning('found color marker in branch={}'.format(
                    branch.encode("utf8")))
                branch = branch[:color_marker_idx]

        # clean string to remove unwanted characters
        branch = branch.replace('/', '--')

        return branch

    @property
    def is_clean(self):
        """
        Returns whether the working copy is clean

        When there are uncommited changes in the working copy return False

        Returns:
            Boolean whether the working copy is clean
        """
        result = False

        command_l = 'git status --untracked --short'.split()
        command = getattr(sh, command_l[0])(command_l[1:])

        lines = command.stdout.decode('utf8').splitlines()
        for line in lines:
            line = line.rstrip()
            print_error('{}'.format(line))

        if not lines:
            result = True

        return result

    @property
    def is_calver(self):
        return(is_calver(self.version, self.args.calver_format))

    @property
    def is_semver(self):
        return(is_semver(self.version))

    @property
    def version(self):
        try:
            command = sh.git(*shlex.split('describe --tags --always'))
        except sh.ErrorReturnCode_128:
            return None
        else:
            version = command.stdout.decode('utf8').strip()

            # if the branch flag was given,
            # check to see if we are on a tagged commit
            if self.args.branch:
                try:
                    command = sh.git(*shlex.split('describe --tags --exact-match'))
                except sh.ErrorReturnCode_128:
                    # not an exact match, so append the branch
                    version = '{}-{}'.format(version, self.branch)

            return version

    @classmethod
    def setup_subparser(cls, subcommand):
        parser = subcommand.add_parser('version', help=cls.__doc__)

        parser.set_defaults(cls=cls)
        parser.add_argument(
            '--bump', action='store_true',
            help='perform a version bump, by default the current version is displayed'
        )
        parser.add_argument(
            '--patch', action='store_true', default=True,
            help='bump the patch version, this is the default bump if one is not specified'
        )
        parser.add_argument(
            '--minor', action='store_true',
            help='bump the minor version and reset patch back to 0'
        )
        parser.add_argument(
            '--major', action='store_true',
            help='bump the major version and reset minor and patch back to 0'
        )
        parser.add_argument(
            '--set',
            help='set version to the given version'
        )
        parser.add_argument(
            '--semver', action='store_true',
            help='only print out if the current tag is a semantic version, or exit 1'
        )
        parser.add_argument(
            '--calver', action='store_true',
            help='only print out if the current tag is a calendar version, or exit 1'
        )
        parser.add_argument(
            '--calver-format', action='store_true', default='%Y%m.%d',
            help='set the calver format (ex: \'%Y%m.%d\')'
        )
        parser.add_argument(
            '--rc', action='store_true',
            help='when bumping, generate a release candidate tag instead of a proper version'
        )
        parser.add_argument(
            '-m', '--message',
            help='set the git tag message on the command line'
        )
        parser.add_argument(
            '--no-branch', action='store_false', dest='branch',
            help='do not append branch to the version when current commit is not tagged'
        )

    def get_next_version(self, version):
        # split the version and int'ify major, minor, and patch
        split_version = version.split('-', 1)[0].split('.', 3)
        for i in range(3):
            split_version[i] = int(split_version[i])

        if self.args.major:
            split_version[MAJOR] += 1
            split_version[MINOR] = 0
            split_version[PATCH] = 0
        elif self.args.minor:
            split_version[MINOR] += 1
            split_version[PATCH] = 0
        elif self.args.patch:
            split_version[PATCH] += 1

        return split_version[:3]

    def get_next_calver_version(self, version):
        # split the current date
        now = datetime.now().strftime(self.args.calver_format)
        split_calver = now.split('.', 2)
        for i in range(2):
            split_calver[i] = int(split_calver[i])

        # split the version and int'ify major, minor, and patch
        split_version = version.split('-', 1)[0].split('.', 3)
        for i in range(3):
            split_version[i] = int(split_version[i])

        # don't allow major/minor
        if self.args.major:
            raise VersionError('''
                You can not bump to a major calver release.
                If you want to override this use `--set --force` instead
                ''')
        elif self.args.minor:
            raise VersionError('''
                You can not bump to a minor calver release.
                If you want to override this use `--set --force` instead
                ''')
        elif self.args.patch:
            # if we are on the same day bump the patch
            # otherwise move to the new date
            if (split_calver == split_version[:2]):
                split_version[PATCH] += 1
                split_calver.append(split_version[PATCH])
            else:
                split_calver.append(0)

        return split_calver[:3]

    def get_next_rc_version(self, version):
        current_rc = int(RC_RE.search(version).group(1))
        next_rc = current_rc + 1
        next_version = version.replace('-rc{}'.format(current_rc), '-rc{}'.format(next_rc))

        return next_version.split('.')

    def bump(self):
        current_version = self.version
        if not current_version:
            current_version = INITIAL_VERSION

        is_rc = re.match(RC_RE, current_version)
        if self.args.rc and is_rc:
            self.logger.info('Latest version is a release candidate, incrementing...')
            next_version = self.get_next_rc_version(current_version)
        elif self.args.calver:
            next_version = self.get_next_calver_version(current_version)
        else:
            split_dashes = current_version.split('-')

            if len(split_dashes) == 1:
                raise VersionError(
                    'Is version={} already bumped?'.format(current_version))

            current_version = split_dashes[0]
            next_version = self.get_next_version(current_version)

        if self.args.rc and not is_rc:
            self.logger.info('Latest version is not a release candidate, generating initial rc tag...')
            next_version[-1] = str(next_version[-1]) + '-rc1'

        return next_version

    def check_bump(self):
        """
        Check to see if a bump request is being made
        """
        if not self.args.bump:
            return False

        return self.bump()

    def check_set(self):
        """
        Check to see if the version is being set
        """
        if not self.args.set:
            return None

        # if there's a calver flag, can only set to a correct calver version
        if self.args.calver:
            if not is_calver(self.args.set, self.args.calver_format):
                raise VersionError(
                    'Trying to set a non-calver version: {}'.format(self.args.set))

        return self.args.set.split('.')

    def get_tag_command(self, new_version):
        tag_command = 'git tag -a '

        if self.args.message:
            tag_command += '-m {} '.format(json.dumps(self.args.message))

        tag_command += new_version
        return tag_command

    def run(self):
        if not self.is_clean:
            print_error('Abort: working copy not clean.')

            return 1

        current_version = self.version

        # check to see if an explicit version is being set
        new_version = self.check_set()
        if not new_version:
            # otherwise, see if the version is being bumped
            try:
                new_version = self.check_bump()
            except VersionError as exc:
                print_error(exc)

                return 1

        status = 0

        if new_version is False:
            if current_version:
                print(self.version)
            else:
                next_version = self.get_next_version(INITIAL_VERSION)
                print_error('No version found, use --bump to set to {}'.format(
                    self.stringify(next_version)
                ))

                status = 1
        else:
            version_str = self.stringify(new_version)
            tag_command = self.get_tag_command(version_str)
            os.system(tag_command)

            print(version_str)

        if self.args.semver:
            if not self.is_semver:
                return 1

        if self.args.calver:
            if not self.is_calver:
                return 1

        return status

    def stringify(self, version):
        return '.'.join([str(x) for x in version])
<commit_msg>add linter ignore for non-critical issue failing build<commit_after>from __future__ import absolute_import, print_function

from datetime import datetime
import json
import logging
import os
import re
import sh
import shlex

import sys

from .exceptions import BranchError, VersionError

'''
    Uses a slightly modified version of this regex
    https://regex101.com/r/E0iVVS/2
'''
SEMVER_RE = re.compile('''
                       ^(?P<VersionTripple>
                            (?P<Major>0|[1-9][0-9]*)\.
                            (?P<Minor>0|[1-9][0-9]*)\.
                            (?P<Patch>0|[1-9][0-9]*)
                        ){1}
                        (?P<Tags>(?:\-
                            (?P<Prerelease>
                                (?:(?=[0]{1}[0-9A-Za-z-]{0})(?:[0]{1})|(?=[1-9]{1}[0-9]*[A-Za-z]{0})(?:[0-9]+)|(?=[0-9]*[A-Za-z-]+[0-9A-Za-z-]*)(?:[0-9A-Za-z-]+)){1}(?:\.(?=[0]{1}[0-9A-Za-z-]{0})(?:[0]{1})|\.(?=[1-9]{1}[0-9]*[A-Za-z]{0})(?:[0-9]+)|\.(?=[0-9]*[A-Za-z-]+[0-9A-Za-z-]*)(?:[0-9A-Za-z-]+))*){1}
                            ){0,1}(?:\+
                            (?P<Build>
                                (?:[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*
                            ))
                        ){0,1})$
                       ''', re.VERBOSE)

RC_RE = re.compile(r'.*-rc(\d+).*')

INITIAL_VERSION = '0.0.0'

MAJOR = 0
MINOR = 1
PATCH = 2

def print_error(buf):
    print(buf, file=sys.stderr)

def is_calver(calver_version, calver_format):
    # ex: 201809.25 from 201809.25.1-rc
    d = '.'.join(calver_version.split('.', 2)[:2])
    try:
        datetime.strptime(d, calver_format)
    except AttributeError:
        return False
    except ValueError:
        return False
    else:
        return True

def is_semver(semver_version):
    return SEMVER_RE.match(semver_version) is not None

class GitVersion(object):
    """
    Get and set git version tag
    """
    def __init__(self, args=None):
        self.args = args

    @property
    def logger(self):
        return logging.getLogger('{}.{}'.format(
            __name__,
            self.__class__.__name__))

    @property
    def branch(self):
        branch = os.environ.get('GIT_BRANCH')
        if branch is None:
            command = sh.git(*shlex.split('rev-parse --abbrev-ref HEAD'))
            lines = command.stdout.decode('utf8').strip().splitlines()
            branch = lines[0].strip()

            color_marker_idx = branch.find('\x1b')
            if color_marker_idx >= 0:
                self.logger.warning('found color marker in branch={}'.format(
                    branch.encode("utf8")))
                branch = branch[:color_marker_idx]

        # clean string to remove unwanted characters
        branch = branch.replace('/', '--')

        return branch

    @property
    def is_clean(self):
        """
        Returns whether the working copy is clean

        When there are uncommited changes in the working copy return False

        Returns:
            Boolean whether the working copy is clean
        """
        result = False

        command_l = 'git status --untracked --short'.split()
        command = getattr(sh, command_l[0])(command_l[1:])

        lines = command.stdout.decode('utf8').splitlines()
        for line in lines:
            line = line.rstrip()
            print_error('{}'.format(line))

        if not lines:
            result = True

        return result

    @property
    def is_calver(self):
        return(is_calver(self.version, self.args.calver_format))

    @property
    def is_semver(self):
        return(is_semver(self.version))

    @property
    def version(self):
        try:
            command = sh.git(*shlex.split('describe --tags --always'))
        except sh.ErrorReturnCode_128:  # pylint: disable=E1101
            return None
        else:
            version = command.stdout.decode('utf8').strip()

            # if the branch flag was given,
            # check to see if we are on a tagged commit
            if self.args.branch:
                try:
                    command = sh.git(*shlex.split('describe --tags --exact-match'))
                except sh.ErrorReturnCode_128:  # pylint: disable=E1101
                    # not an exact match, so append the branch
                    version = '{}-{}'.format(version, self.branch)

            return version

    @classmethod
    def setup_subparser(cls, subcommand):
        parser = subcommand.add_parser('version', help=cls.__doc__)

        parser.set_defaults(cls=cls)
        parser.add_argument(
            '--bump', action='store_true',
            help='perform a version bump, by default the current version is displayed'
        )
        parser.add_argument(
            '--patch', action='store_true', default=True,
            help='bump the patch version, this is the default bump if one is not specified'
        )
        parser.add_argument(
            '--minor', action='store_true',
            help='bump the minor version and reset patch back to 0'
        )
        parser.add_argument(
            '--major', action='store_true',
            help='bump the major version and reset minor and patch back to 0'
        )
        parser.add_argument(
            '--set',
            help='set version to the given version'
        )
        parser.add_argument(
            '--semver', action='store_true',
            help='only print out if the current tag is a semantic version, or exit 1'
        )
        parser.add_argument(
            '--calver', action='store_true',
            help='only print out if the current tag is a calendar version, or exit 1'
        )
        parser.add_argument(
            '--calver-format', action='store_true', default='%Y%m.%d',
            help='set the calver format (ex: \'%Y%m.%d\')'
        )
        parser.add_argument(
            '--rc', action='store_true',
            help='when bumping, generate a release candidate tag instead of a proper version'
        )
        parser.add_argument(
            '-m', '--message',
            help='set the git tag message on the command line'
        )
        parser.add_argument(
            '--no-branch', action='store_false', dest='branch',
            help='do not append branch to the version when current commit is not tagged'
        )

    def get_next_version(self, version):
        # split the version and int'ify major, minor, and patch
        split_version = version.split('-', 1)[0].split('.', 3)
        for i in range(3):
            split_version[i] = int(split_version[i])

        if self.args.major:
            split_version[MAJOR] += 1
            split_version[MINOR] = 0
            split_version[PATCH] = 0
        elif self.args.minor:
            split_version[MINOR] += 1
            split_version[PATCH] = 0
        elif self.args.patch:
            split_version[PATCH] += 1

        return split_version[:3]

    def get_next_calver_version(self, version):
        # split the current date
        now = datetime.now().strftime(self.args.calver_format)
        split_calver = now.split('.', 2)
        for i in range(2):
            split_calver[i] = int(split_calver[i])

        # split the version and int'ify major, minor, and patch
        split_version = version.split('-', 1)[0].split('.', 3)
        for i in range(3):
            split_version[i] = int(split_version[i])

        # don't allow major/minor
        if self.args.major:
            raise VersionError('''
                You can not bump to a major calver release.
                If you want to override this use `--set --force` instead
                ''')
        elif self.args.minor:
            raise VersionError('''
                You can not bump to a minor calver release.
                If you want to override this use `--set --force` instead
                ''')
        elif self.args.patch:
            # if we are on the same day bump the patch
            # otherwise move to the new date
            if (split_calver == split_version[:2]):
                split_version[PATCH] += 1
                split_calver.append(split_version[PATCH])
            else:
                split_calver.append(0)

        return split_calver[:3]

    def get_next_rc_version(self, version):
        current_rc = int(RC_RE.search(version).group(1))
        next_rc = current_rc + 1
        next_version = version.replace('-rc{}'.format(current_rc), '-rc{}'.format(next_rc))

        return next_version.split('.')

    def bump(self):
        current_version = self.version
        if not current_version:
            current_version = INITIAL_VERSION

        is_rc = re.match(RC_RE, current_version)
        if self.args.rc and is_rc:
            self.logger.info('Latest version is a release candidate, incrementing...')
            next_version = self.get_next_rc_version(current_version)
        elif self.args.calver:
            next_version = self.get_next_calver_version(current_version)
        else:
            split_dashes = current_version.split('-')

            if len(split_dashes) == 1:
                raise VersionError(
                    'Is version={} already bumped?'.format(current_version))

            current_version = split_dashes[0]
            next_version = self.get_next_version(current_version)

        if self.args.rc and not is_rc:
            self.logger.info('Latest version is not a release candidate, generating initial rc tag...')
            next_version[-1] = str(next_version[-1]) + '-rc1'

        return next_version

    def check_bump(self):
        """
        Check to see if a bump request is being made
        """
        if not self.args.bump:
            return False

        return self.bump()

    def check_set(self):
        """
        Check to see if the version is being set
        """
        if not self.args.set:
            return None

        # if there's a calver flag, can only set to a correct calver version
        if self.args.calver:
            if not is_calver(self.args.set, self.args.calver_format):
                raise VersionError(
                    'Trying to set a non-calver version: {}'.format(self.args.set))

        return self.args.set.split('.')

    def get_tag_command(self, new_version):
        tag_command = 'git tag -a '

        if self.args.message:
            tag_command += '-m {} '.format(json.dumps(self.args.message))

        tag_command += new_version
        return tag_command

    def run(self):
        if not self.is_clean:
            print_error('Abort: working copy not clean.')

            return 1

        current_version = self.version

        # check to see if an explicit version is being set
        new_version = self.check_set()
        if not new_version:
            # otherwise, see if the version is being bumped
            try:
                new_version = self.check_bump()
            except VersionError as exc:
                print_error(exc)

                return 1

        status = 0

        if new_version is False:
            if current_version:
                print(self.version)
            else:
                next_version = self.get_next_version(INITIAL_VERSION)
                print_error('No version found, use --bump to set to {}'.format(
                    self.stringify(next_version)
                ))

                status = 1
        else:
            version_str = self.stringify(new_version)
            tag_command = self.get_tag_command(version_str)
            os.system(tag_command)

            print(version_str)

        if self.args.semver:
            if not self.is_semver:
                return 1

        if self.args.calver:
            if not self.is_calver:
                return 1

        return status

    def stringify(self, version):
        return '.'.join([str(x) for x in version])
<|endoftext|>


<commit_before>from .geometry import Vector

import pygame
import OpenGL.GL as gl
import math

class Controls:
    def __init__(self):
        pass

    def apply(self):
        pass

    def update(self, time = 10):
        pass

class TrackBallControls(Controls):
    def __init__(self):
        super().__init__()
        self.vertex = Vector()
        self.theta = 0

    def apply(self):
        gl.glRotatef(self.theta * 180 / math.pi, self.vertex.x, self.vertex.y, self.vertex.z)

    def update(self, time = 10):

        if not pygame.mouse.get_pressed()[0]:
            return

        coeff = 0.001
        move = pygame.mouse.get_rel()

        dV = Vector(move[1] * time * coeff, move[0] * time * coeff, 0)
        dTheta = dV.norm2()

        if abs(dTheta) < 0.00001:
            return

        dV.normalize()

        cosT2 = math.cos(self.theta / 2)
        sinT2 = math.sin(self.theta / 2)
        cosDT2 = math.cos(dTheta / 2)
        sinDT2 = math.sin(dTheta / 2)

        A = cosT2 * sinDT2 * dV + cosDT2 * sinT2 * self.vertex + sinDT2 * sinT2 * Vector.cross_product(dV, self.vertex)

        self.theta = 2 * math.acos(cosT2 * cosDT2 - sinT2 * sinDT2 * Vector.dot(dV, self.vertex))

        self.vertex = A
        self.vertex.normalize()

class OrbitControls(Controls):
    def __init__(self):
        super().__init__()
        self.phi = 0
        self.theta = 0
        self.scale_log = 0

    def apply(self):
        scale = math.exp(self.scale_log)
        gl.glScalef(scale, scale, scale)
        gl.glRotatef(self.theta * 180 / math.pi, 1.0, 0.0, 0.0)
        gl.glRotatef(self.phi * 180 / math.pi, 0.0, 1.0, 0.0)

    def apply_event(self, event):
        if event.type == pygame.MOUSEBUTTONDOWN:
            # Wheel up
            if event.button == 4:
                self.scale_log += 0.1
            # Wheel down
            elif event.button == 5:
                self.scale_log -= 0.1

    def update(self, time = 10):

        if not pygame.mouse.get_pressed()[0]:
            return

        move = pygame.mouse.get_rel()
        self.theta += move[1] * 0.01
        self.phi += move[0] * 0.01

        self.theta = max(min(self.theta, math.pi / 2), -math.pi / 2)
<commit_msg>Updated controls<commit_after>from .geometry import Vector

import pygame
import OpenGL.GL as gl
import math

class Controls:
    """Abstract class for controls
    """
    def __init__(self):
        pass

    def apply(self):
        """Apply the controls modification to the model view matrix
        """
        pass

    def update(self, time = 10):
        """Update according to the user's inputs
        """
        pass

class TrackBallControls(Controls):
    """Trackball controls

    Simple trackball controls"""

    def __init__(self):
        """Creates a TrackBallControls

        The trackball is centered at the origin
        """
        super().__init__()
        self.vertex = Vector()
        self.theta = 0

    def apply(self):
        """Apply the rotation of the current trackball
        """
        gl.glRotatef(self.theta * 180 / math.pi, self.vertex.x, self.vertex.y, self.vertex.z)

    def update(self, time = 10):
        """Checks the keyboard inputs and update the angle
        """
        if not pygame.mouse.get_pressed()[0]:
            return

        coeff = 0.001
        move = pygame.mouse.get_rel()

        dV = Vector(move[1] * time * coeff, move[0] * time * coeff, 0)
        dTheta = dV.norm2()

        if abs(dTheta) < 0.00001:
            return

        dV.normalize()

        cosT2 = math.cos(self.theta / 2)
        sinT2 = math.sin(self.theta / 2)
        cosDT2 = math.cos(dTheta / 2)
        sinDT2 = math.sin(dTheta / 2)

        A = cosT2 * sinDT2 * dV + cosDT2 * sinT2 * self.vertex + sinDT2 * sinT2 * Vector.cross_product(dV, self.vertex)

        self.theta = 2 * math.acos(cosT2 * cosDT2 - sinT2 * sinDT2 * Vector.dot(dV, self.vertex))

        self.vertex = A
        self.vertex.normalize()

class OrbitControls(Controls):
    """Simple OrbitControls

    Similar to TrackBallControls but the up vector is preserved"""

    def __init__(self):
        """Creates an OrbitControls with null angles
        """
        super().__init__()
        self.phi = 0
        self.theta = 0
        self.scale_log = 0

    def apply(self):
        scale = math.exp(self.scale_log)
        gl.glScalef(scale, scale, scale)
        gl.glRotatef(self.theta * 180 / math.pi, 1.0, 0.0, 0.0)
        gl.glRotatef(self.phi * 180 / math.pi, 0.0, 1.0, 0.0)

    def apply_event(self, event):
        """Manages the wheel event

        :param event: a pyevent
        """
        if event.type == pygame.MOUSEBUTTONDOWN:
            # Wheel up
            if event.button == 4:
                self.scale_log += 0.1
            # Wheel down
            elif event.button == 5:
                self.scale_log -= 0.1

    def update(self, time = 10):

        if not pygame.mouse.get_pressed()[0]:
            return

        move = pygame.mouse.get_rel()
        self.theta += move[1] * 0.01
        self.phi += move[0] * 0.01

        self.theta = max(min(self.theta, math.pi / 2), -math.pi / 2)
<|endoftext|>