<commit_before>#!/usr/bin/env python
# encoding: utf-8

import sys
import re
import argparse
import fileinput
import copy
import itertools

from argparse import RawDescriptionHelpFormatter

from patterns import main_patterns

class Buffer(object):
    BUFFER_SIZE = 30
    """Class for buffering input"""

    def __init__(self):
        self.buf = [''] * self.BUFFER_SIZE
        self.line_number = 0

    def add(self, line):
        self.buf.append(line)
        self.buf = self.buf[1:self.BUFFER_SIZE+1]
        self.line_number += 1

def show_snippets(input_stream, patterns=main_patterns,
                  output_stream=sys.stdout):
    """Show snippets for patterns

    :input_stream: input stream
    :patterns: patterns that need to look
    :returns: SnippetsQueue

    """

    LINES_ABOVE = 3
    LINES_BELOW = 3
    SNIPPETS_TO_SHOW = 5

    class Snippet(object):
        def __init__(self, lines_above, pattern, line_number):
            self.text = copy.copy(lines_above)
            self.pattern = pattern
            self.line_number = line_number

        def full(self):
            return len(self.text) >= LINES_ABOVE + LINES_BELOW + 1

        def add(self, line):
            self.text.append(line)

        def show(self):
            i = 0
            for line in self.text:
                if i == LINES_ABOVE:
                    output_stream.write('| {} ==>'.format(self.line_number))
                else:
                    if re.search(self.pattern, line):
                        output_stream.write('|-->')
                    else:
                        output_stream.write('|>')
                output_stream.write(line)
                i += 1

            output_stream.write('-' * 80 + '\n')

    class SnippetsQueue(object):
        def __init__(self):
            self.new_snippets = []
            self.ready_snippets = {}
            self.pattern_used = {}

        def push(self, snippet):
            if not self.pattern_used.get(pattern, False):
                self.new_snippets.append(snippet)
            self.pattern_used[pattern] = True

        def add(self, line):
            for snippet in self.new_snippets:
                if snippet.full():
                    self.make_ready(snippet)
                else:
                    snippet.add(line)

        def make_ready(self, snippet):
            self.new_snippets.remove(snippet)
            pattern = snippet.pattern
            try:
                self.ready_snippets[pattern].append(snippet)
            except KeyError:
                self.ready_snippets[pattern] = [snippet]
            self.pattern_used[pattern] = False

        def make_all_ready(self):
            for snippet in self.new_snippets:
                make_ready(snippet)

        def show(self):
            for pattern, snippets in self.ready_snippets.iteritems():
                print """\
*********************************************************************************
pattern: "{}"
*********************************************************************************\
""".format(pattern)
                for snippet in snippets[:SNIPPETS_TO_SHOW]:
                    snippet.show()

            pass

    snippets_queue = SnippetsQueue()

    lines_above = [''] * LINES_ABOVE

    line_number = 1
    for line in input_stream:
        lines_above.append(line)
        lines_above = lines_above[1:LINES_ABOVE+1]
        for pattern in patterns:
            if re.search(pattern, line):
                snippet = Snippet(lines_above, pattern, line_number)
                snippets_queue.push(snippet)
        snippets_queue.add(line)
        line_number += 1

    snippets_queue.make_all_ready()
    snippets_queue.show()

    return snippets_queue

def main():
    parser = argparse.ArgumentParser(description=\
    """
        Parse file[s]\n\n
        examlpe: ./direlog.py file[s]
    """, formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('file', nargs='+', default=[],
                        help='file[s] to do some work')
    parser.add_argument('--snippets', action='store_const', const=True)
    args = parser.parse_args()

    def input_stream_generator(): return fileinput.input(args.file)

    # for line in input_stream_generator():
        # sys.stdout.write(line)
    if args.snippets:
        show_snippets(input_stream_generator())

    pass

if __name__ == '__main__':
    main()
<commit_msg>Add PatternBuffer and make_escaped(string)<commit_after>#!/usr/bin/env python
# encoding: utf-8

import sys
import re
import argparse
import fileinput
import copy
import itertools

from argparse import RawDescriptionHelpFormatter

from patterns import main_patterns

BUFFER_SIZE = 30

class Buffer(object):
    """Class for buffering input"""

    def __init__(self):
        self.buf = [''] * BUFFER_SIZE
        self.line_scrolled = 0

    def add(self, line):
        self.buf.append(line)
        self.buf = self.buf[1:BUFFER_SIZE+1]
        self.line_scrolled += 1

class PatternBuffer(Buffer):

    """Add patterns support to Buffer"""

    def __init__(self):
        super(PatternBuffer, self).__init__(self)

def make_escaped(string):
    """Make escaped pattern from string

    :string: string to be escaped
    :returns: pattern

    """

    return re.escape(string.replace('\n', 'NSLPH')).replace('NSLPH', '\n')

def show_snippets(input_stream, patterns=main_patterns,
                  output_stream=sys.stdout):
    """Show snippets for patterns

    :input_stream: input stream
    :patterns: patterns that need to look
    :returns: SnippetsQueue

    """

    LINES_ABOVE = 3
    LINES_BELOW = 3
    SNIPPETS_TO_SHOW = 5

    class Snippet(object):
        def __init__(self, lines_above, pattern, line_number):
            self.text = copy.copy(lines_above)
            self.pattern = pattern
            self.line_number = line_number

        def full(self):
            return len(self.text) >= LINES_ABOVE + LINES_BELOW + 1

        def add(self, line):
            self.text.append(line)

        def show(self):
            i = 0
            for line in self.text:
                if i == LINES_ABOVE:
                    output_stream.write('| {} ==>'.format(self.line_number))
                else:
                    if re.search(self.pattern, line):
                        output_stream.write('|-->')
                    else:
                        output_stream.write('|>')
                output_stream.write(line)
                i += 1

            output_stream.write('-' * 80 + '\n')

    class SnippetsQueue(object):
        def __init__(self):
            self.new_snippets = []
            self.ready_snippets = {}
            self.pattern_used = {}

        def push(self, snippet):
            if not self.pattern_used.get(pattern, False):
                self.new_snippets.append(snippet)
            self.pattern_used[pattern] = True

        def add(self, line):
            for snippet in self.new_snippets:
                if snippet.full():
                    self.make_ready(snippet)
                else:
                    snippet.add(line)

        def make_ready(self, snippet):
            self.new_snippets.remove(snippet)
            pattern = snippet.pattern
            try:
                self.ready_snippets[pattern].append(snippet)
            except KeyError:
                self.ready_snippets[pattern] = [snippet]
            self.pattern_used[pattern] = False

        def make_all_ready(self):
            for snippet in self.new_snippets:
                make_ready(snippet)

        def show(self):
            for pattern, snippets in self.ready_snippets.iteritems():
                print """\
*********************************************************************************
pattern: "{}"
*********************************************************************************\
""".format(pattern)
                for snippet in snippets[:SNIPPETS_TO_SHOW]:
                    snippet.show()

            pass

    snippets_queue = SnippetsQueue()

    lines_above = [''] * LINES_ABOVE

    line_number = 1
    for line in input_stream:
        lines_above.append(line)
        lines_above = lines_above[1:LINES_ABOVE+1]

        for pattern in patterns:
            if re.search(pattern, line):
                snippet = Snippet(lines_above, pattern, line_number)
                snippets_queue.push(snippet)

        snippets_queue.add(line)
        line_number += 1

    snippets_queue.make_all_ready()
    snippets_queue.show()

    return snippets_queue

def main():
    parser = argparse.ArgumentParser(description=\
    """
        Parse file[s]\n\n
        examlpe: ./direlog.py file[s]
    """, formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('file', nargs='+', default=[],
                        help='file[s] to do some work')
    parser.add_argument('--snippets', action='store_const', const=True)
    args = parser.parse_args()

    def input_stream_generator(): return fileinput.input(args.file)

    # for line in input_stream_generator():
        # sys.stdout.write(line)
    if args.snippets:
        show_snippets(input_stream_generator())

    pass

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

from django.db import models
from django.utils.translation import ugettext_lazy as _

from opps.archives.models import get_file_path
from opps.containers.models import Container

class Document(Container):
    archive = models.FileField(
        upload_to=get_file_path,
        max_length=255,
        verbose_name=_(u'Archive'))

    class Meta:
        verbose_name = _('Document')
        verbose_name_plural = _('Documents')
        ordering = ['-date_available']
<commit_msg>Fix get_absolute_url<commit_after># -*- coding:utf-8 -*-

from django.db import models
from django.utils.translation import ugettext_lazy as _

from opps.archives.models import get_file_path
from opps.containers.models import Container

class Document(Container):
    archive = models.FileField(
        upload_to=get_file_path,
        max_length=255,
        verbose_name=_(u'Archive'))

    class Meta:
        verbose_name = _('Document')
        verbose_name_plural = _('Documents')
        ordering = ['-date_available']

    def get_absolute_url(self):
        return u'/{}'.format(self.archive.url)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Example of how to add a user to Active Directory using,
# LDAP Directory management, wrapper for python-ldap (http://www.python-ldap.org)
#
#
# Derak Berreyesa
# github.com/derak
#####################################################################

import sys, json
import ldap
import ldap.modlist as modlist

import directory

result = {}

firstname   = 'Joe'
lastname    = 'Blow'
username    = 'jblow'
password    = 'some_secret_password'

email = username + '@mycompany.com'
displayName = '%s %s' % (firstname, lastname)

# The dn of our new entry/object
dn="cn=%s,%s" % (displayName, 'ou=Users,dc=ad,dc=mycompany,dc=com')

# A dict to help build the "body" of the object
attrs = {}
attrs['objectclass'] = ['top','person','organizationalPerson','user']
attrs['cn'] = str(displayName)
attrs['sAMAccountname'] = str(username)
attrs['userPassword'] = str(password)
attrs['givenName'] = str(firstname)
attrs['sn'] = str(lastname)
attrs['displayName'] = str(displayName)
attrs['userPrincipalName'] = "%s@ad.mycompany.com" % username
attrs['mail'] = str(email)

# Some flags for userAccountControl property
SCRIPT                    = 1
ACCOUNTDISABLE            = 2
HOMEDIR_REQUIRED          = 8
PASSWD_NOTREQD            = 32
NORMAL_ACCOUNT            = 512
DONT_EXPIRE_PASSWORD      = 65536
TRUSTED_FOR_DELEGATION    = 524288
PASSWORD_EXPIRED          = 8388608

# userAccountControl can be edited later by calling the user_mod method,
# but it is needed for creating an account

# this works!
attrs['userAccountControl'] = str(NORMAL_ACCOUNT + ACCOUNTDISABLE + DONT_EXPIRE_PASSWORD)

## this works w/o secure
##attrs['userAccountControl'] = str(PASSWD_NOTREQD + NORMAL_ACCOUNT + DONT_EXPIRE_PASSWORD)

## this does not work :-(
##attrs['userAccountControl'] = str(NORMAL_ACCOUNT + DONT_EXPIRE_PASSWORD)

# put all of your group dn's into a list to pass to add_user_to_groups()
group_dn_list = ['cn=Engineers,ou=Groups,dc=ad,dc=mycompnay,dc=com',
                 'cn=Interns,ou=Groups,dc=ad,dc=mycompnay,dc=com',
                 'cn=ProjectManagers,ou=Groups,dc=ad,dc=mycompnay,dc=com',
                 'cn=Executives,ou=Groups,dc=ad,dc=mycompnay,dc=com'
                ]

# create a Directory instance and start doing work
d = directory.Directory();
d.connect('ldaps://ad.mycompnay.com', 'AD\admin_user', 'admin_password')
d.add_user(dn, attrs)
d.add_user_to_groups(dn, group_dn_list)
d.set_password(dn, password)
d.enable_user(dn, NORMAL_ACCOUNT + DONT_EXPIRE_PASSWORD)
d.disconnect()

result = d.get_result()
result['status'] = 'Success!'

# send result to stdout (or to PHP)
print json.dumps(result)

<commit_msg>enable_user to modify_user<commit_after>#!/usr/bin/python
#
# Example of how to add a user to Active Directory using,
# LDAP Directory management, wrapper for python-ldap (http://www.python-ldap.org)
#
#
# Derak Berreyesa
# github.com/derak
#####################################################################

import sys, json
import ldap
import ldap.modlist as modlist

import directory

result = {}

firstname   = 'Joe'
lastname    = 'Blow'
username    = 'jblow'
password    = 'some_secret_password'

email = username + '@mycompany.com'
displayName = '%s %s' % (firstname, lastname)

# The dn of our new entry/object
dn="cn=%s,%s" % (displayName, 'ou=Users,dc=ad,dc=mycompany,dc=com')

# A dict to help build the "body" of the object
attrs = {}
attrs['objectclass'] = ['top','person','organizationalPerson','user']
attrs['cn'] = str(displayName)
attrs['sAMAccountname'] = str(username)
attrs['userPassword'] = str(password)
attrs['givenName'] = str(firstname)
attrs['sn'] = str(lastname)
attrs['displayName'] = str(displayName)
attrs['userPrincipalName'] = "%s@ad.mycompany.com" % username
attrs['mail'] = str(email)

# Some flags for userAccountControl property
SCRIPT                    = 1
ACCOUNTDISABLE            = 2
HOMEDIR_REQUIRED          = 8
PASSWD_NOTREQD            = 32
NORMAL_ACCOUNT            = 512
DONT_EXPIRE_PASSWORD      = 65536
TRUSTED_FOR_DELEGATION    = 524288
PASSWORD_EXPIRED          = 8388608

# userAccountControl can be edited later by calling the user_mod method,
# but it is needed for creating an account

# this works!
attrs['userAccountControl'] = str(NORMAL_ACCOUNT + ACCOUNTDISABLE + DONT_EXPIRE_PASSWORD)

## this works w/o secure
##attrs['userAccountControl'] = str(PASSWD_NOTREQD + NORMAL_ACCOUNT + DONT_EXPIRE_PASSWORD)

## this does not work :-(
##attrs['userAccountControl'] = str(NORMAL_ACCOUNT + DONT_EXPIRE_PASSWORD)

# put all of your group dn's into a list to pass to add_user_to_groups()
group_dn_list = ['cn=Engineers,ou=Groups,dc=ad,dc=mycompnay,dc=com',
                 'cn=Interns,ou=Groups,dc=ad,dc=mycompnay,dc=com',
                 'cn=ProjectManagers,ou=Groups,dc=ad,dc=mycompnay,dc=com',
                 'cn=Executives,ou=Groups,dc=ad,dc=mycompnay,dc=com'
                ]

# create a Directory instance and start doing work
d = directory.Directory();
d.connect('ldaps://ad.mycompnay.com', 'AD\admin_user', 'admin_password')
d.add_user(dn, attrs)
d.add_user_to_groups(dn, group_dn_list)
d.set_password(dn, password)
d.modify_user(dn, NORMAL_ACCOUNT + DONT_EXPIRE_PASSWORD)
d.disconnect()

result = d.get_result()
result['status'] = 'Success!'

# send result to stdout (or to PHP)
print json.dumps(result)

<|endoftext|>


<commit_before>from django.db import models
from django.shortcuts import resolve_url as r
from django.contrib.auth.models import User
from django.template.defaultfilters import slugify
from orcamentos.core.models import TimeStampedModel, Address
from .managers import CustomerManager, PersonManager, SellerManager
from orcamentos.utils.lists import GENDER, TREATMENT, PHONE_TYPE, PERSON_TYPE, CUSTOMER_TYPE

class People(TimeStampedModel, Address):
    gender = models.CharField(
        u'gênero',
        max_length=1,
        choices=GENDER,
        default='I',
        null=True,
        blank=True
    )
    treatment = models.CharField(
        'tratamento',
        max_length=4,
        choices=TREATMENT,
        null=True,
        blank=True
    )
    slug = models.SlugField('slug', blank=True)
    photo = models.URLField('foto', null=True, blank=True)
    birthday = models.DateTimeField('nascimento', null=True, blank=True)
    company = models.CharField('empresa', max_length=50, null=True, blank=True)
    department = models.CharField('departamento', max_length=50, blank=True)
    cpf = models.CharField(
        'CPF',
        max_length=11,
        unique=True,
        null=True,
        blank=True
    )
    rg = models.CharField(
        'RG',
        max_length=11,
        null=True,
        blank=True
    )
    cnpj = models.CharField(
        'CNPJ',
        max_length=14,
        unique=True,
        null=True,
        blank=True
    )
    ie = models.CharField(
        u'inscrição estadual',
        max_length=12,
        null=True,
        blank=True
    )
    active = models.BooleanField('ativo', default=True)
    blocked = models.BooleanField('bloqueado', default=False)

    class Meta:
        abstract = True

    def __str__(self):
        return ' '.join(filter(None, [self.get_treatment_display(), self.first_name, self.last_name]))

    full_name = property(__str__)

class Person(People):
    first_name = models.CharField('nome', max_length=50)
    last_name = models.CharField(
        'sobrenome',
        max_length=50,
        null=True,
        blank=True
    )
    email = models.EmailField(null=True, blank=True)
    occupation = models.ForeignKey(
        'Occupation',
        verbose_name='cargo',
        related_name='person_occupation',
        on_delete=models.CASCADE,
        null=True,
        blank=True
    )
    person_type = models.CharField(
        'cliente ou contato',
        max_length=1,
        choices=PERSON_TYPE,
        default='p'
    )
    customer_type = models.CharField(
        'tipo de cliente',
        max_length=1,
        choices=CUSTOMER_TYPE,
        blank=True
    )

    objects = PersonManager()

    class Meta:
        ordering = ['first_name']
        verbose_name = 'contato'
        verbose_name_plural = 'contatos'

    def __str__(self):
        return self.full_name

    def get_absolute_url(self):
        return r('crm:person_detail', slug=self.slug)

    def save(self):
        self.fullname = '{} {}'.format(self.first_name, self.last_name)
        self.slug = slugify(self.fullname)
        super(Person, self).save()

class PhonePerson(models.Model):
    phone = models.CharField('telefone', max_length=20, blank=True)
    person = models.ForeignKey('Person', on_delete=models.CASCADE)
    phone_type = models.CharField(
        'tipo',
        max_length=3,
        choices=PHONE_TYPE,
        default='pri'
    )

class Customer(Person):
    objects = CustomerManager()

    class Meta:
        proxy = True
        verbose_name = 'cliente'
        verbose_name_plural = 'clientes'

    def get_absolute_url(self):
        return r('crm:customer_detail', slug=self.slug)

class Employee(People, User):
    occupation = models.ForeignKey(
        'Occupation',
        verbose_name='cargo',
        related_name='employee_occupation',
        on_delete=models.CASCADE,
        null=True,
        blank=True
    )
    internal = models.BooleanField('interno', default=True)
    commissioned = models.BooleanField('comissionado', default=True)
    commission = models.DecimalField(
        u'comissão',
        max_digits=6,
        decimal_places=2,
        default=0.01
    )
    date_release = models.DateTimeField(
        u'data de saída',
        null=True,
        blank=True
    )

    class Meta:
        ordering = ['username']
        verbose_name = u'funcionário'
        verbose_name_plural = u'funcionários'

class PhoneEmployee(models.Model):
    phone = models.CharField('telefone', max_length=20, blank=True)
    employee = models.ForeignKey('Employee', on_delete=models.CASCADE)
    phone_type = models.CharField(
        'tipo',
        max_length=3,
        choices=PHONE_TYPE,
        default='pri'
    )

class Occupation(models.Model):
    occupation = models.CharField('cargo', max_length=50, unique=True)

    class Meta:
        ordering = ['occupation']
        verbose_name = 'cargo'
        verbose_name_plural = 'cargos'

    def __str__(self):
        return self.occupation

class Seller(Employee):
    objects = SellerManager()

    class Meta:
        proxy = True
        verbose_name = 'vendedor'
        verbose_name_plural = 'vendedores'
<commit_msg>Criar Contatos para Cliente. Fix #75<commit_after>from django.db import models
from django.shortcuts import resolve_url as r
from django.contrib.auth.models import User
from django.template.defaultfilters import slugify
from orcamentos.core.models import TimeStampedModel, Address
from .managers import CustomerManager, PersonManager, SellerManager
from orcamentos.utils.lists import GENDER, TREATMENT, PHONE_TYPE, PERSON_TYPE, CUSTOMER_TYPE

class People(TimeStampedModel, Address):
    gender = models.CharField(
        u'gênero',
        max_length=1,
        choices=GENDER,
        default='I',
        null=True,
        blank=True
    )
    treatment = models.CharField(
        'tratamento',
        max_length=4,
        choices=TREATMENT,
        null=True,
        blank=True
    )
    slug = models.SlugField('slug', blank=True)
    photo = models.URLField('foto', null=True, blank=True)
    birthday = models.DateTimeField('nascimento', null=True, blank=True)
    company = models.CharField('empresa', max_length=50, null=True, blank=True)
    department = models.CharField('departamento', max_length=50, blank=True)
    cpf = models.CharField(
        'CPF',
        max_length=11,
        unique=True,
        null=True,
        blank=True
    )
    rg = models.CharField(
        'RG',
        max_length=11,
        null=True,
        blank=True
    )
    cnpj = models.CharField(
        'CNPJ',
        max_length=14,
        unique=True,
        null=True,
        blank=True
    )
    ie = models.CharField(
        u'inscrição estadual',
        max_length=12,
        null=True,
        blank=True
    )
    active = models.BooleanField('ativo', default=True)
    blocked = models.BooleanField('bloqueado', default=False)

    class Meta:
        abstract = True

    def __str__(self):
        return ' '.join(filter(None, [self.get_treatment_display(), self.first_name, self.last_name]))

    full_name = property(__str__)

class Person(People):
    first_name = models.CharField('nome', max_length=50)
    last_name = models.CharField(
        'sobrenome',
        max_length=50,
        null=True,
        blank=True
    )
    email = models.EmailField(null=True, blank=True)
    occupation = models.ForeignKey(
        'Occupation',
        verbose_name='cargo',
        related_name='person_occupation',
        on_delete=models.CASCADE,
        null=True,
        blank=True
    )
    person_type = models.CharField(
        'cliente ou contato',
        max_length=1,
        choices=PERSON_TYPE,
        default='p'
    )
    customer_type = models.CharField(
        'tipo de cliente',
        max_length=1,
        choices=CUSTOMER_TYPE,
        blank=True
    )

    objects = PersonManager()

    class Meta:
        ordering = ['first_name']
        verbose_name = 'contato'
        verbose_name_plural = 'contatos'

    def __str__(self):
        return self.full_name

    def get_absolute_url(self):
        return r('crm:person_detail', slug=self.slug)

    def save(self):
        self.fullname = '{} {}'.format(self.first_name, self.last_name)
        self.slug = slugify(self.fullname)
        super(Person, self).save()

class PhonePerson(models.Model):
    phone = models.CharField('telefone', max_length=20, blank=True)
    person = models.ForeignKey('Person', on_delete=models.CASCADE)
    phone_type = models.CharField(
        'tipo',
        max_length=3,
        choices=PHONE_TYPE,
        default='pri'
    )

class Customer(Person):
    objects = CustomerManager()

    class Meta:
        proxy = True
        verbose_name = 'cliente'
        verbose_name_plural = 'clientes'

    def get_absolute_url(self):
        return r('crm:customer_detail', slug=self.slug)

class Employee(People, User):
    occupation = models.ForeignKey(
        'Occupation',
        verbose_name='cargo',
        related_name='employee_occupation',
        on_delete=models.CASCADE,
        null=True,
        blank=True
    )
    internal = models.BooleanField('interno', default=True)
    commissioned = models.BooleanField('comissionado', default=True)
    commission = models.DecimalField(
        u'comissão',
        max_digits=6,
        decimal_places=2,
        default=0.01
    )
    date_release = models.DateTimeField(
        u'data de saída',
        null=True,
        blank=True
    )

    class Meta:
        ordering = ['username']
        verbose_name = u'funcionário'
        verbose_name_plural = u'funcionários'

class PhoneEmployee(models.Model):
    phone = models.CharField('telefone', max_length=20, blank=True)
    employee = models.ForeignKey('Employee', on_delete=models.CASCADE)
    phone_type = models.CharField(
        'tipo',
        max_length=3,
        choices=PHONE_TYPE,
        default='pri'
    )

class Occupation(models.Model):
    occupation = models.CharField('cargo', max_length=50, unique=True)

    class Meta:
        ordering = ['occupation']
        verbose_name = 'cargo'
        verbose_name_plural = 'cargos'

    def __str__(self):
        return self.occupation

class Seller(Employee):
    objects = SellerManager()

    class Meta:
        proxy = True
        verbose_name = 'vendedor'
        verbose_name_plural = 'vendedores'

class ContactBase(Active):
    name = models.CharField('Nome', max_length=100, null=True, blank=True)
    email = models.EmailField('E-mail', null=True, blank=True)
    phone = models.CharField('Telefone', max_length=20, null=True, blank=True)
    phone_type = models.CharField(
        'tipo',
        max_length=3,
        choices=PHONE_TYPE,
        default='pri',
    )

    class Meta:
        abstract = True

    def __str__(self):
        return self.name

class CompanyContact(ContactBase):
    customer = models.ForeignKey('Customer', on_delete=models.CASCADE)

    class Meta:
        verbose_name = 'Contato Cliente'
        verbose_name_plural = 'Contatos Clientes'
<|endoftext|>


<commit_before>from services import ServiceHost, ServiceClient, api, publish, request, listen

class IdentityService(ServiceHost):

    @api
    def create(self, user_name, password):
        return user_id

class IdentityClient(ServiceClient):

    @request
    def create(self, user_name, password):
        return locals()
<commit_msg>add @publish/@subscribe signature in example<commit_after>from services import ServiceHost, ServiceClient, api, publish, request, subscribe

class IdentityService(ServiceHost):
    @api
    def create(self, user_name, password):
        return user_name

    @publish
    def password_changed(self, user_name):
        pass

class IdentityClient(ServiceClient):
    @request
    def create(self, user_name, password):
        return locals()

    @subscribe
    def password_changed(self, user_name):
        pass<|endoftext|>


<commit_before>#
# Author: Stanislaw Adaszewski, 2016
# License: Simplified 2-clause BSD
# Website: http://algoholic.eu
#
# Create and compare directory tree snapshots
#

from argparse import ArgumentParser
import sys
import os
from stat import S_ISDIR
import gzip

def create_parser():
    parser = ArgumentParser()
    parser.add_argument('--lst', type=str, help='List contents of'
        ' existing snapshot')
    parser.add_argument('--snap', type=str, help='Create snapshot of'
        ' specified directory tree')
    parser.add_argument('--compare', type=str, help='Compare two'
        ' specified snapshots', nargs=2)
    parser.add_argument('--out', type=str, help='Specifies output'
        ' file name, defaults to dirsnap.out.gz', default='dirsnap.out.gz')
    parser.add_argument('--maxdepth', help='Limit output path depth',
        type=int)
    parser.add_argument('--nohidden', help='Skip hidden files and'
        ' directories', action='store_true')
    parser.add_argument('--strip', type=int, default=[0, 0], nargs=2,
        help='Strip prefix containing specified number of slashes'
        ' on respective sides from each file name in comparison mode')
    parser.add_argument('--prefix', type=str, default='',
        help='Use only entries starting with given prefix in'
        ' comparison mode')
    return parser

def main():
    parser = create_parser()
    args = parser.parse_args()
    if not ((args.lst is not None) ^ (args.snap is not None) ^ (args.compare is not None)):
        sys.stderr.write('Either --snap or --compare has to be'
            ' specified\n')
        return

    if args.lst is not None:
        lst(args)
    elif args.snap is not None:
        snap(args)
    else:
        comp(args)

def lst(args):
    reader = SnapReader(args.lst)
    while True:
        e = reader.read()
        if e is None: break
        output_diff(args, e[2], '%s %d' % (e[0], e[1]))

def snap(args):
    fout = gzip.open(args.out, 'wb')
    fout.write('%s\n' % args.snap)
    Q = [args.snap]
    while len(Q) > 0:
        dirname = Q.pop(0)
        st = os.lstat(dirname)
        if S_ISDIR(st.st_mode):
            L = sorted(os.listdir(dirname))
            L = map(lambda x: os.path.join(dirname, x), L)
            fout.write('D %d %s\n' % (len(L), os.path.split(dirname)[1]))
            Q = L + Q
        else:
            fout.write('F %d %s\n' % (st.st_size, os.path.split(dirname)[1]))
    fout.close()

class SnapReader():
    def __init__(self, fname):
        self.f = gzip.open(fname, 'rb')
        self.basedir = self.f.readline()[:-1]
        self.stack = [[1, self.basedir]]
        self.unread_buf = []

    def close(self):
        self.f.close()

    def read(self):
        if len(self.unread_buf) > 0:
            return self.unread_buf.pop()
        line = self.f.readline()
        if line == '':
            return None
        n = len(line)
        line = line[:-1]
        line = line.split(' ')
        typ = line[0]
        sz = int(line[1])
        name = ' '.join(line[2:])
        while len(self.stack) > 0 and self.stack[-1][0] <= 0:
            self.stack.pop()
        self.stack[-1][0] -= 1
        if len(self.stack) > 0:
            name = os.path.join(self.stack[-1][1], name)
        if typ == 'D':
            self.stack.append([sz, name])
        return [typ, sz, name, n]

    def unread(self, e):
        # self.f.seek(-e[3], 1)
        self.unread_buf.append(e)

def output_diff(args, name, side):
    path = name.split(os.sep)
    if args.maxdepth is not None and len(path) > args.maxdepth + 1:
        return
    if args.nohidden and len(filter(lambda x: x.startswith('.'), path)) > 0:
        return
    print '%s %s' % (side, name)

def path_manip(path, args, side):
    strip = args.strip[0] if side == 'L' else args.strip[1]
    path = os.path.sep.join(path.split(os.path.sep)[strip:])
    if not path.startswith(args.prefix): path = ''
    return path

def comp(args):
    fin1 = SnapReader(args.compare[0])
    fin2 = SnapReader(args.compare[1])
    while True:
        e1 = fin1.read()
        # print e1
        # os.stat(e1[2])
        if e1 is None: break
        # print 'e1:', e1
        # os.stat(e1[2])
        # if : continue
        e1[2] = path_manip(e1[2], args, 'L')
        if e1[2] == '': continue
        while True:
            e2 = fin2.read()
            if e2 is not None:
                # if : e2[2] = ''
                e2[2] = path_manip(e2[2], args, 'R')
                # os.path.sep.join(e2[2].split(os.path.sep)[args.strip:])
                if e2[2] == '': continue
            if e2 is None:
                output_diff(args, e1[2], 'L')
                # print 'L %s' % e1[2]
                break
            elif e2[2] == e1[2]: # match
                # print e1[2], e2[2]
                break
            elif e2[2] < e1[2]: # only in fin2
                # print 'R %s' % e2[2]
                output_diff(args, e2[2], 'R')
                pass
            else: # only in fin1
                # print 'L %s' % e1[2]
                output_diff(args, e1[2], 'L')
                fin2.unread(e2)
                break
    while True:
        e2 = fin2.read()
        if e2 is None: break
        e2[2] = path_manip(e2[2], args, 'R')
        if e2[2] == '': continue
        # print 'R %s' % e2[2]
        output_diff(args, e2[2], 'R')
    fin1.close()
    fin2.close()

if __name__ == '__main__':
    main()
<commit_msg>Fix again.<commit_after>#
# Author: Stanislaw Adaszewski, 2016
# License: Simplified 2-clause BSD
# Website: http://algoholic.eu
#
# Create and compare directory tree snapshots
#

from argparse import ArgumentParser
import sys
import os
from stat import S_ISDIR
import gzip

def create_parser():
    parser = ArgumentParser()
    parser.add_argument('--lst', type=str, help='List contents of'
        ' existing snapshot')
    parser.add_argument('--snap', type=str, help='Create snapshot of'
        ' specified directory tree')
    parser.add_argument('--compare', type=str, help='Compare two'
        ' specified snapshots', nargs=2)
    parser.add_argument('--out', type=str, help='Specifies output'
        ' file name, defaults to dirsnap.out.gz', default='dirsnap.out.gz')
    parser.add_argument('--maxdepth', help='Limit output path depth',
        type=int)
    parser.add_argument('--nohidden', help='Skip hidden files and'
        ' directories', action='store_true')
    parser.add_argument('--strip', type=int, default=[0, 0], nargs=2,
        help='Strip prefix containing specified number of slashes'
        ' on respective sides from each file name in comparison mode')
    parser.add_argument('--prefix', type=str, default='',
        help='Use only entries starting with given prefix in'
        ' comparison mode')
    return parser

def main():
    parser = create_parser()
    args = parser.parse_args()
    if not ((args.lst is not None) ^ (args.snap is not None) ^ (args.compare is not None)):
        sys.stderr.write('Either --snap, --list or --compare has to be'
            ' specified\n')
        return

    # print args.strip

    if args.lst is not None:
        lst(args)
    elif args.snap is not None:
        snap(args)
    else:
        comp(args)

def lst(args):
    reader = SnapReader(args.lst)
    while True:
        e = reader.read()
        if e is None: break
        output_diff(args, e[2], '%s %d' % (e[0], e[1]))

def snap(args):
    fout = gzip.open(args.out, 'wb')
    fout.write('%s\n' % args.snap)
    Q = [args.snap]
    while len(Q) > 0:
        dirname = Q.pop(0)
        st = os.lstat(dirname)
        if S_ISDIR(st.st_mode):
            L = sorted(os.listdir(dirname))
            L = map(lambda x: os.path.join(dirname, x), L)
            fout.write('D %d %s\n' % (len(L), os.path.split(dirname)[1]))
            Q = L + Q
        else:
            fout.write('F %d %s\n' % (st.st_size, os.path.split(dirname)[1]))
    fout.close()

class SnapReader():
    def __init__(self, fname):
        self.f = gzip.open(fname, 'rb')
        self.basedir = self.f.readline()[:-1]
        self.stack = [[1, self.basedir]]
        self.unread_buf = []

    def close(self):
        self.f.close()

    def read(self):
        if len(self.unread_buf) > 0:
            return self.unread_buf.pop()
        line = self.f.readline()
        if line == '':
            return None
        n = len(line)
        line = line[:-1]
        line = line.split(' ')
        typ = line[0]
        sz = int(line[1])
        name = ' '.join(line[2:])
        while len(self.stack) > 0 and self.stack[-1][0] <= 0:
            self.stack.pop()
        self.stack[-1][0] -= 1
        if len(self.stack) > 0:
            name = os.path.join(self.stack[-1][1], name)
        if typ == 'D':
            self.stack.append([sz, name])
        return (typ, sz, name, n)

    def unread(self, e):
        # self.f.seek(-e[3], 1)
        self.unread_buf.append(e)

def output_diff(args, name, side):
    path = name.split(os.sep)
    if args.maxdepth is not None and len(path) > args.maxdepth + 1:
        return
    if args.nohidden and len(filter(lambda x: x.startswith('.'), path)) > 0:
        return
    print '%s %s' % (side, name)

def path_manip(in_path, args, side):
    strip = args.strip[0] if side == 'L' else args.strip[1]
    path = os.path.sep.join(in_path.split(os.path.sep)[strip:])
    if not path.startswith(args.prefix): path = ''
    # if path == '':
        # print in_path, 'turned to empty path'
    return path

def comp(args):
    fin1 = SnapReader(args.compare[0])
    fin2 = SnapReader(args.compare[1])
    while True:
        e1 = fin1.read()
        # print e1
        # os.stat(e1[2])
        if e1 is None: break
        # print 'e1:', e1
        # os.stat(e1[2])
        # if : continue
        path1 = path_manip(e1[2], args, 'L')
        if path1 == '': continue
        while True:
            e2 = fin2.read()
            if e2 is not None:
                # if : e2[2] = ''
                path2 = path_manip(e2[2], args, 'R')
                # os.path.sep.join(e2[2].split(os.path.sep)[args.strip:])
                if path2 == '': continue
            if e2 is None:
                output_diff(args, path1, 'L')
                # print 'L %s' % e1[2]
                break
            elif path1 == path2: # match
                # print e1[2], e2[2]
                break
            elif path2 < path1: # only in fin2
                # print 'R %s' % e2[2]
                output_diff(args, path2, 'R')
                pass
            else: # only in fin1
                # print 'L %s' % e1[2]
                output_diff(args, path1, 'L')
                fin2.unread(e2)
                break
    while True:
        e2 = fin2.read()
        if e2 is None: break
        path2 = path_manip(e2[2], args, 'R')
        if path2 == '': continue
        # print 'R %s' % e2[2]
        output_diff(args, path2, 'R')
    fin1.close()
    fin2.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from fpgaperf import *

families = [
    'xc7'
]

devices = [
    'a35t',
    'z010'
]

packages = [
    'cpg236-1',
    'csg324-1',
    'clg400-1'
]

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description=
        'Exhaustively try project-toolchain combinations'
    )
    parser.add_argument('--family', default='all', help='device family')
    parser.add_argument('--device', default='all', help='device')
    parser.add_argument('--package', default='all', help='device package')
    parser.add_argument('--project', default='all', help='run given project only (default: all)')
    parser.add_argument('--toolchain', default='all', help='run given toolchain only (default: all)')
    parser.add_argument('--out-prefix', default='build', help='output directory prefix (default: build)')
    parser.add_argument('--dry', action='store_true', help='print commands, don\'t invoke')
    parser.add_argument('--fail', action='store_true', help='fail on error')
    parser.add_argument('--verbose', action='store_true', help='verbose output')
    args = parser.parse_args()

    print('Running exhaustive project-toolchain search')

    if(args.project != "all"):
        # Only specified project
        selected_projects = [args.project]
    else:
        # All projects
        selected_projects = get_projects()

    if(args.toolchain != "all"):
        selected_toolchains = [args.toolchain]
    else:
        selected_toolchains = get_toolchains()

    if(args.family != "all"):
        selected_families = [args.familes]
    else:
        selected_families = families

    if(args.device != "all"):
        selected_devices = [args.device]
    else:
        selected_devices = devices

    if(args.package != "all"):
        selected_packages = [args.package]
    else:
        selected_packages = packages

    for project in selected_projects:
        for toolchain in selected_toolchains:
            for family in selected_families:
                for device in selected_devices:
                    for package in selected_packages:
                        if(get_pcf(project, family, device, package, toolchain) is not None):
                            run(
                                family,
                                device,
                                package,
                                toolchain,
                                project,
                                None, #out_dir
                                args.out_prefix,
                                None, #strategy
                                None, #carry
                                None, #seed
                                None, #build
                                args.verbose
                                )

if __name__ == '__main__':
    main()

<commit_msg>exhaust: read family/device/package from directory structure<commit_after>#!/usr/bin/env python3

from fpgaperf import *
import sow

# to find data files
root_dir = os.path.dirname(os.path.abspath(__file__))
src_dir = root_dir + '/src'

def get_families(project):
    return matching_pattern(
        src_dir + '/' + project + '/*/',
        '.*\/(.*)\/$'
    )

def get_devices(project, family):
    return matching_pattern(
        src_dir + '/' + project + '/' + family + '/*/',
        '.*\/([^/_]*)(?:_?)(?:[^/_]*)\/$'
    )

def get_packages(project, family, device):
    return matching_pattern(
        src_dir + '/' + project + '/' + family + '/' + device + '*/',
        '.*\/(?:[^/_]*' + device + ')(?:_?)([^/_]*)\/$'
    )

def get_reports(out_prefix):
    return matching_pattern(
        root_dir + '/' + out_prefix + '/*/meta.json',
        '(.*)'
    )

def user_selected(option):
    return [option] if option else None

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description=
        'Exhaustively try project-toolchain combinations'
    )
    parser.add_argument('--family', default=None, help='device family')
    parser.add_argument('--device', default=None, help='device')
    parser.add_argument('--package', default=None, help='device package')
    parser.add_argument('--project', default=None, help='run given project only (default: all)')
    parser.add_argument('--toolchain', default=None, help='run given toolchain only (default: all)')
    parser.add_argument('--out-prefix', default='build', help='output directory prefix (default: build)')
    parser.add_argument('--dry', action='store_true', help='print commands, don\'t invoke')
    parser.add_argument('--fail', action='store_true', help='fail on error')
    parser.add_argument('--verbose', action='store_true', help='verbose output')
    args = parser.parse_args()

    print('Running exhaustive project-toolchain search')

    # Always check if given option was overriden by user's argument
    # if not - run all available tests
    for project in user_selected(args.project) or get_projects():

        for toolchain in user_selected(args.toolchain) or get_toolchains():

            for family in user_selected(args.family) or get_families(project):

                for device in user_selected(args.device) or get_devices(project, family):

                    for package in user_selected(args.package) or get_packages(project, family, device):
                        # Only run a test if PCF constraint file is present
                        # the other (SDC, XDC) files are optional
                        if(get_pcf(project, family, device, package, toolchain) is not None):
                            run(
                                family,
                                device,
                                package,
                                toolchain,
                                project,
                                None, #out_dir
                                args.out_prefix,
                                None, #strategy
                                None, #carry
                                None, #seed
                                None, #build
                                args.verbose
                            )

    # Combine results of all the tests
    print('Merging results')
    merged_dict = {}

    get_reports(args.out_prefix)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import enf
import execnet
import sys
import textwrap

# Shared "worker" functions.
def square(n):
    return n * n
def pid():
    import os
    return os.getpid()

def example_1():
    """
    Simplest possible example: run a small computation in local
    interpreters.
    """
    group = execnet.Group(['popen'] * 2)
    with enf.GatewayExecutor(group) as executor:
        futures = [executor.submit(square, n) for n in range(5)]
        for future in futures:
            print future.result()

def example_2():
    """
    Get the PID of the interpreters used to prove that execnet
    is actually forking them.
    """
    group = execnet.Group(['popen'] * 3)
    with enf.GatewayExecutor(group) as executor:
        futures = [executor.submit(pid) for i in range(10)]
        for future in futures:
            print future.result()

def example_3():
    """
    Use concurrent.futures' implementation of map() to concisely
    run DOALL computations.
    """
    group = execnet.Group(['popen'] * 2)
    with enf.GatewayExecutor(group) as executor:
        for res in executor.map(square, range(10)):
            print res

if __name__ == '__main__':
    for arg in sys.argv[1:]:
        example = eval('example_%i' % int(arg.strip()))
        print textwrap.dedent(example.__doc__).strip()
        example()
<commit_msg>condor cluster example<commit_after>import enf
import execnet
import sys
import textwrap
import subprocess
import re

# Shared "worker" functions.
def square(n):
    return n * n
def pid():
    import os
    return os.getpid()
def hostinfo():
    import subprocess
    return subprocess.check_output('hostname; uname -a', shell=True)

# Magic for Condor stuff.
def idle_condor_hosts():
    status = subprocess.check_output('condor_status', shell=True)
    hostnames = re.findall(r'@(\S*)', status)
    hostnames = set(hostnames) # Uniquify.
    return hostnames
def condor_group(num=None):
    gwspecs = ['ssh=%s' % name for name in idle_condor_hosts()]
    if num:
        gwspecs = gwspecs[:num]
    return execnet.Group(gwspecs)

def example_1():
    """
    Simplest possible example: run a small computation in local
    interpreters.
    """
    group = execnet.Group(['popen'] * 2)
    with enf.GatewayExecutor(group) as executor:
        futures = [executor.submit(square, n) for n in range(5)]
        for future in futures:
            print future.result()

def example_2():
    """
    Get the PID of the interpreters used to prove that execnet
    is actually forking them.
    """
    group = execnet.Group(['popen'] * 3)
    with enf.GatewayExecutor(group) as executor:
        futures = [executor.submit(pid) for i in range(10)]
        for future in futures:
            print future.result()

def example_3():
    """
    Use concurrent.futures' implementation of map() to concisely
    run DOALL computations.
    """
    group = execnet.Group(['popen'] * 2)
    with enf.GatewayExecutor(group) as executor:
        for res in executor.map(square, range(10)):
            print res

def example_4():
    """Distribute tasks across the cluster!"""
    group = condor_group(4)
    with enf.GatewayExecutor(group) as executor:
        futures = [executor.submit(hostinfo) for i in range(5)]
        for future in futures:
            print future.result()

if __name__ == '__main__':
    for arg in sys.argv[1:]:
        example = eval('example_%i' % int(arg.strip()))
        print textwrap.dedent(example.__doc__).strip()
        example()
<|endoftext|>


<commit_before>'''The purpose of this program is to find some optimal parameter
   settings for a given program. If you expose parameters to the
   command line, but don't know what the optimal default value
   should be, this program will distribute the parameter
   possibilities over a cluster of machines, and rank the results
   based on user defined criteria.

   dispynode.py must be run on all machines in the cluster prior to
   running this program. The machines will be identified
   automatically, and made available for processing.

   The user must specify a program, which is assumed to be network
   accessible to all nodes of the cluster. The user must also specify
   a argument file. This file is assumed to be a text file with all
   of the parameter sets the user wants to test, one per line. The
   program will handle distribution over the cluster, and returning
   the results.

   If the user needs additional functionality, they can derive the
   JobRunner class with some additional functionality specific to
   their needs.
'''
class JobRunner() :

    def __init__(self, jobID, prog, argList) :
        '''Setup some common information for the class. This object is
           distributed with the job to the target node on the cluster. It
           is used as a conduit for passing information around the cluster
           and collecting results later.

           prog     - program to run (assumed to be network accessible)
           argList  - unique list of arguments to run on this node
           jobID    - unique identifier for this job (numeric)
           hostname - name of the machine this job was ran on
           stdout   - standard output generated by the subprocess
           stderr   - standard error generated by the subprocess
           results  - results should contain a metric for which the host
                      can later rank the job results.
        '''
        self.prog = prog
        self.argList = argList if not isinstance(argList, str) \
                       else argList.split()
        self.jobID = jobID
        self.hostname = None
        self.stdout = None
        self.stderr = None
        self.result = None

    def __str__(self) :
        return 'Job [' + str(self.jobID) + ']\n' + \
               '\tProgram  = ' + str(self.prog) + '\n' + \
               '\tArgList  = ' + str(self.argList) + '\n' + \
               '\tHostName = ' + str(self.hostname) + '\n' + \
               '\tStdOut   = ' + str(self.stdout) + '\n' + \
               '\tStdErr   = ' + str(self.stderr) + '\n' + \
               '\tResult   = ' + str(self.result) + '\n'

    # run this command as a subprocess on the node
    def _runSubProcess(self) :
        from subprocess import Popen
        import socket
        self.hostname = socket.gethostname()
        (self.stdout, self.stderr) = Popen(args=[self.prog] + self.argList,
                                           stdout=Popen.PIPE, stderr=Popen.PIPE,
                                           shell=False).communicate()

    # collect the results for this run
    def _collectResults(self) :
        # by default we assume the program returns a numeric metric
        # for ranking to standard output
        self.result = float(self.stdout)

    def run(self) :
        self._runSubProcess()
        self._collectResults()

    @staticmethod
    def rank(jobs) :
        lowestJob = None
        for job in jobs :
            if lowestJob is None or job.result < lowestJob.result :
                lowestJob = job
        return lowestJob

def disCompute(jobRunner) :
    # run the provided class
    jobRunner.run()
    return "here"
    #return jobRunner.hostname

if __name__ == '__main__':
    import dispy, dispy.httpd, argparse, logging

    parser = argparse.ArgumentParser()
    parser.add_argument('--prog', dest='prog',
                        help='Program to run in the subprocess.')
    parser.add_argument('--args', dest='args',
                        help='File containing all program variations (one per line)')
    parser.add_argument('--override', dest='jobRunner',
                        help='Python file containing a derived JobRunner')
    options = parser.parse_args()

    # setup a typedef for the new runner
    JR = JobRunner
    if options.jobRunner is not None :
        cls = options.jobRunner
        imp = __import__(cls)
        JR = imp.cls

    # setup the logger
    log = logging.getLogger('dispy: ' + options.prog)
    log.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(levelname)s - %(message)s')
    stream = logging.StreamHandler()
    stream.setLevel(logging.DEBUG)
    stream.setFormatter(formatter)
    log.addHandler(stream)

    # open the argument list for parsing
    with open(options.args, 'r') as f :
        argSets = f.readlines()

    # create the cluster for this program launch
    cluster = dispy.JobCluster(disCompute, depends=[JR])

    # start a monitor for the cluster at http://localhost:8181
    jobMonitor = dispy.httpd.DispyHTTPServer(cluster, host='localhost')

    # submit the jobs to the cluster --
    # this submits each argument set individually, and stores the jobs
    # so we can later recall their outputs and status. There is no
    # assignment here to schedule on the next available node.
    jobs = []
    for ii in range(len(argSets)) :
        log.info('Adding Job[' + str(ii) + ']')

        # create an object to perform the work on the node
        runner = JR(ii, options.prog, argSets[ii])
        job = cluster.submit(runner)
        job.id = runner
        jobs.append(job)

    # wait for the cluster to finish
    cluster.wait()

    # print the statistic of the clustered run
    log.info(cluster.stats())

    for job in jobs :
        log.info('Job[' + str(job.id.jobID) + ']: ' + str(job.result))

    # rank the result and return the best
    bestJob = JR.rank(jobs)
    log.info('The best job was: \n' + str(bestJob.result))

    # cleanup
    jobMonitor.shutdown()
    cluster.close()<commit_msg>redirect stdout so we can log the cluster status. Also pickling the object for results<commit_after>'''The purpose of this program is to find some optimal parameter
   settings for a given program. If you expose parameters to the
   command line, but don't know what the optimal default value
   should be, this program will distribute the parameter
   possibilities over a cluster of machines, and rank the results
   based on user defined criteria.

   dispynode.py must be run on all machines in the cluster prior to
   running this program. The machines will be identified
   automatically, and made available for processing.

   The user must specify a program, which is assumed to be network
   accessible to all nodes of the cluster. The user must also specify
   a argument file. This file is assumed to be a text file with all
   of the parameter sets the user wants to test, one per line. The
   program will handle distribution over the cluster, and returning
   the results.

   If the user needs additional functionality, they can derive the
   JobRunner class with some additional functionality specific to
   their needs.
'''
class JobRunner() :

    def __init__(self, jobID, prog, argList) :
        '''Setup some common information for the class. This object is
           distributed with the job to the target node on the cluster. It
           is used as a conduit for passing information around the cluster
           and collecting results later.

           prog     - program to run (assumed to be network accessible)
           argList  - unique list of arguments to run on this node
           jobID    - unique identifier for this job (numeric)
           hostname - name of the machine this job was ran on
           stdout   - standard output generated by the subprocess
           stderr   - standard error generated by the subprocess
           results  - results should contain a metric for which the host
                      can later rank the job results.
        '''
        self.prog = prog
        self.argList = argList if not isinstance(argList, str) \
                       else argList.split()
        self.jobID = jobID
        self.hostname = None
        self.stdout = None
        self.stderr = None
        self.result = None

    def __str__(self) :
        return 'Job [' + str(self.jobID) + ']\n' + \
               '\tProgram  = ' + str(self.prog) + '\n' + \
               '\tArgList  = ' + str(self.argList) + '\n' + \
               '\tHostName = ' + str(self.hostname) + '\n' + \
               '\tStdOut   = ' + str(self.stdout) + '\n' + \
               '\tStdErr   = ' + str(self.stderr) + '\n' + \
               '\tResult   = ' + str(self.result) + '\n'

    # run this command as a subprocess on the node
    def _runSubProcess(self) :
        from subprocess import Popen
        import socket
        self.hostname = socket.gethostname()
        (self.stdout, self.stderr) = Popen(args=[self.prog] + self.argList,
                                           stdout=Popen.PIPE, stderr=Popen.PIPE,
                                           shell=False).communicate()

    # collect the results for this run
    def _collectResults(self) :
        # by default we assume the program returns a numeric metric
        # for ranking to standard output
        self.result = float(self.stdout)

    def run(self) :
        self._runSubProcess()
        self._collectResults()

    @staticmethod
    def rank(jobs) :
        lowestJob = None
        for job in jobs :
            if lowestJob is None or job.result < lowestJob.result :
                lowestJob = job
        return lowestJob

def disCompute(jobRunner) :
    import pickle
    # run the provided class
    jobRunner.run()
    return pickle.dump(jobRunner)

if __name__ == '__main__':
    import dispy, dispy.httpd, argparse, logging, sys
    from cStringIO import StringIO

    parser = argparse.ArgumentParser()
    parser.add_argument('--prog', dest='prog',
                        help='Program to run in the subprocess.')
    parser.add_argument('--args', dest='args',
                        help='File containing all program variations (one per line)')
    parser.add_argument('--override', dest='jobRunner',
                        help='Python file containing a derived JobRunner')
    options = parser.parse_args()

    # setup a typedef for the new runner
    JR = JobRunner
    if options.jobRunner is not None :
        cls = options.jobRunner
        imp = __import__(cls)
        JR = imp.cls

    # setup the logger
    log = logging.getLogger('dispy: ' + options.prog)
    log.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(levelname)s - %(message)s')
    stream = logging.StreamHandler()
    stream.setLevel(logging.DEBUG)
    stream.setFormatter(formatter)
    log.addHandler(stream)

    # open the argument list for parsing
    with open(options.args, 'r') as f :
        argSets = f.readlines()

    # create the cluster for this program launch
    cluster = dispy.JobCluster(disCompute, depends=[JR])

    # start a monitor for the cluster at http://localhost:8181
    jobMonitor = dispy.httpd.DispyHTTPServer(cluster, host='localhost')

    # submit the jobs to the cluster --
    # this submits each argument set individually, and stores the jobs
    # so we can later recall their outputs and status. There is no
    # assignment here to schedule on the next available node.
    jobs = []
    for ii in range(len(argSets)) :
        log.info('Adding Job[' + str(ii) + ']')

        # create an object to perform the work on the node
        runner = JR(ii, options.prog, argSets[ii])
        job = cluster.submit(runner)
        job.id = runner
        jobs.append(job)

    # wait for the cluster to finish
    cluster.wait()

    # print the statistic of the clustered run --
    # We have no access to their method's print, so we temporarily redirect
    # stdout to a string, so we can capture the status in our logger.
    temp = sys.stdout
    sys.stdout = StringIO()
    cluster.stats()
    log.info(sys.stdout.getvalue())
    sys.stdout = temp

    for job in jobs :
        log.info('Job[' + str(job.id.jobID) + ']: ' + str(job.result))

    # rank the result and return the best
    bestJob = JR.rank(jobs)
    log.info('The best job was: \n' + str(bestJob.result))

    # cleanup
    jobMonitor.shutdown()
    cluster.close()<|endoftext|>


<commit_before>#!/usr/bin/python
import tempfile

from virtualenvapi.manage import VirtualEnvironment

def example(path=tempfile.mkdtemp('virtualenv.test')):
    print('Env path', path)
    env = VirtualEnvironment(path)

    print('django 1.5 installed?', env.is_installed('django==1.5'))

    print('mezzanine installed?', env.is_installed('mezzanine'))
    env.install('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))

    print(env.installed_packages)

    payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
    print('cartridge-payments installed?', env.is_installed(payments_repo))
    env.install(payments_repo)
    print('cartridge-payments installed?', env.is_installed(payments_repo))
    print(env.installed_packages)

    env.uninstall('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))
    print(env.installed_packages)

    pkgs = env.search('requests')
    print('search for \'requests\':')
    print(len(pkgs), 'found:')
    print(pkgs)

if __name__ == '__main__':
    example()
<commit_msg>Fixed package specification in example script: #10<commit_after>#!/usr/bin/python
import tempfile

from virtualenvapi.manage import VirtualEnvironment

def example(path=tempfile.mkdtemp('virtualenv.test')):
    print('Env path', path)
    env = VirtualEnvironment(path)

    print('django 1.5 installed?', env.is_installed('django==1.5'))

    print('mezzanine installed?', env.is_installed('mezzanine'))
    env.install('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))

    print(env.installed_packages)

    payments_repo = 'git+git://github.com/sjkingo/cartridge-payments.git'
    payments_pkg = payments_repo.split('/')[-1].replace('.git', '')
    print('cartridge-payments installed?', env.is_installed(payments_pkg))
    env.install(payments_repo)
    print('cartridge-payments installed?', env.is_installed(payments_pkg))
    print(env.installed_packages)

    env.uninstall('mezzanine')
    print('mezzanine installed?', env.is_installed('mezzanine'))
    print(env.installed_packages)

    pkgs = env.search('requests')
    print('search for \'requests\':')
    print(len(pkgs), 'found:')
    print(pkgs)

if __name__ == '__main__':
    example()
<|endoftext|>


<commit_before>#This file is part of Tryton and Nereid.  The COPYRIGHT file at the top level of
#this repository contains the full copyright notices and license terms.
{
    'name': 'Nereid Catalog',
    'version': '2.4.0.1dev',
    'author': 'Openlabs Technologies & Consulting (P) LTD',
    'email': 'info@openlabs.co.in',
    'website': 'http://www.openlabs.co.in/',
    'description': '''Nereid Catalog''',
    'depends': [
        'product',
        'nereid',
    ],
    'xml': [
        'product.xml',
        'urls.xml',
    ],
    'translation': [
    ],
}
<commit_msg>Bumped version to 2.4.0.1<commit_after>#This file is part of Tryton and Nereid.  The COPYRIGHT file at the top level of
#this repository contains the full copyright notices and license terms.
{
    'name': 'Nereid Catalog',
    'version': '2.4.0.1',
    'author': 'Openlabs Technologies & Consulting (P) LTD',
    'email': 'info@openlabs.co.in',
    'website': 'http://www.openlabs.co.in/',
    'description': '''Nereid Catalog''',
    'depends': [
        'product',
        'nereid',
    ],
    'xml': [
        'product.xml',
        'urls.xml',
    ],
    'translation': [
    ],
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2016, Major Hayden
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Analyzes the differences between two OpenStack-Ansible commits."""
import argparse
import json
import logging
import os
import sys

from git import Repo
import jinja2
import requests
import yaml

def create_parser():
    """Setup argument Parsing."""
    description = """OpenStack-Ansible Release Diff Generator
----------------------------------------

Finds changes in OpenStack projects and OpenStack-Ansible roles between two
commits in OpenStack-Ansible.

"""

    parser = argparse.ArgumentParser(
        usage='%(prog)s',
        description=description,
        epilog='Licensed "Apache 2.0"',
        formatter_class=argparse.RawTextHelpFormatter
    )
    parser.add_argument(
        'old_commit',
        action='store',
        nargs=1,
        help="Git SHA of the older commit",
    )
    parser.add_argument(
        'new_commit',
        action='store',
        nargs=1,
        help="Git SHA of the newer commit",
    )
    parser.add_argument(
        '--debug',
        action='store_true',
        default=False,
        help="Enable debug output",
    )
    parser.add_argument(
        '-d', '--directory',
        action='store',
        default="~/.osa-differ",
        help="Git repo storage directory (default: ~/.osa-differ)",
    )
    parser.add_argument(
        '-u', '--update',
        action='store_true',
        default=False,
        help="Fetch latest changes to repo",
    )
    display_opts = parser.add_argument_group("Limit scope")
    display_opts.add_argument(
        "--skip-projects",
        action="store_true",
        help="Skip checking for changes in OpenStack projects"
    )
    display_opts.add_argument(
        "--skip-roles",
        action="store_true",
        help="Skip checking for changes in OpenStack-Ansible roles"
    )
    output_desc = ("Note: Output is always printed to stdout unless "
                   "--quiet is provided.")
    output_opts = parser.add_argument_group('Output options', output_desc)
    output_opts.add_argument(
        '--quiet',
        action='store_true',
        default=False,
        help="Do not output to stdout",
    )
    output_opts.add_argument(
        '--gist',
        action='store_true',
        default=False,
        help="Output into a GitHub Gist",
    )
    output_opts.add_argument(
        '--file',
        metavar="FILENAME",
        action='store',
        help="Output to a file",
    )
    return parser

def get_commits(repo_dir, old_commit, new_commit, hide_merges=True):
    """Find all commits between two commit SHAs."""
    repo = Repo(repo_dir)
    commits = repo.iter_commits(rev="{0}..{1}".format(old_commit, new_commit))
    if hide_merges:
        return [x for x in commits if not x.summary.startswith("Merge ")]
    else:
        return list(commits)

def get_commit_url(repo_url):
    """Determine URL to view commits for repo."""
    if "github.com" in repo_url:
        return repo_url
    if "git.openstack.org" in repo_url:
        uri = '/'.join(repo_url.split('/')[-2:])
        return "https://github.com/{0}".format(uri)

    # If it didn't match these conditions, just return it.
    return repo_url

def get_logger(debug=False):
    """Set up the logger."""
    log_format = "%(asctime)s - %(levelname)s - %(message)s"
    logging.basicConfig(level=logging.WARNING, format=log_format)
    logger = logging.getLogger(__name__)

    if debug:
        logger.setLevel(logging.DEBUG)

    return logger

def get_projects(osa_repo_dir, yaml_files, commit):
    """Get all projects from multiple YAML files."""
    yaml_parsed = []
    for yaml_file in yaml_files:
        # Check out the correct commit SHA from the repository
        repo = Repo(osa_repo_dir)
        repo.head.reference = repo.commit(commit)
        repo.head.reset(index=True, working_tree=True)

        filename = "{0}/{1}".format(osa_repo_dir, yaml_file)
        with open(filename, 'r') as f:
            yaml_parsed.append(yaml.load(f))

    merged_dicts = {k: v for d in yaml_parsed for k, v in d.items()}

    return merged_dicts

def get_roles(osa_repo_dir, commit):
    """Read OSA role information at a particular commit."""
    repo = Repo(osa_repo_dir)
    repo.head.reference = repo.commit(commit)
    repo.head.reset(index=True, working_tree=True)

    filename = "{0}/ansible-role-requirements.yml".format(osa_repo_dir)
    with open(filename, 'r') as f:
        roles_yaml = yaml.load(f)

    return roles_yaml

def parse_arguments():
    """Parse arguments."""
    parser = create_parser()
    args = parser.parse_args()
    return args

def post_gist(report_data, old_sha, new_sha):
    """Post the report to a GitHub Gist and return the URL of the gist."""
    payload = {
        "description": ("Changes in OpenStack-Ansible between "
                        "{0} and {1}".format(old_sha, new_sha)),
        "public": True,
        "files": {
            "osa-diff-{0}-{1}.rst".format(old_sha, new_sha): {
                "content": report_data
            }
        }
    }
    url = "https://api.github.com/gists"
    r = requests.post(url, data=json.dumps(payload))
    response = r.json()
    return response['html_url']

def prepare_storage_dir(storage_directory):
    """Prepare the storage directory."""
    storage_directory = os.path.expanduser(storage_directory)
    if not os.path.exists(storage_directory):
        os.mkdir(storage_directory)

    return storage_directory

def render_template(template_file, template_vars):
    """Render a jinja template."""
    # Load our Jinja templates
    template_dir = "{0}/templates".format(
        os.path.dirname(os.path.abspath(__file__))
    )
    jinja_env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(template_dir),
        trim_blocks=True
    )
    rendered = jinja_env.get_template(template_file).render(template_vars)

    return rendered

def repo_clone(repo_dir, repo_url):
    """Clone repository to this host."""
    repo = Repo.clone_from(repo_url, repo_dir)
    return repo

def repo_pull(repo_dir, repo_url, fetch=False):
    """Reset repository and optionally update it."""
    # Make sure the repository is reset to the master branch.
    repo = Repo(repo_dir)
    repo.heads.master.checkout()
    repo.head.reset(index=True, working_tree=True)

    # Only get the latest updates if requested.
    if fetch:
        repo.remotes['origin'].pull()

    return repo

def update_repo(repo_dir, repo_url, fetch=False):
    """Clone the repo if it doesn't exist already, otherwise update it."""
    repo_exists = os.path.exists(repo_dir)
    if repo_exists:
        repo = repo_pull(repo_dir, repo_url, fetch)
    else:
        repo = repo_clone(repo_dir, repo_url)

    return repo

def valid_commit(repo_dir, commit):
    """Test if a commit is valid for the repository."""
    repo = Repo(repo_dir)
    try:
        commit = repo.commit(commit)
    except:
        return False

    return True

def run_osa_differ():
    """The script starts here."""
    # Get our arguments from the command line
    args = parse_arguments()

    # Configure logging
    logger = get_logger(args.debug)

    # Create the storage directory if it doesn't exist already.
    try:
        storage_directory = prepare_storage_dir(args.storage_directory)
    except OSError:
        print("ERROR: Couldn't create the storage directory {0}. "
              "Please create it manually.".format(args.storage_directory))
        sys.exit(1)

    osa_repo_url = "https://git.openstack.org/openstack/openstack-ansible"
    osa_repo_dir = "{0}/openstack-ansible".format(storage_directory)
    update_repo(osa_repo_dir, osa_repo_url)

    # Store the commits we were provided
    osa_old_commit = args.old_commit[0]
    osa_new_commit = args.new_commit[0]

    # Are these commits valid?
    if (not valid_commit(osa_repo_dir, osa_old_commit) or
       not valid_commit(osa_repo_dir, osa_new_commit)):
        print("The range of commits provided can't be found within the "
              "OpenStack-Ansible repository:\n\n"
              "\t{0}..{1}".format(osa_old_commit, osa_new_commit))
        sys.exit(1)

    # Are there any commits between the two commits that were provided?
    commits = get_commits(osa_repo_dir, osa_old_commit, osa_new_commit)
    if len(commits) == 0:
        # The user might have gotten their commits out of order. Let's flip
        # the order of the commits and try again.
        commits = get_commits(osa_repo_dir, osa_new_commit, osa_old_commit)
        if len(commits) == 0:
            # Okay, so there really are no commits between the two commits
            # provided by the user. :)
            print("No commits were found in the commit range provided:\n\n"
                  "\t{0}..{1}\n\n".format(osa_old_commit, osa_new_commit))
            sys.exit(1)
        else:
            logger.warning("Commits provided in reverse order; "
                           "flipping these variables now")
            osa_old_commit, osa_new_commit = osa_new_commit, osa_old_commit

    # Start off our report with a header and our OpenStack-Ansible commits.
    template_vars = {
        'args': args,
        'repo': 'openstack-ansible',
        'commits': commits,
        'commit_base_url': get_commit_url(osa_repo_url),
        'old_sha': osa_old_commit,
        'new_sha': osa_new_commit
    }
    report = render_template('offline-header.j2', template_vars)

    # Get the list of OpenStack roles from the newer and older commits.
    role_yaml = get_roles(osa_repo_dir, osa_old_commit)
    role_yaml_latest = get_roles(osa_repo_dir, osa_new_commit)

    # Don't loop through the roles if the user asked us not to do so.
    if args.skip_roles:
        role_yaml_latest = []
    else:
        report += ("OpenStack-Ansible Roles\n"
                   "-----------------------")

    for role in role_yaml_latest:
        # Prepare our repo directory and clone the repo if needed. Only pull
        # if the user requests it.
        repo_dir = "{0}/{1}".format(storage_directory, role['name'])
        repo_url = role['src']
        update_repo(repo_dir, repo_url, args.update)

        # Get the commit SHAs for this role from the older and newer
        # OpenStack-Ansible commits.
        role_old_sha = next(x['version'] for x in role_yaml
                            if x['name'] == role['name'])
        role_new_sha = role['version']

        # Loop through the commits and render our template.
        commits = get_commits(repo_dir, role_old_sha, role_new_sha)
        template_vars = {
            'repo': role['name'],
            'commits': commits,
            'commit_base_url': get_commit_url(repo_url),
            'old_sha': role_old_sha,
            'new_sha': role_new_sha
        }
        rst = render_template('offline-repo-changes.j2', template_vars)
        report += rst

    # Get the list of OpenStack projects from newer commit and older commit.
    yaml_files = [
        'playbooks/defaults/repo_packages/openstack_services.yml',
        'playbooks/defaults/repo_packages/openstack_other.yml'
    ]
    project_yaml = get_projects(osa_repo_dir, yaml_files, osa_old_commit)
    project_yaml_latest = get_projects(osa_repo_dir, yaml_files,
                                       osa_new_commit)

    # Narrow down a list of projects from the latest YAML we retrieved.
    projects = sorted([x[:-9] for x in project_yaml_latest.keys()
                      if x.endswith('git_repo')])

    # Don't loop through the projects if the user asked us not to do so.
    if args.skip_projects:
        project_yaml_latest = []
    else:
        report += ("OpenStack Projects\n"
                   "------------------")

    # Loop through each project and find changes. If we've been asked to update
    # the repository, let's do that too.
    logger.info("Looping through projects to check for changes")
    for project in projects:
        # Prepare our repo directory and clone the repo if needed. Only pull
        # if the user requests it.
        repo_dir = "{0}/{1}".format(storage_directory, project)
        repo_url = project_yaml_latest['{0}_git_repo'.format(project)]
        update_repo(repo_dir, repo_url, args.update)

        # Get the commit SHAs for this project from the older and newer
        # OpenStack-Ansible commits.
        key_for_sha = "{0}_git_install_branch".format(project)
        project_old_sha = project_yaml[key_for_sha]
        project_new_sha = project_yaml_latest[key_for_sha]

        # Loop through the commits and render our template.
        commits = get_commits(repo_dir, project_old_sha, project_new_sha)
        template_vars = {
            'repo': project,
            'commits': commits,
            'commit_base_url': get_commit_url(repo_url),
            'old_sha': project_old_sha,
            'new_sha': project_new_sha
        }
        rst = render_template('offline-repo-changes.j2', template_vars)

    # Print the report to stdout unless the user specified --quiet.
    if not args.quiet:
        print(report)

    if args.gist:
        gist_url = post_gist(report, osa_old_commit, osa_new_commit)
        print("Report posted to GitHub Gist: {0}".format(gist_url))

    if args.file is not None:
        print("Report written to file: {0}".format(args.file))
        with open(args.file, 'w') as f:
            f.write(report)

if __name__ == "__main__":
    run_osa_differ()
<commit_msg>Fix validation<commit_after>#!/usr/bin/env python
# Copyright 2016, Major Hayden
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Analyzes the differences between two OpenStack-Ansible commits."""
import argparse
import json
import logging
import os
import sys

from git import Repo
import jinja2
import requests
import yaml

def create_parser():
    """Setup argument Parsing."""
    description = """OpenStack-Ansible Release Diff Generator
----------------------------------------

Finds changes in OpenStack projects and OpenStack-Ansible roles between two
commits in OpenStack-Ansible.

"""

    parser = argparse.ArgumentParser(
        usage='%(prog)s',
        description=description,
        epilog='Licensed "Apache 2.0"',
        formatter_class=argparse.RawTextHelpFormatter
    )
    parser.add_argument(
        'old_commit',
        action='store',
        nargs=1,
        help="Git SHA of the older commit",
    )
    parser.add_argument(
        'new_commit',
        action='store',
        nargs=1,
        help="Git SHA of the newer commit",
    )
    parser.add_argument(
        '--debug',
        action='store_true',
        default=False,
        help="Enable debug output",
    )
    parser.add_argument(
        '-d', '--directory',
        action='store',
        default="~/.osa-differ",
        help="Git repo storage directory (default: ~/.osa-differ)",
    )
    parser.add_argument(
        '-u', '--update',
        action='store_true',
        default=False,
        help="Fetch latest changes to repo",
    )
    display_opts = parser.add_argument_group("Limit scope")
    display_opts.add_argument(
        "--skip-projects",
        action="store_true",
        help="Skip checking for changes in OpenStack projects"
    )
    display_opts.add_argument(
        "--skip-roles",
        action="store_true",
        help="Skip checking for changes in OpenStack-Ansible roles"
    )
    output_desc = ("Note: Output is always printed to stdout unless "
                   "--quiet is provided.")
    output_opts = parser.add_argument_group('Output options', output_desc)
    output_opts.add_argument(
        '--quiet',
        action='store_true',
        default=False,
        help="Do not output to stdout",
    )
    output_opts.add_argument(
        '--gist',
        action='store_true',
        default=False,
        help="Output into a GitHub Gist",
    )
    output_opts.add_argument(
        '--file',
        metavar="FILENAME",
        action='store',
        help="Output to a file",
    )
    return parser

def get_commits(repo_dir, old_commit, new_commit, hide_merges=True):
    """Find all commits between two commit SHAs."""
    repo = Repo(repo_dir)
    commits = repo.iter_commits(rev="{0}..{1}".format(old_commit, new_commit))
    if hide_merges:
        return [x for x in commits if not x.summary.startswith("Merge ")]
    else:
        return list(commits)

def get_commit_url(repo_url):
    """Determine URL to view commits for repo."""
    if "github.com" in repo_url:
        return repo_url
    if "git.openstack.org" in repo_url:
        uri = '/'.join(repo_url.split('/')[-2:])
        return "https://github.com/{0}".format(uri)

    # If it didn't match these conditions, just return it.
    return repo_url

def get_logger(debug=False):
    """Set up the logger."""
    log_format = "%(asctime)s - %(levelname)s - %(message)s"
    logging.basicConfig(level=logging.WARNING, format=log_format)
    logger = logging.getLogger(__name__)

    if debug:
        logger.setLevel(logging.DEBUG)

    return logger

def get_projects(osa_repo_dir, yaml_files, commit):
    """Get all projects from multiple YAML files."""
    yaml_parsed = []
    for yaml_file in yaml_files:
        # Check out the correct commit SHA from the repository
        repo = Repo(osa_repo_dir)
        repo.head.reference = repo.commit(commit)
        repo.head.reset(index=True, working_tree=True)

        filename = "{0}/{1}".format(osa_repo_dir, yaml_file)
        with open(filename, 'r') as f:
            yaml_parsed.append(yaml.load(f))

    merged_dicts = {k: v for d in yaml_parsed for k, v in d.items()}

    return merged_dicts

def get_roles(osa_repo_dir, commit):
    """Read OSA role information at a particular commit."""
    repo = Repo(osa_repo_dir)
    repo.head.reference = repo.commit(commit)
    repo.head.reset(index=True, working_tree=True)

    filename = "{0}/ansible-role-requirements.yml".format(osa_repo_dir)
    with open(filename, 'r') as f:
        roles_yaml = yaml.load(f)

    return roles_yaml

def parse_arguments():
    """Parse arguments."""
    parser = create_parser()
    args = parser.parse_args()
    return args

def post_gist(report_data, old_sha, new_sha):
    """Post the report to a GitHub Gist and return the URL of the gist."""
    payload = {
        "description": ("Changes in OpenStack-Ansible between "
                        "{0} and {1}".format(old_sha, new_sha)),
        "public": True,
        "files": {
            "osa-diff-{0}-{1}.rst".format(old_sha, new_sha): {
                "content": report_data
            }
        }
    }
    url = "https://api.github.com/gists"
    r = requests.post(url, data=json.dumps(payload))
    response = r.json()
    return response['html_url']

def prepare_storage_dir(storage_directory):
    """Prepare the storage directory."""
    storage_directory = os.path.expanduser(storage_directory)
    if not os.path.exists(storage_directory):
        os.mkdir(storage_directory)

    return storage_directory

def render_template(template_file, template_vars):
    """Render a jinja template."""
    # Load our Jinja templates
    template_dir = "{0}/templates".format(
        os.path.dirname(os.path.abspath(__file__))
    )
    jinja_env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(template_dir),
        trim_blocks=True
    )
    rendered = jinja_env.get_template(template_file).render(template_vars)

    return rendered

def repo_clone(repo_dir, repo_url):
    """Clone repository to this host."""
    repo = Repo.clone_from(repo_url, repo_dir)
    return repo

def repo_pull(repo_dir, repo_url, fetch=False):
    """Reset repository and optionally update it."""
    # Make sure the repository is reset to the master branch.
    repo = Repo(repo_dir)
    repo.heads.master.checkout()
    repo.head.reset(index=True, working_tree=True)

    # Only get the latest updates if requested.
    if fetch:
        repo.remotes['origin'].pull()

    return repo

def update_repo(repo_dir, repo_url, fetch=False):
    """Clone the repo if it doesn't exist already, otherwise update it."""
    repo_exists = os.path.exists(repo_dir)
    if repo_exists:
        repo = repo_pull(repo_dir, repo_url, fetch)
    else:
        repo = repo_clone(repo_dir, repo_url)

    return repo

def validate_commits(repo_dir, commits):
    """Test if a commit is valid for the repository."""
    repo = Repo(repo_dir)
    for commit in commits:
        try:
            commit = repo.commit(commit)
        except:
            raise Exception("Commit {0} could not be found".format(commit))

    return True

def validate_commit_range(repo_dir, old_commit, new_commit):
    """Check if commit range is valid. Flip it if needed."""
    # Are there any commits between the two commits that were provided?
    commits = get_commits(repo_dir, old_commit, new_commit)
    if len(commits) == 0:
        # The user might have gotten their commits out of order. Let's flip
        # the order of the commits and try again.
        commits = get_commits(repo_dir, new_commit, old_commit)
        if len(commits) == 0:
            # Okay, so there really are no commits between the two commits
            # provided by the user. :)
            msg = "The commit range provided is invalid."
            raise Exception(msg)
        else:
            return 'flip'

    return True

def run_osa_differ():
    """The script starts here."""
    # Get our arguments from the command line
    args = parse_arguments()

    # Configure logging
    logger = get_logger(args.debug)

    # Create the storage directory if it doesn't exist already.
    try:
        storage_directory = prepare_storage_dir(args.directory)
    except OSError:
        print("ERROR: Couldn't create the storage directory {0}. "
              "Please create it manually.".format(args.directory))
        sys.exit(1)

    osa_old_commit = args.old_commit[0]
    osa_new_commit = args.new_commit[0]
    osa_repo_url = "https://git.openstack.org/openstack/openstack-ansible"
    osa_repo_dir = "{0}/openstack-ansible".format(storage_directory)
    update_repo(osa_repo_dir, osa_repo_url)

    # Are these commits valid?
    validate_commits(osa_repo_dir, [osa_old_commit, osa_new_commit])

    # Do we have a valid commit range?
    validate_commit_range(osa_repo_dir, osa_old_commit, osa_new_commit)

    # Get the commits in the range
    commits = get_commits(osa_repo_dir, osa_old_commit, osa_new_commit)

    # Start off our report with a header and our OpenStack-Ansible commits.
    template_vars = {
        'args': args,
        'repo': 'openstack-ansible',
        'commits': commits,
        'commit_base_url': get_commit_url(osa_repo_url),
        'old_sha': osa_old_commit,
        'new_sha': osa_new_commit
    }
    report = render_template('offline-header.j2', template_vars)

    # Get the list of OpenStack roles from the newer and older commits.
    role_yaml = get_roles(osa_repo_dir, osa_old_commit)
    role_yaml_latest = get_roles(osa_repo_dir, osa_new_commit)

    # Don't loop through the roles if the user asked us not to do so.
    if args.skip_roles:
        role_yaml_latest = []
    else:
        report += ("OpenStack-Ansible Roles\n"
                   "-----------------------")

    for role in role_yaml_latest:
        # Prepare our repo directory and clone the repo if needed. Only pull
        # if the user requests it.
        repo_dir = "{0}/{1}".format(storage_directory, role['name'])
        repo_url = role['src']
        update_repo(repo_dir, repo_url, args.update)

        # Get the commit SHAs for this role from the older and newer
        # OpenStack-Ansible commits.
        role_old_sha = next(x['version'] for x in role_yaml
                            if x['name'] == role['name'])
        role_new_sha = role['version']

        # Loop through the commits and render our template.
        commits = get_commits(repo_dir, role_old_sha, role_new_sha)
        template_vars = {
            'repo': role['name'],
            'commits': commits,
            'commit_base_url': get_commit_url(repo_url),
            'old_sha': role_old_sha,
            'new_sha': role_new_sha
        }
        rst = render_template('offline-repo-changes.j2', template_vars)
        report += rst

    # Get the list of OpenStack projects from newer commit and older commit.
    yaml_files = [
        'playbooks/defaults/repo_packages/openstack_services.yml',
        'playbooks/defaults/repo_packages/openstack_other.yml'
    ]
    project_yaml = get_projects(osa_repo_dir, yaml_files, osa_old_commit)
    project_yaml_latest = get_projects(osa_repo_dir, yaml_files,
                                       osa_new_commit)

    # Narrow down a list of projects from the latest YAML we retrieved.
    projects = sorted([x[:-9] for x in project_yaml_latest.keys()
                      if x.endswith('git_repo')])

    # Don't loop through the projects if the user asked us not to do so.
    if args.skip_projects:
        project_yaml_latest = []
    else:
        report += ("OpenStack Projects\n"
                   "------------------")

    # Loop through each project and find changes. If we've been asked to update
    # the repository, let's do that too.
    logger.info("Looping through projects to check for changes")
    for project in projects:
        # Prepare our repo directory and clone the repo if needed. Only pull
        # if the user requests it.
        repo_dir = "{0}/{1}".format(storage_directory, project)
        repo_url = project_yaml_latest['{0}_git_repo'.format(project)]
        update_repo(repo_dir, repo_url, args.update)

        # Get the commit SHAs for this project from the older and newer
        # OpenStack-Ansible commits.
        key_for_sha = "{0}_git_install_branch".format(project)
        project_old_sha = project_yaml[key_for_sha]
        project_new_sha = project_yaml_latest[key_for_sha]

        # Loop through the commits and render our template.
        commits = get_commits(repo_dir, project_old_sha, project_new_sha)
        template_vars = {
            'repo': project,
            'commits': commits,
            'commit_base_url': get_commit_url(repo_url),
            'old_sha': project_old_sha,
            'new_sha': project_new_sha
        }
        rst = render_template('offline-repo-changes.j2', template_vars)

    # Print the report to stdout unless the user specified --quiet.
    if not args.quiet:
        print(report)

    if args.gist:
        gist_url = post_gist(report, osa_old_commit, osa_new_commit)
        print("Report posted to GitHub Gist: {0}".format(gist_url))

    if args.file is not None:
        print("Report written to file: {0}".format(args.file))
        with open(args.file, 'w') as f:
            f.write(report)

if __name__ == "__main__":
    run_osa_differ()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
##################################################################################
#
#    Copyright 2014-2017 Félix Brezo and Yaiza Rubio (i3visio, contacto@i3visio.com)
#
#	OSRFramework is free software: you can redistribute it and/or modify
#	it under the terms of the GNU Affero General Public License as published by
#	the Free Software Foundation, either version 3 of the License, or
#	(at your option) any later version.
#
#	This program is distributed in the hope that it will be useful,
#	but WITHOUT ANY WARRANTY; without even the implied warranty of
#	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#	GNU Affero General Public License for more details.
#
#	You should have received a copy of the GNU Affero General Public License
#	along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##################################################################################

import osrframework.utils.logger

# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework")
__version__="0.16.6"
<commit_msg>Update version number<commit_after># -*- coding: utf-8 -*-
#
##################################################################################
#
#    Copyright 2014-2017 Félix Brezo and Yaiza Rubio (i3visio, contacto@i3visio.com)
#
#	OSRFramework is free software: you can redistribute it and/or modify
#	it under the terms of the GNU Affero General Public License as published by
#	the Free Software Foundation, either version 3 of the License, or
#	(at your option) any later version.
#
#	This program is distributed in the hope that it will be useful,
#	but WITHOUT ANY WARRANTY; without even the implied warranty of
#	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#	GNU Affero General Public License for more details.
#
#	You should have received a copy of the GNU Affero General Public License
#	along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##################################################################################

import osrframework.utils.logger

# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework")
__version__="0.16.7"
<|endoftext|>