<commit_before># coding: utf-8
import logging
import requests
import xmltodict

from .configs import Config
from .configs import AbstractConfig
from .utils import parse_date, is_valid_email, is_valid_cpf

logger = logging.getLogger()

class PagSeguroNotificationResponse(object):
    def __init__(self, xml, config=None):
        self.xml = xml
        self.config = config or {}
        self.parse_xml(xml)

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug(
                "Cannot parse the returned xml '{0}' -> '{1}'".format(xml, e)
            )
            parsed = {}

        transaction = parsed.get('transaction', {})
        for k, v in transaction.iteritems():
            setattr(self, k, v)

class PagSeguroCheckoutResponse(object):
    def __init__(self, xml, config=None):
        self.xml = xml
        self.config = config or {}
        self.code = None
        self.date = None
        self.errors = None
        self.payment_url = None
        logger.debug(self.__dict__)
        self.parse_xml(xml)

    def parse_xml(self, xml):
        """ parse returned data """
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug(
                "Cannot parse the returned xml '{0}' -> '{1}'".format(xml, e)
            )
            parsed = {}

        if 'errors' in parsed:
            self.errors = parsed['errors']['error']
            return

        checkout = parsed.get('checkout', {})
        self.code = checkout.get('code')
        self.date = parse_date(checkout.get('date'))

        self.payment_url = self.config.PAYMENT_URL % self.code

class PagSeguroTransactionSearchResult(object):

    current_page = None
    total_pages = None
    results_in_page = None
    transactions = []

    def __init__(self, xml, config=None):
        self.xml = xml
        self.config = config or {}
        self.parse_xml(xml)

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug(
                "Cannot parse the returned xml '{0}' -> '{1}'".format(xml, e)
            )
            parsed = {}

        search_result = parsed.get('transactionSearchResult', {})
        self.transactions = search_result.get('transactions', {})
        self.transactions = self.transactions.get('transaction', [])
        if not isinstance(self.transactions, list):
            self.transactions = [self.transactions]
        self.current_page = search_result.get('currentPage', None)
        if self.current_page is not None:
            self.current_page = int(self.current_page)
        self.results_in_page = search_result.get('resultsInThisPage', None)
        if self.results_in_page is not None:
            self.results_in_page = int(self.results_in_page)
        self.total_pages = search_result.get('totalPages', None)
        if self.total_pages is not None:
            self.total_pages = int(self.total_pages)

class PagSeguro(object):
    """ Pag Seguro V2 wrapper """

    PAC = 1
    SEDEX = 2
    NONE = 3

    def __init__(self, email, token, data=None, config=None):
        self.config = config or Config()
        assert isinstance(self.config, AbstractConfig), "Config should be instance of AbstractConfig"
        self.data = {}
        self.data['email'] = email
        self.data['token'] = token

        if data and isinstance(data, dict):
            self.data.update(data)

        self.items = []
        self.sender = {}
        self.shipping = {}
        self._reference = ""
        self.extra_amount = None
        self.redirect_url = None
        self.notification_url = None
        self.abandon_url = None

    def build_checkout_params(self, **kwargs):
        """ build a dict with params """
        params = kwargs or {}
        if self.sender:
            params['senderName'] = self.sender.get('name')
            params['senderAreaCode'] = self.sender.get('area_code')
            params['senderPhone'] = self.sender.get('phone')
            params['senderEmail'] = is_valid_email(self.sender.get('email'))
            params['senderCPF'] = is_valid_cpf(self.sender.get('cpf'))
            params['senderBornDate'] = self.sender.get('born_date')

        if self.shipping:
            params['shippingType'] = self.shipping.get('type')
            params['shippingAddressStreet'] = self.shipping.get('street')
            params['shippingAddressNumber'] = self.shipping.get('number')
            params['shippingAddressComplement'] = self.shipping.get(
                'complement'
            )
            params['shippingAddressDistrict'] = self.shipping.get('district')
            params['shippingAddressPostalCode'] = self.shipping.get(
                'postal_code'
            )
            params['shippingAddressCity'] = self.shipping.get('city')
            params['shippingAddressState'] = self.shipping.get('state')
            params['shippingAddressCountry'] = self.shipping.get('country',
                                                                 'BRA')

        if self.shipping and self.shipping.get('cost'):
            params['shippingCost'] = self.shipping.get('cost')

        if self.extra_amount:
            params['extraAmount'] = self.extra_amount

        params['reference'] = self.reference

        if self.redirect_url:
            params['redirectURL'] = self.redirect_url

        if self.notification_url:
            params['notificationURL'] = self.notification_url

        if self.abandon_url:
            params['abandonURL'] = self.abandon_url

        for i, item in enumerate(self.items, 1):
            params['itemId%s' % i] = item.get('id')
            params['itemDescription%s' % i] = item.get('description')
            params['itemAmount%s' % i] = item.get('amount')
            params['itemQuantity%s' % i] = item.get('quantity')
            params['itemWeight%s' % i] = item.get('weight')
            params['itemShippingCost%s' % i] = item.get('shipping_cost')

        self.data.update(params)
        self.clean_none_params()

    def clean_none_params(self):
        copy = dict(self.data)
        for k, v in copy.iteritems():
            if not v:
                del self.data[k]

    @property
    def reference_prefix(self):
        return self.config.REFERENCE_PREFIX or "%s"

    @reference_prefix.setter
    def reference_prefix(self, value):
        self.config.REFERENCE_PREFIX = (value or "") + "%s"

    @property
    def reference(self):
        return self.reference_prefix % self._reference

    @reference.setter
    def reference(self, value):
        if not isinstance(value, (str, unicode)):
            value = str(value)
        if value.startswith(self.reference_prefix):
            value = value[len(self.reference_prefix):]
        self._reference = value

    def get(self, url):
        """ do a get transaction """
        return requests.get(url, params=self.data, headers=self.config.HEADERS)

    def post(self, url):
        """ do a post request """
        return requests.post(url, data=self.data, headers=self.config.HEADERS)

    def checkout(self, **kwargs):
        """ create a pagseguro checkout """
        self.data['currency'] = self.config.CURRENCY
        self.build_checkout_params(**kwargs)
        response = self.post(url=self.config.CHECKOUT_URL)
        return PagSeguroCheckoutResponse(response.content, config=self.config)

    def check_notification(self, code):
        """ check a notification by its code """
        response = self.get(url=self.config.NOTIFICATION_URL % code)
        return PagSeguroNotificationResponse(response.content, self.config)

    def check_transaction(self, code):
        """ check a transaction by its code """
        response = self.get(url=self.config.TRANSACTION_URL % code)
        return PagSeguroNotificationResponse(response.content, self.config)

    def query_transactions(self, initial_date, final_date, page=None,
                           max_results=None):
        """ query transaction by date range """
        last_page = False
        results = []
        while last_page is False:
            search_result = self._consume_query_transactions(
                initial_date, final_date, page, max_results
            )
            results.extend(search_result.transactions)
            if search_result.current_page is None or \
               search_result.total_pages is None or \
               search_result.current_page == search_result.total_pages:
                last_page = True
            else:
                page = search_result.current_page + 1

        return results

    def _consume_query_transactions(self, initial_date, final_date, page=None,
                                    max_results=None):
        querystring = {
            'initialDate': initial_date.strftime('%Y-%m-%dT%H:%M'),
            'finalDate': final_date.strftime('%Y-%m-%dT%H:%M'),
            'page': page,
            'maxPageResults': max_results,
        }
        self.data.update(querystring)
        self.clean_none_params()
        response = self.get(url=self.config.QUERY_TRANSACTION_URL)
        return PagSeguroTransactionSearchResult(response.content, self.config)

    def add_item(self, **kwargs):
        self.items.append(kwargs)
<commit_msg>Fix PEP8<commit_after># coding: utf-8
import logging
import requests
import xmltodict

from .configs import Config
from .configs import AbstractConfig
from .utils import parse_date, is_valid_email, is_valid_cpf

logger = logging.getLogger()

class PagSeguroNotificationResponse(object):
    def __init__(self, xml, config=None):
        self.xml = xml
        self.config = config or {}
        self.parse_xml(xml)

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug(
                "Cannot parse the returned xml '{0}' -> '{1}'".format(xml, e)
            )
            parsed = {}

        transaction = parsed.get('transaction', {})
        for k, v in transaction.iteritems():
            setattr(self, k, v)

class PagSeguroCheckoutResponse(object):
    def __init__(self, xml, config=None):
        self.xml = xml
        self.config = config or {}
        self.code = None
        self.date = None
        self.errors = None
        self.payment_url = None
        logger.debug(self.__dict__)
        self.parse_xml(xml)

    def parse_xml(self, xml):
        """ parse returned data """
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug(
                "Cannot parse the returned xml '{0}' -> '{1}'".format(xml, e)
            )
            parsed = {}

        if 'errors' in parsed:
            self.errors = parsed['errors']['error']
            return

        checkout = parsed.get('checkout', {})
        self.code = checkout.get('code')
        self.date = parse_date(checkout.get('date'))

        self.payment_url = self.config.PAYMENT_URL % self.code

class PagSeguroTransactionSearchResult(object):

    current_page = None
    total_pages = None
    results_in_page = None
    transactions = []

    def __init__(self, xml, config=None):
        self.xml = xml
        self.config = config or {}
        self.parse_xml(xml)

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug(
                "Cannot parse the returned xml '{0}' -> '{1}'".format(xml, e)
            )
            parsed = {}

        search_result = parsed.get('transactionSearchResult', {})
        self.transactions = search_result.get('transactions', {})
        self.transactions = self.transactions.get('transaction', [])
        if not isinstance(self.transactions, list):
            self.transactions = [self.transactions]
        self.current_page = search_result.get('currentPage', None)
        if self.current_page is not None:
            self.current_page = int(self.current_page)
        self.results_in_page = search_result.get('resultsInThisPage', None)
        if self.results_in_page is not None:
            self.results_in_page = int(self.results_in_page)
        self.total_pages = search_result.get('totalPages', None)
        if self.total_pages is not None:
            self.total_pages = int(self.total_pages)

class PagSeguro(object):
    """ Pag Seguro V2 wrapper """

    PAC = 1
    SEDEX = 2
    NONE = 3

    def __init__(self, email, token, data=None, config=None):
        self.config = config or Config()
        assert isinstance(self.config, AbstractConfig), \
            "Config should be instance of AbstractConfig"
        self.data = {}
        self.data['email'] = email
        self.data['token'] = token

        if data and isinstance(data, dict):
            self.data.update(data)

        self.items = []
        self.sender = {}
        self.shipping = {}
        self._reference = ""
        self.extra_amount = None
        self.redirect_url = None
        self.notification_url = None
        self.abandon_url = None

    def build_checkout_params(self, **kwargs):
        """ build a dict with params """
        params = kwargs or {}
        if self.sender:
            params['senderName'] = self.sender.get('name')
            params['senderAreaCode'] = self.sender.get('area_code')
            params['senderPhone'] = self.sender.get('phone')
            params['senderEmail'] = is_valid_email(self.sender.get('email'))
            params['senderCPF'] = is_valid_cpf(self.sender.get('cpf'))
            params['senderBornDate'] = self.sender.get('born_date')

        if self.shipping:
            params['shippingType'] = self.shipping.get('type')
            params['shippingAddressStreet'] = self.shipping.get('street')
            params['shippingAddressNumber'] = self.shipping.get('number')
            params['shippingAddressComplement'] = self.shipping.get(
                'complement'
            )
            params['shippingAddressDistrict'] = self.shipping.get('district')
            params['shippingAddressPostalCode'] = self.shipping.get(
                'postal_code'
            )
            params['shippingAddressCity'] = self.shipping.get('city')
            params['shippingAddressState'] = self.shipping.get('state')
            params['shippingAddressCountry'] = self.shipping.get('country',
                                                                 'BRA')

        if self.shipping and self.shipping.get('cost'):
            params['shippingCost'] = self.shipping.get('cost')

        if self.extra_amount:
            params['extraAmount'] = self.extra_amount

        params['reference'] = self.reference

        if self.redirect_url:
            params['redirectURL'] = self.redirect_url

        if self.notification_url:
            params['notificationURL'] = self.notification_url

        if self.abandon_url:
            params['abandonURL'] = self.abandon_url

        for i, item in enumerate(self.items, 1):
            params['itemId%s' % i] = item.get('id')
            params['itemDescription%s' % i] = item.get('description')
            params['itemAmount%s' % i] = item.get('amount')
            params['itemQuantity%s' % i] = item.get('quantity')
            params['itemWeight%s' % i] = item.get('weight')
            params['itemShippingCost%s' % i] = item.get('shipping_cost')

        self.data.update(params)
        self.clean_none_params()

    def clean_none_params(self):
        copy = dict(self.data)
        for k, v in copy.iteritems():
            if not v:
                del self.data[k]

    @property
    def reference_prefix(self):
        return self.config.REFERENCE_PREFIX or "%s"

    @reference_prefix.setter
    def reference_prefix(self, value):
        self.config.REFERENCE_PREFIX = (value or "") + "%s"

    @property
    def reference(self):
        return self.reference_prefix % self._reference

    @reference.setter
    def reference(self, value):
        if not isinstance(value, (str, unicode)):
            value = str(value)
        if value.startswith(self.reference_prefix):
            value = value[len(self.reference_prefix):]
        self._reference = value

    def get(self, url):
        """ do a get transaction """
        return requests.get(url, params=self.data, headers=self.config.HEADERS)

    def post(self, url):
        """ do a post request """
        return requests.post(url, data=self.data, headers=self.config.HEADERS)

    def checkout(self, **kwargs):
        """ create a pagseguro checkout """
        self.data['currency'] = self.config.CURRENCY
        self.build_checkout_params(**kwargs)
        response = self.post(url=self.config.CHECKOUT_URL)
        return PagSeguroCheckoutResponse(response.content, config=self.config)

    def check_notification(self, code):
        """ check a notification by its code """
        response = self.get(url=self.config.NOTIFICATION_URL % code)
        return PagSeguroNotificationResponse(response.content, self.config)

    def check_transaction(self, code):
        """ check a transaction by its code """
        response = self.get(url=self.config.TRANSACTION_URL % code)
        return PagSeguroNotificationResponse(response.content, self.config)

    def query_transactions(self, initial_date, final_date, page=None,
                           max_results=None):
        """ query transaction by date range """
        last_page = False
        results = []
        while last_page is False:
            search_result = self._consume_query_transactions(
                initial_date, final_date, page, max_results
            )
            results.extend(search_result.transactions)
            if search_result.current_page is None or \
               search_result.total_pages is None or \
               search_result.current_page == search_result.total_pages:
                last_page = True
            else:
                page = search_result.current_page + 1

        return results

    def _consume_query_transactions(self, initial_date, final_date, page=None,
                                    max_results=None):
        querystring = {
            'initialDate': initial_date.strftime('%Y-%m-%dT%H:%M'),
            'finalDate': final_date.strftime('%Y-%m-%dT%H:%M'),
            'page': page,
            'maxPageResults': max_results,
        }
        self.data.update(querystring)
        self.clean_none_params()
        response = self.get(url=self.config.QUERY_TRANSACTION_URL)
        return PagSeguroTransactionSearchResult(response.content, self.config)

    def add_item(self, **kwargs):
        self.items.append(kwargs)
<|endoftext|>


<commit_before>import logging

from sqlalchemy import Column, INT
from sqlalchemy import ForeignKey
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
from sqlalchemy_utc import UtcDateTime

from pajbot import utils
from pajbot.managers.db import Base

log = logging.getLogger(__name__)

class UserDuelStats(Base):
    __tablename__ = "user_duel_stats"

    user_id = Column(INT, ForeignKey("user.id", ondelete="CASCADE"), primary_key=True, autoincrement=False)
    duels_won = Column(INT, nullable=False)
    duels_total = Column(INT, nullable=False)
    points_won = Column(INT, nullable=False)
    points_lost = Column(INT, nullable=False)
    last_duel = Column(UtcDateTime(), nullable=True)
    current_streak = Column(INT, nullable=False)
    longest_winstreak = Column(INT, nullable=False)
    longest_losestreak = Column(INT, nullable=False)

    def __init__(self, *args, **kwargs):
        self.duels_won = 0
        self.duels_total = 0
        self.points_won = 0
        self.points_lost = 0
        self.last_duel = None
        self.current_streak = 0
        self.longest_winstreak = 0
        self.longest_losestreak = 0

        super().__init__(*args, **kwargs)

    user = relationship("User", cascade="save-update, merge", lazy="joined", back_populates="duel_stats")

    @hybrid_property
    def duels_lost(self):
        return self.duels_total - self.duels_won

    @hybrid_property
    def winrate(self):
        return self.duels_won * 100 / self.duels_total

    @hybrid_property
    def profit(self):
        return self.points_won - self.points_lost

    def won(self, points_won):
        self.duels_won += 1
        self.duels_total += 1
        self.points_won += points_won
        self.last_duel = utils.now()

        if self.current_streak > 0:
            self.current_streak += 1
        else:
            self.current_streak = 1
        if self.current_streak > self.longest_winstreak:
            self.longest_winstreak = self.current_streak

    def lost(self, points_lost):
        self.duels_total += 1
        self.points_lost += points_lost
        self.last_duel = utils.now()

        if self.current_streak < 0:
            self.current_streak -= 1
        else:
            self.current_streak = -1
        if abs(self.current_streak) > self.longest_losestreak:
            self.longest_losestreak = self.current_streak
<commit_msg>Fix crash introduced by #519 oops<commit_after>import logging

from sqlalchemy import Column, INT
from sqlalchemy import ForeignKey
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
from sqlalchemy_utc import UtcDateTime

from pajbot import utils
from pajbot.managers.db import Base

log = logging.getLogger(__name__)

class UserDuelStats(Base):
    __tablename__ = "user_duel_stats"

    user_id = Column(INT, ForeignKey("user.id", ondelete="CASCADE"), primary_key=True, autoincrement=False)
    duels_won = Column(INT, nullable=False)
    duels_total = Column(INT, nullable=False)
    points_won = Column(INT, nullable=False)
    points_lost = Column(INT, nullable=False)
    last_duel = Column(UtcDateTime(), nullable=True)
    current_streak = Column(INT, nullable=False)
    longest_winstreak = Column(INT, nullable=False)
    longest_losestreak = Column(INT, nullable=False)

    def __init__(self, *args, **kwargs):
        self.duels_won = 0
        self.duels_total = 0
        self.points_won = 0
        self.points_lost = 0
        self.last_duel = None
        self.current_streak = 0
        self.longest_winstreak = 0
        self.longest_losestreak = 0

        super().__init__(*args, **kwargs)

    user = relationship("User", cascade="save-update, merge", lazy="joined", back_populates="_duel_stats")

    @hybrid_property
    def duels_lost(self):
        return self.duels_total - self.duels_won

    @hybrid_property
    def winrate(self):
        return self.duels_won * 100 / self.duels_total

    @hybrid_property
    def profit(self):
        return self.points_won - self.points_lost

    def won(self, points_won):
        self.duels_won += 1
        self.duels_total += 1
        self.points_won += points_won
        self.last_duel = utils.now()

        if self.current_streak > 0:
            self.current_streak += 1
        else:
            self.current_streak = 1
        if self.current_streak > self.longest_winstreak:
            self.longest_winstreak = self.current_streak

    def lost(self, points_lost):
        self.duels_total += 1
        self.points_lost += points_lost
        self.last_duel = utils.now()

        if self.current_streak < 0:
            self.current_streak -= 1
        else:
            self.current_streak = -1
        if abs(self.current_streak) > self.longest_losestreak:
            self.longest_losestreak = self.current_streak
<|endoftext|>


<commit_before>from collections import OrderedDict
from itertools import groupby
import logging
from optparse import make_option
import pprint
from django.conf import settings
from django.core.management import BaseCommand
from bs4 import BeautifulSoup
from unidecode import unidecode
from bilanci.utils.comuni import FLMapper
from bilanci.models import CodiceVoce, ValoreBilancio, Voce
from territori.models import Territorio, Contesto, ObjectDoesNotExist

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('--dry-run',
                    dest='dryrun',
                    action='store_true',
                    default=False,
                    help='Set the dry-run command mode: nothing is written on the db'),

        make_option('--file',
                    dest='input_file',
                    default='',
                    help='Input Xml file to parse'),
        make_option('--append',
                    dest='append',
                    action='store_true',
                    default=False,
                    help='Use the log file appending instead of overwriting (used when launching shell scripts)'),
    )

    help = 'Import xml bilancio files into pg db'

    logger = logging.getLogger('management')
    comuni_dicts = {}
    dryrun = None
    territorio = None
    tipo_certificato = None
    anno = None
    colonne_regroup = None
    codici_regroup = None
    popolazione_residente = None

    def import_context(self,):
        sep="."
        popolazione_str = self.colonne_regroup[('01','001','1')]['valore']
        popolazione_residente = int(popolazione_str.split(sep, 1)[0])

        try:
            contesto_db = Contesto.objects.get(anno = self.anno, territorio = self.territorio)

        except ObjectDoesNotExist:
            new_contesto = Contesto()
            new_contesto.anno = self.anno
            new_contesto.territorio = self.territorio
            new_contesto.bil_popolazione_residente = popolazione_residente

            if not self.dryrun:
                new_contesto.save()
        else:

            if contesto_db.bil_popolazione_residente != popolazione_residente:
                self.logger.error("Popolazione residente year:{0} in xml file ({1}) != pop.residente stored in db ({2})". \
                    format(self.anno, contesto_db.bil_popolazione_residente, popolazione_residente))

        return popolazione_residente

    def set_popolazione_residente(self, ):

        if self.tipo_certificato == 'consuntivo':
            # import territorio context data from xml file and set self.popolazione_residente
            self.popolazione_residente = self.import_context()
        else:
            try:
                contesto_db = Contesto.objects.get(anno = self.anno, territorio = self.territorio)
            except ObjectDoesNotExist:
                self.logger.error(u"Context not present for territorio {0} year {1}, cannot compute per capita values!".format(
                    self.territorio.denominazione, self.anno
                ))
            else:
                self.popolazione_residente = contesto_db.bil_popolazione_residente

    def create_mapping(self, bilancio):

        ##
        # creates a colonne mapping from the xml file
        ##

        colonne_xml = bilancio.find_all('colonne')
        colonne_list = []
        for colonne_element in colonne_xml:

            for colonna_element in colonne_element.contents:
                if colonna_element != u'\n':

                    colonne_list.append({
                        'voce':colonna_element.parent['voce'],
                        'quadro':colonna_element.parent['quadro'],
                        'colonna': colonna_element['num'],
                        'valore': colonna_element.contents[0]
                        })

        colonne_list_keygen = lambda x: (x['quadro'], x['voce'], x['colonna'])
        self.colonne_regroup = dict((k,list(v)[0]) for k,v in groupby(colonne_list, key=colonne_list_keygen))

        ##
        # CodiceVoce: get all elements for anno, tipo_certificato and regroup by voce__slug
        ##
        codici = CodiceVoce.get_bilancio_codes(anno=int(self.anno), tipo_certificato=self.tipo_certificato).order_by('voce__slug')
        codici_keygen = lambda x: x.voce.slug
        self.codici_regroup = dict((k,list(v)) for k,v in groupby(codici, key=codici_keygen))

        if self.tipo_certificato == 'consuntivo':
            # add cassa branch
            entrate_cassa = list(Voce.objects.get(slug = 'consuntivo-entrate-cassa').get_descendants(include_self=True))
            spese_cassa = list(Voce.objects.get(slug = 'consuntivo-spese-cassa').get_descendants(include_self=True).exclude(slug__startswith='consuntivo-spese-cassa-spese-somma-funzioni'))
            entrate_cassa.extend(spese_cassa)

            self.add_branch(node_set=entrate_cassa, is_cassa=True)

            # set somma_funzioni_ set for consuntivo
            funzioni_cassa = list(Voce.objects.get(slug = 'consuntivo-spese-cassa-spese-somma-funzioni').get_descendants(include_self=True))
            funzioni_impegni = list(Voce.objects.get(slug = 'consuntivo-spese-impegni-spese-somma-funzioni').get_descendants(include_self=True))
            funzioni_cassa.extend(funzioni_impegni)
            somma_funzioni_set = funzioni_cassa
        else:
            # set somma_funzioni_ set for preventivo
            funzioni_preventivo = list(Voce.objects.get(slug = 'preventivo-spese-spese-somma-funzioni').get_descendants(include_self=True))
            somma_funzioni_set = funzioni_preventivo

        # add somma_funzioni branch
        self.add_branch(node_set=somma_funzioni_set, is_cassa=False)

    def add_branch(self, node_set, is_cassa=False):
        # add branches (somma funzioni, cassa) to the codici regroup dict
        # to do so asks the model which are the slugs of each Voce which is a descendant of the rootnode
        # example: for Voce
        for node in node_set:
            if is_cassa:
                composition = node.get_components_cassa()
            else:
                composition = node.get_components_somma_funzioni()

            # pprint.pprint(node.slug)
            # pprint.pprint(composition)
            if composition is None:
                self.logger.error("{0} returned None for composition".format(node.slug))
                return

            try:
                 x = self.codici_regroup[composition[0].slug]
                 x.extend(self.codici_regroup[composition[1].slug])
                 self.codici_regroup[node.slug] = x
            except KeyError:
                self.logger.error("Cannot compute {0}: slug missing in codici_regroup".format(node.slug, ))

    def import_bilancio(self, bilancio):

        self.create_mapping(bilancio)
        self.set_popolazione_residente()

        for voce_slug, codice_list in self.codici_regroup.iteritems():
            xml_code_found = True
            self.logger.debug(u"Getting data for {0}".format(voce_slug))

            valore_totale = 0
            # adds up all the xml codice needed to calculate a single voce
            for codice in codice_list:
                pprint.pprint(codice)
                if (codice.quadro_cod, codice.voce_cod, codice.colonna_cod) not in self.colonne_regroup.keys():
                    self.logger.error(u"Code {0}-{1}-{2} not found in XML file! Cannot compute {3} Voce".\
                        format(codice.quadro_cod, codice.voce_cod, codice.colonna_cod, voce_slug))
                    xml_code_found = False
                    break

                valore_string = unicode(self.colonne_regroup[(codice.quadro_cod, codice.voce_cod, codice.colonna_cod)]['valore'])
                if valore_string.lower() != 's' and valore_string.lower() != 'n':
                    valore_totale += float(valore_string)

            if xml_code_found:
                self.logger.info(u"Write {0}:{1} on db".format(voce_slug, valore_totale))
                vb = ValoreBilancio()
                vb.voce = Voce.objects.get(slug = voce_slug)
                vb.anno = int(self.anno)
                vb.territorio = self.territorio
                vb.valore = valore_totale

                if self.popolazione_residente is not None:
                    vb.valore_procapite = valore_totale/float(self.popolazione_residente)

                if not self.dryrun:
                    vb.save()

    def handle(self, *args, **options):
        verbosity = options['verbosity']
        if verbosity == '0':
            self.logger.setLevel(logging.ERROR)
        elif verbosity == '1':
            self.logger.setLevel(logging.WARNING)
        elif verbosity == '2':
            self.logger.setLevel(logging.INFO)
        elif verbosity == '3':
            self.logger.setLevel(logging.DEBUG)

        self.dryrun = options['dryrun']
        input_file = options['input_file']
        popolazione_residente = None

        if options['append'] is True:
            self.logger = logging.getLogger('management_append')

        # open input file
        try:
            bilancio = BeautifulSoup(open(input_file),"xml")
        except IOError:
            self.logger.error("File {0} not found".format(input_file))
            return

        # get finloc, year, tipo bilancio
        certificato = bilancio.certificato
        self.anno = certificato['anno']
        tipo_certificato_code = certificato['tipoCertificato']

        valuta = certificato['tipoValuta']
        if valuta.lower() != 'e':
            self.logger.error("Valuta is not EURO")
            return

        if tipo_certificato_code == "C":
            self.tipo_certificato = 'consuntivo'
        else:
            self.tipo_certificato = 'preventivo'

        # identifies the Comune from the finloc code
        codiceente = certificato['codiceEnte']
        mapper = FLMapper(settings.LISTA_COMUNI_PATH)
        codfinloc = mapper.get_cities(codiceente)[0]
        self.territorio = Territorio.objects.get(cod_finloc = codfinloc)
        self.logger.info(u"Comune: {0}".format(self.territorio.cod_finloc))

        # import bilancio data into Postgres db, calculate per capita values
        self.import_bilancio(bilancio)<commit_msg>Xml2pg: added logging for errors<commit_after>from collections import OrderedDict
from itertools import groupby
import logging
from optparse import make_option
import pprint
from django.conf import settings
from django.core.management import BaseCommand
from bs4 import BeautifulSoup
from unidecode import unidecode
from bilanci.utils import gdocs
from bilanci.utils.comuni import FLMapper
from bilanci.models import CodiceVoce, ValoreBilancio, Voce
from territori.models import Territorio, Contesto, ObjectDoesNotExist

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('--dry-run',
                    dest='dryrun',
                    action='store_true',
                    default=False,
                    help='Set the dry-run command mode: nothing is written on the db'),

        make_option('--file',
                    dest='input_file',
                    default='',
                    help='Input Xml file to parse'),
        make_option('--append',
                    dest='append',
                    action='store_true',
                    default=False,
                    help='Use the log file appending instead of overwriting (used when launching shell scripts)'),
    )

    help = 'Import xml bilancio files into pg db'

    logger = logging.getLogger('management')
    comuni_dicts = {}
    composition_errors = []
    dryrun = None
    territorio = None
    tipo_certificato = None
    anno = None
    colonne_regroup = None
    codici_regroup = None
    popolazione_residente = None

    def import_context(self,):
        sep="."
        popolazione_str = self.colonne_regroup[('01','001','1')]['valore']
        popolazione_residente = int(popolazione_str.split(sep, 1)[0])

        try:
            contesto_db = Contesto.objects.get(anno = self.anno, territorio = self.territorio)

        except ObjectDoesNotExist:
            new_contesto = Contesto()
            new_contesto.anno = self.anno
            new_contesto.territorio = self.territorio
            new_contesto.bil_popolazione_residente = popolazione_residente

            if not self.dryrun:
                new_contesto.save()
        else:

            if contesto_db.bil_popolazione_residente != popolazione_residente:
                self.logger.error("Popolazione residente year:{0} in xml file ({1}) != pop.residente stored in db ({2})". \
                    format(self.anno, contesto_db.bil_popolazione_residente, popolazione_residente))

        return popolazione_residente

    def set_popolazione_residente(self, ):

        if self.tipo_certificato == 'consuntivo':
            # import territorio context data from xml file and set self.popolazione_residente
            self.popolazione_residente = self.import_context()
        else:
            try:
                contesto_db = Contesto.objects.get(anno = self.anno, territorio = self.territorio)
            except ObjectDoesNotExist:
                self.logger.error(u"Context not present for territorio {0} year {1}, cannot compute per capita values!".format(
                    self.territorio.denominazione, self.anno
                ))
            else:
                self.popolazione_residente = contesto_db.bil_popolazione_residente

    def create_mapping(self, bilancio):

        ##
        # creates a colonne mapping from the xml file
        ##

        colonne_xml = bilancio.find_all('colonne')
        colonne_list = []
        for colonne_element in colonne_xml:

            for colonna_element in colonne_element.contents:
                if colonna_element != u'\n':

                    colonne_list.append({
                        'voce':colonna_element.parent['voce'],
                        'quadro':colonna_element.parent['quadro'],
                        'colonna': colonna_element['num'],
                        'valore': colonna_element.contents[0]
                        })

        colonne_list_keygen = lambda x: (x['quadro'], x['voce'], x['colonna'])
        self.colonne_regroup = dict((k,list(v)[0]) for k,v in groupby(colonne_list, key=colonne_list_keygen))

        ##
        # CodiceVoce: get all elements for anno, tipo_certificato and regroup by voce__slug
        ##
        codici = CodiceVoce.get_bilancio_codes(anno=int(self.anno), tipo_certificato=self.tipo_certificato).order_by('voce__slug')
        codici_keygen = lambda x: x.voce.slug
        self.codici_regroup = dict((k,list(v)) for k,v in groupby(codici, key=codici_keygen))

        if self.tipo_certificato == 'consuntivo':
            # add cassa branch
            entrate_cassa = list(Voce.objects.get(slug = 'consuntivo-entrate-cassa').get_descendants(include_self=True))
            spese_cassa = list(Voce.objects.get(slug = 'consuntivo-spese-cassa').get_descendants(include_self=True).exclude(slug__startswith='consuntivo-spese-cassa-spese-somma-funzioni'))
            entrate_cassa.extend(spese_cassa)

            self.add_branch(node_set=entrate_cassa, is_cassa=True)

            # set somma_funzioni_ set for consuntivo
            funzioni_cassa = list(Voce.objects.get(slug = 'consuntivo-spese-cassa-spese-somma-funzioni').get_descendants(include_self=True))
            funzioni_impegni = list(Voce.objects.get(slug = 'consuntivo-spese-impegni-spese-somma-funzioni').get_descendants(include_self=True))
            funzioni_cassa.extend(funzioni_impegni)
            somma_funzioni_set = funzioni_cassa
        else:
            # set somma_funzioni_ set for preventivo
            funzioni_preventivo = list(Voce.objects.get(slug = 'preventivo-spese-spese-somma-funzioni').get_descendants(include_self=True))
            somma_funzioni_set = funzioni_preventivo

        # add somma_funzioni branch
        self.add_branch(node_set=somma_funzioni_set, is_cassa=False)

    def add_branch(self, node_set, is_cassa=False):
        # add branches (somma funzioni, cassa) to the codici regroup dict
        # to do so asks the model which are the slugs of each Voce which is a descendant of the rootnode
        # example: for Voce
        for node in node_set:
            if is_cassa:
                composition = node.get_components_cassa()
            else:
                composition = node.get_components_somma_funzioni()

            # pprint.pprint(node.slug)
            # pprint.pprint(composition)
            if composition is None or len(composition)==0:
                self.logger.error("{0} returned None for composition".format(node.slug))
                self.composition_errors.append(node.slug)
                return

            try:
                 x = self.codici_regroup[composition[0].slug]
                 x.extend(self.codici_regroup[composition[1].slug])
                 self.codici_regroup[node.slug] = x
            except KeyError:
                self.logger.error("Cannot compute {0}: slug missing in codici_regroup".format(node.slug, ))
                self.composition_errors.append(node.slug)

    def import_bilancio(self, bilancio):

        self.create_mapping(bilancio)
        self.set_popolazione_residente()

        for voce_slug, codice_list in self.codici_regroup.iteritems():
            xml_code_found = True
            self.logger.debug(u"Getting data for {0}".format(voce_slug))

            valore_totale = 0
            # adds up all the xml codice needed to calculate a single voce
            for codice in codice_list:
                if (codice.quadro_cod, codice.voce_cod, codice.colonna_cod) not in self.colonne_regroup.keys():
                    self.logger.error(u"Code {0}-{1}-{2} not found in XML file! Cannot compute {3} Voce".\
                        format(codice.quadro_cod, codice.voce_cod, codice.colonna_cod, voce_slug))
                    xml_code_found = False
                    break

                valore_string = unicode(self.colonne_regroup[(codice.quadro_cod, codice.voce_cod, codice.colonna_cod)]['valore'])
                if valore_string.lower() != 's' and valore_string.lower() != 'n':
                    valore_totale += float(valore_string)

            if xml_code_found:
                self.logger.info(u"Write {0}:{1} on db".format(voce_slug, valore_totale))
                vb = ValoreBilancio()
                vb.voce = Voce.objects.get(slug = voce_slug)
                vb.anno = int(self.anno)
                vb.territorio = self.territorio
                vb.valore = valore_totale

                if self.popolazione_residente is not None:
                    vb.valore_procapite = valore_totale/float(self.popolazione_residente)

                if not self.dryrun:
                    vb.save()

    def handle(self, *args, **options):
        verbosity = options['verbosity']
        if verbosity == '0':
            self.logger.setLevel(logging.ERROR)
        elif verbosity == '1':
            self.logger.setLevel(logging.WARNING)
        elif verbosity == '2':
            self.logger.setLevel(logging.INFO)
        elif verbosity == '3':
            self.logger.setLevel(logging.DEBUG)

        self.dryrun = options['dryrun']
        input_file = options['input_file']
        popolazione_residente = None

        if options['append'] is True:
            self.logger = logging.getLogger('management_append')

        # open input file
        try:
            bilancio = BeautifulSoup(open(input_file),"xml")
        except IOError:
            self.logger.error("File {0} not found".format(input_file))
            return

        # get finloc, year, tipo bilancio
        certificato = bilancio.certificato
        self.anno = certificato['anno']
        tipo_certificato_code = certificato['tipoCertificato']

        valuta = certificato['tipoValuta']
        if valuta.lower() != 'e':
            self.logger.error("Valuta is not EURO")
            return

        if tipo_certificato_code == "C":
            self.tipo_certificato = 'consuntivo'
        else:
            self.tipo_certificato = 'preventivo'

        # identifies the Comune from the finloc code
        codiceente = certificato['codiceEnte']
        mapper = FLMapper(settings.LISTA_COMUNI_PATH)
        codfinloc = mapper.get_cities(codiceente)[0]
        self.territorio = Territorio.objects.get(cod_finloc = codfinloc)
        self.logger.info(u"Comune: {0}".format(self.territorio.cod_finloc))

        # import bilancio data into Postgres db, calculate per capita values
        self.import_bilancio(bilancio)

        if len(self.composition_errors) > 0:

            # create a list of lists to be written with unicodewriter
            errors_to_write = [[nms] for nms in self.composition_errors]
            bilancio_type_year = self.tipo_certificato+"_"+self.anno
            composition_filename = "{0}_composition_errors".format(bilancio_type_year,)
            log_base_dir = "{0}/".format(settings.REPO_ROOT)
            gdocs.write_to_csv(path_name='log', contents={composition_filename:errors_to_write},csv_base_dir=log_base_dir)
            self.logger.warning("{0} VOCE SLUG FROM BILANCIO TREE GAVE COMPOSITION ERRORS ".format(len(self.composition_errors)))
            self.logger.warning("{0}log/{1}.csv file written for check".format(log_base_dir,composition_filename))
<|endoftext|>


<commit_before>#!/usr/bin/python

# Author: Disa Mhembere, Johns Hopkins University
# Separated: 10/2/2012
# Load up an adjacency matrix given G_fn, lcc & roiRoot

import mrpaths
import argparse
import mrcap.lcc as lcc
from getBaseName import getBaseName
import os
import sys

def loadAdjMat(G_fn, lcc_fn, roiRootName = None):
  '''
  Load adjacency matrix given lcc_fn & G_fn. lcc has z-indicies corresponding to the lcc :
  G_fn - the .mat file holding graph
  lcc_fn - the largest connected component .npy z-ordering
  '''
  print "Loading adjacency matrix..."
  if not roiRootName:
    roiRootName =  os.path.join(getRoiRoot(G_fn),getBaseName(G_fn))
    #import pdb; pdb.set_trace()
  vcc = lcc.ConnectedComponent(fn = lcc_fn)
  try:
    fg = lcc._load_fibergraph(roiRootName , G_fn)
    G = vcc.induced_subgraph(fg.spcscmat)
    G = G+G.T # Symmetrize
  except:
    print "****Problem loading real lcc & graph****"
    sys.exit(-1)

  return G

def getRoiRoot(G_fn):
  '''
  Get the roi root name form G_fn
  G_fn - full filename of graph (e.g of format /{User}/{disa}/{graphs}/filename_fiber.mat)
  * {} - Not necessary
  '''
  roiRoot = ''
  for i in G_fn.split('/')[1:-3]:
    roiRoot = os.path.join(roiRoot, i)
  roiRoot = os.path.join(roiRoot, "base", "roi")
  return roiRoot

def main():

    parser = argparse.ArgumentParser(description='Calculate Max Avg Degree estimate as max eigenvalue for biggraphs')
    parser.add_argument('G_fn', action='store',help='Full filename sparse graph (.mat)')
    parser.add_argument('lcc_fn', action='store',help='Full filename of largest connected component (.npy)')
    parser.add_argument('roiRootName', action='store',help='Full path of roi director + baseName')

    result = parser.parse_args()
    getMaxAveDegree(result.G_fn, result.lcc_fn, result.roiRootName)

if __name__ == '__main__':
  main()

<commit_msg>Loading adj mat echos<commit_after>#!/usr/bin/python

# Author: Disa Mhembere, Johns Hopkins University
# Separated: 10/2/2012
# Load up an adjacency matrix given G_fn, lcc & roiRoot

import mrpaths
import argparse
import mrcap.lcc as lcc
from getBaseName import getBaseName
import os
import sys

def loadAdjMat(G_fn, lcc_fn, roiRootName = None):
  '''
  Load adjacency matrix given lcc_fn & G_fn. lcc has z-indicies corresponding to the lcc :
  G_fn - the .mat file holding graph
  lcc_fn - the largest connected component .npy z-ordering
  '''
  print "Loading adjacency matrix..."
  if not roiRootName:
    roiRootName =  os.path.join(getRoiRoot(G_fn),getBaseName(G_fn))
    #import pdb; pdb.set_trace()
  vcc = lcc.ConnectedComponent(fn = lcc_fn)
  try:
    fg = lcc._load_fibergraph(roiRootName , G_fn)
    G = vcc.induced_subgraph(fg.spcscmat)
    G = G+G.T # Symmetrize
  except:
    if not os.path.exists(roiRootName):
      print "%s Doesn't exist" % roiRootName

    if not os.path.exists(lcc_fn):
      print "%s Doesn't exist" % lcc_fn

    if not os.path.exists(G_fn):
      print "%s Doesn't exist" % G_fn

    if os.path.exist(roiRootName) and os.path.exist(lcc_fn) and os.path.exist(G_fn):
      print "****Some wild Problem loading real lcc & graph****"
    sys.exit(-1)

  return G

def getRoiRoot(G_fn):
  '''
  Get the roi root name form G_fn
  G_fn - full filename of graph (e.g of format /{User}/{disa}/{graphs}/filename_fiber.mat)
  * {} - Not necessary
  '''
  roiRoot = ''
  for i in G_fn.split('/')[1:-3]:
    roiRoot = os.path.join(roiRoot, i)
  roiRoot = os.path.join(roiRoot, "base", "roi")
  return roiRoot

def main():

    parser = argparse.ArgumentParser(description='Calculate Max Avg Degree estimate as max eigenvalue for biggraphs')
    parser.add_argument('G_fn', action='store',help='Full filename sparse graph (.mat)')
    parser.add_argument('lcc_fn', action='store',help='Full filename of largest connected component (.npy)')
    parser.add_argument('roiRootName', action='store',help='Full path of roi director + baseName')

    result = parser.parse_args()
    getMaxAveDegree(result.G_fn, result.lcc_fn, result.roiRootName)

if __name__ == '__main__':
  main()

<|endoftext|>


<commit_before>import events
import requests
from errors import PanoplyException
import traceback

# abstract DataSource object
class DataSource(events.Emitter):

    source = None
    options = None

    # data source constructor
    def __init__(self, source, options = {}, events = {}):
        super(DataSource, self).__init__(events)

        if 'destination' not in source:
            source['destination'] = source['type']

        self.source = source
        self.options = options

    # log a message
    def log(self, *msgs):
        if "logger" in self.options:
            self.options["logger"](msgs)
        else:
            print(msgs)

    def progress(self, loaded, total, msg=''):
        self.fire("progress", {
            "loaded": loaded,
            "total": total,
            "msg": msg
        })

# a decorator used to invalidate the access_token for oauth based data sources
# this should be used on every method in the data source that fetches data
# from the server (controlled by this oauth), and that needs to be invalidated
def invalidate_token(refresh_url, callback=None,
                     keys=['access_token', 'refresh_token']):
    def _invalidate_token(f):
        def wrapper(*args):
            self = args[0]
            try:
                return f(*args)
            except:
                try:
                    self.log("Reinvalidating access_token...")
                    self.source[keys[0]] = None

                    # get a new token from refresh_url
                    token = self.source.get('refresh_token')
                    r = requests.post(
                        refresh_url,
                        data = dict(self.options['refresh'], **{keys[1]:token}))
                    self.source[keys[0]] = r.json()[keys[0]]

                    # save the new token in the database
                    changes = {keys[0]: self.source[keys[0]]}
                    self.fire("source-change", changes)

                    # notify the callback that a new token was issued
                    if callback:
                        if callable(callback):
                            _callback = callback
                        else:
                            _callback = getattr(self, callback)
                        _callback(self.source.get(keys[0]))

                    return f(*args)
                except:
                    # make sure the real exception is logged
                    self.log(traceback.format_exc())
                    self.log("Error: Access token can't be invalidated." +
                          " The user would have to re-authenticate")
                    # raise a non-retryable exception
                    raise PanoplyException(
                        'access token could not be refreshed',
                        retryable=False)
        return wrapper
    return _invalidate_token
<commit_msg>cleanup<commit_after>import events
import requests
from errors import PanoplyException
import traceback

# abstract DataSource object
class DataSource(events.Emitter):

    source = None
    options = None

    # data source constructor
    def __init__(self, source, options = {}, events = {}):
        super(DataSource, self).__init__(events)

        if 'destination' not in source:
            source['destination'] = source['type']

        self.source = source
        self.options = options

    # log a message
    def log(self, *msgs):
        if "logger" in self.options:
            self.options["logger"](msgs)
        else:
            print(msgs)

    def progress(self, loaded, total, msg=''):
        self.fire("progress", {
            "loaded": loaded,
            "total": total,
            "msg": msg
        })

# a decorator used to invalidate the access_token for oauth based data sources
# this should be used on every method in the data source that fetches data
# from the server (controlled by this oauth), and that needs to be invalidated
def invalidate_token(refresh_url, callback=None,
                     keys=['access_token', 'refresh_token']):
    def _invalidate_token(f):
        def wrapper(*args):
            self = args[0]
            try:
                return f(*args)
            except:
                try:
                    self.log("Reinvalidating access_token...")
                    self.source[keys[0]] = None

                    # get a new token from refresh_url
                    token = self.source.get(keys[1])
                    r = requests.post(
                        refresh_url,
                        data = dict(self.options['refresh'], **{keys[1]:token}))
                    self.source[keys[0]] = r.json()[keys[0]]

                    # save the new token in the database
                    changes = {keys[0]: self.source[keys[0]]}
                    self.fire("source-change", changes)

                    # notify the callback that a new token was issued
                    if callback:
                        if callable(callback):
                            _callback = callback
                        else:
                            _callback = getattr(self, callback)
                        _callback(self.source.get(keys[0]))

                    return f(*args)
                except:
                    # make sure the real exception is logged
                    self.log(traceback.format_exc())
                    self.log("Error: Access token can't be invalidated." +
                          " The user would have to re-authenticate")
                    # raise a non-retryable exception
                    raise PanoplyException(
                        'access token could not be refreshed',
                        retryable=False)
        return wrapper
    return _invalidate_token
<|endoftext|>


<commit_before>#!/usr/bin/python
# encoding: utf-8
"""

@author: xuk1
@license: (C) Copyright 2013-2017
@contact: kai.a.xu@intel.com
@file: parallelprocessing.py
@time: 9/15/2017 15:43
@desc:

"""
import os
import time
from functools import partial
from multiprocessing import Pool

import pandas as pd

from cluster import Cluster
from component.factory import AttribFactory
from utils.commonOps import get_paths

def get_node_attrib_data_by_time(attrib, start, end, file_path):
    """
    Get data of a given attribute within a given time period
    :param attrib: input attribute
    :param start: list of start timestamp
    :param end: list of end timestamp, should be the same length of start
    :return: dict that contains avg value and all raw data of all the timestamp pair
    """
    if attrib.lower() in AttribFactory.node_attrib.keys():
        attrib_file = file_path + os.sep + AttribFactory.node_attrib[attrib.lower()]
        if os.path.isfile(attrib_file):
            return AttribFactory.create_attrib(attrib, attrib_file).get_data_by_time(start, end)
        else:
            print 'node does not have attribute {0}'.format(attrib)
            exit(-1)
    else:
        print 'Node does not have attrib: {0} defined, defined attributions are: {1}, will exit...' \
            .format(attrib, AttribFactory.node_attrib.keys())
        exit(-1)

def get_cluster_attrib_data(pat_path, start, end, save_raw, attrib):
    """
    Get the avg data of a given attrib of the cluster.
    :param pat_path: pat path of raw data
    :param start: list of start timestamp
    :param end: list of end timestamp
    :param save_raw: whether to save the raw avg data
    :param attrib: attrib intended to calc, which could be cpu, disk, network, mem
    :return: data frame that contains the avg attrib data of the cluster
    """
    pat_path = pat_path + os.sep + 'instruments'
    if os.path.exists(pat_path):
        nodes = get_paths(pat_path)
    else:
        print 'Path: {0} does not exist, will exit...'.format(pat_path)
        exit(-1)

    pool = Pool(processes=None)
    func = partial(get_node_attrib_data_by_time, attrib, start, end)
    p = pool.map(func, nodes)

    tmp_avg = pd.DataFrame()
    tmp_all = pd.DataFrame()

    for i in range(len(p)):
        tmp_avg = tmp_avg.append(p[i][0])  # avg data
        if save_raw:
            tmp_all = tmp_all.append(p[i][1])  # all raw data
    if save_raw:
        raw_path = pat_path + os.sep + attrib + '.csv'
        tmp_all.index = pd.to_datetime(tmp_all.index, unit='s')
        tmp_all.to_csv(raw_path, sep=',')

    avg = pd.DataFrame()
    for i in range(len(start)):
        avg = avg.append(tmp_avg.loc[i].mean(axis=0), ignore_index=True)
    return avg

def get_cluster_data_by_time(pat_path, start, end, save_raw):
    """
    Get the avg data of all the attrib of the cluster.
    :param pat_path: pat path of raw data
    :param start: list of start timestamp
    :param end: list of end timestamp
    :param save_raw: whether to save the raw avg data
    :return: dict that contains all the avg attrib data of the cluster
    """
    cluster = Cluster(pat_path)
    cluster_avg = {}

    for attr in cluster.attrib:
        cluster_avg[attr] = get_cluster_attrib_data(pat_path, start, end, save_raw, attr)
    if 'cpu' in cluster_avg.keys():
        cluster_avg['cpu'].insert(0, '1-%idle', 100 - cluster_avg['cpu']['%idle'])  # add column '1-%idle' to the result
    return cluster_avg

if __name__ == '__main__':
    pat_path = 'C:\\Users\\xuk1\PycharmProjects\\tmp_data\pat_spark163_1TB_r1'
    tic = time.time()
    # pat_path = 'C:\\Users\\xuk1\\PycharmProjects\\tmp_data\\pat_cdh511_HoS_27workers_2699v4_72vcores_PCIe_30T_4S_r1'
    # pat_path = sys.argv[1]
    # nodes = Cluster(pat_path).nodes
    # attrib = 'cpu'
    # start = [0, 1505148392, 1505148392, 1505148392]
    # end = [0, 1505272675, 1505272675, 1505272675]
    start = [0, 1487687161, 1487687176]
    end = [0, 1487687170, 1487687185]
    # get_cluster_attrib_data(nodes, attrib, start, end)

    get_cluster_data_by_time(pat_path, start, end, False)
    toc = time.time()
    print 'Processing elapsed time: {0}'.format(toc - tic)
<commit_msg>sort output result<commit_after>#!/usr/bin/python
# encoding: utf-8
"""

@author: xuk1
@license: (C) Copyright 2013-2017
@contact: kai.a.xu@intel.com
@file: parallelprocessing.py
@time: 9/15/2017 15:43
@desc:

"""
import os
import time
from functools import partial
from multiprocessing import Pool
from collections import OrderedDict

import pandas as pd

from cluster import Cluster
from component.factory import AttribFactory
from utils.commonOps import get_paths
# from db.write_db import write_df

def get_node_attrib_data_by_time(attrib, start, end, file_path):
    """
    Get data of a given attribute within a given time period
    :param attrib: input attribute
    :param start: list of start timestamp
    :param end: list of end timestamp, should be the same length of start
    :return: dict that contains avg value and all raw data of all the timestamp pair
    """
    if attrib.lower() in AttribFactory.node_attrib.keys():
        attrib_file = file_path + os.sep + AttribFactory.node_attrib[attrib.lower()]
        if os.path.isfile(attrib_file):
            return AttribFactory.create_attrib(attrib, attrib_file).get_data_by_time(start, end)
        else:
            print 'node does not have attribute {0}'.format(attrib)
            exit(-1)
    else:
        print 'Node does not have attrib: {0} defined, defined attributions are: {1}, will exit...' \
            .format(attrib, AttribFactory.node_attrib.keys())
        exit(-1)

def get_cluster_attrib_data(pat_path, start, end, save_raw, attrib):
    """
    Get the avg data of a given attrib of the cluster.
    :param pat_path: pat path of raw data
    :param start: list of start timestamp
    :param end: list of end timestamp
    :param save_raw: whether to save the raw avg data
    :param attrib: attrib intended to calc, which could be cpu, disk, network, mem
    :return: data frame that contains the avg attrib data of the cluster
    """
    pat_path = pat_path + os.sep + 'instruments'
    if os.path.exists(pat_path):
        nodes = get_paths(pat_path)
    else:
        print 'Path: {0} does not exist, will exit...'.format(pat_path)
        exit(-1)

    pool = Pool(processes=None)
    func = partial(get_node_attrib_data_by_time, attrib, start, end)
    p = pool.map(func, nodes)

    tmp_avg = pd.DataFrame()
    tmp_all = pd.DataFrame()

    for i in range(len(p)):
        tmp_avg = tmp_avg.append(p[i][0])  # avg data
        if save_raw:
            tmp_all = tmp_all.append(p[i][1])  # all raw data
    if save_raw:
        raw_path = pat_path + os.sep + attrib + '.csv'
        # tmp_all.index = pd.to_datetime(tmp_all.index, unit='s')
        tmp_all.to_csv(raw_path, sep=',')
    # if save_raw:
    #     tmp_all.index = pd.to_datetime(tmp_all.index, unit='s')
    #     write_df(tmp_all, 'pat_avg_large', attrib)

    avg = pd.DataFrame()
    for i in range(len(start)):
        avg = avg.append(tmp_avg.loc[i].mean(axis=0), ignore_index=True)
    return avg

def get_cluster_data_by_time(pat_path, start, end, save_raw):
    """
    Get the avg data of all the attrib of the cluster.
    :param pat_path: pat path of raw data
    :param start: list of start timestamp
    :param end: list of end timestamp
    :param save_raw: whether to save the raw avg data
    :return: dict that contains all the avg attrib data of the cluster
    """
    cluster = Cluster(pat_path)
    cluster_avg = OrderedDict()

    for attr in cluster.attrib:
        cluster_avg[attr] = get_cluster_attrib_data(pat_path, start, end, save_raw, attr)
    if 'cpu' in cluster_avg.keys():
        cluster_avg['cpu'].insert(0, '1-%idle', 100 - cluster_avg['cpu']['%idle'])  # add column '1-%idle' to the result
    return cluster_avg

if __name__ == '__main__':
    pat_path = 'C:\\Users\\xuk1\PycharmProjects\\tmp_data\pat_spark163_1TB_r1'
    tic = time.time()
    # pat_path = 'C:\\Users\\xuk1\\PycharmProjects\\tmp_data\\pat_cdh511_HoS_27workers_2699v4_72vcores_PCIe_30T_4S_r1'
    # pat_path = sys.argv[1]
    # nodes = Cluster(pat_path).nodes
    # attrib = 'cpu'
    # start = [0, 1505148392, 1505148392, 1505148392]
    # end = [0, 1505272675, 1505272675, 1505272675]
    start = [0, 1487687161, 1487687176]
    end = [0, 1487687170, 1487687185]
    # get_cluster_attrib_data(nodes, attrib, start, end)

    get_cluster_data_by_time(pat_path, start, end, True)
    toc = time.time()
    print 'Processing elapsed time: {0}'.format(toc - tic)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import os
import json

class c:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	END = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'

def header(app_path):
	print(c.UNDERLINE + c.HEADER + c.BOLD + "YUNOHOST APP PACKAGE CHECKER\n" + c.END)
	print("App packaging documentation: https://yunohost.org/#/packaging_apps")
	print("App package example: https://github.com/YunoHost/example_ynh\n")
	print("Checking " + c.BOLD + app_path + c.END + " package\n")

def print_right(str):
	print(c.OKGREEN + "✔", str, c.END)

def print_wrong(str):
	print(c.FAIL + "✘", str, c.END)

def check_files_exist(app_path):
	"""
	Check files exist
	"""
	print (c.BOLD + c.HEADER + ">>>> MISSING FILES <<<<" + c.END);
	fname = ("manifest.json", "scripts/install", "scripts/remove", \
	"scripts/upgrade", "scripts/backup", "scripts/restore", "LICENSE", "README.md")
	i = 0;
	while (i < len(fname)):
		if (check_file_exist(app_path + "/" + fname[i])): print_right(fname[i])
		else: print_wrong(fname[i])
		i+=1

def check_file_exist(file_path):
	return 1 if os.path.isfile(file_path) else 0

def read_file(file_path):
	with open(file_path) as f:
		file = f.read().splitlines()
	return file

def check_manifest(manifest):
	print (c.BOLD + c.HEADER + "\n>>>> MANIFEST <<<<" + c.END);
	"""
	Check if there is no comma syntax issue
	"""
	try:
		with open(manifest, encoding='utf-8') as data_file:
			manifest = json.loads(data_file.read())
		print_right("Manifest syntax is good.")
	except: print_wrong("There is a syntax (comma) issue in manifest.json. Can't check manifest."); return
	i, fields = 0, ("name", "id", "description", "url", "license", \
	"maintainer", "multi_instance", "services", "arguments")
	while (i < len(fields)):
		if fields[i] in manifest: print_right("\"" + fields[i] + "\" fields is present")
		else: print_wrong("\"" + fields[i] + "\" field is missing")
		i+=1

# Check under array
#		manifest["description"]["en"]
#		manifest["maintainer"]["name"]
#		manifest["maintainer"]["email"]
#		manifest["arguments"]["install"]
	"""
	Check values in keys
	"""
	if "license" in manifest and manifest["license"] != "free" and manifest["license"] != "non-free":
		print_wrong("You should specify 'free' or 'non-free' software package in the license field.")
	elif "license" in manifest: print_right("'licence' key value is good.")
	if "multi_instance" in manifest and manifest["multi_instance"] != "true" and manifest["multi_instance"] != "false":
		print_wrong("multi_instance field must use 'true' or 'false' value.");
	if "services" in manifest:
		services = ("nginx", "php5-fpm", "mysql", "uwsgi", "metronome", "postfix", "dovecot") #, "rspamd", "rmilter")
		i = 0
		while (i < len(manifest["services"])):
			if manifest["services"][i] not in services:
				print_wrong(manifest["services"][i] + " service doesn't exist")
			i+=1
	if "install" in manifest["arguments"]:
		types = ("domain", "path", "password")
		i = 0
		while (i < len(types)):
			j = 0
			while (j < len(manifest["arguments"]["install"])):
				if types[i] == manifest["arguments"]["install"][j]["name"]:
					if "type" not in manifest["arguments"]["install"][j]:
						print("You should specify " + types[i] + " type: \"type\": \"" + types[i] + "\",")
				j+=1
			i+=1

def check_script(path, script_name):
	script_path = path + "/scripts/" + script_name
	if check_file_exist(script_path) == 0: return
	print (c.BOLD + c.HEADER + "\n>>>>", scripts[i].upper(), "SCRIPT <<<<" + c.END);
	script = read_file(script_path)
	check_script_header_presence(script)
	check_sudo_prefix_commands(script)
	check_verifications_done_before_modifying_system(script)
	if "wget" in script or "curl" in script:
		print("You should not fetch sources from internet with curl or wget for security reasons.")

def check_script_header_presence(script):
	if "#!/bin/bash" in script[0]: print_right("Script contain at first line \"#!/bin/bash\"")
	else: print_wrong("Script must contain at first line \"#!/bin/bash\"")

def check_sudo_prefix_commands(script):
	"""
	Check if commands are prefix with "sudo"
	"""
	cmd = ("rm", "chown", "chmod", "apt-get", "apt", \
	"service", "yunohost", "find" "swapon", "mkswap", "useradd")#, "dd") cp, mkdir
	i, ok = 0, 1
	while i < len(script):
		j = 0
		while j < len(cmd):
			if cmd[j] + " " in script[i] and "sudo " + cmd[j] + " " not in script[i] \
			and "yunohost service" not in script[i] and "-exec " + cmd[j] not in script[i] \
			and ".service" not in script[i] and script[i][0] != '#':
				print(c.FAIL + "✘ Line ", i + 1, "you should add \"sudo\" before this command line:", c.END)
				print(script[i]); ok = 0
			j+=1
		i+=1
	if ok == 1: print_right("All commands are prefix with \"sudo\".")

def check_verifications_done_before_modifying_system(script):
	"""
	Check if verifications are done before modifying the system
	"""
	ex, i = 0, 0
	while i < len(script):
		if "exit" in script[i]: ex = i
		i+=1
	cmd = ("cp", "mkdir", "rm", "chown", "chmod", "apt-get", "apt", "service", \
	"find", "sed", "mysql", "swapon", "mount", "dd", "mkswap", "useradd")#"yunohost"
	i, ok = 0, 1
	while i < len(script):
		if i >= ex: break
		j = 0
		while (j < len(cmd)):
			if cmd[j] in script[i] and script[i][0] != '#': ok = 0
			j+=1
		i+=1
	if ok == 0:
		print(c.FAIL + "✘ At line", ex + 1, "'exit' command is executed with system modification before.")
		print("This system modification is an issue if a verification exit the script.")
		print("You should move this verification before any system modification." + c.END);
	else: print_right("Verifications (with exit commands) are done before any system modification.")

if __name__ == '__main__':
	os.system("clear")
	if len(sys.argv) != 2: print("Give one app package path."); exit()
	app_path = sys.argv[1]
	header(app_path)
	check_files_exist(app_path)
	check_manifest(app_path + "/manifest.json")
	i, scripts = 0, ("install", "remove", "upgrade", "backup", "restore")
	while i < len(scripts):
		check_script(app_path, scripts[i])
		i+=1

"""
## Todo ##
* Si nginx dans les services du manifest, vérifier :
	* présence de /conf/nginx.conf
	* sudo service reload nginx dans les scripts install, remove, upgrade, restore (backup n’est pas nécessaire)

* Helper propositions
	if "apt" in install: print("You should use this helper: \"sudo yunohost \".")

* use jsonchema to check the manifest
https://github.com/YunoHost/yunotest/blob/master/apps_tests/manifest_schema.json
https://github.com/YunoHost/yunotest/blob/master/apps_tests/__init__.py
"""
<commit_msg>[enh] Handle 'user' type for key name 'user' and 'admin'.<commit_after>#!/usr/bin/env python3

import sys
import os
import json

class c:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	END = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'

def header(app_path):
	print(c.UNDERLINE + c.HEADER + c.BOLD + "YUNOHOST APP PACKAGE CHECKER\n" + c.END)
	print("App packaging documentation: https://yunohost.org/#/packaging_apps")
	print("App package example: https://github.com/YunoHost/example_ynh\n")
	print("Checking " + c.BOLD + app_path + c.END + " package\n")

def print_right(str):
	print(c.OKGREEN + "✔", str, c.END)

def print_wrong(str):
	print(c.FAIL + "✘", str, c.END)

def check_files_exist(app_path):
	"""
	Check files exist
	"""
	print (c.BOLD + c.HEADER + ">>>> MISSING FILES <<<<" + c.END);
	fname = ("manifest.json", "scripts/install", "scripts/remove", \
	"scripts/upgrade", "scripts/backup", "scripts/restore", "LICENSE", "README.md")
	i = 0;
	while (i < len(fname)):
		if (check_file_exist(app_path + "/" + fname[i])): print_right(fname[i])
		else: print_wrong(fname[i])
		i+=1

def check_file_exist(file_path):
	return 1 if os.path.isfile(file_path) else 0

def read_file(file_path):
	with open(file_path) as f:
		file = f.read().splitlines()
	return file

def check_manifest(manifest):
	print (c.BOLD + c.HEADER + "\n>>>> MANIFEST <<<<" + c.END);
	"""
	Check if there is no comma syntax issue
	"""
	try:
		with open(manifest, encoding='utf-8') as data_file:
			manifest = json.loads(data_file.read())
		print_right("Manifest syntax is good.")
	except: print_wrong("There is a syntax (comma) issue in manifest.json. Can't check manifest."); return
	i, fields = 0, ("name", "id", "description", "url", "license", \
	"maintainer", "multi_instance", "services", "arguments")
	while (i < len(fields)):
		if fields[i] in manifest: print_right("\"" + fields[i] + "\" fields is present")
		else: print_wrong("\"" + fields[i] + "\" field is missing")
		i+=1

# Check under array
#		manifest["description"]["en"]
#		manifest["maintainer"]["name"]
#		manifest["maintainer"]["email"]
#		manifest["arguments"]["install"]
	"""
	Check values in keys
	"""
	if "license" in manifest and manifest["license"] != "free" and manifest["license"] != "non-free":
		print_wrong("You should specify 'free' or 'non-free' software package in the license field.")
	elif "license" in manifest: print_right("'licence' key value is good.")
	if "multi_instance" in manifest and manifest["multi_instance"] != "true" and manifest["multi_instance"] != "false":
		print_wrong("multi_instance field must use 'true' or 'false' value.");
	if "services" in manifest:
		services = ("nginx", "php5-fpm", "mysql", "uwsgi", "metronome", "postfix", "dovecot") #, "rspamd", "rmilter")
		i = 0
		while (i < len(manifest["services"])):
			if manifest["services"][i] not in services:
				print_wrong(manifest["services"][i] + " service doesn't exist")
			i+=1
	if "install" in manifest["arguments"]:
		types = ("user", "domain", "path", "password", "user", "admin")
		i = 0
		while (i < len(types)):
			j = 0
			while (j < len(manifest["arguments"]["install"])):
				if types[i] == manifest["arguments"]["install"][j]["name"]:
					if "type" not in manifest["arguments"]["install"][j]:
						print("You should specify the type of the key with", end =" ")
						print(types[i - 1]) if i == 5 else print(types[i])
				j+=1
			i+=1

def check_script(path, script_name):
	script_path = path + "/scripts/" + script_name
	if check_file_exist(script_path) == 0: return
	print (c.BOLD + c.HEADER + "\n>>>>", scripts[i].upper(), "SCRIPT <<<<" + c.END);
	script = read_file(script_path)
	check_script_header_presence(script)
	check_sudo_prefix_commands(script)
	check_verifications_done_before_modifying_system(script)
	if "wget" in script or "curl" in script:
		print("You should not fetch sources from internet with curl or wget for security reasons.")

def check_script_header_presence(script):
	if "#!/bin/bash" in script[0]: print_right("Script contain at first line \"#!/bin/bash\"")
	else: print_wrong("Script must contain at first line \"#!/bin/bash\"")

def check_sudo_prefix_commands(script):
	"""
	Check if commands are prefix with "sudo"
	"""
	cmd = ("rm", "chown", "chmod", "apt-get", "apt", \
	"service", "yunohost", "find" "swapon", "mkswap", "useradd")#, "dd") cp, mkdir
	i, ok = 0, 1
	while i < len(script):
		j = 0
		while j < len(cmd):
			if cmd[j] + " " in script[i] and "sudo " + cmd[j] + " " not in script[i] \
			and "yunohost service" not in script[i] and "-exec " + cmd[j] not in script[i] \
			and ".service" not in script[i] and script[i][0] != '#':
				print(c.FAIL + "✘ Line ", i + 1, "you should add \"sudo\" before this command line:", c.END)
				print(script[i]); ok = 0
			j+=1
		i+=1
	if ok == 1: print_right("All commands are prefix with \"sudo\".")

def check_verifications_done_before_modifying_system(script):
	"""
	Check if verifications are done before modifying the system
	"""
	ex, i = 0, 0
	while i < len(script):
		if "exit" in script[i]: ex = i
		i+=1
	cmd = ("cp", "mkdir", "rm", "chown", "chmod", "apt-get", "apt", "service", \
	"find", "sed", "mysql", "swapon", "mount", "dd", "mkswap", "useradd")#"yunohost"
	i, ok = 0, 1
	while i < len(script):
		if i >= ex: break
		j = 0
		while (j < len(cmd)):
			if cmd[j] in script[i] and script[i][0] != '#': ok = 0
			j+=1
		i+=1
	if ok == 0:
		print(c.FAIL + "✘ At line", ex + 1, "'exit' command is executed with system modification before.")
		print("This system modification is an issue if a verification exit the script.")
		print("You should move this verification before any system modification." + c.END);
	else: print_right("Verifications (with exit commands) are done before any system modification.")

if __name__ == '__main__':
	os.system("clear")
	if len(sys.argv) != 2: print("Give one app package path."); exit()
	app_path = sys.argv[1]
	header(app_path)
	check_files_exist(app_path)
	check_manifest(app_path + "/manifest.json")
	i, scripts = 0, ("install", "remove", "upgrade", "backup", "restore")
	while i < len(scripts):
		check_script(app_path, scripts[i])
		i+=1

"""
## Todo ##
* Si nginx dans les services du manifest, vérifier :
	* présence de /conf/nginx.conf
	* sudo service reload nginx dans les scripts install, remove, upgrade, restore (backup n’est pas nécessaire)

* Helper propositions
	if "apt" in install: print("You should use this helper: \"sudo yunohost \".")

* use jsonchema to check the manifest
https://github.com/YunoHost/yunotest/blob/master/apps_tests/manifest_schema.json
https://github.com/YunoHost/yunotest/blob/master/apps_tests/__init__.py
"""
<|endoftext|>


<commit_before>#! /usr/bin/env python
# encoding: UTF-8

"""
This tool can help to reduce the memory usage in very large builds featuring many tasks with after/before attributes.
It may also improve the overall build time by decreasing the amount of iterations over tasks.

Usage:
def options(opt):
	opt.load('mem_reducer')
"""

import itertools
from waflib import Utils, Task, Runner

class SetOfTasks(object):
	"""Wraps a set and a task which has a list of other sets.
	The interface is meant to mimic the interface of set. Add missing functions as needed.
	"""
	def __init__(self, owner):
		self._set = owner.run_after
		self._owner = owner

	def __iter__(self):
		for g in self._owner.run_after_groups:
			#print len(g)
			for task in g:
				yield task
		for task in self._set:
			yield task

	def add(self, obj):
		self._set.add(obj)

	def update(self, obj):
		self._set.update(obj)

def set_precedence_constraints(tasks):
	cstr_groups = Utils.defaultdict(list)
	for x in tasks:
		x.run_after = SetOfTasks(x)
		x.run_after_groups = []
		x.waiting_sets = []

		h = x.hash_constraints()
		cstr_groups[h].append(x)

	# create sets which can be reused for all tasks
	for k in cstr_groups.iterkeys():
		cstr_groups[k] = set(cstr_groups[k])

	# this list should be short
	for key1, key2 in itertools.combinations(cstr_groups.iterkeys(), 2):
		group1 = cstr_groups[key1]
		group2 = cstr_groups[key2]
		# get the first entry of the set
		t1 = iter(group1).next()
		t2 = iter(group2).next()

		# add the constraints based on the comparisons
		if Task.is_before(t1, t2):
			for x in group2:
				x.run_after_groups.append(group1)
			for k in group1:
				k.waiting_sets.append(group1)
		elif Task.is_before(t2, t1):
			for x in group1:
				x.run_after_groups.append(group2)
			for k in group2:
				k.waiting_sets.append(group2)

Task.set_precedence_constraints = set_precedence_constraints

def get_out(self):
	tsk = self.out.get()
	if not self.stop:
		self.add_more_tasks(tsk)
	self.count -= 1
	self.dirty = True

	# shrinking sets
	try:
		ws = tsk.waiting_sets
	except AttributeError:
		pass
	else:
		for k in ws:
			k.remove(tsk)

	return tsk
Runner.Parallel.get_out = get_out

def skip(self, tsk):
	tsk.hasrun = Task.SKIPPED

	# shrinking sets
	try:
		ws = tsk.waiting_sets
	except AttributeError:
		pass
	else:
		for k in ws:
			k.remove(tsk)
Runner.Parallel.skip = skip

<commit_msg>make mem_reducer.py ready for python 3<commit_after>#! /usr/bin/env python
# encoding: UTF-8

"""
This tool can help to reduce the memory usage in very large builds featuring many tasks with after/before attributes.
It may also improve the overall build time by decreasing the amount of iterations over tasks.

Usage:
def options(opt):
	opt.load('mem_reducer')
"""

import itertools
from waflib import Utils, Task, Runner

class SetOfTasks(object):
	"""Wraps a set and a task which has a list of other sets.
	The interface is meant to mimic the interface of set. Add missing functions as needed.
	"""
	def __init__(self, owner):
		self._set = owner.run_after
		self._owner = owner

	def __iter__(self):
		for g in self._owner.run_after_groups:
			#print len(g)
			for task in g:
				yield task
		for task in self._set:
			yield task

	def add(self, obj):
		self._set.add(obj)

	def update(self, obj):
		self._set.update(obj)

def set_precedence_constraints(tasks):
	cstr_groups = Utils.defaultdict(list)
	for x in tasks:
		x.run_after = SetOfTasks(x)
		x.run_after_groups = []
		x.waiting_sets = []

		h = x.hash_constraints()
		cstr_groups[h].append(x)

	# create sets which can be reused for all tasks
	for k in cstr_groups.keys():
		cstr_groups[k] = set(cstr_groups[k])

	# this list should be short
	for key1, key2 in itertools.combinations(cstr_groups.keys(), 2):
		group1 = cstr_groups[key1]
		group2 = cstr_groups[key2]
		# get the first entry of the set
		t1 = next(iter(group1))
		t2 = next(iter(group2))

		# add the constraints based on the comparisons
		if Task.is_before(t1, t2):
			for x in group2:
				x.run_after_groups.append(group1)
			for k in group1:
				k.waiting_sets.append(group1)
		elif Task.is_before(t2, t1):
			for x in group1:
				x.run_after_groups.append(group2)
			for k in group2:
				k.waiting_sets.append(group2)

Task.set_precedence_constraints = set_precedence_constraints

def get_out(self):
	tsk = self.out.get()
	if not self.stop:
		self.add_more_tasks(tsk)
	self.count -= 1
	self.dirty = True

	# shrinking sets
	try:
		ws = tsk.waiting_sets
	except AttributeError:
		pass
	else:
		for k in ws:
			k.remove(tsk)

	return tsk
Runner.Parallel.get_out = get_out

def skip(self, tsk):
	tsk.hasrun = Task.SKIPPED

	# shrinking sets
	try:
		ws = tsk.waiting_sets
	except AttributeError:
		pass
	else:
		for k in ws:
			k.remove(tsk)
Runner.Parallel.skip = skip

<|endoftext|>


<commit_before>#!/usr/bin/env python

#################################
#				@author: Mahmoud Shaheen			#
#				 MedicalBox IOT Project				#
#				   			Pinout							#
#################################

#for controlling GPIO

import data

import RPi.GPIO as GPIO           # import RPi.GPIO module
GPIO.setmode(GPIO.BCM)            # choose BCM or BOARD
GPIO.setwarnings(False)
set a pins as an output
GPIO.setup(data.switch1, GPIO.OUT)
GPIO.setup(data.switch2, GPIO.OUT)
GPIO.setup(data.switch3, GPIO.OUT)
GPIO.setup(data.switch4, GPIO.OUT)
GPIO.setup(data.switch5, GPIO.OUT)
GPIO.setup(data.switch6, GPIO.OUT)
GPIO.setup(data.switch7, GPIO.OUT)
GPIO.setup(data.switch8, GPIO.OUT)

def gpioSwitches(state1, state2, state3, state4, state5, state6, state7, state8):
	GPIO.output(data.switch1, state1)       # set pin value to switch state
	GPIO.output(data.switch2, state2)       # set pin value to switch state
	GPIO.output(data.switch3, state3)       # set pin value to switch state
	GPIO.output(data.switch4, state4)       # set pin value to switch state
	GPIO.output(data.switch5, state5)       # set pin value to switch state
	GPIO.output(data.switch6, state6)       # set pin value to switch state
	GPIO.output(data.switch7, state7)       # set pin value to switch state
	GPIO.output(data.switch8, state8)       # set pin value to switch state
	print "gpio should be " + str(state1) + ", " +  str(state2)  + ", " +  str(state3)  + ", " +  str(state4)  + ", " +  str(state5)  + ", " +  str(state6)  + ", " +  str(state7)  + ", " +  str(state8) <commit_msg>typo fix<commit_after>#!/usr/bin/env python

#################################
#				@author: Mahmoud Shaheen			#
#				 MedicalBox IOT Project				#
#				   			Pinout							#
#################################

#for controlling GPIO

import data

import RPi.GPIO as GPIO           # import RPi.GPIO module
GPIO.setmode(GPIO.BCM)            # choose BCM or BOARD
GPIO.setwarnings(False)
#set a pins as an output
GPIO.setup(data.switch1, GPIO.OUT)
GPIO.setup(data.switch2, GPIO.OUT)
GPIO.setup(data.switch3, GPIO.OUT)
GPIO.setup(data.switch4, GPIO.OUT)
GPIO.setup(data.switch5, GPIO.OUT)
GPIO.setup(data.switch6, GPIO.OUT)
GPIO.setup(data.switch7, GPIO.OUT)
GPIO.setup(data.switch8, GPIO.OUT)

def gpioSwitches(state1, state2, state3, state4, state5, state6, state7, state8):
	GPIO.output(data.switch1, state1)       # set pin value to switch state
	GPIO.output(data.switch2, state2)       # set pin value to switch state
	GPIO.output(data.switch3, state3)       # set pin value to switch state
	GPIO.output(data.switch4, state4)       # set pin value to switch state
	GPIO.output(data.switch5, state5)       # set pin value to switch state
	GPIO.output(data.switch6, state6)       # set pin value to switch state
	GPIO.output(data.switch7, state7)       # set pin value to switch state
	GPIO.output(data.switch8, state8)       # set pin value to switch state
	print "gpio should be " + str(state1) + ", " +  str(state2)  + ", " +  str(state3)  + ", " +  str(state4)  + ", " +  str(state5)  + ", " +  str(state6)  + ", " +  str(state7)  + ", " +  str(state8) <|endoftext|>


<commit_before>"""

"""

"""
test_cookiejar.py
websocket - WebSocket client library for Python

Copyright 2021 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import unittest

from websocket._cookiejar import SimpleCookieJar

class CookieJarTest(unittest.TestCase):
    def testAdd(self):
        cookie_jar = SimpleCookieJar()
        cookie_jar.add("")
        self.assertFalse(cookie_jar.jar, "Cookie with no domain should not be added to the jar")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b")
        self.assertFalse(cookie_jar.jar, "Cookie with no domain should not be added to the jar")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; domain=.abc")
        self.assertTrue(".abc" in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; domain=abc")
        self.assertTrue(".abc" in cookie_jar.jar)
        self.assertTrue("abc" not in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")
        self.assertEqual(cookie_jar.get(None), "")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        cookie_jar.add("e=f; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d; e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        cookie_jar.add("e=f; domain=.abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d; e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        cookie_jar.add("e=f; domain=xyz")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("xyz"), "e=f")
        self.assertEqual(cookie_jar.get("something"), "")

    def testSet(self):
        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b")
        self.assertFalse(cookie_jar.jar, "Cookie with no domain should not be added to the jar")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; domain=.abc")
        self.assertTrue(".abc" in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; domain=abc")
        self.assertTrue(".abc" in cookie_jar.jar)
        self.assertTrue("abc" not in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        cookie_jar.set("e=f; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        cookie_jar.set("e=f; domain=.abc")
        self.assertEqual(cookie_jar.get("abc"), "e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        cookie_jar.set("e=f; domain=xyz")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("xyz"), "e=f")
        self.assertEqual(cookie_jar.get("something"), "")

    def testGet(self):
        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc.com")
        self.assertEqual(cookie_jar.get("abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("x.abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("abc.com.es"), "")
        self.assertEqual(cookie_jar.get("xabc.com"), "")

        cookie_jar.set("a=b; c=d; domain=.abc.com")
        self.assertEqual(cookie_jar.get("abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("x.abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("abc.com.es"), "")
        self.assertEqual(cookie_jar.get("xabc.com"), "")

if __name__ == "__main__":
    unittest.main()
<commit_msg>Fix linting<commit_after>"""

"""

"""
test_cookiejar.py
websocket - WebSocket client library for Python

Copyright 2021 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import unittest

from websocket._cookiejar import SimpleCookieJar

class CookieJarTest(unittest.TestCase):
    def testAdd(self):
        cookie_jar = SimpleCookieJar()
        cookie_jar.add("")
        self.assertFalse(cookie_jar.jar, "Cookie with no domain should not be added to the jar")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b")
        self.assertFalse(cookie_jar.jar, "Cookie with no domain should not be added to the jar")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; domain=.abc")
        self.assertTrue(".abc" in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; domain=abc")
        self.assertTrue(".abc" in cookie_jar.jar)
        self.assertTrue("abc" not in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")
        self.assertEqual(cookie_jar.get(None), "")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        cookie_jar.add("e=f; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d; e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        cookie_jar.add("e=f; domain=.abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d; e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.add("a=b; c=d; domain=abc")
        cookie_jar.add("e=f; domain=xyz")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("xyz"), "e=f")
        self.assertEqual(cookie_jar.get("something"), "")

    def testSet(self):
        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b")
        self.assertFalse(cookie_jar.jar, "Cookie with no domain should not be added to the jar")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; domain=.abc")
        self.assertTrue(".abc" in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; domain=abc")
        self.assertTrue(".abc" in cookie_jar.jar)
        self.assertTrue("abc" not in cookie_jar.jar)

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        cookie_jar.set("e=f; domain=abc")
        self.assertEqual(cookie_jar.get("abc"), "e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        cookie_jar.set("e=f; domain=.abc")
        self.assertEqual(cookie_jar.get("abc"), "e=f")

        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc")
        cookie_jar.set("e=f; domain=xyz")
        self.assertEqual(cookie_jar.get("abc"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("xyz"), "e=f")
        self.assertEqual(cookie_jar.get("something"), "")

    def testGet(self):
        cookie_jar = SimpleCookieJar()
        cookie_jar.set("a=b; c=d; domain=abc.com")
        self.assertEqual(cookie_jar.get("abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("x.abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("abc.com.es"), "")
        self.assertEqual(cookie_jar.get("xabc.com"), "")

        cookie_jar.set("a=b; c=d; domain=.abc.com")
        self.assertEqual(cookie_jar.get("abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("x.abc.com"), "a=b; c=d")
        self.assertEqual(cookie_jar.get("abc.com.es"), "")
        self.assertEqual(cookie_jar.get("xabc.com"), "")

if __name__ == "__main__":
    unittest.main()

<|endoftext|>