<commit_before># -*- coding: utf-8 -*-
from barbecue import vnmax
from logging import getLogger
from schematics.exceptions import ValidationError
from openprocurement.api.utils import (
    context_unpack, generate_id, get_now,
    set_ownership as api_set_ownership
)
from openprocurement.tender.core.utils import (
    save_tender, apply_patch, calculate_business_date
)
from openprocurement.tender.openua.constants import TENDERING_EXTRA_PERIOD
from openprocurement.tender.openua.utils import (
    check_complaint_status, has_unanswered_questions,
    has_unanswered_complaints
)
from openprocurement.tender.openeu.utils import (
    all_bids_are_reviewed, prepare_qualifications
)
from openprocurement.tender.openeu.constants import (
    PREQUALIFICATION_COMPLAINT_STAND_STILL as COMPLAINT_STAND_STILL
)
from openprocurement.tender.competitivedialogue.constants import (
    MINIMAL_NUMBER_OF_BIDS
)
from openprocurement.tender.core.events import TenderInitializeEvent

LOGGER = getLogger(__name__)

def patch_eu(self):
    """Tender Edit (partial)

            For example here is how procuring entity can change number of items to be procured and total Value of a tender:

            .. sourcecode:: http

                PATCH /tenders/4879d3f8ee2443169b5fbbc9f89fa607 HTTP/1.1
                Host: example.com
                Accept: application/json

                {
                    "data": {
                        "value": {
                            "amount": 600
                        },
                        "itemsToBeProcured": [
                            {
                                "quantity": 6
                            }
                        ]
                    }
                }

            And here is the response to be expected:

            .. sourcecode:: http

                HTTP/1.0 200 OK
                Content-Type: application/json

                {
                    "data": {
                        "id": "4879d3f8ee2443169b5fbbc9f89fa607",
                        "tenderID": "UA-64e93250be76435397e8c992ed4214d1",
                        "dateModified": "2014-10-27T08:12:34.956Z",
                        "value": {
                            "amount": 600
                        },
                        "itemsToBeProcured": [
                            {
                                "quantity": 6
                            }
                        ]
                    }
                }

            """
    tender = self.context
    if self.request.authenticated_role != 'Administrator' and tender.status in ['complete', 'unsuccessful',
                                                                                'cancelled']:
        self.request.errors.add('body', 'data', 'Can\'t update tender in current ({}) status'.format(tender.status))
        self.request.errors.status = 403
        return
    data = self.request.validated['data']
    if self.request.authenticated_role == 'tender_owner' and 'status' in data and \
            data['status'] not in ['active.pre-qualification.stand-still', 'active.stage2.waiting', tender.status]:
        self.request.errors.add('body', 'data', 'Can\'t update tender status')
        self.request.errors.status = 403
        return

    if self.request.authenticated_role == 'tender_owner' \
            and self.request.validated['tender_status'] == 'active.tendering':
        if 'tenderPeriod' in data and 'endDate' in data['tenderPeriod']:
            self.request.validated['tender'].tenderPeriod.import_data(data['tenderPeriod'])
            if calculate_business_date(get_now(), TENDERING_EXTRA_PERIOD, self.request.validated['tender']) > \
                    self.request.validated['tender'].tenderPeriod.endDate:
                self.request.errors.add('body', 'data', 'tenderPeriod should be extended by {0.days} days'.format(
                    TENDERING_EXTRA_PERIOD))
                self.request.errors.status = 403
                return
            # import pdb; pdb.set_trace()
            self.request.registry.notify(TenderInitializeEvent(self.request.validated['tender']))
            self.request.validated['data']["enquiryPeriod"] = self.request.validated[
                'tender'].enquiryPeriod.serialize()

    apply_patch(self.request, save=False, src=self.request.validated['tender_src'])
    if self.request.authenticated_role == 'chronograph':
        check_status(self.request)
    elif self.request.authenticated_role == 'tender_owner' and tender.status == 'active.tendering':
        tender.invalidate_bids_data()
    elif self.request.authenticated_role == 'tender_owner' and \
            self.request.validated['tender_status'] == 'active.pre-qualification' and \
            tender.status == "active.pre-qualification.stand-still":
        if all_bids_are_reviewed(self.request):
            tender.qualificationPeriod.endDate = calculate_business_date(get_now(), COMPLAINT_STAND_STILL,
                                                                         self.request.validated['tender'])
            tender.check_auction_time()
        else:
            self.request.errors.add('body', 'data', 'Can\'t switch to \'active.pre-qualification.stand-still\' while not all bids are qualified')
            self.request.errors.status = 403
            return
    elif self.request.authenticated_role == 'tender_owner' and \
            self.request.validated['tender_status'] == 'active.pre-qualification' and \
            tender.status != "active.pre-qualification.stand-still":
        self.request.errors.add('body', 'data', 'Can\'t update tender status')
        self.request.errors.status = 403
        return

    save_tender(self.request)
    self.LOGGER.info('Updated tender {}'.format(tender.id),
                     extra=context_unpack(self.request, {'MESSAGE_ID': 'tender_patch'}))
    return {'data': tender.serialize(tender.status)}

def validate_unique_bids(bids):
    """ Return Bool
        True if number of unique identifier id biggest then MINIMAL_NUMBER_OF_BIDS
        else False
    """
    return len(set(bid['tenderers'][0]['identifier']['id'] for bid in bids)) >= MINIMAL_NUMBER_OF_BIDS

def check_initial_bids_count(request):
    tender = request.validated['tender']
    if tender.lots:
        [setattr(i.auctionPeriod, 'startDate', None) for i in tender.lots if i.numberOfBids < MINIMAL_NUMBER_OF_BIDS and i.auctionPeriod and i.auctionPeriod.startDate]
        for i in tender.lots:
            # gather all bids by lot id
            bids = [bid
                    for bid in tender.bids
                    if i.id in [i_lot.relatedLot for i_lot in bid.lotValues
                                if i_lot.status in ["active", "pending"]] and bid.status in ["active", "pending"]]

            if i.numberOfBids < MINIMAL_NUMBER_OF_BIDS or not validate_unique_bids(bids) and i.status == 'active':
                setattr(i, 'status', 'unsuccessful')
                for bid_index, bid in enumerate(tender.bids):
                    for lot_index, lot_value in enumerate(bid.lotValues):
                        if lot_value.relatedLot == i.id:
                            setattr(tender.bids[bid_index].lotValues[lot_index], 'status', 'unsuccessful')

        if not set([i.status for i in tender.lots]).difference(set(['unsuccessful', 'cancelled'])):
            LOGGER.info('Switched tender {} to {}'.format(tender.id, 'unsuccessful'),
                        extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_unsuccessful'}))
            tender.status = 'unsuccessful'
    elif tender.numberOfBids < MINIMAL_NUMBER_OF_BIDS or not validate_unique_bids(tender.bids):
        LOGGER.info('Switched tender {} to {}'.format(tender.id, 'unsuccessful'),
                    extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_unsuccessful'}))
        tender.status = 'unsuccessful'

def validate_features_custom_weight(self, data, features, max_sum):
    if features and data['lots'] and any([
        round(vnmax([
            i
            for i in features
            if i.featureOf == 'tenderer' or i.featureOf == 'lot' and i.relatedItem == lot['id'] or i.featureOf == 'item' and i.relatedItem in [j.id for j in data['items'] if j.relatedLot == lot['id']]
        ]), 15) > max_sum
        for lot in data['lots']
    ]):
        raise ValidationError(u"Sum of max value of all features for lot should be less then or equal to {:.0f}%".format(max_sum * 100))
    elif features and not data['lots'] and round(vnmax(features), 15) > max_sum:
        raise ValidationError(u"Sum of max value of all features should be less then or equal to {:.0f}%".format(max_sum * 100))

def check_status(request):
    tender = request.validated['tender']
    now = get_now()

    if tender.status == 'active.tendering' and tender.tenderPeriod.endDate <= now and \
            not has_unanswered_complaints(tender) and not has_unanswered_questions(tender):
        for complaint in tender.complaints:
            check_complaint_status(request, complaint)
        LOGGER.info('Switched tender {} to {}'.format(tender['id'], 'active.pre-qualification'),
                    extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_active.pre-qualification'}))
        tender.status = 'active.pre-qualification'
        tender.qualificationPeriod = type(tender).qualificationPeriod({'startDate': now})
        check_initial_bids_count(request)
        prepare_qualifications(request)
        return

    elif tender.status == 'active.pre-qualification.stand-still' and tender.qualificationPeriod and tender.qualificationPeriod.endDate <= now and not any([
        i.status in tender.block_complaint_status
        for q in tender.qualifications
        for i in q.complaints
    ]):
        LOGGER.info('Switched tender {} to {}'.format(tender['id'], 'active.stage2.pending'),
                    extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_active_stage2_pending'}))
        tender.status = 'active.stage2.pending'
        check_initial_bids_count(request)
        return

def set_ownership(item):
    item.owner_token = generate_id()

def prepare_shortlistedFirms(shortlistedFirms):
    """ Make list with keys
        key = {identifier_id}_{identifier_scheme}_{lot_id}
    """
    all_keys = set()
    for firm in shortlistedFirms:
        key = u"{firm_id}_{firm_scheme}".format(firm_id=firm['identifier']['id'],
                                                firm_scheme=firm['identifier']['scheme'])
        if firm.get('lots'):
            keys = set([u"{key}_{lot_id}".format(key=key, lot_id=lot['id']) for lot in firm.get('lots')])
        else:
            keys = set([key])
        all_keys |= keys
    return all_keys

def prepare_author(obj):
    """ Make key
        {author.identifier.id}_{author.identifier.scheme}
        or
        {author.identifier.id}_{author.identifier.scheme}_{lot.id}
        if obj has relatedItem and questionOf != tender or obj has relatedLot than
    """
    base_key = u"{id}_{scheme}".format(scheme=obj['author']['identifier']['scheme'],
                                       id=obj['author']['identifier']['id'])
    if obj.get('relatedLot') or (obj.get('relatedItem') and obj.get('questionOf') == 'lot'):
        base_key = u"{base_key}_{lotId}".format(base_key=base_key,
                                                lotId=obj.get('relatedLot') or obj.get('relatedItem'))
    return base_key

def prepare_bid_identifier(bid):
    """ Make list with keys
        key = {identifier_id}_{identifier_scheme}_{lot_id}
    """
    all_keys = set()
    for tenderer in bid['tenderers']:
        key = u"{id}_{scheme}".format(id=tenderer['identifier']['id'],
                                      scheme=tenderer['identifier']['scheme'])
        if bid.get('lotValues'):
            keys = set([u"{key}_{lot_id}".format(key=key,
                                                 lot_id=lot['relatedLot'])
                        for lot in bid.get('lotValues')])
        else:
            keys = set([key])
        all_keys |= keys
    return all_keys

def stage2_bid_post(self):
    tender = self.request.validated['tender']
    if self.request.validated['tender_status'] != 'active.tendering':
        self.request.errors.add('body', 'data', 'Can\'t add bid in current ({}) tender status'.format(
            self.request.validated['tender_status']))
        self.request.errors.status = 403
        return
    if tender.tenderPeriod.startDate and \
            get_now() < tender.tenderPeriod.startDate or \
            get_now() > tender.tenderPeriod.endDate:
        self.request.errors.add('body', 'data',
                                'Bid can be added only during the tendering period: from ({}) to ({}).'.format(
                                    tender.tenderPeriod.startDate, tender.tenderPeriod.endDate))
        self.request.errors.status = 403
        return
    bid = self.request.validated['bid']
    firm_keys = prepare_shortlistedFirms(tender.shortlistedFirms)
    bid_keys = prepare_bid_identifier(bid)
    if not (bid_keys <= firm_keys):
        self.request.errors.add('body', 'data', 'Firm can\'t create bid')
        self.request.errors.status = 403
        return
    if bid.status not in self.allowed_bid_status_on_create:
        self.request.errors.add('body', 'data',
                                'Bid can be added only with status: {}.'.format(self.allowed_bid_status_on_create))
        self.request.errors.status = 403
        return
    tender.modified = False
    api_set_ownership(bid, self.request)
    tender.bids.append(bid)
    if save_tender(self.request):
        self.LOGGER.info('Created tender bid {}'.format(bid.id),
                         extra=context_unpack(self.request, {'MESSAGE_ID': 'tender_bid_create'},
                                              {'bid_id': bid.id}))
        self.request.response.status = 201
        self.request.response.headers['Location'] = self.request.route_url('{}:Tender Bids'.format(tender.procurementMethodType), tender_id=tender.id,
                                                                           bid_id=bid['id'])
        return {
            'data': bid.serialize('view'),
            'access': {
                'token': bid.owner_token
            }
        }
<commit_msg>Move validators<commit_after># -*- coding: utf-8 -*-
from barbecue import vnmax
from logging import getLogger
from schematics.exceptions import ValidationError
from openprocurement.api.utils import (
    context_unpack, generate_id, get_now, error_handler,
    set_ownership as api_set_ownership
)
from openprocurement.tender.core.utils import (
    save_tender, apply_patch, calculate_business_date
)
from openprocurement.tender.openua.constants import TENDERING_EXTRA_PERIOD
from openprocurement.tender.openua.utils import (
    check_complaint_status, has_unanswered_questions,
    has_unanswered_complaints
)
from openprocurement.tender.openeu.utils import (
    all_bids_are_reviewed, prepare_qualifications
)
from openprocurement.tender.openeu.constants import (
    PREQUALIFICATION_COMPLAINT_STAND_STILL as COMPLAINT_STAND_STILL
)
from openprocurement.tender.competitivedialogue.constants import (
    MINIMAL_NUMBER_OF_BIDS
)
from openprocurement.tender.core.events import TenderInitializeEvent

LOGGER = getLogger(__name__)

def patch_eu(self):
    """Tender Edit (partial)

            For example here is how procuring entity can change number of items to be procured and total Value of a tender:

            .. sourcecode:: http

                PATCH /tenders/4879d3f8ee2443169b5fbbc9f89fa607 HTTP/1.1
                Host: example.com
                Accept: application/json

                {
                    "data": {
                        "value": {
                            "amount": 600
                        },
                        "itemsToBeProcured": [
                            {
                                "quantity": 6
                            }
                        ]
                    }
                }

            And here is the response to be expected:

            .. sourcecode:: http

                HTTP/1.0 200 OK
                Content-Type: application/json

                {
                    "data": {
                        "id": "4879d3f8ee2443169b5fbbc9f89fa607",
                        "tenderID": "UA-64e93250be76435397e8c992ed4214d1",
                        "dateModified": "2014-10-27T08:12:34.956Z",
                        "value": {
                            "amount": 600
                        },
                        "itemsToBeProcured": [
                            {
                                "quantity": 6
                            }
                        ]
                    }
                }

            """
    tender = self.context
    data = self.request.validated['data']
    # TODO use tender configurator instead of TENDERING_EXTRA_PERIOD
    if self.request.authenticated_role == 'tender_owner' \
            and self.request.validated['tender_status'] == 'active.tendering':
        if 'tenderPeriod' in data and 'endDate' in data['tenderPeriod']:
            self.request.validated['tender'].tenderPeriod.import_data(data['tenderPeriod'])
            if calculate_business_date(get_now(), TENDERING_EXTRA_PERIOD, self.request.validated['tender']) > \
                    self.request.validated['tender'].tenderPeriod.endDate:
                self.request.errors.add('body', 'data', 'tenderPeriod should be extended by {0.days} days'.format(
                    TENDERING_EXTRA_PERIOD))
                self.request.errors.status = 403
                raise error_handler(self.request.errors)
            # import pdb; pdb.set_trace()
            self.request.registry.notify(TenderInitializeEvent(self.request.validated['tender']))
            self.request.validated['data']["enquiryPeriod"] = self.request.validated[
                'tender'].enquiryPeriod.serialize()

    apply_patch(self.request, save=False, src=self.request.validated['tender_src'])
    if self.request.authenticated_role == 'chronograph':
        check_status(self.request)
    elif self.request.authenticated_role == 'tender_owner' and tender.status == 'active.tendering':
        tender.invalidate_bids_data()
    elif self.request.authenticated_role == 'tender_owner' and \
            self.request.validated['tender_status'] == 'active.pre-qualification' and \
            tender.status == "active.pre-qualification.stand-still":
        if all_bids_are_reviewed(self.request):
            tender.qualificationPeriod.endDate = calculate_business_date(get_now(), COMPLAINT_STAND_STILL,
                                                                         self.request.validated['tender'])
            tender.check_auction_time()
        else:
            self.request.errors.add('body', 'data', 'Can\'t switch to \'active.pre-qualification.stand-still\' while not all bids are qualified')
            self.request.errors.status = 403
            raise error_handler(self.request.errors)
    elif self.request.authenticated_role == 'tender_owner' and \
            self.request.validated['tender_status'] == 'active.pre-qualification' and \
            tender.status != "active.pre-qualification.stand-still":
        self.request.errors.add('body', 'data', 'Can\'t update tender status')
        self.request.errors.status = 403
        raise error_handler(self.request.errors)

    save_tender(self.request)
    self.LOGGER.info('Updated tender {}'.format(tender.id),
                     extra=context_unpack(self.request, {'MESSAGE_ID': 'tender_patch'}))
    return {'data': tender.serialize(tender.status)}

def validate_unique_bids(bids):
    """ Return Bool
        True if number of unique identifier id biggest then MINIMAL_NUMBER_OF_BIDS
        else False
    """
    return len(set(bid['tenderers'][0]['identifier']['id'] for bid in bids)) >= MINIMAL_NUMBER_OF_BIDS

def check_initial_bids_count(request):
    tender = request.validated['tender']
    if tender.lots:
        [setattr(i.auctionPeriod, 'startDate', None) for i in tender.lots if i.numberOfBids < MINIMAL_NUMBER_OF_BIDS and i.auctionPeriod and i.auctionPeriod.startDate]
        for i in tender.lots:
            # gather all bids by lot id
            bids = [bid
                    for bid in tender.bids
                    if i.id in [i_lot.relatedLot for i_lot in bid.lotValues
                                if i_lot.status in ["active", "pending"]] and bid.status in ["active", "pending"]]

            if i.numberOfBids < MINIMAL_NUMBER_OF_BIDS or not validate_unique_bids(bids) and i.status == 'active':
                setattr(i, 'status', 'unsuccessful')
                for bid_index, bid in enumerate(tender.bids):
                    for lot_index, lot_value in enumerate(bid.lotValues):
                        if lot_value.relatedLot == i.id:
                            setattr(tender.bids[bid_index].lotValues[lot_index], 'status', 'unsuccessful')

        if not set([i.status for i in tender.lots]).difference(set(['unsuccessful', 'cancelled'])):
            LOGGER.info('Switched tender {} to {}'.format(tender.id, 'unsuccessful'),
                        extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_unsuccessful'}))
            tender.status = 'unsuccessful'
    elif tender.numberOfBids < MINIMAL_NUMBER_OF_BIDS or not validate_unique_bids(tender.bids):
        LOGGER.info('Switched tender {} to {}'.format(tender.id, 'unsuccessful'),
                    extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_unsuccessful'}))
        tender.status = 'unsuccessful'

def validate_features_custom_weight(self, data, features, max_sum):
    if features and data['lots'] and any([
        round(vnmax([
            i
            for i in features
            if i.featureOf == 'tenderer' or i.featureOf == 'lot' and i.relatedItem == lot['id'] or i.featureOf == 'item' and i.relatedItem in [j.id for j in data['items'] if j.relatedLot == lot['id']]
        ]), 15) > max_sum
        for lot in data['lots']
    ]):
        raise ValidationError(u"Sum of max value of all features for lot should be less then or equal to {:.0f}%".format(max_sum * 100))
    elif features and not data['lots'] and round(vnmax(features), 15) > max_sum:
        raise ValidationError(u"Sum of max value of all features should be less then or equal to {:.0f}%".format(max_sum * 100))

def check_status(request):
    tender = request.validated['tender']
    now = get_now()

    if tender.status == 'active.tendering' and tender.tenderPeriod.endDate <= now and \
            not has_unanswered_complaints(tender) and not has_unanswered_questions(tender):
        for complaint in tender.complaints:
            check_complaint_status(request, complaint)
        LOGGER.info('Switched tender {} to {}'.format(tender['id'], 'active.pre-qualification'),
                    extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_active.pre-qualification'}))
        tender.status = 'active.pre-qualification'
        tender.qualificationPeriod = type(tender).qualificationPeriod({'startDate': now})
        check_initial_bids_count(request)
        prepare_qualifications(request)
        return

    elif tender.status == 'active.pre-qualification.stand-still' and tender.qualificationPeriod and tender.qualificationPeriod.endDate <= now and not any([
        i.status in tender.block_complaint_status
        for q in tender.qualifications
        for i in q.complaints
    ]):
        LOGGER.info('Switched tender {} to {}'.format(tender['id'], 'active.stage2.pending'),
                    extra=context_unpack(request, {'MESSAGE_ID': 'switched_tender_active_stage2_pending'}))
        tender.status = 'active.stage2.pending'
        check_initial_bids_count(request)
        return

def set_ownership(item):
    item.owner_token = generate_id()

def prepare_shortlistedFirms(shortlistedFirms):
    """ Make list with keys
        key = {identifier_id}_{identifier_scheme}_{lot_id}
    """
    all_keys = set()
    for firm in shortlistedFirms:
        key = u"{firm_id}_{firm_scheme}".format(firm_id=firm['identifier']['id'],
                                                firm_scheme=firm['identifier']['scheme'])
        if firm.get('lots'):
            keys = set([u"{key}_{lot_id}".format(key=key, lot_id=lot['id']) for lot in firm.get('lots')])
        else:
            keys = set([key])
        all_keys |= keys
    return all_keys

def prepare_author(obj):
    """ Make key
        {author.identifier.id}_{author.identifier.scheme}
        or
        {author.identifier.id}_{author.identifier.scheme}_{lot.id}
        if obj has relatedItem and questionOf != tender or obj has relatedLot than
    """
    base_key = u"{id}_{scheme}".format(scheme=obj['author']['identifier']['scheme'],
                                       id=obj['author']['identifier']['id'])
    if obj.get('relatedLot') or (obj.get('relatedItem') and obj.get('questionOf') == 'lot'):
        base_key = u"{base_key}_{lotId}".format(base_key=base_key,
                                                lotId=obj.get('relatedLot') or obj.get('relatedItem'))
    return base_key

def prepare_bid_identifier(bid):
    """ Make list with keys
        key = {identifier_id}_{identifier_scheme}_{lot_id}
    """
    all_keys = set()
    for tenderer in bid['tenderers']:
        key = u"{id}_{scheme}".format(id=tenderer['identifier']['id'],
                                      scheme=tenderer['identifier']['scheme'])
        if bid.get('lotValues'):
            keys = set([u"{key}_{lot_id}".format(key=key,
                                                 lot_id=lot['relatedLot'])
                        for lot in bid.get('lotValues')])
        else:
            keys = set([key])
        all_keys |= keys
    return all_keys

def stage2_bid_post(self):
    tender = self.request.validated['tender']
    bid = self.request.validated['bid']
    # TODO can't move validator because of self.allowed_bid_status_on_create
    if bid.status not in self.allowed_bid_status_on_create:
        self.request.errors.add('body', 'data',
                                'Bid can be added only with status: {}.'.format(self.allowed_bid_status_on_create))
        self.request.errors.status = 403
        return
    tender.modified = False
    api_set_ownership(bid, self.request)
    tender.bids.append(bid)
    if save_tender(self.request):
        self.LOGGER.info('Created tender bid {}'.format(bid.id),
                         extra=context_unpack(self.request, {'MESSAGE_ID': 'tender_bid_create'},
                                              {'bid_id': bid.id}))
        self.request.response.status = 201
        self.request.response.headers['Location'] = self.request.route_url('{}:Tender Bids'.format(tender.procurementMethodType), tender_id=tender.id,
                                                                           bid_id=bid['id'])
        return {
            'data': bid.serialize('view'),
            'access': {
                'token': bid.owner_token
            }
        }
<|endoftext|>


<commit_before>from django.template import Node, NodeList
from django.template import TemplateSyntaxError
from django.template import Library
from solid.util.logs import getLogger
from satchmo.product.models import Product
from satchmo.configuration import config_value

log = getLogger('satchmo_sss')

register = Library()

@register.inclusion_tag('recentlist/_recently_viewed.html')
def recentlyviewed(recent, slug=""):
    """Build a list of recent products, skipping the current one if given."""
    if slug:
        recent = [r for r in recent if slug != r.slug]

    rmax = config_value('SHOP','RECENT_MAX')
    if len(recent) > rmax:
        recent = recent[:rmax]
    return {
        'recent_products' : recent,
    }
<commit_msg>Fix a broken import. Closes #332<commit_after>from django.template import Node, NodeList
from django.template import TemplateSyntaxError
from django.template import Library
from logging import getLogger
from satchmo.product.models import Product
from satchmo.configuration import config_value

log = getLogger('satchmo_sss')

register = Library()

@register.inclusion_tag('recentlist/_recently_viewed.html')
def recentlyviewed(recent, slug=""):
    """Build a list of recent products, skipping the current one if given."""
    if slug:
        recent = [r for r in recent if slug != r.slug]

    rmax = config_value('SHOP','RECENT_MAX')
    if len(recent) > rmax:
        recent = recent[:rmax]
    return {
        'recent_products' : recent,
    }
<|endoftext|>


<commit_before># tools.py

import collections
import operator
import zlib
from itertools import permutations, groupby, starmap

from . import _compat

__all__ = ['Unique', 'max_len', 'maximal', 'lazyproperty']

class Unique(_compat.MutableSet):
    """Unique items preserving order.

    >>> Unique([3, 2, 1, 3, 2, 1, 0])
    Unique([3, 2, 1, 0])
    """

    @classmethod
    def _fromargs(cls, _seen, _items):
        inst = super(Unique, cls).__new__(cls)
        inst._seen = _seen
        inst._items = _items
        return inst

    def __init__(self, iterable=()):
        self._seen = seen = set()
        add = seen.add
        self._items = [item for item in iterable
            if item not in seen and not add(item)]

    def copy(self):
        return self._fromargs(self._seen.copy(), self._items[:])

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __contains__(self, item):
        return item in self._seen

    def __repr__(self):
        items = repr(self._items) if self._items else ''
        return '%s(%s)' % (self.__class__.__name__, items)

    def add(self, item):
        if item not in self._seen:
            self._seen.add(item)
            self._items.append(item)

    def discard(self, item):
        if item in self._seen:
            self._seen.remove(item)
            self._items.remove(item)

    def replace(self, item, new_item):
        """Replace an item preserving order.

        >>> u = Unique([0, 1, 2])
        >>> u.replace(1, 'spam')
        >>> u
        Unique([0, 'spam', 2])

        >>> u.replace('eggs', 1)
        Traceback (most recent call last):
            ...
        ValueError: 'eggs' is not in list

        >>> u.replace('spam', 0)
        Traceback (most recent call last):
            ...
        ValueError: 0 already in list
        """
        if new_item in self._seen:
            raise ValueError('%r already in list' % new_item)
        idx = self._items.index(item)
        self._seen.remove(item)
        self._seen.add(new_item)
        self._items[idx] = new_item

    def move(self, item, new_index):
        """Move an item to the given position.

        >>> u = Unique(['spam', 'eggs'])
        >>> u.move('spam', 1)
        >>> u
        Unique(['eggs', 'spam'])

        >>> u.move('ham', 0)
        Traceback (most recent call last):
            ...
        ValueError: 'ham' is not in list
        """
        idx = self._items.index(item)
        if idx != new_index:
            item = self._items.pop(idx)
            self._items.insert(new_index, item)

    def issuperset(self, items):
        """Return whether this collection contains all items.

        >>> Unique(['spam', 'eggs']).issuperset(['spam', 'spam', 'spam'])
        True
        """
        return all(_compat.map(self._seen.__contains__, items))

    def rsub(self, items):
        """Return order preserving unique items not in this collection.

        >>> Unique(['spam']).rsub(['ham', 'spam', 'eggs'])
        Unique(['ham', 'eggs'])
        """
        ignore = self._seen
        seen = set()
        add = seen.add
        items = [i for i in items
                 if i not in ignore and i not in seen and not add(i)]
        return self._fromargs(seen, items)

def max_len(iterable, minimum=0):
    """Return the len() of the longest item in ``iterable`` or ``minimum``.

    >>> max_len(['spam', 'ham'])
    4

    >>> max_len([])
    0

    >>> max_len(['ham'], 4)
    4
    """
    try:
        result = max(map(len, iterable))
    except ValueError:
        result = minimum
    return minimum if result < minimum else result

def maximal(iterable, comparison=operator.lt, _groupkey=operator.itemgetter(0)):
    """Yield the unique maximal elements from ``iterable`` using ``comparison``.

    >>> list(maximal([1, 2, 3, 3]))
    [3]

    >>> list(maximal([1]))
    [1]
    """
    iterable = set(iterable)
    if len(iterable) < 2:
        return iterable
    return (item for item, pairs
        in groupby(permutations(iterable, 2), key=_groupkey)
        if not any(starmap(comparison, pairs)))

class lazyproperty(object):
    """Non-data descriptor caching the computed result as instance attribute.

    >>> class Spam(object):
    ...     @lazyproperty
    ...     def eggs(self):
    ...         return 'spamspamspam'

    >>> spam=Spam(); spam.eggs
    'spamspamspam'

    >>> spam.eggs='eggseggseggs'; spam.eggs
    'eggseggseggs'

    >>> Spam().eggs
    'spamspamspam'

    >>> Spam.eggs  # doctest: +ELLIPSIS
    <...lazyproperty object at 0x...>
    """

    def __init__(self, fget):
        self.fget = fget
        for attr in ('__module__', '__name__', '__doc__'):
            setattr(self, attr, getattr(fget, attr))

    def __get__(self, instance, owner):
        if instance is None:
            return self
        result = instance.__dict__[self.__name__] = self.fget(instance)
        return result

def crc32_hex(value):
    """

    >>> crc32_hex(b'spam')
    '43daff3d'
    """
    return '%x' % (zlib.crc32(value) & 0xffffffff)
<commit_msg>add crc32_hex() docstring and register<commit_after># tools.py

import collections
import operator
import zlib
from itertools import permutations, groupby, starmap

from . import _compat

__all__ = ['Unique', 'max_len', 'maximal', 'lazyproperty', 'crc32_hex']

class Unique(_compat.MutableSet):
    """Unique items preserving order.

    >>> Unique([3, 2, 1, 3, 2, 1, 0])
    Unique([3, 2, 1, 0])
    """

    @classmethod
    def _fromargs(cls, _seen, _items):
        inst = super(Unique, cls).__new__(cls)
        inst._seen = _seen
        inst._items = _items
        return inst

    def __init__(self, iterable=()):
        self._seen = seen = set()
        add = seen.add
        self._items = [item for item in iterable
            if item not in seen and not add(item)]

    def copy(self):
        return self._fromargs(self._seen.copy(), self._items[:])

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __contains__(self, item):
        return item in self._seen

    def __repr__(self):
        items = repr(self._items) if self._items else ''
        return '%s(%s)' % (self.__class__.__name__, items)

    def add(self, item):
        if item not in self._seen:
            self._seen.add(item)
            self._items.append(item)

    def discard(self, item):
        if item in self._seen:
            self._seen.remove(item)
            self._items.remove(item)

    def replace(self, item, new_item):
        """Replace an item preserving order.

        >>> u = Unique([0, 1, 2])
        >>> u.replace(1, 'spam')
        >>> u
        Unique([0, 'spam', 2])

        >>> u.replace('eggs', 1)
        Traceback (most recent call last):
            ...
        ValueError: 'eggs' is not in list

        >>> u.replace('spam', 0)
        Traceback (most recent call last):
            ...
        ValueError: 0 already in list
        """
        if new_item in self._seen:
            raise ValueError('%r already in list' % new_item)
        idx = self._items.index(item)
        self._seen.remove(item)
        self._seen.add(new_item)
        self._items[idx] = new_item

    def move(self, item, new_index):
        """Move an item to the given position.

        >>> u = Unique(['spam', 'eggs'])
        >>> u.move('spam', 1)
        >>> u
        Unique(['eggs', 'spam'])

        >>> u.move('ham', 0)
        Traceback (most recent call last):
            ...
        ValueError: 'ham' is not in list
        """
        idx = self._items.index(item)
        if idx != new_index:
            item = self._items.pop(idx)
            self._items.insert(new_index, item)

    def issuperset(self, items):
        """Return whether this collection contains all items.

        >>> Unique(['spam', 'eggs']).issuperset(['spam', 'spam', 'spam'])
        True
        """
        return all(_compat.map(self._seen.__contains__, items))

    def rsub(self, items):
        """Return order preserving unique items not in this collection.

        >>> Unique(['spam']).rsub(['ham', 'spam', 'eggs'])
        Unique(['ham', 'eggs'])
        """
        ignore = self._seen
        seen = set()
        add = seen.add
        items = [i for i in items
                 if i not in ignore and i not in seen and not add(i)]
        return self._fromargs(seen, items)

def max_len(iterable, minimum=0):
    """Return the len() of the longest item in ``iterable`` or ``minimum``.

    >>> max_len(['spam', 'ham'])
    4

    >>> max_len([])
    0

    >>> max_len(['ham'], 4)
    4
    """
    try:
        result = max(map(len, iterable))
    except ValueError:
        result = minimum
    return minimum if result < minimum else result

def maximal(iterable, comparison=operator.lt, _groupkey=operator.itemgetter(0)):
    """Yield the unique maximal elements from ``iterable`` using ``comparison``.

    >>> list(maximal([1, 2, 3, 3]))
    [3]

    >>> list(maximal([1]))
    [1]
    """
    iterable = set(iterable)
    if len(iterable) < 2:
        return iterable
    return (item for item, pairs
        in groupby(permutations(iterable, 2), key=_groupkey)
        if not any(starmap(comparison, pairs)))

class lazyproperty(object):
    """Non-data descriptor caching the computed result as instance attribute.

    >>> class Spam(object):
    ...     @lazyproperty
    ...     def eggs(self):
    ...         return 'spamspamspam'

    >>> spam=Spam(); spam.eggs
    'spamspamspam'

    >>> spam.eggs='eggseggseggs'; spam.eggs
    'eggseggseggs'

    >>> Spam().eggs
    'spamspamspam'

    >>> Spam.eggs  # doctest: +ELLIPSIS
    <...lazyproperty object at 0x...>
    """

    def __init__(self, fget):
        self.fget = fget
        for attr in ('__module__', '__name__', '__doc__'):
            setattr(self, attr, getattr(fget, attr))

    def __get__(self, instance, owner):
        if instance is None:
            return self
        result = instance.__dict__[self.__name__] = self.fget(instance)
        return result

def crc32_hex(data):
    """Return unsigned CRC32 of binary data as hex-encoded string.

    >>> crc32_hex(b'spam')
    '43daff3d'
    """
    return '%x' % (zlib.crc32(data) & 0xffffffff)
<|endoftext|>


<commit_before>import unittest
import sys
import re
import os
from py65.memory import ObservableMemory

class ObservableMemoryTests(unittest.TestCase):

    # __setitem__

    def test___setitem__with_no_listeners_changes_memory(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        mem[0xC000] = 0xAB
        self.assertEqual(0xAB, subject[0xC000])

    def test___setitem__ignores_listeners_returning_none(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener_1(address, value):
            return None

        def listener_2(address, value):
            return None

        mem.register_listener([0xC000], listener_1)
        mem.register_listener([0xC000], listener_2)

        mem[0xC000] = 0xAB
        self.assertEqual(0xAB, subject[0xC000])

    def test___setitem__uses_result_of_last_listener(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener_1(address, value):
            return 0x01

        def listener_2(address, value):
            return 0x02

        mem.register_listener([0xC000], listener_1)
        mem.register_listener([0xC000], listener_2)

        mem[0xC000] = 0xAB
        self.assertEqual(0x02, subject[0xC000])

    # register_listener

    def test_register_listener_covers_all_addresses_in_range(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener(address, value):
            return 0xAB

        mem.register_listener(range(0xC000, 0xC001+1), listener)

        mem[0xC000] = 0xAB
        mem[0xC001] = 0xAB
        self.assertEqual(0xAB, subject[0xC001])
        self.assertEqual(0xAB, subject[0xC001])

    def test_register_listener_does_not_register_same_listener_twice(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        try:
            self._count = 0

            def listener(address, value):
                self._count += 1

            mem.register_listener([0xC000], listener)
            mem.register_listener([0xC000], listener)

            mem[0xC000] = 0xAB
            self.assertEqual(1, self._count)
        finally:
            delattr(self, '_count')

    # __getitem__

    def test___getitem__with_no_providers_changes_memory(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        subject[0xC000] = 0xAB
        self.assertEqual(0xAB, mem[0xC000])

    def test___getitem__ignores_providers_returning_none(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def provider_1(address):
            return None

        def provider_2(address):
            return None

        mem.register_provider([0xC000], provider_1)
        mem.register_provider([0xC000], provider_2)

        mem[0xC000] = 0xAB
        self.assertEqual(0xAB, subject[0xC000])

    def test___getitem__uses_result_of_last_provider(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def provider_1(address):
            return None

        def provider_2(address):
            return None

        mem.register_provider([0xC000], provider_1)
        mem.register_provider([0xC000], provider_2)

        subject[0xC000] = 0xAB
        self.assertEqual(0xAB, mem[0xC000])

    # __getattr__

    def test__getattr__proxies_subject(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)
        self.assertEqual(subject.count, mem.count)

    # write

    def test_write_directly_writes_values_to_subject(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener(address, value):
            return 0xFF
        mem.register_listener([0xC000,0xC001], listener)

        mem.write(0xC000, [0x01, 002])
        self.assertEqual(0x01, subject[0xC000])
        self.assertEqual(0x02, subject[0xC001])

    # Test Helpers

    def _make_subject(self):
        subject = 0x10000 * [0x00]
        return subject

<commit_msg>Replaces a range by an xrange<commit_after>import unittest
import sys
import re
import os
from py65.memory import ObservableMemory

class ObservableMemoryTests(unittest.TestCase):

    # __setitem__

    def test___setitem__with_no_listeners_changes_memory(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        mem[0xC000] = 0xAB
        self.assertEqual(0xAB, subject[0xC000])

    def test___setitem__ignores_listeners_returning_none(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener_1(address, value):
            return None

        def listener_2(address, value):
            return None

        mem.register_listener([0xC000], listener_1)
        mem.register_listener([0xC000], listener_2)

        mem[0xC000] = 0xAB
        self.assertEqual(0xAB, subject[0xC000])

    def test___setitem__uses_result_of_last_listener(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener_1(address, value):
            return 0x01

        def listener_2(address, value):
            return 0x02

        mem.register_listener([0xC000], listener_1)
        mem.register_listener([0xC000], listener_2)

        mem[0xC000] = 0xAB
        self.assertEqual(0x02, subject[0xC000])

    # register_listener

    def test_register_listener_covers_all_addresses_in_range(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener(address, value):
            return 0xAB

        mem.register_listener(xrange(0xC000, 0xC001+1), listener)

        mem[0xC000] = 0xAB
        mem[0xC001] = 0xAB
        self.assertEqual(0xAB, subject[0xC001])
        self.assertEqual(0xAB, subject[0xC001])

    def test_register_listener_does_not_register_same_listener_twice(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        try:
            self._count = 0

            def listener(address, value):
                self._count += 1

            mem.register_listener([0xC000], listener)
            mem.register_listener([0xC000], listener)

            mem[0xC000] = 0xAB
            self.assertEqual(1, self._count)
        finally:
            delattr(self, '_count')

    # __getitem__

    def test___getitem__with_no_providers_changes_memory(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        subject[0xC000] = 0xAB
        self.assertEqual(0xAB, mem[0xC000])

    def test___getitem__ignores_providers_returning_none(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def provider_1(address):
            return None

        def provider_2(address):
            return None

        mem.register_provider([0xC000], provider_1)
        mem.register_provider([0xC000], provider_2)

        mem[0xC000] = 0xAB
        self.assertEqual(0xAB, subject[0xC000])

    def test___getitem__uses_result_of_last_provider(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def provider_1(address):
            return None

        def provider_2(address):
            return None

        mem.register_provider([0xC000], provider_1)
        mem.register_provider([0xC000], provider_2)

        subject[0xC000] = 0xAB
        self.assertEqual(0xAB, mem[0xC000])

    # __getattr__

    def test__getattr__proxies_subject(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)
        self.assertEqual(subject.count, mem.count)

    # write

    def test_write_directly_writes_values_to_subject(self):
        subject = self._make_subject()
        mem = ObservableMemory(subject=subject)

        def listener(address, value):
            return 0xFF
        mem.register_listener([0xC000,0xC001], listener)

        mem.write(0xC000, [0x01, 002])
        self.assertEqual(0x01, subject[0xC000])
        self.assertEqual(0x02, subject[0xC001])

    # Test Helpers

    def _make_subject(self):
        subject = 0x10000 * [0x00]
        return subject

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Opennamed
    ~~~~~
    :copyright: (c) 2014 by Openname.org
    :license: MIT, see LICENSE for more details.
"""

import argparse
import zerorpc
import daemon
import sys

import config
import logging

log = logging.getLogger()
log.setLevel(logging.DEBUG if config.DEBUG else logging.INFO)
console = logging.StreamHandler()
console.setLevel(logging.DEBUG if config.DEBUG else logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
log.addHandler(console)

from bitcoinrpc.authproxy import AuthServiceProxy

config_options = 'https://' + config.BITCOIND_USER + ':' + \
    config.BITCOIND_PASSWD + '@' + config.BITCOIND_SERVER + ':' + \
    str(config.BITCOIND_PORT)

bitcoind = AuthServiceProxy(config_options)

class OpennamedRPC(object):
    """ opennamed rpc
    """
    def getinfo(self):
        info = bitcoind.getinfo()
        reply = {}
        reply['blocks'] = info['blocks']
        return reply

def run_server():
    """ run the server
    """
    try:
        server = zerorpc.Server(OpennamedRPC())
        server.bind('tcp://' + config.LISTEN_IP + ':' + config.DEFAULT_PORT)
        server.run()
    except Exception as e:
        log.debug(e)
        log.info('Exiting opennamed server')

def run_opennamed():
    """ run opennamed
    """
    parser = argparse.ArgumentParser(
        description='Openname Core Daemon version {}'.format(config.VERSION))

    parser.add_argument(
        '--bitcoind-server',
        help='the hostname or IP address of the bitcoind RPC server')
    parser.add_argument(
        '--bitcoind-port', type=int, help='the bitcoind RPC port to connect to')
    parser.add_argument(
        '--bitcoind-user', help='the username for bitcoind RPC server')
    parser.add_argument(
        '--bitcoind-passwd', help='the password for bitcoind RPC server')

    subparsers = parser.add_subparsers(
        dest='action', help='the action to be taken')

    parser_server = subparsers.add_parser(
        'start', help='start the opennamed server')
    parser_server.add_argument(
        '--foreground', action='store_true',
        help='start the opennamed server in foreground')
    parser_server = subparsers.add_parser(
        'stop', help='stop the opennamed server')

    #print default help message, if no argument is given
    if len(sys.argv)==1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()

    if args.action == 'start':
        if args.foreground:
            log.info('Starting opennamed server in foreground')
            run_server()
        else:
            log.info('Starting opennamed server')
            with daemon.DaemonContext():
                run_server()
    elif args.action == 'stop':
        log.info('Stopping opennamed server')
        #quick hack to kill a background daemon
        import subprocess, signal
        import os
        p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
        out, err = p.communicate()

        for line in out.splitlines():
            if 'opennamed' in line:
                pid = int(line.split(None, 1)[0])
                os.kill(pid, signal.SIGTERM)

if __name__ == '__main__':
    run_opennamed()
<commit_msg>small edits for pep8<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Opennamed
    ~~~~~
    :copyright: (c) 2014 by Openname.org
    :license: MIT, see LICENSE for more details.
"""

import argparse
import zerorpc
import daemon
import sys

import config
import logging

log = logging.getLogger()
log.setLevel(logging.DEBUG if config.DEBUG else logging.INFO)
console = logging.StreamHandler()
console.setLevel(logging.DEBUG if config.DEBUG else logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
log.addHandler(console)

from bitcoinrpc.authproxy import AuthServiceProxy

config_options = 'https://' + config.BITCOIND_USER + ':' + \
    config.BITCOIND_PASSWD + '@' + config.BITCOIND_SERVER + ':' + \
    str(config.BITCOIND_PORT)

bitcoind = AuthServiceProxy(config_options)

class OpennamedRPC(object):
    """ opennamed rpc
    """
    def getinfo(self):
        info = bitcoind.getinfo()
        reply = {}
        reply['blocks'] = info['blocks']
        return reply

def run_server():
    """ run the server
    """
    try:
        server = zerorpc.Server(OpennamedRPC())
        server.bind('tcp://' + config.LISTEN_IP + ':' + config.DEFAULT_PORT)
        server.run()
    except Exception as e:
        log.debug(e)
        log.info('Exiting opennamed server')

def run_opennamed():
    """ run opennamed
    """
    parser = argparse.ArgumentParser(
        description='Openname Core Daemon version {}'.format(config.VERSION))

    parser.add_argument(
        '--bitcoind-server',
        help='the hostname or IP address of the bitcoind RPC server')
    parser.add_argument(
        '--bitcoind-port', type=int,
        help='the bitcoind RPC port to connect to')
    parser.add_argument(
        '--bitcoind-user',
        help='the username for bitcoind RPC server')
    parser.add_argument(
        '--bitcoind-passwd',
        help='the password for bitcoind RPC server')
    subparsers = parser.add_subparsers(
        dest='action', help='the action to be taken')
    parser_server = subparsers.add_parser(
        'start',
        help='start the opennamed server')
    parser_server.add_argument(
        '--foreground', action='store_true',
        help='start the opennamed server in foreground')
    parser_server = subparsers.add_parser(
        'stop',
        help='stop the opennamed server')

    # Print default help message, if no argument is given
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()

    if args.action == 'start':
        if args.foreground:
            log.info('Starting opennamed server in foreground')
            run_server()
        else:
            log.info('Starting opennamed server')
            with daemon.DaemonContext():
                run_server()
    elif args.action == 'stop':
        log.info('Stopping opennamed server')
        # Quick hack to kill a background daemon
        import subprocess
        import signal
        import os
        p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
        out, err = p.communicate()

        for line in out.splitlines():
            if 'opennamed' in line:
                pid = int(line.split(None, 1)[0])
                os.kill(pid, signal.SIGTERM)

if __name__ == '__main__':
    run_opennamed()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2021 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>
import copy
import unittest
import numpy as np
from openquake.hazardlib.imt import SA, PGA
from openquake.hazardlib import const
from openquake.hazardlib.tests.gsim.utils import BaseGSIMTestCase
from openquake.hazardlib.gsim.base import RuptureContext
from openquake.hazardlib.gsim.cauzzi_2014 import CauzziEtAl2014RhypoGermany
from openquake.hazardlib.gsim.derras_2014 import DerrasEtAl2014RhypoGermany
from openquake.hazardlib.gsim.akkar_2014 import AkkarEtAlRhyp2014
from openquake.hazardlib.gsim.bindi_2014 import BindiEtAl2014Rhyp
from openquake.hazardlib.gsim.bindi_2017 import BindiEtAl2017Rhypo

# First Test the Cauzzi et al. (2014) and Derras et al., (2014) adjustments
# using test tables generated from the FRISK implementation

class CauzziEtAl2014RhypTestCase(BaseGSIMTestCase):
    """
    Implements the Cauzzi et al. (2014) Rhypo adjusted GMPE test

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = CauzziEtAl2014RhypoGermany

    MEAN_FILE = "germany_2018/C14_RHYPO_ADJUSTMENT_MEAN.csv"

    def test_mean(self):
        self.check(self.MEAN_FILE, max_discrep_percentage=0.1)

class CauzziEtAl2014RhypFiniteRuptureTestCase(BaseGSIMTestCase):
    """
    Implements the Cauzzi et al. (2014) Rhypo adjusted GMPE test for the case
    of a finite rupture (i.e. should use rrup rather than rhypo)

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = CauzziEtAl2014RhypoGermany

    MEAN_FILE = "germany_2018/C14_RHYPO_FINITE_RUPTURE_MEAN.csv"

    def test_mean(self):
        self.check(self.MEAN_FILE, max_discrep_percentage=0.1)

class DerrasEtAl2014RhypTestCase(CauzziEtAl2014RhypTestCase):
    """
    Implements the Derras et al. (2014) Rhypo adjusted GMPE test

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = DerrasEtAl2014RhypoGermany
    MEAN_FILE = "germany_2018/D14_RHYPO_ADJUSTMENT_MEAN.csv"

class DerrasEtAl2014RhypFiniteRuptureTestCase(CauzziEtAl2014RhypTestCase):
    """
    Implements the Derras et al. (2014) Rhypo adjusted GMPE test for the case
    of a finite rupture (i.e. should use rjb rather than rhypo)

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = DerrasEtAl2014RhypoGermany
    MEAN_FILE = "germany_2018/D14_RHYPO_FINITE_RUPTURE_MEAN.csv"

class GermanyStressParameterAdjustmentTestCase(unittest.TestCase):
    """
    Test suite to verify the stress parameter adjustments for GMPEs adopted
    within the German national seismic hazard model.
    """
    def setUp(self):
        """
        Setup with a set of distances and site paramwters
        """
        self.imts = [PGA(), SA(0.1), SA(0.2), SA(0.5), SA(1.0), SA(2.0)]
        self.mags = [4.5, 5.5, 6.5, 7.5]
        self.rakes = [-90., 0., 90.]
        self.ctx = RuptureContext()
        self.ctx.rhypo = np.array([5., 10., 20., 50., 100.])
        self.ctx.vs30 = 800.0 * np.ones(5)

    def check_gmpe_adjustments(self, adj_gmpe_set, original_gmpe):
        """
        Takes a set of three adjusted GMPEs representing the "low", "middle"
        and "high" stress drop adjustments for Germany and compares them
        against the original "target" GMPE for a variety of magnitudes
        and styles of fauling.
        """
        low_gsim, mid_gsim, high_gsim = adj_gmpe_set
        tot_std = [const.StdDev.TOTAL]
        for imt in self.imts:
            for mag in self.mags:
                for rake in self.rakes:
                    ctx = copy.copy(self.ctx)
                    ctx.mag = mag
                    ctx.rake = rake
                    ctx.hypo_depth = 10.
                    ctx.width = 0.0001
                    # Get "original" values
                    mean = original_gmpe.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    mean = np.exp(mean)
                    # Get "low" adjustments (0.75 times the original)
                    low_mean = low_gsim.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    np.testing.assert_array_almost_equal(
                        np.exp(low_mean) / mean, 0.75 * np.ones_like(low_mean))

                    # Get "middle" adjustments (1.25 times the original)
                    mid_mean = mid_gsim.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    np.testing.assert_array_almost_equal(
                        np.exp(mid_mean) / mean, 1.25 * np.ones_like(mid_mean))

                    # Get "high" adjustments (1.5 times the original)
                    high_mean = high_gsim.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    np.testing.assert_array_almost_equal(
                        np.exp(high_mean) / mean,
                        1.5 * np.ones_like(high_mean))

    def test_akkar_germany_adjustments(self):
        adj_gmpes = [AkkarEtAlRhyp2014(adjustment_factor=0.75),
                     AkkarEtAlRhyp2014(adjustment_factor=1.25),
                     AkkarEtAlRhyp2014(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, AkkarEtAlRhyp2014())

    def test_bindi_2014_germany_adjustments(self):
        adj_gmpes = [BindiEtAl2014Rhyp(adjustment_factor=0.75),
                     BindiEtAl2014Rhyp(adjustment_factor=1.25),
                     BindiEtAl2014Rhyp(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, BindiEtAl2014Rhyp())

    def test_cauzzi_germany_adjustments(self):
        adj_gmpes = [CauzziEtAl2014RhypoGermany(adjustment_factor=0.75),
                     CauzziEtAl2014RhypoGermany(adjustment_factor=1.25),
                     CauzziEtAl2014RhypoGermany(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, CauzziEtAl2014RhypoGermany())

    def test_derras_germany_adjustments(self):
        adj_gmpes = [DerrasEtAl2014RhypoGermany(adjustment_factor=0.75),
                     DerrasEtAl2014RhypoGermany(adjustment_factor=1.25),
                     DerrasEtAl2014RhypoGermany(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, DerrasEtAl2014RhypoGermany())

    def test_bindi_2017_germany_adjustments(self):
        adj_gmpes = [BindiEtAl2017Rhypo(adjustment_factor=0.75),
                     BindiEtAl2017Rhypo(adjustment_factor=1.25),
                     BindiEtAl2017Rhypo(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, BindiEtAl2017Rhypo())
<commit_msg>Fixed test<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2021 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>
import copy
import unittest
import numpy as np
from openquake.hazardlib.imt import SA, PGA
from openquake.hazardlib import const
from openquake.hazardlib.tests.gsim.utils import BaseGSIMTestCase
from openquake.hazardlib.gsim.base import RuptureContext
from openquake.hazardlib.gsim.cauzzi_2014 import CauzziEtAl2014RhypoGermany
from openquake.hazardlib.gsim.derras_2014 import DerrasEtAl2014RhypoGermany
from openquake.hazardlib.gsim.akkar_2014 import AkkarEtAlRhyp2014
from openquake.hazardlib.gsim.bindi_2014 import BindiEtAl2014Rhyp
from openquake.hazardlib.gsim.bindi_2017 import BindiEtAl2017Rhypo

# First Test the Cauzzi et al. (2014) and Derras et al., (2014) adjustments
# using test tables generated from the FRISK implementation

class CauzziEtAl2014RhypTestCase(BaseGSIMTestCase):
    """
    Implements the Cauzzi et al. (2014) Rhypo adjusted GMPE test

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = CauzziEtAl2014RhypoGermany

    MEAN_FILE = "germany_2018/C14_RHYPO_ADJUSTMENT_MEAN.csv"

    def test_mean(self):
        self.check(self.MEAN_FILE, max_discrep_percentage=0.1)

class CauzziEtAl2014RhypFiniteRuptureTestCase(BaseGSIMTestCase):
    """
    Implements the Cauzzi et al. (2014) Rhypo adjusted GMPE test for the case
    of a finite rupture (i.e. should use rrup rather than rhypo)

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = CauzziEtAl2014RhypoGermany

    MEAN_FILE = "germany_2018/C14_RHYPO_FINITE_RUPTURE_MEAN.csv"

    def test_mean(self):
        self.check(self.MEAN_FILE, max_discrep_percentage=0.1)

class DerrasEtAl2014RhypTestCase(CauzziEtAl2014RhypTestCase):
    """
    Implements the Derras et al. (2014) Rhypo adjusted GMPE test

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = DerrasEtAl2014RhypoGermany
    MEAN_FILE = "germany_2018/D14_RHYPO_ADJUSTMENT_MEAN.csv"

class DerrasEtAl2014RhypFiniteRuptureTestCase(CauzziEtAl2014RhypTestCase):
    """
    Implements the Derras et al. (2014) Rhypo adjusted GMPE test for the case
    of a finite rupture (i.e. should use rjb rather than rhypo)

    Test tables for corrected GMPE verified against original FRISK version
    """
    GSIM_CLASS = DerrasEtAl2014RhypoGermany
    MEAN_FILE = "germany_2018/D14_RHYPO_FINITE_RUPTURE_MEAN.csv"

class GermanyStressParameterAdjustmentTestCase(unittest.TestCase):
    """
    Test suite to verify the stress parameter adjustments for GMPEs adopted
    within the German national seismic hazard model.
    """
    def setUp(self):
        """
        Setup with a set of distances and site paramwters
        """
        self.imts = [PGA(), SA(0.1), SA(0.2), SA(0.5), SA(1.0), SA(2.0)]
        self.mags = [4.5, 5.5, 6.5, 7.5]
        self.rakes = [-90., 0., 90.]
        self.ctx = RuptureContext()
        self.ctx.sids = np.arange(5)
        self.ctx.rhypo = np.array([5., 10., 20., 50., 100.])
        self.ctx.vs30 = 800.0 * np.ones(5)

    def check_gmpe_adjustments(self, adj_gmpe_set, original_gmpe):
        """
        Takes a set of three adjusted GMPEs representing the "low", "middle"
        and "high" stress drop adjustments for Germany and compares them
        against the original "target" GMPE for a variety of magnitudes
        and styles of fauling.
        """
        low_gsim, mid_gsim, high_gsim = adj_gmpe_set
        tot_std = [const.StdDev.TOTAL]
        for imt in self.imts:
            for mag in self.mags:
                for rake in self.rakes:
                    ctx = copy.copy(self.ctx)
                    ctx.mag = mag
                    ctx.rake = rake
                    ctx.hypo_depth = 10.
                    ctx.width = 0.0001
                    # Get "original" values
                    mean = original_gmpe.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    mean = np.exp(mean)
                    # Get "low" adjustments (0.75 times the original)
                    low_mean = low_gsim.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    np.testing.assert_array_almost_equal(
                        np.exp(low_mean) / mean, 0.75 * np.ones_like(low_mean))

                    # Get "middle" adjustments (1.25 times the original)
                    mid_mean = mid_gsim.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    np.testing.assert_array_almost_equal(
                        np.exp(mid_mean) / mean, 1.25 * np.ones_like(mid_mean))

                    # Get "high" adjustments (1.5 times the original)
                    high_mean = high_gsim.get_mean_and_stddevs(
                        ctx, ctx, ctx, imt, tot_std)[0]
                    np.testing.assert_array_almost_equal(
                        np.exp(high_mean) / mean,
                        1.5 * np.ones_like(high_mean))

    def test_akkar_germany_adjustments(self):
        adj_gmpes = [AkkarEtAlRhyp2014(adjustment_factor=0.75),
                     AkkarEtAlRhyp2014(adjustment_factor=1.25),
                     AkkarEtAlRhyp2014(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, AkkarEtAlRhyp2014())

    def test_bindi_2014_germany_adjustments(self):
        adj_gmpes = [BindiEtAl2014Rhyp(adjustment_factor=0.75),
                     BindiEtAl2014Rhyp(adjustment_factor=1.25),
                     BindiEtAl2014Rhyp(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, BindiEtAl2014Rhyp())

    def test_cauzzi_germany_adjustments(self):
        adj_gmpes = [CauzziEtAl2014RhypoGermany(adjustment_factor=0.75),
                     CauzziEtAl2014RhypoGermany(adjustment_factor=1.25),
                     CauzziEtAl2014RhypoGermany(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, CauzziEtAl2014RhypoGermany())

    def test_derras_germany_adjustments(self):
        adj_gmpes = [DerrasEtAl2014RhypoGermany(adjustment_factor=0.75),
                     DerrasEtAl2014RhypoGermany(adjustment_factor=1.25),
                     DerrasEtAl2014RhypoGermany(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, DerrasEtAl2014RhypoGermany())

    def test_bindi_2017_germany_adjustments(self):
        adj_gmpes = [BindiEtAl2017Rhypo(adjustment_factor=0.75),
                     BindiEtAl2017Rhypo(adjustment_factor=1.25),
                     BindiEtAl2017Rhypo(adjustment_factor=1.50)]
        self.check_gmpe_adjustments(adj_gmpes, BindiEtAl2017Rhypo())
<|endoftext|>


<commit_before>from thrift.transport.TSocket import TSocket
from satori.ars.thrift import ThriftReader, ThriftClient
from satori.ars import ars2py

transport = TSocket(host='localhost', port=38889)
client = ThriftClient(transport)
client.start(bootstrap=True)
ars2py.process(client.contracts)

globals().update(ars2py.classes)

def start_console():
    import code
    import readline
    console = code.InteractiveConsole()
    console.runcode('import satori.client.common as satori_classes')
    print
    print 'satori.client.common is imported as satori_classes'
    print
    console.interact()

<commit_msg>Changes towards separate per-thread connections.<commit_after>from thrift.transport.TSocket import TSocket
from satori.ars.thrift import ThriftReader, ThriftClient
from satori.ars import ars2py
from threading import local

transport = TSocket(host='localhost', port=38889)
client = ThriftClient(transport)
client.start(bootstrap=True)

globals().update(ars2py.process(client.contracts))

#class Classes(local):
#    def __init__(self, host, port):
#        self._port = _port
#        self._host = _host
#
#        self._transport = None
#        self._client = None

def start_console():
    import code
    import readline
    console = code.InteractiveConsole()
    console.runcode('import satori.client.common as satori_classes')
    print
    print 'satori.client.common is imported as satori_classes'
    print
    console.interact()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
1. Pre-process the English corpus:
read the ".conll" format mixed pipeline source file (the english version)
create two files:
a) Original raw english corpus line by line
b) POS-tagged and lemmatized english

2. Pre-process the Swedish:

3. Create the final parallel file
read lines of both swedish and english lemmatized and POS-tagged file
write them in fast-align format <line_en> ||| <line_sv>

"""
__author__ = 'mehdi'

import sys, os

# CONLL word format:
ID, FORM, LEMMA, PLEMMA, POS, PPOS, FEAT, PFEAT, HEAD, PHEAD, DEPREL, PDEPREL = range(12)

# Swedish lemmatized format:
SV_POS, SV_TOKEN = range(2)

if __name__ == "__main__":

    if len(sys.argv) < 3:
        sys.exit('Please set english conll file and swedish lemmatized file: %s <en_conll_file> <sv_lemmatized_file>' % sys.argv[0])

    en_file_path = os.path.normpath(sys.argv[1])
    sv_file_path = os.path.normpath(sys.argv[2])

    en_file_path, en_file_name = os.path.split(en_file_path)
    sv_file_path, sv_file_name = os.path.split(sv_file_path)

    # read form source file:
    # open the English CONLL file, extract raw sentences,
    en_file = open(os.path.join(en_file_path, en_file_name))

    # sentence-by-sentence output corpus, ready to be aligned with parallel corpora:
    out_en_file = open(os.path.join(en_file_path, en_file_name + ".out.txt"), "w")

    # raw word forms corpus. (human readable)
    out_en_raw_file = open(os.path.join(en_file_path, en_file_name + ".out.raw.txt"), "w")
    sentence = []
    raw = ""
    lemma = ""

    # read from source file:
    for line in en_file:
        # each line contain a CONLL formatted word.
        # a new line means the previous sentence has ended.
        if line[0] == "\n":
            # join all words in the sentence
            out_en_file.write(" ".join(sentence) + "\n")
            out_en_raw_file.write(raw.strip() + "\n")
            sentence = []
            raw = ""
            lemma = ""
        else:
            word = line.split("\t")
            if word[LEMMA] not in [".", ",", "!", "?", ":", ";",
                                   "'", "''", '"', '""', "(", ")",
                                   "{", "}", "/", "`", "``"] and \
                word[PLEMMA] not in [":"] and \
                word[PFEAT] != 'punct':
                sentence.append(word[LEMMA] + ".." + word[PLEMMA])
                raw += " " + word[FORM]
                lemma += " " + word[LEMMA]

    # save unsaved files, and close opened file.
    out_en_raw_file.close()
    out_en_file.close()
    en_file.close()

    # open the sentence-by-sentence source file
    en_file = open(os.path.join(en_file_path, en_file_name + ".out.txt"))

    # Swedish source file
    sv_file = open(os.path.join(sv_file_path, sv_file_name))

    # The sentence-by-sentence Swedish output file:
    out_sv_file = open(os.path.join(sv_file_path, sv_file_name + ".out.txt"), "w")

    # The paralleled sentences (ready for fast-align):
    out_file = open(os.path.join(en_file_path, en_file_name + "__" + sv_file_name + ".out.parallel.txt"), "w")

    # re-initialize variables:
    sentence = []

    # read from the Swedish file:
    for line in sv_file:
        # a new line means the previous sentence has ended.
        if line[0] == "\n":
            # glue all the swedish words in one sentence and write them on the file.
            swedish = " ".join(sentence)
            out_sv_file.write(swedish + "\n")

            # read the english sentence :
            english = en_file.readline().strip()

            # if both sentences are available then write them in parallel file:
            if len(swedish) > 1 and len(english) > 1:
                out_file.write("%s ||| %s\n" % (english, swedish))

            sentence = []
        else:
            # split the line with <tab>-separator (Swedish formatted file)
            word = line.split("\t")
            if word[SV_POS] not in ['MID', 'MAD', 'PAD']:
                sentence.append(word[SV_TOKEN].strip())

    out_file.close()
    out_sv_file.close()
    sv_file.close()
    en_file.close()

<commit_msg>CD, ..nl => #NUMBER#<commit_after># -*- coding: utf-8 -*-
"""
1. Pre-process the English corpus:
read the ".conll" format mixed pipeline source file (the english version)
create two files:
a) Original raw english corpus line by line
b) POS-tagged and lemmatized english

2. Pre-process the Swedish:

3. Create the final parallel file
read lines of both swedish and english lemmatized and POS-tagged file
write them in fast-align format <line_en> ||| <line_sv>

"""
__author__ = 'mehdi'

import sys, os

# CONLL word format:
ID, FORM, LEMMA, PLEMMA, POS, PPOS, FEAT, PFEAT, HEAD, PHEAD, DEPREL, PDEPREL = range(12)

# Swedish lemmatized format:
SV_POS, SV_TOKEN = range(2)

if __name__ == "__main__":

    if len(sys.argv) < 3:
        sys.exit('Please set english conll file and swedish lemmatized file: %s <en_conll_file> <sv_lemmatized_file>' % sys.argv[0])

    en_file_path = os.path.normpath(sys.argv[1])
    sv_file_path = os.path.normpath(sys.argv[2])

    en_file_path, en_file_name = os.path.split(en_file_path)
    sv_file_path, sv_file_name = os.path.split(sv_file_path)

    # read form source file:
    # open the English CONLL file, extract raw sentences,
    en_file = open(os.path.join(en_file_path, en_file_name))

    # sentence-by-sentence output corpus, ready to be aligned with parallel corpora:
    out_en_file = open(os.path.join(en_file_path, en_file_name + ".out.txt"), "w")

    # raw word forms corpus. (human readable)
    out_en_raw_file = open(os.path.join(en_file_path, en_file_name + ".out.raw.txt"), "w")
    sentence = []
    raw = ""

    # read from source file:
    for line in en_file:
        # each line contain a CONLL formatted word.
        # a new line means the previous sentence has ended.
        if line[0] == "\n":
            # join all words in the sentence
            out_en_file.write(" ".join(sentence) + "\n")
            out_en_raw_file.write(raw.strip() + "\n")
            sentence = []
            raw = ""
            lemma = ""
        else:
            word = line.split("\t")
            if word[LEMMA] not in [".", ",", "!", "?", ":", ";",
                                   "'", "''", '"', '""', "(", ")",
                                   "{", "}", "/", "`", "``"] and \
                word[PLEMMA] not in [":"] and \
                word[PFEAT] != 'punct':

                if word[PLEMMA] == "CD":
                    sentence.append("#NUMBER#")
                    raw += " #NUMBER#"
                else:
                    sentence.append(word[LEMMA] + ".." + word[PLEMMA])
                    raw += " " + word[FORM]

    # save unsaved files, and close opened file.
    out_en_raw_file.close()
    out_en_file.close()
    en_file.close()

    # open the sentence-by-sentence source file
    en_file = open(os.path.join(en_file_path, en_file_name + ".out.txt"))

    # Swedish source file
    sv_file = open(os.path.join(sv_file_path, sv_file_name))

    # The sentence-by-sentence Swedish output file:
    out_sv_file = open(os.path.join(sv_file_path, sv_file_name + ".out.txt"), "w")

    # The paralleled sentences (ready for fast-align):
    out_file = open(os.path.join(en_file_path, en_file_name + "__" + sv_file_name + ".out.parallel.txt"), "w")

    # re-initialize variables:
    sentence = []

    # read from the Swedish file:
    for line in sv_file:
        # a new line means the previous sentence has ended.
        if line[0] == "\n":
            # glue all the swedish words in one sentence and write them on the file.
            swedish = " ".join(sentence)
            out_sv_file.write(swedish + "\n")

            # read the english sentence :
            english = en_file.readline().strip()

            # if both sentences are available then write them in parallel file:
            # if len(swedish) > 1 and len(english) > 1:
            #     out_file.write("%s ||| %s\n" % (english, swedish))

            out_file.write("%s ||| %s\n" % (english, swedish))

            sentence = []
        else:
            # split the line with <tab>-separator (Swedish formatted file)
            word = line.split("\t")
            if word[SV_POS] not in ['MID', 'MAD', 'PAD']:
                if word[SV_TOKEN][-4:] == "..nl":
                    sentence.append("#NUMBER#")
                else:
                    sentence.append(word[SV_TOKEN].strip())

    out_file.close()
    out_sv_file.close()
    sv_file.close()
    en_file.close()

<|endoftext|>


<commit_before>from elasticsearch_api import ElasticsearchAPI
from source_manager import SourceManager
from stat_calculator import StatCalculator
from stoppable_worker import ClosableQueue, StoppableWorker, TimedWorker, LoggingWorker

class StatManager:
    """
    Uses source_manager to keep track of all the sources.
    Uses stat calculator to calculate deltas from the data_rows it got from the sources

    pushes delta to elasticsearch

    creates the timestamp which all the delta_rows will use.
    """

    def __init__(self, config_manager,
                 source_manager_cls=SourceManager,
                 elastic_api_cls=ElasticsearchAPI,
                 stat_calculator=StatCalculator):

        collector_config = config_manager.get_config('perf-collector')
        print('Starting collection, polling interval %s seconds' % collector_config['polling-interval-seconds'])

        delta_queue = ClosableQueue()
        elastic_queue = ClosableQueue()
        result_queue = ClosableQueue()

        source_manager = source_manager_cls(delta_queue)
        source_manager.load_sources(config_manager)

        elastic_api = elastic_api_cls.from_config_manager(config_manager)
        elastic_api.init_indexes_for(source_manager.sources)

        self.source_thread = TimedWorker(
                source_manager.process_all_sources,
                collector_config.get_int('polling-interval-seconds'))
        self.delta_worker = StoppableWorker(stat_calculator.calculate_collection_delta, delta_queue, elastic_queue)
        self.elastic_worker = StoppableWorker(elastic_api.consume_collection, elastic_queue, result_queue)
        self.log_completed_worker = LoggingWorker(result_queue.get, result_queue)

        self.threads = [
            self.source_thread, self.delta_worker, self.elastic_worker, self.log_completed_worker
        ]

    def run(self):
        for thread in self.threads:
            thread.start()

        self.source_thread.join()
        # TODO: can we stop gracefully?<commit_msg>updated comment<commit_after>from elasticsearch_api import ElasticsearchAPI
from source_manager import SourceManager
from stat_calculator import StatCalculator
from stoppable_worker import ClosableQueue, StoppableWorker, TimedWorker, LoggingWorker

class StatManager:
    """
    Coordinates the threads and queues used to get data from the sources and consume them in elastic search.

    """

    def __init__(self, config_manager,
                 source_manager_cls=SourceManager,
                 elastic_api_cls=ElasticsearchAPI,
                 stat_calculator=StatCalculator):

        collector_config = config_manager.get_config('perf-collector')
        print('Starting collection, polling interval %s seconds' % collector_config['polling-interval-seconds'])

        delta_queue = ClosableQueue()
        elastic_queue = ClosableQueue()
        result_queue = ClosableQueue()

        source_manager = source_manager_cls(delta_queue)
        source_manager.load_sources(config_manager)

        elastic_api = elastic_api_cls.from_config_manager(config_manager)
        elastic_api.init_indexes_for(source_manager.sources)

        self.source_thread = TimedWorker(
                source_manager.process_all_sources,
                collector_config.get_int('polling-interval-seconds'))
        self.delta_worker = StoppableWorker(stat_calculator.calculate_collection_delta, delta_queue, elastic_queue)
        self.elastic_worker = StoppableWorker(elastic_api.consume_collection, elastic_queue, result_queue)
        self.log_completed_worker = LoggingWorker(result_queue.get, result_queue)

        self.threads = [
            self.source_thread, self.delta_worker, self.elastic_worker, self.log_completed_worker
        ]

    def run(self):
        for thread in self.threads:
            thread.start()

        self.source_thread.join()
        # TODO: can we stop gracefully?<|endoftext|>


<commit_before>import xml.etree.ElementTree as ET
import urllib2
import logging
import re
from copy import deepcopy

VOURL = "https://myosg.grid.iu.edu/vosummary/xml?summary_attrs_showfield_of_science=on&all_vos=on&active=on&active_value=1&oasis_value=1&sort_key=name"

class OIMVOInfo(object):
    """
    Class to obtain and store VO info from OIM
    """
    # Put any non-nested element before the last item in this list.  The last
    # item is a dict that holds all nested elements and the structure
    wanted_attributes = ['LongName', 'Name',        # These two are examples - we don't actually need them
                         {'FieldsOfScience':        # We actually want these
                              {'PrimaryFields': 'Field',
                               'SecondaryFields': 'Field'
                               }
                          }
                         ]
    vodict = {}

    def __init__(self):
        self.xmlfile = self.__get_oim_info()
        if self.xmlfile:
            self.__get_vo_info()

    def __get_oim_info(self):
        """
        Download VO info from OIM
        :return file oim_xml: File-like object with OIM VO info in XML format
        """
        try:
            oim_xml = urllib2.urlopen(VOURL)
            logging.info('Downloaded OIM VO info')
        except (urllib2.HTTPError, urllib2.URLError) as e:
            logging.error(e)
            return None
        return oim_xml

    def __get_vo_info(self):
        """
        Parse XML file and store VO Information we want in vodict

        :return: None
        """
        pathattr = re.compile('^.//(\w+)$')     # Top level elements
        pathattr_rec = re.compile('^.//.+/(\w+)/.+$')   # Nested Elements

        root = ET.parse(self.xmlfile)
        root = root.getroot()

        for voelt in root:
            VOName = voelt.find('Name').text.lower()
            if VOName not in self.vodict:
                self.vodict[VOName] = {}
            cur_dict = self.vodict[VOName]

            # Top level xpaths
            xpaths = ['.//' + attr for attr in self.wanted_attributes[:-1]]

            # Nested xpaths
            def recurse_attrs(cur_level, cur_list=[], final_list=[]):
                if not isinstance(cur_level, dict):   # We're at end of nesting
                    cur_list.append(cur_level)
                    final_list.append(cur_list)     # Add the list to the final output
                else:       # Process current level, go down to next level
                    for key in cur_level:
                        nowlist = deepcopy(cur_list)
                        nowlist.append(key)
                        recurse_attrs(cur_level[key], nowlist, final_list)
                return final_list

            xpath_list_raw = recurse_attrs(self.wanted_attributes[-1])

            xpaths.extend(['.//' + '/'.join(pathlist)
                           for pathlist in xpath_list_raw])

            # Look for info in XML file
            for path in xpaths:
                # Grab whichever regex returns a match
                m = filter(None, (pat.match(path) for pat in (pathattr, pathattr_rec)))
                assert len(m) == 1  # One or other should be a match, not both

                if m:
                    key = m[0].group(1)
                else:
                    continue    # Skip this key
                lst = [elt.text for elt in voelt.findall(path)] # Search XML file using xpath

                if len(lst) > 1:
                    cur_dict[key] = lst  # Add to dict
                elif len(lst) == 1:
                    cur_dict[key] = lst[0]

        logging.info('Constructed OIM VO dictionary')
        return

    def parse_doc(self, doc):
        """
        Parse corrected raw non-Payload doc, write VO FOS into the
        OIM_FieldofScience field.

        :param dict doc: GRACC Record
        :return dict doc:  GRACC record with OIM_FOS either added or corrected
        """
        if 'ResourceType' in doc and doc['ResourceType'] != 'Payload':
            if 'VOName' in doc and doc['VOName'].lower() in self.vodict:
                newfos = self.vodict[doc['VOName'].lower()]['PrimaryFields']
                doc['OIM_FieldOfScience'] = newfos
                logging.info('Set FOS to {0}'.format(newfos))
        return doc
<commit_msg>Added check for PrimaryFields in vodict<commit_after>import xml.etree.ElementTree as ET
import urllib2
import logging
import re
from copy import deepcopy

VOURL = "https://myosg.grid.iu.edu/vosummary/xml?summary_attrs_showfield_of_science=on&all_vos=on&active=on&active_value=1&oasis_value=1&sort_key=name"

class OIMVOInfo(object):
    """
    Class to obtain and store VO info from OIM
    """
    # Put any non-nested element before the last item in this list.  The last
    # item is a dict that holds all nested elements and the structure
    wanted_attributes = ['LongName', 'Name',        # These two are examples - we don't actually need them
                         {'FieldsOfScience':        # We actually want these
                              {'PrimaryFields': 'Field',
                               'SecondaryFields': 'Field'
                               }
                          }
                         ]
    vodict = {}

    def __init__(self):
        self.xmlfile = self.__get_oim_info()
        if self.xmlfile:
            self.__get_vo_info()

    def __get_oim_info(self):
        """
        Download VO info from OIM
        :return file oim_xml: File-like object with OIM VO info in XML format
        """
        try:
            oim_xml = urllib2.urlopen(VOURL)
            logging.info('Downloaded OIM VO info')
        except (urllib2.HTTPError, urllib2.URLError) as e:
            logging.error(e)
            return None
        return oim_xml

    def __get_vo_info(self):
        """
        Parse XML file and store VO Information we want in vodict

        :return: None
        """
        pathattr = re.compile('^.//(\w+)$')     # Top level elements
        pathattr_rec = re.compile('^.//.+/(\w+)/.+$')   # Nested Elements

        root = ET.parse(self.xmlfile)
        root = root.getroot()

        for voelt in root:
            VOName = voelt.find('Name').text.lower()
            if VOName not in self.vodict:
                self.vodict[VOName] = {}
            cur_dict = self.vodict[VOName]

            # Top level xpaths
            xpaths = ['.//' + attr for attr in self.wanted_attributes[:-1]]

            # Nested xpaths
            def recurse_attrs(cur_level, cur_list=[], final_list=[]):
                if not isinstance(cur_level, dict):   # We're at end of nesting
                    cur_list.append(cur_level)
                    final_list.append(cur_list)     # Add the list to the final output
                else:       # Process current level, go down to next level
                    for key in cur_level:
                        nowlist = deepcopy(cur_list)
                        nowlist.append(key)
                        recurse_attrs(cur_level[key], nowlist, final_list)
                return final_list

            xpath_list_raw = recurse_attrs(self.wanted_attributes[-1])

            xpaths.extend(['.//' + '/'.join(pathlist)
                           for pathlist in xpath_list_raw])

            # Look for info in XML file
            for path in xpaths:
                # Grab whichever regex returns a match
                m = filter(None, (pat.match(path) for pat in (pathattr, pathattr_rec)))
                assert len(m) == 1  # One or other should be a match, not both

                if m:
                    key = m[0].group(1)
                else:
                    continue    # Skip this key
                lst = [elt.text for elt in voelt.findall(path)] # Search XML file using xpath

                if len(lst) > 1:
                    cur_dict[key] = lst  # Add to dict
                elif len(lst) == 1:
                    cur_dict[key] = lst[0]

        logging.info('Constructed OIM VO dictionary')
        return

    def parse_doc(self, doc):
        """
        Parse corrected raw non-Payload doc, write VO FOS into the
        OIM_FieldofScience field.

        :param dict doc: GRACC Record
        :return dict doc:  GRACC record with OIM_FOS either added or corrected
        """
        if 'ResourceType' in doc and doc['ResourceType'] != 'Payload':
            if 'VOName' in doc and doc['VOName'].lower() in self.vodict:
                curvodict = self.vodict[doc['VOName'].lower()]
                if 'PrimaryFields' in curvodict:
                    newfos = curvodict['PrimaryFields']
                doc['OIM_FieldOfScience'] = newfos
                logging.info('Set FOS to {0}'.format(newfos))
        return doc

if __name__ == '__main__':
    v = OIMVOInfo()
    testdoc = {
            "OIM_Organization": "Georgia Institute of Technology",
            "ResourceType": "Batch",
            "VOName": "osg",
            "RawProjectName": "VERITAS",
            "OIM_Department": "School of Physics & Center for Relativistic Astrophysics",
            "ReportableVOName": "osg",
            "RawVOName": "/osg/LocalGroup=users",
            "ProjectName": "VERITAS",
            "OIM_FieldOfScience": "Astrophysics",
        }
    print v.parse_doc(testdoc)
<|endoftext|>