<commit_before>import logging
import re

from django import forms
from django.utils.translation import ugettext_lazy as _
from sentry.plugins import Plugin
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute

import sentry_restricted_github

logger = logging.getLogger('sentry.plugin.' + __name__)

class Form(forms.Form):
    # TODO: validate repo
    proj_url = forms.CharField(
        label=_('Project URL'),
        widget=forms.TextInput(attrs={'placeholder': 'e.g. github.com/stripe/sentry-restricted-github'}),
        help_text=_('Enter your project url.'))

class RestrictedGithubPlugin(Plugin):
    title = _('Restricted Github')
    slug = 'restricted_github'
    description = "Create Github tickets."
    version = sentry_restricted_github.VERSION

    author = 'Stripe'
    author_url = 'https://github.com/stripe/sentry-restricted-github'

    conf_title = title
    conf_key = 'restricted_github'
    project_conf_form = Form

    def is_configured(self, project, **kwargs):
        return bool(self.get_option('proj_url', project))

    def widget(self, request, group, **kwargs):
        event = group.get_latest_event()
        return self.render('sentry_restricted_github/widget.html', {
            'is_configured': self.is_configured(group.project),
            'proj_url': self._get_github_url(group),
            'title': self._get_group_title(request, group, event),
            'desc': self._get_group_description(request, group, event),
        })

    def _get_group_body(self, request, group, event, **kwargs):
        interface = (event.interfaces.get('exception') or
                     event.interfaces.get('sentry.interfaces.Exception'))
        if interface:
            return safe_execute(interface.to_string, event)
        return

    def _get_group_description(self, request, group, event):
        output = [
            absolute_uri(group.get_absolute_url()),
        ]
        body = self._get_group_body(request, group, event)
        if body:
            output.extend([
                '',
                '```',
                body,
                '```',
            ])
        return '\n'.join(output)

    def _get_group_title(self, request, group, event):
        return event.error()

    def _get_github_url(self, group):
        url = self.get_option('proj_url', group.project)
        if url is None:
            return None
        match = re.match('github.com[/:]([-\w]+/[-\w]+)', url)
        if match:
            return 'https://github.com/%s' % match.groups()[0]
        else:
            return None
<commit_msg>Add interface that it can get stacktraces from<commit_after>import logging
import re

from django import forms
from django.utils.translation import ugettext_lazy as _
from sentry.plugins import Plugin
from sentry.utils.http import absolute_uri
from sentry.utils.safe import safe_execute

import sentry_restricted_github

logger = logging.getLogger('sentry.plugin.' + __name__)

class Form(forms.Form):
    # TODO: validate repo
    proj_url = forms.CharField(
        label=_('Project URL'),
        widget=forms.TextInput(attrs={'placeholder': 'e.g. github.com/stripe/sentry-restricted-github'}),
        help_text=_('Enter your project url.'))

class RestrictedGithubPlugin(Plugin):
    title = _('Restricted Github')
    slug = 'restricted_github'
    description = "Create Github tickets."
    version = sentry_restricted_github.VERSION

    author = 'Stripe'
    author_url = 'https://github.com/stripe/sentry-restricted-github'

    conf_title = title
    conf_key = 'restricted_github'
    project_conf_form = Form

    def is_configured(self, project, **kwargs):
        return bool(self.get_option('proj_url', project))

    def widget(self, request, group, **kwargs):
        event = group.get_latest_event()
        return self.render('sentry_restricted_github/widget.html', {
            'is_configured': self.is_configured(group.project),
            'proj_url': self._get_github_url(group),
            'title': self._get_group_title(request, group, event),
            'desc': self._get_group_description(request, group, event),
        })

    def _get_group_body(self, request, group, event, **kwargs):
        interface = (event.interfaces.get('exception') or
                     event.interfaces.get('sentry.interfaces.Exception') or
                     event.interfaces.get('sentry.interfaces.Stacktrace'))
        if interface:
            return safe_execute(interface.to_string, event)
        return

    def _get_group_description(self, request, group, event):
        output = [
            absolute_uri(group.get_absolute_url()),
        ]
        body = self._get_group_body(request, group, event)
        if body:
            output.extend([
                '',
                '```',
                body,
                '```',
            ])
        return '\n'.join(output)

    def _get_group_title(self, request, group, event):
        return event.error()

    def _get_github_url(self, group):
        url = self.get_option('proj_url', group.project)
        if url is None:
            return None
        match = re.match('github.com[/:]([-\w]+/[-\w]+)', url)
        if match:
            return 'https://github.com/%s' % match.groups()[0]
        else:
            return None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import pandas as pd
from sqlalchemy import and_, extract
from datetime import datetime

from flask import (Blueprint, render_template)
from flask.ext import restful
from flask.ext.restful import fields
from flask.ext.restful.reqparse import RequestParser

from .models import Revenue
from gastosabertos.extensions import db

parser = RequestParser()
parser.add_argument('page', type=int, default=0)
parser.add_argument('per_page_num', type=int, default=100)
parser.add_argument('years', type=int, action='append')

receita = Blueprint('receita', __name__,
                    template_folder='templates',
                    static_folder='static',
                    static_url_path='/receita/static')

receita_api = restful.Api(receita, prefix="/api/v1")

class Date(fields.Raw):
    def format(self, value):
        return str(value)

revenue_fields = { 'id': fields.Integer()
                 , 'date': Date
                 , 'description': fields.String()
                 , 'code': fields.String()
                 , 'monthly_predicted': fields.Float()
                 , 'monthly_outcome': fields.Float() }

class RevenueApi(restful.Resource):

    @restful.marshal_with(revenue_fields)
    def get(self):
        # Extract the argumnets in GET request
        args = parser.parse_args()
        page = args['page']
        per_page_num = args['per_page_num']
        years = args['years']

        # Create the query
        revenue_data = db.session.query(Revenue)

        if years:
            # Filter years
            if len(years) == 2:
                year_min = "{}-1-1".format(years[0])
                year_max = "{}-12-31".format(years[1])
                revenue_data = revenue_data.filter(and_(Revenue.date >= datetime.strptime(year_min, '%Y-%m-%d')),
                                                        Revenue.date <= datetime.strptime(year_max, '%Y-%m-%d'))
            elif len(years) == 1:
                year = str(years[0])
                revenue_data = revenue_data.filter(extract('year', Revenue.date) == year)

        # Limit que number of results per page
        revenue_data = revenue_data.offset(page*per_page_num)\
                            .limit(per_page_num)\

        return revenue_data.all()

receita_api.add_resource(RevenueApi, '/receita/hello')

# csv_receita = os.path.join(receita.root_path, 'static', 'receita-2008-01.csv')
# df_receita = pd.read_csv(csv_receita, encoding='utf8')

years = range(2008, 2015)

def get_year_data(year):
    # TODO correct real year, an not month 01
    csv_receita = os.path.join(
        receita.root_path,
        'static',
        'receita-%s-01.csv' %
        year)
    return pd.read_csv(csv_receita, encoding='utf8').iterrows()

@receita.route('/receita/<int:year>')
@receita.route('/receita/<int:year>/<int:page>')
def receita_table(year, page=0):
    receita_data = get_year_data(year)
    return render_template(
        'fulltable.html',
        receita_data=receita_data,
        years=years)

@receita.route('/sankey/<path:filename>')
def sankey(filename):
    return send_from_directory('sankey', filename)
<commit_msg>Add Grouped Revenue endpoint API<commit_after># -*- coding: utf-8 -*-

import os
import pandas as pd
from sqlalchemy import and_, extract, func
from datetime import datetime

from flask import (Blueprint, render_template)
from flask.ext import restful
from flask.ext.restful import fields
from flask.ext.restful.reqparse import RequestParser

from .models import Revenue
from gastosabertos.extensions import db

# Blueprint for Receita
receita = Blueprint('receita', __name__,
                    template_folder='templates',
                    static_folder='static',
                    static_url_path='/receita/static')

# Create the restful API
receita_api = restful.Api(receita, prefix="/api/v1")

class Date(fields.Raw):
    def format(self, value):
        return str(value)

# Parser for RevenueAPI arguments
revenue_list_parser = RequestParser()
revenue_list_parser.add_argument('page', type=int, default=0)
revenue_list_parser.add_argument('per_page_num', type=int, default=100)
revenue_list_parser.add_argument('years', type=int, action='append')

# Fields for RevenueAPI data marshal
revenue_fields = { 'id': fields.Integer()
                 , 'date': Date
                 , 'description': fields.String()
                 , 'code': fields.String()
                 , 'monthly_predicted': fields.Float()
                 , 'monthly_outcome': fields.Float() }

class RevenueApi(restful.Resource):

    @restful.marshal_with(revenue_fields)
    def get(self):
        # Extract the argumnets in GET request
        args = revenue_list_parser.parse_args()
        page = args['page']
        per_page_num = args['per_page_num']
        years = args['years']

        # Create the query
        revenue_data = db.session.query(Revenue)

        if years:
            # Filter years
            if len(years) == 2:
                year_min = "{}-1-1".format(years[0])
                year_max = "{}-12-31".format(years[1])
                revenue_data = revenue_data.filter(and_(Revenue.date >= datetime.strptime(year_min, '%Y-%m-%d')),
                                                        Revenue.date <= datetime.strptime(year_max, '%Y-%m-%d'))
            elif len(years) == 1:
                year = str(years[0])
                revenue_data = revenue_data.filter(extract('year', Revenue.date) == year)

        # Limit que number of results per page
        revenue_data = revenue_data.offset(page*per_page_num)\
                            .limit(per_page_num)\

        return revenue_data.all()

# Revenues levels Dict for GroupedRevenueApi
revenue_levels = {}
revenue_levels[0] = Revenue.economical_category
revenue_levels[1] = Revenue.economical_subcategory

# Parser for GroupedRevenueApi
grouped_revenue_parser = RequestParser()
grouped_revenue_parser.add_argument('level', type=int, default=0)
grouped_revenue_parser.add_argument('years', type=int, action='append')

# Fields for GroupedRevenueApi data marshal
grouped_fields = { 'category_code': fields.String
                 , 'Total Predicted': fields.Float()
                 , 'Total Outcome': fields.Float() }

class GroupedRevenueApi(restful.Resource):

    def get(self):
        args = grouped_revenue_parser.parse_args()
        years = args['years']
        level = args['level']

        # Create the query
        revenue_query_base = db.session.query(Revenue.economical_category,
                                              func.sum(Revenue.monthly_predicted).label('Total predicted'),
                                              func.sum(Revenue.monthly_outcome).label('Total outcome'))

        if not years:
            revenue_data = revenue_query_base.group_by(revenue_levels[level])
            revenue_grouped = [{'category_code': rev[0], 'total pref': str(rev[1]), 'total outcome': str(rev[2])} for rev in revenue_data.all()]
            return revenue_grouped

        revenue_grouped = {}
        for year in years:
            year = str(year)
            revenue_data = revenue_query_base.filter(extract('year', Revenue.date) == year)\
                                       .group_by(revenue_levels[level])

            revenue_grouped[year] = [{'category_code': rev[0], 'total pref': str(rev[1]), 'total outcome': str(rev[2])} for rev in revenue_data.all()]

        return revenue_grouped

receita_api.add_resource(RevenueApi, '/receita/list')
receita_api.add_resource(GroupedRevenueApi, '/receita/grouped')

# csv_receita = os.path.join(receita.root_path, 'static', 'receita-2008-01.csv')
# df_receita = pd.read_csv(csv_receita, encoding='utf8')

years = range(2008, 2015)

def get_year_data(year):
    # TODO correct real year, an not month 01
    csv_receita = os.path.join(
        receita.root_path,
        'static',
        'receita-%s-01.csv' %
        year)
    return pd.read_csv(csv_receita, encoding='utf8').iterrows()

@receita.route('/receita/<int:year>')
@receita.route('/receita/<int:year>/<int:page>')
def receita_table(year, page=0):
    receita_data = get_year_data(year)
    return render_template(
        'fulltable.html',
        receita_data=receita_data,
        years=years)

@receita.route('/sankey/<path:filename>')
def sankey(filename):
    return send_from_directory('sankey', filename)
<|endoftext|>


<commit_before>'''
module containing constants used by qc reporting tools
'''
# GENERIC

LIMIT_ALL_JSON = '&limit=all&format=json'

# Only download needed fields.
EXPERIMENT_FIELDS_QUERY = (
    '&field=@id'
    '&field=accession'
    '&field=status'
    '&field=award.rfa'
    '&field=date_created'
    '&field=target.name'
    '&field=biosample_term_name'
    '&field=biosample_type'
    '&field=replication_type'
    '&field=lab.name'
)

FILE_FIELDS_QUERY = (
    '&field=@id'
    '&field=accession'
    '&field=date_created'
    '&field=status'
    '&field=dataset'
    '&field=assembly'
    '&field=step_run'
    '&field=quality_metrics'
    '&field=notes'
)

# HISTONE SPECIFIC

HISTONE_PEAK_FILES_QUERY = (
    '/search/?type=File'
    '&output_type=replicated+peaks'
    '&output_type=stable+peaks'
    '&lab.title=ENCODE+Processing+Pipeline'
    '&file_format=bed'
    '&status=released'
    '&status=in+progress'
    '&status=uploading'
)

HISTONE_CHIP_EXPERIMENTS_QUERY = (
    '/search/?type=Experiment'
    '&assay_title=ChIP-seq'
    '&target.investigated_as=histone'
    '&award.project=ENCODE'
    '&status=released'
    '&status=in+progress'
    '&status=submitted'
)

HISTONE_QC_FIELDS = [
    'nreads',
    'nreads_in_peaks',
    'npeak_overlap',
    'Fp',
    'Ft',
    'F1',
    'F2',
    'quality_metric_of'
]

# RNA SPECIFIC

RNA_EXPERIMENTS_QUERY = (
    '/search/?type=Experiment'
    '&award.project=ENCODE'
    '&award.project=Roadmap'
    '&assay_slims=Transcription'
    '&assay_title=polyA+RNA-seq'
    '&assay_title=total+RNA-seq'
    '&assay_title=small+RNA-seq'
    '&assay_title=single+cell+RNA-seq'
    '&assay_title=microRNA-seq'
    '&assay_title=CRISPRi+RNA-seq'
    '&assay_title=CRISPR+RNA-seq'
    '&assay_title=polyA+depleted+RNA-seq'
    '&status=released'
    '&status=in+progress'
    '&status=submitted'
)

RNA_QUANTIFICATION_FILES_QUERY = (
    '/search/?type=File'
    '&lab.title=ENCODE+Processing+Pipeline'
    '&output_type=gene+quantifications'
    '&status=released'
    '&file_format=tsv'
    '&status=released'
    '&status=in+progress'
    '&status=uploading'
)

RNA_MAPPING_FILES_QUERY = (
    '/search/?type=File'
    '&lab.title=ENCODE+Processing+Pipeline'
    '&output_type=alignments'
    '&output_type=transcriptome+alignments'
    '&file_format=bam'
    '&status=released'
    '&status=in+progress'
    '&status=uploading'
)
<commit_msg>Remove duplicate status<commit_after>'''
module containing constants used by qc reporting tools
'''
# GENERIC

LIMIT_ALL_JSON = '&limit=all&format=json'

# Only download needed fields.
EXPERIMENT_FIELDS_QUERY = (
    '&field=@id'
    '&field=accession'
    '&field=status'
    '&field=award.rfa'
    '&field=date_created'
    '&field=target.name'
    '&field=biosample_term_name'
    '&field=biosample_type'
    '&field=replication_type'
    '&field=lab.name'
)

FILE_FIELDS_QUERY = (
    '&field=@id'
    '&field=accession'
    '&field=date_created'
    '&field=status'
    '&field=dataset'
    '&field=assembly'
    '&field=step_run'
    '&field=quality_metrics'
    '&field=notes'
)

# HISTONE SPECIFIC

HISTONE_PEAK_FILES_QUERY = (
    '/search/?type=File'
    '&output_type=replicated+peaks'
    '&output_type=stable+peaks'
    '&lab.title=ENCODE+Processing+Pipeline'
    '&file_format=bed'
    '&status=released'
    '&status=in+progress'
    '&status=uploading'
)

HISTONE_CHIP_EXPERIMENTS_QUERY = (
    '/search/?type=Experiment'
    '&assay_title=ChIP-seq'
    '&target.investigated_as=histone'
    '&award.project=ENCODE'
    '&status=released'
    '&status=in+progress'
    '&status=submitted'
)

HISTONE_QC_FIELDS = [
    'nreads',
    'nreads_in_peaks',
    'npeak_overlap',
    'Fp',
    'Ft',
    'F1',
    'F2',
    'quality_metric_of'
]

# RNA SPECIFIC

RNA_EXPERIMENTS_QUERY = (
    '/search/?type=Experiment'
    '&award.project=ENCODE'
    '&award.project=Roadmap'
    '&assay_slims=Transcription'
    '&assay_title=polyA+RNA-seq'
    '&assay_title=total+RNA-seq'
    '&assay_title=small+RNA-seq'
    '&assay_title=single+cell+RNA-seq'
    '&assay_title=microRNA-seq'
    '&assay_title=CRISPRi+RNA-seq'
    '&assay_title=CRISPR+RNA-seq'
    '&assay_title=polyA+depleted+RNA-seq'
    '&status=released'
    '&status=in+progress'
    '&status=submitted'
)

RNA_QUANTIFICATION_FILES_QUERY = (
    '/search/?type=File'
    '&lab.title=ENCODE+Processing+Pipeline'
    '&output_type=gene+quantifications'
    '&file_format=tsv'
    '&status=released'
    '&status=in+progress'
    '&status=uploading'
)

RNA_MAPPING_FILES_QUERY = (
    '/search/?type=File'
    '&lab.title=ENCODE+Processing+Pipeline'
    '&output_type=alignments'
    '&output_type=transcriptome+alignments'
    '&file_format=bam'
    '&status=released'
    '&status=in+progress'
    '&status=uploading'
)
<|endoftext|>


<commit_before>import fileinput
import time

file_path = '/media/ABB4-4F3A/DATALOG.TXT'

def split_in_blocks(txt_file, pattern):
	'''
	Find the last appears of the text that indicate a new flight and divide in the number of blocks generated by the rocket

	Return: A list that contains all the different blocks of data.
	'''
	num_times_find_pattern = []
	for num_line, line in enumerate(fileinput.input(txt_file)):
		if pattern in line:
			num_times_find_pattern.append(num_line)
	blocks_of_data = []
	with open(txt_file) as f:
		lines = f.readlines()
		for num_header_line in num_times_find_pattern:
			if num_header_line == 0:
				num_header_line_prev = num_header_line
			else:
				block_lines = lines[num_header_line_prev + 1 : num_header_line - 1]
				blocks_of_data.append(block_lines)
				num_header_line_prev = num_header_line
		block_lines = lines[num_header_line_prev + 1 : num_line + 1]
		blocks_of_data.append(block_lines)
	return blocks_of_data

def	find_millis_average(concrete_blocks_dict):
	for position, millis in enumerate(concrete_blocks_dict):
		millis_interval_list = []
		if position != 0:
			millis_interval = millis - millis_prev
			#print millis_interval
			millis_interval_list.append(millis_interval)
		millis_prev = millis
	millis_average = sum(millis_interval_list) / position
	millis_max = max(millis_interval_list)
	millis_min = min(millis_interval_list)
	print "Interval: ",millis_interval
	print "MAX: ",millis_max
	print "MIN: ",millis_min

def manage_data_from_blocks(blocks):
	blocks_dict = {}
	for block_number, block in enumerate(blocks):
		blocks_dict['millis_%s' % block_number] = []
		for line in block:
			line_list = line.strip().split(",")
			#print line_list
			timestamp_list = int(line_list[0])
			#print timestamp_list
			blocks_dict['millis_%s' % block_number].append(timestamp_list)
	for key in blocks_dict:
		find_millis_average(blocks_dict[key])

	#print blocks_dict

#start = time.time()
blocks = split_in_blocks(file_path, "millis")
manage_data_from_blocks(blocks)
#stop = time.time()
#total_time = stop -start
#print total_time<commit_msg>Added way to pass from dict to list and recognize the real initial order<commit_after>import fileinput
import time

file_path = '/media/ABB4-4F3A/DATALOG.TXT'

def split_in_blocks(txt_file, pattern):
	'''
	Find the last appears of the text that indicate a new flight and divide in the number of blocks generated by the rocket

	Return: A list that contains all the different blocks of data.
	'''
	num_times_find_pattern = []
	for num_line, line in enumerate(fileinput.input(txt_file)):
		if pattern in line:
			num_times_find_pattern.append(num_line)
	blocks_of_data = []
	with open(txt_file) as f:
		lines = f.readlines()
		for num_header_line in num_times_find_pattern:
			if num_header_line == 0:
				num_header_line_prev = num_header_line
			else:
				block_lines = lines[num_header_line_prev + 1 : num_header_line - 1]
				blocks_of_data.append(block_lines)
				num_header_line_prev = num_header_line
		block_lines = lines[num_header_line_prev + 1 : num_line + 1]
		blocks_of_data.append(block_lines)
	return blocks_of_data

def process_millis(concrete_blocks_dict):
	for position, millis in enumerate(concrete_blocks_dict):
		millis_interval_list = []
		if position != 0:
			millis_interval = millis - millis_prev
			#print millis_interval
			millis_interval_list.append(millis_interval)
		millis_prev = millis
	millis_average = sum(millis_interval_list) / position
	millis_max = max(millis_interval_list)
	millis_min = min(millis_interval_list)
	print "Interval: ",millis_interval
	print "MAX: ",millis_max
	print "MIN: ",millis_min

def manage_data_from_blocks(blocks):
	blocks_dict = {}
	for block_number, block in enumerate(blocks):
		blocks_dict['millis_%s' % block_number] = []
		for line in block:
			line_list = line.strip().split(",")
			#print line_list
			timestamp_list = int(line_list[0])
			#print timestamp_list
			blocks_dict['millis_%s' % block_number].append(timestamp_list)
	for key in blocks_dict:
		process_millis(blocks_dict[key])

	#print blocks_dict

#start = time.time()
blocks = split_in_blocks(file_path, "millis")
manage_data_from_blocks(blocks)
#stop = time.time()
#total_time = stop -start
#print total_time<|endoftext|>


<commit_before># This file is part of the GOsa framework.
#
#  http://gosa-project.org
#
# Copyright:
#  (C) 2016 GONICUS GmbH, Germany, http://www.gonicus.de
#
# See the LICENSE file in the project's top-level directory for details.

import unittest
import pytest
from gosa.backend.plugins.samba.domain import *

class SambaGuiMethodsTestCase(unittest.TestCase):

    @unittest.mock.patch.object(Environment, "getInstance")
    @unittest.mock.patch.object(PluginRegistry, 'getInstance')
    def test_getSambaPassword(self, mockedRegistry, mockedEnv):

        # mockup ACL resolver
        mockedRegistry.return_value.check.return_value = True

        # mockup the environment
        mockedEnv.return_value.domain = "testdomain"

        with unittest.mock.patch('gosa.backend.plugins.samba.domain.ObjectProxy', autoSpec=True, create=True) as m:
            # run the test
            user = m.return_value
            methods = SambaGuiMethods()
            methods.setSambaPassword("username", "dn", "password")
            assert user.sambaNTPassword is not None
            assert user.sambaLMPassword is not None
            assert user.commit.called is True
            assert m.called is True

        # test with ACL.check for sambaNTPassword is False
        mockedRegistry.return_value.check.return_value = False

        with unittest.mock.patch('gosa.backend.plugins.samba.domain.ObjectProxy', create=True):
            # run the test
            methods = SambaGuiMethods()
            with pytest.raises(ACLException):
                methods.setSambaPassword("username", "dn", "password")

        # test with ACL.check for sambaLMPassword is False
        def check(user, topic, flags, base):
            return not topic == "testdomain.objects.User.attributes.sambaLMPassword"
        mockedRegistry.return_value.check.side_effect = check

        with unittest.mock.patch('gosa.backend.plugins.samba.domain.ObjectProxy', create=True):
            # run the test
            methods = SambaGuiMethods()
            with pytest.raises(ACLException):
                methods.setSambaPassword("username", "dn", "password")

    @unittest.mock.patch.object(PluginRegistry, 'getInstance')
    def test_getSambaDomainInformation(self, mockedInstance):
        # mock the whole lookup in the ObjectIndex to return True
        mockedInstance.return_value.search.return_value = unittest.mock.MagicMock()

        methods = SambaGuiMethods()
        target = unittest.mock.MagicMock()
        res = methods.getSambaDomainInformation("username", target)
        # this is just a check that the method is callable so we do not really check the output here
        assert len(res) > 0

@unittest.mock.patch.object(PluginRegistry, 'getInstance')
def test_IsValidSambaDomainName(mockedInstance):
    # mock the whole lookup in the ObjectIndex to return True
    mockedInstance.return_value.search.return_value = [1]

    check = IsValidSambaDomainName(None)

    (res, errors) = check.process(None, None, ["test"])
    assert res is True
    assert len(errors) == 0

    # mockup everything to return False
    mockedInstance.return_value.search.return_value = []

    (res, errors) = check.process(None, None, ["test"])
    assert res is False
    assert len(errors) == 1
<commit_msg>some test data for domain information test added<commit_after># This file is part of the GOsa framework.
#
#  http://gosa-project.org
#
# Copyright:
#  (C) 2016 GONICUS GmbH, Germany, http://www.gonicus.de
#
# See the LICENSE file in the project's top-level directory for details.

from unittest import TestCase, mock
import pytest
from gosa.backend.plugins.samba.domain import *
from tests.GosaTestCase import slow, gosaContext

class SambaGuiMethodsTestCase(TestCase):

    @mock.patch.object(Environment, "getInstance")
    @mock.patch.object(PluginRegistry, 'getInstance')
    def test_getSambaPassword(self, mockedRegistry, mockedEnv):

        # mockup ACL resolver
        mockedRegistry.return_value.check.return_value = True

        # mockup the environment
        mockedEnv.return_value.domain = "testdomain"

        with mock.patch('gosa.backend.plugins.samba.domain.ObjectProxy', autoSpec=True, create=True) as m:
            # run the test
            user = m.return_value
            methods = SambaGuiMethods()
            methods.setSambaPassword("username", "dn", "password")
            assert user.sambaNTPassword is not None
            assert user.sambaLMPassword is not None
            assert user.commit.called is True
            assert m.called is True

        # test with ACL.check for sambaNTPassword is False
        mockedRegistry.return_value.check.return_value = False

        with mock.patch('gosa.backend.plugins.samba.domain.ObjectProxy', create=True):
            # run the test
            methods = SambaGuiMethods()
            with pytest.raises(ACLException):
                methods.setSambaPassword("username", "dn", "password")

        # test with ACL.check for sambaLMPassword is False
        def check(user, topic, flags, base):
            return not topic == "testdomain.objects.User.attributes.sambaLMPassword"
        mockedRegistry.return_value.check.side_effect = check

        with mock.patch('gosa.backend.plugins.samba.domain.ObjectProxy', create=True):
            # run the test
            methods = SambaGuiMethods()
            with pytest.raises(ACLException):
                methods.setSambaPassword("username", "dn", "password")

    @mock.patch.object(PluginRegistry, 'getInstance')
    def test_getSambaDomainInformation(self, mockedInstance):
        # mock the whole lookup in the ObjectIndex to return True
        mockedInstance.return_value.search.return_value = [{"sambaMinPwdLength": 6,
                                                            "sambaPwdHistoryLength": 10,
                                                            "sambaMaxPwdAge": 10,
                                                            "sambaMinPwdAge": 1,
                                                            "sambaLockoutDuration": 60,
                                                            "sambaRefuseMachinePwdChange": False,
                                                            "sambaLogonToChgPwd": True,
                                                            "sambaLockoutThreshold": 30,
                                                            "sambaBadPasswordTime": 2147483647}]

        methods = SambaGuiMethods()
        target = mock.MagicMock()
        target.sambaDomainName = 'DEFAULT'
        res = methods.getSambaDomainInformation("username", target)
        # this is just a check that the method is callable so we do not really check the output here
        assert len(res) > 0

@mock.patch.object(PluginRegistry, 'getInstance')
def test_IsValidSambaDomainName(mockedInstance):
    # mock the whole lookup in the ObjectIndex to return True
    mockedInstance.return_value.search.return_value = [1]

    check = IsValidSambaDomainName(None)

    (res, errors) = check.process(None, None, ["test"])
    assert res is True
    assert len(errors) == 0

    # mockup everything to return False
    mockedInstance.return_value.search.return_value = []

    (res, errors) = check.process(None, None, ["test"])
    assert res is False
    assert len(errors) == 1
<|endoftext|>


<commit_before>import imaplib
import unittest2 as unittest
from mock import Mock, patch

class ImapWorkerTest(unittest.TestCase):
    def _worker_with_fake_imap(self):
        from tinymail.imap_worker import ImapWorker, ConnectionErrorWrapper
        worker = ImapWorker()
        imap_conn = Mock(spec=imaplib.IMAP4)
        worker.conn = ConnectionErrorWrapper(imap_conn)
        return worker, imap_conn

    @patch('tinymail.imap_worker.imaplib')
    def test_connect(self, mock_imaplib):
        mock_conn = Mock(spec=imaplib.IMAP4)
        mock_imaplib.IMAP4_SSL.return_value = mock_conn
        mock_conn.login.return_value = ('OK', [])

        from tinymail.imap_worker import ImapWorker
        worker = ImapWorker()
        worker.connect('test_host', 'test_login', 'test_pass')

        mock_imaplib.IMAP4_SSL.assert_called_once_with('test_host')
        mock_conn.login.assert_called_once_with('test_login', 'test_pass')

    def test_get_mailbox_names(self):
        worker, imap_conn = self._worker_with_fake_imap()
        imap_conn.list.return_value = ('OK', [
            r'(\HasNoChildren) "." INBOX',
            r'(\HasNoChildren) "." "fol1"',
            r'(\HasChildren) "." "fol2"',
            r'(\HasNoChildren) "." "fol2.sub"',
        ])

        names = worker.get_mailbox_names()

        self.assertEqual(sorted(names), ['INBOX', 'fol1', 'fol2', 'fol2.sub'])

    def test_crash_on_non_ascii_folder(self):
        from tinymail.imap_worker import ImapWorkerError
        worker, imap_conn = self._worker_with_fake_imap()
        imap_conn.list.return_value = ('OK', ['(\\HasNoChildren) "." "f\xb3"'])

        msg = "Non-ascii mailbox names not supported"
        self.assertRaisesRegexp(ImapWorkerError, msg, worker.get_mailbox_names)

    def test_get_messages_in_folder(self):
        worker, imap_conn = self._worker_with_fake_imap()
        imap_conn.select.return_value = ('OK', [])
        imap_conn.status.return_value = ('OK', [
            '"fol1" (MESSAGES 3 RECENT 1 UIDNEXT 14 '
                    'UIDVALIDITY 1300189203 UNSEEN 1)'])
        imap_conn.fetch.return_value = ('OK', [
            '1 (UID 6 FLAGS (\\Seen))',
            '2 (UID 8 FLAGS (\\Answered \\Seen))',
            '3 (UID 13 FLAGS ())',
        ])

        mbox_status, message_data = worker.get_messages_in_folder('fol1')

        imap_conn.select.assert_called_once_with('fol1', readonly=True)
        imap_conn.status.assert_called_once_with(
            'fol1', '(MESSAGES UIDNEXT UIDVALIDITY)')
        imap_conn.fetch.assert_called_once_with('1:*', '(UID FLAGS)')

        self.assertEqual(mbox_status, {
            'UIDNEXT': 14,
            'UIDVALIDITY': 1300189203,
            'MESSAGES': 3,
            'UNSEEN': 1,
            'RECENT': 1,
        })
        self.assertEqual(message_data, {
            6: {'index': 1, 'flags': set([r'\Seen'])},
            8: {'index': 2, 'flags': set([r'\Answered', r'\Seen'])},
            13: {'index': 3, 'flags': set()},
        })

    def test_open_mailbox_for_writing(self):
        worker, imap_conn = self._worker_with_fake_imap()
        imap_conn.select.return_value = ('OK', [])
        imap_conn.status.return_value = ('OK', [
            '"fol1" (MESSAGES 1 RECENT 1 UIDNEXT 14 '
                    'UIDVALIDITY 1300189203 UNSEEN 0)'])
        imap_conn.fetch.return_value = ('OK', ['1 (UID 6 FLAGS (\\Seen))'])

        worker.get_messages_in_folder('fol1', readonly=False)

        imap_conn.select.assert_called_once_with('fol1', readonly=False)

    def test_get_message_headers(self):
        worker, imap_conn = self._worker_with_fake_imap()
        worker.message_index = {31: 1, 32: 2, 35: 5}
        hdr = ('From: somebody@example.com\r\n'
               'To: somebody_else@example.com\r\n'
               'Subject: One test message!\r\n'
               '\r\n')
        imap_conn.fetch.return_value = ('OK', [
            ('1 (BODY[HEADER] {%s} FLAGS (\\Seen)' % len(hdr), hdr), ')',
            ('2 (BODY[HEADER] {%s} FLAGS (\\Seen)' % len(hdr), hdr), ')',
            ('5 (BODY[HEADER] {%s} FLAGS (\\Seen)' % len(hdr), hdr), ')',
        ])

        header_by_index = worker.get_message_headers([31, 32, 35])

        imap_conn.fetch.assert_called_once_with('1,2,5',
                                                '(FLAGS BODY.PEEK[HEADER])')
        self.assertEqual(header_by_index, {31: hdr, 32: hdr, 35: hdr})

    def test_get_message_body(self):
        worker, imap_conn = self._worker_with_fake_imap()
        worker.message_index = {22: 5}
        response_data = [('5 (RFC822 {7}', 'ZE BODY'), ')']
        imap_conn.fetch.return_value = ('OK', response_data)

        message_body = worker.get_message_body(22)

        imap_conn.fetch.assert_called_once_with('5', '(RFC822)')
        self.assertEqual(message_body, 'ZE BODY')

    def test_add_flag(self):
        worker, imap_conn = self._worker_with_fake_imap()
        worker.message_index = {31: 1, 32: 2, 35: 5}
        imap_conn.store.return_value = ('OK', [])

        worker.change_flag([31, 32, 35], 'add', '\\Seen')

        imap_conn.store.assert_called_once_with('1,2,5', '+FLAGS', '\\Seen')

    def test_remove_flag(self):
        worker, imap_conn = self._worker_with_fake_imap()
        worker.message_index = {31: 1, 32: 2, 35: 5}
        imap_conn.store.return_value = ('OK', [])

        worker.change_flag([31, 32, 35], 'del', '\\Flagged')

        imap_conn.store.assert_called_once_with('1,2,5', '-FLAGS', '\\Flagged')

    def test_close(self):
        worker, imap_conn = self._worker_with_fake_imap()
        imap_conn.close.return_value = ('OK', [])

        worker.close_mailbox()

        imap_conn.close.assert_called_once_with()
<commit_msg>factor out creation of worker and mock IMAP<commit_after>import imaplib
import unittest2 as unittest
from mock import Mock, patch

def worker_with_fake_imap():
    from tinymail.imap_worker import ImapWorker, ConnectionErrorWrapper
    worker = ImapWorker()
    imap_conn = Mock(spec=imaplib.IMAP4)
    worker.conn = ConnectionErrorWrapper(imap_conn)
    return worker, imap_conn

class ImapWorkerTest(unittest.TestCase):
    @patch('tinymail.imap_worker.imaplib')
    def test_connect(self, mock_imaplib):
        mock_conn = Mock(spec=imaplib.IMAP4)
        mock_imaplib.IMAP4_SSL.return_value = mock_conn
        mock_conn.login.return_value = ('OK', [])

        from tinymail.imap_worker import ImapWorker
        worker = ImapWorker()
        worker.connect('test_host', 'test_login', 'test_pass')

        mock_imaplib.IMAP4_SSL.assert_called_once_with('test_host')
        mock_conn.login.assert_called_once_with('test_login', 'test_pass')

    def test_get_mailbox_names(self):
        worker, imap_conn = worker_with_fake_imap()
        imap_conn.list.return_value = ('OK', [
            r'(\HasNoChildren) "." INBOX',
            r'(\HasNoChildren) "." "fol1"',
            r'(\HasChildren) "." "fol2"',
            r'(\HasNoChildren) "." "fol2.sub"',
        ])

        names = worker.get_mailbox_names()

        self.assertEqual(sorted(names), ['INBOX', 'fol1', 'fol2', 'fol2.sub'])

    def test_crash_on_non_ascii_folder(self):
        from tinymail.imap_worker import ImapWorkerError
        worker, imap_conn = worker_with_fake_imap()
        imap_conn.list.return_value = ('OK', ['(\\HasNoChildren) "." "f\xb3"'])

        msg = "Non-ascii mailbox names not supported"
        self.assertRaisesRegexp(ImapWorkerError, msg, worker.get_mailbox_names)

    def test_get_messages_in_folder(self):
        worker, imap_conn = worker_with_fake_imap()
        imap_conn.select.return_value = ('OK', [])
        imap_conn.status.return_value = ('OK', [
            '"fol1" (MESSAGES 3 RECENT 1 UIDNEXT 14 '
                    'UIDVALIDITY 1300189203 UNSEEN 1)'])
        imap_conn.fetch.return_value = ('OK', [
            '1 (UID 6 FLAGS (\\Seen))',
            '2 (UID 8 FLAGS (\\Answered \\Seen))',
            '3 (UID 13 FLAGS ())',
        ])

        mbox_status, message_data = worker.get_messages_in_folder('fol1')

        imap_conn.select.assert_called_once_with('fol1', readonly=True)
        imap_conn.status.assert_called_once_with(
            'fol1', '(MESSAGES UIDNEXT UIDVALIDITY)')
        imap_conn.fetch.assert_called_once_with('1:*', '(UID FLAGS)')

        self.assertEqual(mbox_status, {
            'UIDNEXT': 14,
            'UIDVALIDITY': 1300189203,
            'MESSAGES': 3,
            'UNSEEN': 1,
            'RECENT': 1,
        })
        self.assertEqual(message_data, {
            6: {'index': 1, 'flags': set([r'\Seen'])},
            8: {'index': 2, 'flags': set([r'\Answered', r'\Seen'])},
            13: {'index': 3, 'flags': set()},
        })

    def test_open_mailbox_for_writing(self):
        worker, imap_conn = worker_with_fake_imap()
        imap_conn.select.return_value = ('OK', [])
        imap_conn.status.return_value = ('OK', [
            '"fol1" (MESSAGES 1 RECENT 1 UIDNEXT 14 '
                    'UIDVALIDITY 1300189203 UNSEEN 0)'])
        imap_conn.fetch.return_value = ('OK', ['1 (UID 6 FLAGS (\\Seen))'])

        worker.get_messages_in_folder('fol1', readonly=False)

        imap_conn.select.assert_called_once_with('fol1', readonly=False)

    def test_get_message_headers(self):
        worker, imap_conn = worker_with_fake_imap()
        worker.message_index = {31: 1, 32: 2, 35: 5}
        hdr = ('From: somebody@example.com\r\n'
               'To: somebody_else@example.com\r\n'
               'Subject: One test message!\r\n'
               '\r\n')
        imap_conn.fetch.return_value = ('OK', [
            ('1 (BODY[HEADER] {%s} FLAGS (\\Seen)' % len(hdr), hdr), ')',
            ('2 (BODY[HEADER] {%s} FLAGS (\\Seen)' % len(hdr), hdr), ')',
            ('5 (BODY[HEADER] {%s} FLAGS (\\Seen)' % len(hdr), hdr), ')',
        ])

        header_by_index = worker.get_message_headers([31, 32, 35])

        imap_conn.fetch.assert_called_once_with('1,2,5',
                                                '(FLAGS BODY.PEEK[HEADER])')
        self.assertEqual(header_by_index, {31: hdr, 32: hdr, 35: hdr})

    def test_get_message_body(self):
        worker, imap_conn = worker_with_fake_imap()
        worker.message_index = {22: 5}
        response_data = [('5 (RFC822 {7}', 'ZE BODY'), ')']
        imap_conn.fetch.return_value = ('OK', response_data)

        message_body = worker.get_message_body(22)

        imap_conn.fetch.assert_called_once_with('5', '(RFC822)')
        self.assertEqual(message_body, 'ZE BODY')

    def test_add_flag(self):
        worker, imap_conn = worker_with_fake_imap()
        worker.message_index = {31: 1, 32: 2, 35: 5}
        imap_conn.store.return_value = ('OK', [])

        worker.change_flag([31, 32, 35], 'add', '\\Seen')

        imap_conn.store.assert_called_once_with('1,2,5', '+FLAGS', '\\Seen')

    def test_remove_flag(self):
        worker, imap_conn = worker_with_fake_imap()
        worker.message_index = {31: 1, 32: 2, 35: 5}
        imap_conn.store.return_value = ('OK', [])

        worker.change_flag([31, 32, 35], 'del', '\\Flagged')

        imap_conn.store.assert_called_once_with('1,2,5', '-FLAGS', '\\Flagged')

    def test_close(self):
        worker, imap_conn = worker_with_fake_imap()
        imap_conn.close.return_value = ('OK', [])

        worker.close_mailbox()

        imap_conn.close.assert_called_once_with()
<|endoftext|>


<commit_before># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from contextlib import contextmanager
import logging
import multiprocessing
import os
import subprocess
import time
import math
from threading import Thread
from threading import Lock, Condition

# Python 3 compatibility imports
from six.moves.queue import Empty, Queue
from six.moves import xrange
from six import iteritems

import toil
from toil.batchSystems.abstractBatchSystem import BatchSystemSupport, InsufficientSystemResources

log = logging.getLogger(__name__)

class SingleMachineBatchSystem(BatchSystemSupport):
    """
    The interface for running jobs on a single machine, runs all the jobs you give it as they
    come in, but in parallel.
    """

    @classmethod
    def supportsHotDeployment(cls):
        return False

    @classmethod
    def supportsWorkerCleanup(cls):
        return True

    numCores = multiprocessing.cpu_count()

    minCores = 0.1
    """
    The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
    value. One important invariant of this class is that each worker thread represents a CPU
    requirement of minCores, meaning that we can never run more than numCores / minCores jobs
    concurrently.
    """
    physicalMemory = toil.physicalMemory()

    def __init__(self, config, maxCores, maxMemory, maxDisk):
        if maxCores > self.numCores:
            log.warn('Limiting maxCores to CPU count of system (%i).', self.numCores)
            maxCores = self.numCores
        if maxMemory > self.physicalMemory:
            log.warn('Limiting maxMemory to physically available memory (%i).', self.physicalMemory)
            maxMemory = self.physicalMemory
        self.physicalDisk = toil.physicalDisk(config)
        if maxDisk > self.physicalDisk:
            log.warn('Limiting maxDisk to physically available disk (%i).', self.physicalDisk)
            maxDisk = self.physicalDisk
        super(SingleMachineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
        assert self.maxCores >= self.minCores
        assert self.maxMemory >= 1

        # The scale allows the user to apply a factor to each task's cores requirement, thereby
        # squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
        # (scale > 1).
        self.scale = config.scale
        # Number of worker threads that will be started
        self.numWorkers = int(self.maxCores / self.minCores)
        # A counter to generate job IDs and a lock to guard it
        self.jobIndex = 0
        self.jobIndexLock = Lock()
        # A dictionary mapping IDs of submitted jobs to the command line
        self.jobs = {}
        """
        :type: dict[str,toil.job.JobNode]
        """
        # A queue of jobs waiting to be executed. Consumed by the workers.
        self.inputQueue = Queue()
        # A queue of finished jobs. Produced by the workers.
        self.outputQueue = Queue()
        # A dictionary mapping IDs of currently running jobs to their Info objects
        self.runningJobs = {}
        """
        :type: dict[str,Info]
        """
        # The list of worker threads
        self.workerThreads = []
        """
        :type list[Thread]
        """
        # Variables involved with non-blocking resource acquisition
        self.acquisitionTimeout = 5
        self.acquisitionRetryDelay = 10
        self.aquisitionCondition = Condition()

        # A pool representing available CPU in units of minCores
        self.coreFractions = ResourcePool(self.numWorkers, 'cores', self.acquisitionTimeout)
        # A lock to work around the lack of thread-safety in Python's subprocess module
        self.popenLock = Lock()
        # A pool representing available memory in bytes
        self.memory = ResourcePool(self.maxMemory, 'memory', self.acquisitionTimeout)
        # A pool representing the available space in bytes
        self.disk = ResourcePool(self.maxDisk, 'disk', self.acquisitionTimeout)

        log.debug('Setting up the thread pool with %i workers, '
                 'given a minimum CPU fraction of %f '
                 'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
        for i in xrange(self.numWorkers):
            worker = Thread(target=self.worker, args=(self.inputQueue,))
            self.workerThreads.append(worker)
            worker.start()

    # Note: The input queue is passed as an argument because the corresponding attribute is reset
    # to None in shutdown()

    def worker(self, inputQueue):
        while True:
            args = inputQueue.get()
            if args is None:
                log.debug('Received queue sentinel.')
                break
            jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
            while True:
                try:
                    coreFractions = int(jobCores / self.minCores)
                    log.debug('Acquiring %i bytes of memory from a pool of %s.', jobMemory,
                              self.memory)
                    with self.memory.acquisitionOf(jobMemory):
                        log.debug('Acquiring %i fractional cores from a pool of %s to satisfy a '
                                  'request of %f cores', coreFractions, self.coreFractions,
                                  jobCores)
                        with self.coreFractions.acquisitionOf(coreFractions):
                            with self.disk.acquisitionOf(jobDisk):
                                startTime = time.time() #Time job is started
                                with self.popenLock:
                                    popen = subprocess.Popen(jobCommand,
                                                             shell=True,
                                                             env=dict(os.environ, **environment))
                                statusCode = None
                                info = Info(time.time(), popen, killIntended=False)
                                try:
                                    self.runningJobs[jobID] = info
                                    try:
                                        statusCode = popen.wait()
                                        if 0 != statusCode:
                                            if statusCode != -9 or not info.killIntended:
                                                log.error("Got exit code %i (indicating failure) "
                                                          "from job %s.", statusCode,
                                                          self.jobs[jobID])
                                    finally:
                                        self.runningJobs.pop(jobID)
                                finally:
                                    if statusCode is not None and not info.killIntended:
                                        self.outputQueue.put((jobID, statusCode,
                                                              time.time() - startTime))
                except ResourcePool.AcquisitionTimeoutException as e:
                    log.debug('Could not acquire enough (%s) to run job. Requested: (%s), '
                              'Avaliable: %s. Sleeping for 10s.', e.resource, e.requested,
                              e.available)
                    with self.aquisitionCondition:
                        # Make threads sleep for the given delay, or until another job finishes.
                        # Whichever is sooner.
                        self.aquisitionCondition.wait(timeout=self.acquisitionRetryDelay)
                    continue
                else:
                    log.debug('Finished job. self.coreFractions ~ %s and self.memory ~ %s',
                              self.coreFractions.value, self.memory.value)
                    with self.aquisitionCondition:
                        # Wake up sleeping threads
                        self.aquisitionCondition.notifyAll()
                    break
        log.debug('Exiting worker thread normally.')

    def issueBatchJob(self, jobNode):
        """
        Adds the command and resources to a queue to be run.
        """
        # Round cores to minCores and apply scale
        cores = math.ceil(jobNode.cores * self.scale / self.minCores) * self.minCores
        assert cores <= self.maxCores, ('The job is requesting {} cores, more than the maximum of '
                                        '{} cores this batch system was configured with. Scale is '
                                        'set to {}.'.format(cores, self.maxCores, self.scale))
        assert cores >= self.minCores
        assert jobNode.memory <= self.maxMemory, ('The job is requesting {} bytes of memory, more than '
                                          'the maximum of {} this batch system was configured '
                                          'with.'.format(jobNode.memory, self.maxMemory))

        self.checkResourceRequest(jobNode.memory, cores, jobNode.disk)
        log.debug("Issuing the command: %s with memory: %i, cores: %i, disk: %i" % (
            jobNode.command, jobNode.memory, cores, jobNode.disk))
        with self.jobIndexLock:
            jobID = self.jobIndex
            self.jobIndex += 1
        self.jobs[jobID] = jobNode.command
        self.inputQueue.put((jobNode.command, jobID, cores, jobNode.memory,
                             jobNode.disk, self.environment.copy()))
        return jobID

    def killBatchJobs(self, jobIDs):
        """
        Kills jobs by ID
        """
        log.debug('Killing jobs: {}'.format(jobIDs))
        for jobID in jobIDs:
            if jobID in self.runningJobs:
                info = self.runningJobs[jobID]
                info.killIntended = True
                os.kill(info.popen.pid, 9)
                while jobID in self.runningJobs:
                    pass

    def getIssuedBatchJobIDs(self):
        """
        Just returns all the jobs that have been run, but not yet returned as updated.
        """
        return self.jobs.keys()

    def getRunningBatchJobIDs(self):
        now = time.time()
        return {jobID: now - info.time for jobID, info in iteritems(self.runningJobs)}

    def shutdown(self):
        """
        Cleanly terminate worker threads. Add sentinels to inputQueue equal to maxThreads. Join
        all worker threads.
        """
        # Remove reference to inputQueue (raises exception if inputQueue is used after method call)
        inputQueue = self.inputQueue
        self.inputQueue = None
        for i in xrange(self.numWorkers):
            inputQueue.put(None)
        for thread in self.workerThreads:
            thread.join()
        BatchSystemSupport.workerCleanup(self.workerCleanupInfo)

    def getUpdatedBatchJob(self, maxWait):
        """
        Returns a map of the run jobs and the return value of their processes.
        """
        try:
            item = self.outputQueue.get(timeout=maxWait)
        except Empty:
            return None
        jobID, exitValue, wallTime = item
        jobCommand = self.jobs.pop(jobID)
        log.debug("Ran jobID: %s with exit value: %i", jobID, exitValue)
        return jobID, exitValue, wallTime

    @classmethod
    def getRescueBatchJobFrequency(cls):
        """
        This should not really occur, wihtout an error. To exercise the system we allow it every 90 minutes.
        """
        return 5400

class Info(object):
    # Can't use namedtuple here since killIntended needs to be mutable
    def __init__(self, startTime, popen, killIntended):
        self.time = startTime
        self.popen = popen
        self.killIntended = killIntended

class ResourcePool(object):
    def __init__(self, initial_value, resourceType, timeout):
        super(ResourcePool, self).__init__()
        self.condition = Condition()
        self.value = initial_value
        self.resourceType = resourceType
        self.timeout = timeout

    def acquire(self, amount):
        with self.condition:
            startTime = time.time()
            while amount > self.value:
                if time.time() - startTime >= self.timeout:
                    # This means the thread timed out waiting for the resource.  We exit the nested
                    # context managers in worker to prevent blocking of a resource due to
                    # unavailability of a nested resource request.
                    raise self.AcquisitionTimeoutException(resource=self.resourceType,
                                                           requested=amount, available=self.value)
                # Allow 5 seconds to get the resource, else quit through the above if condition.
                # This wait + timeout is the last thing in the loop such that a request that takes
                # longer than 5s due to multiple wakes under the 5 second threshold are still
                # honored.
                self.condition.wait(timeout=self.timeout)
            self.value -= amount
            self.__validate()

    def release(self, amount):
        with self.condition:
            self.value += amount
            self.__validate()
            self.condition.notifyAll()

    def __validate(self):
        assert 0 <= self.value

    def __str__(self):
        return str(self.value)

    def __repr__(self):
        return "ResourcePool(%i)" % self.value

    @contextmanager
    def acquisitionOf(self, amount):
        self.acquire(amount)
        try:
            yield
        finally:
            self.release(amount)

    class AcquisitionTimeoutException(Exception):
        """
        To be raised when a resource request times out.
        """

        def __init__(self, resource, requested, available):
            """
            Creates an instance of this exception that indicates which resource is insufficient for
            current demands, as well as the amount requested and amount actually available.

            :param str resource: string representing the resource type

            :param int|float requested: the amount of the particular resource requested that resulted
                   in this exception

            :param int|float available: amount of the particular resource actually available
            """
            self.requested = requested
            self.available = available
            self.resource = resource

<commit_msg>Use list of items rather than generator (resolves #1551)<commit_after># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from contextlib import contextmanager
import logging
import multiprocessing
import os
import subprocess
import time
import math
from threading import Thread
from threading import Lock, Condition

# Python 3 compatibility imports
from six.moves.queue import Empty, Queue
from six.moves import xrange

import toil
from toil.batchSystems.abstractBatchSystem import BatchSystemSupport, InsufficientSystemResources

log = logging.getLogger(__name__)

class SingleMachineBatchSystem(BatchSystemSupport):
    """
    The interface for running jobs on a single machine, runs all the jobs you give it as they
    come in, but in parallel.
    """

    @classmethod
    def supportsHotDeployment(cls):
        return False

    @classmethod
    def supportsWorkerCleanup(cls):
        return True

    numCores = multiprocessing.cpu_count()

    minCores = 0.1
    """
    The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
    value. One important invariant of this class is that each worker thread represents a CPU
    requirement of minCores, meaning that we can never run more than numCores / minCores jobs
    concurrently.
    """
    physicalMemory = toil.physicalMemory()

    def __init__(self, config, maxCores, maxMemory, maxDisk):
        if maxCores > self.numCores:
            log.warn('Limiting maxCores to CPU count of system (%i).', self.numCores)
            maxCores = self.numCores
        if maxMemory > self.physicalMemory:
            log.warn('Limiting maxMemory to physically available memory (%i).', self.physicalMemory)
            maxMemory = self.physicalMemory
        self.physicalDisk = toil.physicalDisk(config)
        if maxDisk > self.physicalDisk:
            log.warn('Limiting maxDisk to physically available disk (%i).', self.physicalDisk)
            maxDisk = self.physicalDisk
        super(SingleMachineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
        assert self.maxCores >= self.minCores
        assert self.maxMemory >= 1

        # The scale allows the user to apply a factor to each task's cores requirement, thereby
        # squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
        # (scale > 1).
        self.scale = config.scale
        # Number of worker threads that will be started
        self.numWorkers = int(self.maxCores / self.minCores)
        # A counter to generate job IDs and a lock to guard it
        self.jobIndex = 0
        self.jobIndexLock = Lock()
        # A dictionary mapping IDs of submitted jobs to the command line
        self.jobs = {}
        """
        :type: dict[str,toil.job.JobNode]
        """
        # A queue of jobs waiting to be executed. Consumed by the workers.
        self.inputQueue = Queue()
        # A queue of finished jobs. Produced by the workers.
        self.outputQueue = Queue()
        # A dictionary mapping IDs of currently running jobs to their Info objects
        self.runningJobs = {}
        """
        :type: dict[str,Info]
        """
        # The list of worker threads
        self.workerThreads = []
        """
        :type list[Thread]
        """
        # Variables involved with non-blocking resource acquisition
        self.acquisitionTimeout = 5
        self.acquisitionRetryDelay = 10
        self.aquisitionCondition = Condition()

        # A pool representing available CPU in units of minCores
        self.coreFractions = ResourcePool(self.numWorkers, 'cores', self.acquisitionTimeout)
        # A lock to work around the lack of thread-safety in Python's subprocess module
        self.popenLock = Lock()
        # A pool representing available memory in bytes
        self.memory = ResourcePool(self.maxMemory, 'memory', self.acquisitionTimeout)
        # A pool representing the available space in bytes
        self.disk = ResourcePool(self.maxDisk, 'disk', self.acquisitionTimeout)

        log.debug('Setting up the thread pool with %i workers, '
                 'given a minimum CPU fraction of %f '
                 'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
        for i in xrange(self.numWorkers):
            worker = Thread(target=self.worker, args=(self.inputQueue,))
            self.workerThreads.append(worker)
            worker.start()

    # Note: The input queue is passed as an argument because the corresponding attribute is reset
    # to None in shutdown()

    def worker(self, inputQueue):
        while True:
            args = inputQueue.get()
            if args is None:
                log.debug('Received queue sentinel.')
                break
            jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
            while True:
                try:
                    coreFractions = int(jobCores / self.minCores)
                    log.debug('Acquiring %i bytes of memory from a pool of %s.', jobMemory,
                              self.memory)
                    with self.memory.acquisitionOf(jobMemory):
                        log.debug('Acquiring %i fractional cores from a pool of %s to satisfy a '
                                  'request of %f cores', coreFractions, self.coreFractions,
                                  jobCores)
                        with self.coreFractions.acquisitionOf(coreFractions):
                            with self.disk.acquisitionOf(jobDisk):
                                startTime = time.time() #Time job is started
                                with self.popenLock:
                                    popen = subprocess.Popen(jobCommand,
                                                             shell=True,
                                                             env=dict(os.environ, **environment))
                                statusCode = None
                                info = Info(time.time(), popen, killIntended=False)
                                try:
                                    self.runningJobs[jobID] = info
                                    try:
                                        statusCode = popen.wait()
                                        if 0 != statusCode:
                                            if statusCode != -9 or not info.killIntended:
                                                log.error("Got exit code %i (indicating failure) "
                                                          "from job %s.", statusCode,
                                                          self.jobs[jobID])
                                    finally:
                                        self.runningJobs.pop(jobID)
                                finally:
                                    if statusCode is not None and not info.killIntended:
                                        self.outputQueue.put((jobID, statusCode,
                                                              time.time() - startTime))
                except ResourcePool.AcquisitionTimeoutException as e:
                    log.debug('Could not acquire enough (%s) to run job. Requested: (%s), '
                              'Avaliable: %s. Sleeping for 10s.', e.resource, e.requested,
                              e.available)
                    with self.aquisitionCondition:
                        # Make threads sleep for the given delay, or until another job finishes.
                        # Whichever is sooner.
                        self.aquisitionCondition.wait(timeout=self.acquisitionRetryDelay)
                    continue
                else:
                    log.debug('Finished job. self.coreFractions ~ %s and self.memory ~ %s',
                              self.coreFractions.value, self.memory.value)
                    with self.aquisitionCondition:
                        # Wake up sleeping threads
                        self.aquisitionCondition.notifyAll()
                    break
        log.debug('Exiting worker thread normally.')

    def issueBatchJob(self, jobNode):
        """
        Adds the command and resources to a queue to be run.
        """
        # Round cores to minCores and apply scale
        cores = math.ceil(jobNode.cores * self.scale / self.minCores) * self.minCores
        assert cores <= self.maxCores, ('The job is requesting {} cores, more than the maximum of '
                                        '{} cores this batch system was configured with. Scale is '
                                        'set to {}.'.format(cores, self.maxCores, self.scale))
        assert cores >= self.minCores
        assert jobNode.memory <= self.maxMemory, ('The job is requesting {} bytes of memory, more than '
                                          'the maximum of {} this batch system was configured '
                                          'with.'.format(jobNode.memory, self.maxMemory))

        self.checkResourceRequest(jobNode.memory, cores, jobNode.disk)
        log.debug("Issuing the command: %s with memory: %i, cores: %i, disk: %i" % (
            jobNode.command, jobNode.memory, cores, jobNode.disk))
        with self.jobIndexLock:
            jobID = self.jobIndex
            self.jobIndex += 1
        self.jobs[jobID] = jobNode.command
        self.inputQueue.put((jobNode.command, jobID, cores, jobNode.memory,
                             jobNode.disk, self.environment.copy()))
        return jobID

    def killBatchJobs(self, jobIDs):
        """
        Kills jobs by ID
        """
        log.debug('Killing jobs: {}'.format(jobIDs))
        for jobID in jobIDs:
            if jobID in self.runningJobs:
                info = self.runningJobs[jobID]
                info.killIntended = True
                os.kill(info.popen.pid, 9)
                while jobID in self.runningJobs:
                    pass

    def getIssuedBatchJobIDs(self):
        """
        Just returns all the jobs that have been run, but not yet returned as updated.
        """
        return self.jobs.keys()

    def getRunningBatchJobIDs(self):
        now = time.time()
        return {jobID: now - info.time for jobID, info in self.runningJobs.items()}

    def shutdown(self):
        """
        Cleanly terminate worker threads. Add sentinels to inputQueue equal to maxThreads. Join
        all worker threads.
        """
        # Remove reference to inputQueue (raises exception if inputQueue is used after method call)
        inputQueue = self.inputQueue
        self.inputQueue = None
        for i in xrange(self.numWorkers):
            inputQueue.put(None)
        for thread in self.workerThreads:
            thread.join()
        BatchSystemSupport.workerCleanup(self.workerCleanupInfo)

    def getUpdatedBatchJob(self, maxWait):
        """
        Returns a map of the run jobs and the return value of their processes.
        """
        try:
            item = self.outputQueue.get(timeout=maxWait)
        except Empty:
            return None
        jobID, exitValue, wallTime = item
        jobCommand = self.jobs.pop(jobID)
        log.debug("Ran jobID: %s with exit value: %i", jobID, exitValue)
        return jobID, exitValue, wallTime

    @classmethod
    def getRescueBatchJobFrequency(cls):
        """
        This should not really occur, wihtout an error. To exercise the system we allow it every 90 minutes.
        """
        return 5400

class Info(object):
    # Can't use namedtuple here since killIntended needs to be mutable
    def __init__(self, startTime, popen, killIntended):
        self.time = startTime
        self.popen = popen
        self.killIntended = killIntended

class ResourcePool(object):
    def __init__(self, initial_value, resourceType, timeout):
        super(ResourcePool, self).__init__()
        self.condition = Condition()
        self.value = initial_value
        self.resourceType = resourceType
        self.timeout = timeout

    def acquire(self, amount):
        with self.condition:
            startTime = time.time()
            while amount > self.value:
                if time.time() - startTime >= self.timeout:
                    # This means the thread timed out waiting for the resource.  We exit the nested
                    # context managers in worker to prevent blocking of a resource due to
                    # unavailability of a nested resource request.
                    raise self.AcquisitionTimeoutException(resource=self.resourceType,
                                                           requested=amount, available=self.value)
                # Allow 5 seconds to get the resource, else quit through the above if condition.
                # This wait + timeout is the last thing in the loop such that a request that takes
                # longer than 5s due to multiple wakes under the 5 second threshold are still
                # honored.
                self.condition.wait(timeout=self.timeout)
            self.value -= amount
            self.__validate()

    def release(self, amount):
        with self.condition:
            self.value += amount
            self.__validate()
            self.condition.notifyAll()

    def __validate(self):
        assert 0 <= self.value

    def __str__(self):
        return str(self.value)

    def __repr__(self):
        return "ResourcePool(%i)" % self.value

    @contextmanager
    def acquisitionOf(self, amount):
        self.acquire(amount)
        try:
            yield
        finally:
            self.release(amount)

    class AcquisitionTimeoutException(Exception):
        """
        To be raised when a resource request times out.
        """

        def __init__(self, resource, requested, available):
            """
            Creates an instance of this exception that indicates which resource is insufficient for
            current demands, as well as the amount requested and amount actually available.

            :param str resource: string representing the resource type

            :param int|float requested: the amount of the particular resource requested that resulted
                   in this exception

            :param int|float available: amount of the particular resource actually available
            """
            self.requested = requested
            self.available = available
            self.resource = resource

<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Name:     ghdl.py
# Purpose:
# Author:   Fabien Marteau <fabien.marteau@armadeus.com>
# Created:  24/07/2008
# ----------------------------------------------------------------------------
#  Copyright (2008)  Armadeus Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# ----------------------------------------------------------------------------
# Revision list :
#
# Date       By        Changes
#
# ----------------------------------------------------------------------------

__doc__ = ""
__version__ = "1.0.0"
__versionTime__ = "24/07/2008"
__author__ = "Fabien Marteau <fabien.marteau@armadeus.com>"

from periphondemand.bin.define import *
from periphondemand.bin.define import VHDLEXT

from periphondemand.bin.code.topgen    import TopGen

from periphondemand.bin.utils          import wrappersystem as sy
from periphondemand.bin.utils.settings import Settings
from periphondemand.bin.utils.display  import Display
from periphondemand.bin.utils.error    import Error

import time
import datetime

TAB = "    "
MAKEFILETEMPLATE="ghdlsimulationmakefile"
settings = Settings()
display  = Display()

def header():
    """ return vhdl header
    """
    header = open(settings.path + TEMPLATESPATH+ "/"+HEADERTPL,"r").read()
    header = header.replace("$tpl:author$",settings.author)
    header = header.replace("$tpl:date$",str(datetime.date.today()))
    header = header.replace("$tpl:filename$","Top_"+settings.active_project.getName()+"_tb.vhd")
    header = header.replace("$tpl:abstract$",settings.active_project.getDescription())
    return header

def include():
    include = ""
    platform = settings.active_project.platform
    for library in platform.getLibrariesList():
        for line in library.getDescription().split("\n"):
            include = include+"-- "+line+"\n"
        include = include + "use " +\
                  settings.active_project.simulation_toolchain.getLibrary() +\
                  "." + library.getFileName().replace(VHDLEXT,".all") + ";\n"
    return include

def entity():
    """ return entity code
    """
    entity = "\nentity "+"top_"+\
            settings.active_project.getName()+"_tb is\n"
    entity = entity + "end entity top_"+settings.active_project.getName()+"_tb;\n\n"
    return entity

def architecturehead():
    """ return architecture head VHDL code
    """
    arch = "architecture RTL of "+"top_"+settings.active_project.getName()+\
            "_tb is\n\n"
    return arch

def constant(clockhalfperiode):
    """ return constant declaration
    """
    constant = TAB+"CONSTANT HALF_PERIODE : time := "+str(clockhalfperiode)+" ns;  -- Half clock period\n"
    for instance in settings.active_project.instances:
        for interface in instance.getInterfacesList():
            for register in interface.getRegisterList():
                constant = constant + TAB +\
                        "CONSTANT "+\
                        instance.getInstanceName().upper()+\
                        "_"+\
                        register.getName().upper()+\
                        " : std_logic_vector := x\""+\
                        register.getAbsoluteAddr()+"\";"+\
                        "\n"
    return constant

def signals(portlist):
    """ return signal declaration
    """
    out = ""
    for port in portlist:
        portname = port.getName()
        interfacename = port.getParent().getName()
        instancename = port.getParent().getParent().getInstanceName()

        out = out + TAB +"signal  "+\
                instancename+"_"+portname +\
                " : "
        if port.getMSBConnected() <1:
            out = out + " std_logic;"
        else:
            out = out + " std_logic_vector("+str(port.getMSBConnected())\
                    +" downto 0);"
        out = out + "\n"
    return out

def declareTop(portlist):
    """ declare top component
    """

    out = "\n"+TAB+"component top_"+settings.active_project.getName()
    out = out + "\n" + TAB + "port ("

    for port in portlist:
        portname = port.getName()
        interfacename = port.getParent().getName()
        instancename = port.getParent().getParent().getInstanceName()

        out = out + TAB*2 +\
                instancename+"_"+portname+\
                " : " + port.getDir()
        if port.getMSBConnected()  < 1:
            out = out + " std_logic;"
        else:
            out = out + " std_logic_vector("+str(port.getMSBConnected())+" downto 0);"
        out = out + "\n"
    # Suppress the #!@ last semicolon
    out = out[:-2]
    out = out + "\n"+TAB + ");\n"
    out = out + TAB + "end component top_"+settings.active_project.getName()+";\n"
    return out

def beginarch():
    return "\nbegin\n\n"

def connectTop(portlist):
    out = TAB+"top : top_"+settings.active_project.getName()+"\n"
    out = out + TAB+"port map(\n"

    for port in portlist:
        portname = port.getName()
        interfacename = port.getParent().getName()
        instancename = port.getParent().getParent().getInstanceName()

        # sig declaration
        out = out + TAB*2 +\
                instancename+"_"+portname +\
                " => " +\
                instancename+"_"+portname +\
                ",\n"
    # Suppress the #!@ last comma
    out = out[:-2]
    out = out + "\n"+TAB + ");\n"
    return out

def clock(clockname):
   """ write clock process
   """
   clock = TAB+"clockp : process\n"+TAB+"begin\n"
   clock = clock+TAB*2+clockname+" <= '1';\n"
   clock = clock+TAB*2+"wait for HALF_PERIODE;\n"
   clock = clock+TAB*2+clockname+" <= '0';\n"
   clock = clock+TAB*2+"wait for HALF_PERIODE;\n"
   clock = clock+TAB+"end process clockp;\n"
   return clock

def stimulis():
    """ write a template stimulis processus
    """
    return """
    stimulis : process
    begin
    -- write stimulis here
    wait for 10 us;
    assert false report "End of test" severity error;
    end process stimulis;
    """

def endarch():
        return "end architecture RTL;\n"

def generateTemplate():
    """ generate Template Testbench
    """
    filename = settings.projectpath + SIMULATIONPATH +\
               "/top_" + settings.active_project.getName() +\
               "_tb" + VHDLEXT
    clockportlist = settings.active_project.clock_ports
    if len(clockportlist) == 0:
        raise Error("No external clock signal found",0)
    if len(clockportlist) != 1:
        display.msg("More than one external clock in design",1)
    clockport = clockportlist[0]
    clockname = clockport.getParent().getParent().getInstanceName()+"_"+clockport.getName()

    ###################
    # header
    out = header()
    out = out + include()
    out = out + entity()
    out = out + architecturehead()
    freq = clockport.getDestinationPort().getFreq()
    clockhalfperiode= (1000/float(freq))/2
    out = out + constant(clockhalfperiode)
    portlist =  settings.active_project.platform.getConnectPortsList()
    out = out + signals(portlist)
    out = out + declareTop(portlist)
    out = out + beginarch()
    out = out + connectTop(portlist)

    out = out + stimulis()

    out = out + "\n"
    out = out + clock(clockname)
    out = out + "\n"
    out = out + endarch()
    #######################
    # save file
    if sy.fileExist(filename):
        display.msg("[WARNING] File exist, file renamed in "+filename+"old",0)
        sy.renameFile(filename,filename+"old")
    try:
        file = open(filename,"w")
    except IOError, e:
        raise e
    file.write(out)
    file.close()
    return filename

def generateMakefile():
    """ generate makefile for ghdl
    """
    # include file list:
    srclist =[]
    platform = settings.active_project.platform
    projectname = "top_"+settings.active_project.getName()
    srclist.append(".."+SYNTHESISPATH+"/top_"+settings.active_project.getName()+VHDLEXT)
    for component in settings.active_project.instances:
        if component.getNum() == "0":
            compdir = ".."+SYNTHESISPATH+"/"+component.getName()+"/"
            for hdlfile in component.getHdl_filesList():
                srclist.append(compdir+hdlfile.getFileName().split("/")[-1])

    librarylist = []
    for library in platform.getLibrariesList():
        srclist.append(library.getFileName())
        librarylist.append(library.getFileName())

    makefile = open(settings.path + TEMPLATESPATH+ "/"+MAKEFILETEMPLATE,"r").read()
    makefile = makefile.replace(r'$tpl:projectname$',projectname)
    makefile = makefile.replace(r'$tpl:files$'," ".join(srclist))
    makefile = makefile.replace(r'$tpl:library$'," ".join(librarylist))

    makefilename = settings.projectpath+SIMULATIONPATH+"/Makefile"
    file = open(makefilename,"w")
    file.write(makefile)
    file.close()
    return makefilename

<commit_msg>src/toolchains/simulation/ghdl/ghdl.py: improve code quality<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Name:     ghdl.py
# Purpose:
# Author:   Fabien Marteau <fabien.marteau@armadeus.com>
# Created:  24/07/2008
# ----------------------------------------------------------------------------
#  Copyright (2008)  Armadeus Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# ----------------------------------------------------------------------------
""" Generate simulation code for GHDL """

from periphondemand.bin.define import VHDLEXT
from periphondemand.bin.define import ONETAB
from periphondemand.bin.define import SIMULATIONPATH
from periphondemand.bin.define import SYNTHESISPATH
from periphondemand.bin.define import TEMPLATESPATH
from periphondemand.bin.define import HEADERTPL

from periphondemand.bin.code.topgen import TopGen

from periphondemand.bin.utils import wrappersystem as sy
from periphondemand.bin.utils.settings import Settings
from periphondemand.bin.utils.display import Display
from periphondemand.bin.utils.error import Error

import time
import datetime

MAKEFILETEMPLATE = "ghdlsimulationmakefile"
SETTINGS = Settings()
DISPLAY = Display()

def header():
    """ return vhdl header
    """
    header = open(SETTINGS.path + TEMPLATESPATH + "/" + HEADERTPL, "r").read()
    header = header.replace("$tpl:author$", SETTINGS.author)
    header = header.replace("$tpl:date$", str(datetime.date.today()))
    header = header.replace("$tpl:filename$", "Top_" +
                            SETTINGS.active_project.getName() + "_tb.vhd")
    header = header.replace("$tpl:abstract$",
                            SETTINGS.active_project.getDescription())
    return header

def include():
    include = ""
    platform = SETTINGS.active_project.platform
    for library in platform.getLibrariesList():
        for line in library.getDescription().split("\n"):
            include = include + "-- " + line + "\n"
        include = include + "use " +\
            SETTINGS.active_project.simulation_toolchain.getLibrary() +\
            "." + library.getFileName().replace(VHDLEXT, ".all") + ";\n"
    return include

def entity():
    """ return entity code
    """
    entity = "\nentity " + "top_" +\
        SETTINGS.active_project.getName() + "_tb is\n"
    entity = entity + "end entity top_" +\
        SETTINGS.active_project.getName() + "_tb;\n\n"
    return entity

def architecturehead():
    """ return architecture head VHDL code
    """
    arch = "architecture RTL of " + "top_" +\
        SETTINGS.active_project.getName() + "_tb is\n\n"
    return arch

def constant(clockhalfperiod):
    """ return constant declaration
    """
    constant = ONETAB + "CONSTANT HALF_PERIODE : time := " +\
        str(clockhalfperiod) + " ns;  -- Half clock period\n"
    for instance in SETTINGS.active_project.instances:
        for interface in instance.getInterfacesList():
            for register in interface.getRegisterList():
                constant = constant + ONETAB + "CONSTANT " +\
                    instance.getInstanceName().upper() + "_" +\
                    register.getName().upper() +\
                    " : std_logic_vector := x\"" + register.getAbsoluteAddr() +\
                    "\";\n"
    return constant

def signals(portlist):
    """ return signal declaration
    """
    out = ""
    for port in portlist:
        portname = port.getName()
        interfacename = port.getParent().getName()
        instancename = port.getParent().getParent().getInstanceName()
        out = out + ONETAB + "signal  " +\
            instancename + "_" + portname + " : "
        if port.getMSBConnected() < 1:
            out = out + " std_logic;"
        else:
            out = out + " std_logic_vector(" + str(port.getMSBConnected()) +\
                " downto 0);"
        out = out + "\n"
    return out

def declareTop(portlist):
    """ declare top component
    """

    out = "\n" + ONETAB + "component top_" + SETTINGS.active_project.getName()
    out = out + "\n" + ONETAB + "port ("

    for port in portlist:
        portname = port.getName()
        interfacename = port.getParent().getName()
        instancename = port.getParent().getParent().getInstanceName()
        out = out + ONETAB * 2 +\
            instancename + "_" + portname + \
            " : " + port.getDir()
        if port.getMSBConnected() < 1:
            out = out + " std_logic;"
        else:
            out = out + " std_logic_vector(" +\
                str(port.getMSBConnected()) + " downto 0);"
        out = out + "\n"
    # Suppress the #!@ last semicolon
    out = out[:-2]
    out = out + "\n" + ONETAB + ");\n"
    out = out + ONETAB + "end component top_" +\
        SETTINGS.active_project.getName() + ";\n"
    return out

def beginarch():
    return "\nbegin\n\n"

def connectTop(portlist):
    out = ONETAB + "top : top_" + SETTINGS.active_project.getName() + "\n"
    out = out + ONETAB + "port map(\n"

    for port in portlist:
        portname = port.getName()
        interfacename = port.getParent().getName()
        instancename = port.getParent().getParent().getInstanceName()

        # sig declaration
        out = out + ONETAB * 2 + instancename + "_" + portname +\
            " => " + instancename + "_" + portname + ", \n"
    # Suppress the #!@ last comma
    out = out[:-2]
    out = out + "\n" + ONETAB + ");\n"
    return out

def clock(clockname):
    """ write clock process
    """
    clock = ONETAB + "clockp : process\n" + ONETAB + "begin\n"
    clock = clock + ONETAB * 2 + clockname + " <= '1';\n"
    clock = clock + ONETAB * 2 + "wait for HALF_PERIODE;\n"
    clock = clock + ONETAB * 2 + clockname + " <= '0';\n"
    clock = clock + ONETAB * 2 + "wait for HALF_PERIODE;\n"
    clock = clock + ONETAB + "end process clockp;\n"
    return clock

def stimulis():
    """ write a template stimulis processus
    """
    return """
    stimulis : process
    begin
    -- write stimulis here
    wait for 10 us;
    assert false report "End of test" severity error;
    end process stimulis;
    """

def endarch():
        return "end architecture RTL;\n"

def generateTemplate():
    """ generate Template Testbench
    """
    filename = SETTINGS.projectpath + SIMULATIONPATH +\
        "/top_" + SETTINGS.active_project.getName() +\
        "_tb" + VHDLEXT
    clockportlist = SETTINGS.active_project.clock_ports
    if len(clockportlist) == 0:
        raise Error("No external clock signal found", 0)
    if len(clockportlist) != 1:
        DISPLAY.msg("More than one external clock in design", 1)
    clockport = clockportlist[0]
    clockname = clockport.getParent().getParent().getInstanceName() +\
        "_" + clockport.getName()

    ###################
    # header
    out = header()
    out = out + include()
    out = out + entity()
    out = out + architecturehead()
    freq = clockport.getDestinationPort().getFreq()
    clockhalfperiod = (1000 / float(freq)) / 2
    out = out + constant(clockhalfperiod)
    portlist = SETTINGS.active_project.platform.getConnectPortsList()
    out = out + signals(portlist)
    out = out + declareTop(portlist)
    out = out + beginarch()
    out = out + connectTop(portlist)

    out = out + stimulis()

    out = out + "\n"
    out = out + clock(clockname)
    out = out + "\n"
    out = out + endarch()
    #######################
    # save file
    if sy.fileExist(filename):
        DISPLAY.msg("[WARNING] File exist, file renamed in " +
                    filename + "old", 0)
        sy.renameFile(filename, filename + "old")
    try:
        afile = open(filename, "w")
    except IOError, error:
        raise error
    afile.write(out)
    afile.close()
    return filename

def generateMakefile():
    """ generate makefile for ghdl
    """
    # include file list:
    srclist = []
    platform = SETTINGS.active_project.platform
    projectname = "top_" + SETTINGS.active_project.getName()
    srclist.append(".." + SYNTHESISPATH + "/top_" +
                   SETTINGS.active_project.getName() + VHDLEXT)
    for component in SETTINGS.active_project.instances:
        if component.getNum() == "0":
            compdir = ".." + SYNTHESISPATH + "/" + component.getName() + "/"
            for hdlfile in component.getHdl_filesList():
                srclist.append(compdir + hdlfile.getFileName().split("/")[-1])

    librarylist = []
    for library in platform.getLibrariesList():
        srclist.append(library.getFileName())
        librarylist.append(library.getFileName())

    makefile = open(SETTINGS.path + TEMPLATESPATH +
                    "/" + MAKEFILETEMPLATE, "r").read()
    makefile = makefile.replace(r'$tpl:projectname$', projectname)
    makefile = makefile.replace(r'$tpl:files$', " ".join(srclist))
    makefile = makefile.replace(r'$tpl:library$', " ".join(librarylist))

    makefilename = SETTINGS.projectpath + SIMULATIONPATH + "/Makefile"
    afile = open(makefilename, "w")
    afile.write(makefile)
    afile.close()
    return makefilename
<|endoftext|>