<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
from openerp import netsvc
import logging
from openerp.osv import osv, orm, fields
from datetime import datetime, timedelta
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT, DATETIME_FORMATS_MAP, float_compare)
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _

_logger = logging.getLogger(__name__)

class ProductTemplateAccounting(orm.Model):
    ''' Accounting external fields
    '''

    _inherit = 'product.template'

    _columns = {
        'minimum_qty': fields.float('Min. quantity', digits=(10, 2),
            help="Minimum value for stock"),
        'maximum_qty': fields.float('Max. quantity', digits=(10, 2),
            help="Maximum value for stock"),
        'accounting_qty': fields.float('Accounting quantity', digits=(10, 2),
            help="Accounting existence updated today"),
    }

class SaleOrderLine(orm.Model):
    ''' Add extra field to manage connection with accounting
    '''

    _inherit = 'sale.order.line'

    # Force new order for production
    _order = 'mrp_sequence,order_id,sequence'

    # -------------
    # Button event:
    # -------------
    """def force_fast_creation(self, cr, uid, ids, context=None):
        ''' Force fast creation of wizard passing default elements
        '''
        line_proxy = self.browse(cr, uid, ids, context=context)[0]

        # Create a wizard record (setting default) and simulate button pressure
        wiz_pool = self.pool.get('mrp.production.create.wizard')
        wiz_id = wiz_pool.create(cr, uid, {
            'total': wiz_pool.wiz_total,
            'product_tmpl_id': wiz_proxy.product_id.product_tmpl_id.id,
            #'bom_id' << default (hoping is present!)
            'schedule_from_date': wiz_proxy.wiz_date,
            #'workhour_id': wiz_proxy.wiz_workhour_id.id
            'operation': 'lavoration',
            }, context=context)
        return True"""

    def free_line(self, cr, uid, ids, context=None):
        ''' Free the line from production order
        '''
        return self.write(cr, uid, ids, {
            'mrp_id': False,
            'mrp_sequence': False, # reset order
            }, context=context)

    def close_production(self, cr, uid, ids, context=None):
        ''' Close production
        '''
        # TODO (interact with accounting)
        line_proxy = self.browse(cr, uid, ids, context=context)[0]

        #if line_proxy.product_uom_maked_qty: # partial
        #    pass # TODO close (partial)
        #else: # TODO manage well (now not correct)
        return self.write(cr, uid, ids, {
            'product_uom_maked_qty':
                line_proxy.product_uom_qty,
            'sync_state': 'closed',
            }, context=context)

    def accounting_sync(self, cr, uid, ids, context=context):
        ''' Read all line to sync in accounting and produce it for
            XMLRPC call
        '''
        # Read all line to close
        sol_ids = self.search(cr, uid, [
            ('sync_state', 'in', ('partial', 'closed'))
            ],
            order='order_id', # TODO line sequence?
            context=context, )

        # Write in file:
        temp_file = 'close.txt'
        out = open(temp_file, 'w')
        for line in self.browse(cr, uid, sol_ids, context=context):
            out.write("%10s" % ( # TODO
                line.order_id.name,
                )
        out.close()

        # XMLRPC call for import the file

        return True

    _columns = {
        'mrp_id': fields.many2one(
            'mrp.production', 'Production', ondelete='set null', ),
        'product_uom_maked_qty': fields.float(
            'Maked', digits=(16, 2), ),
        'production_note': fields.char('Note', size=100),

        #'default_code': fields.related('product_id','default_code',
        #    type='char', string='Code'),
        # TODO remove with state?
        #'is_produced': fields.boolean('Is produced', required=False),
        'mrp_sequence': fields.integer('MRP order'),

        'sync_state': fields.selection([
            ('draft', 'Draft'),
            ('partial', 'Partial'),
            ('closed', 'Closed'),
            ('sync', 'Sync'), ],'Sync state', select=True),
        }

    _defaults = {
        'sync_state': lambda *x: 'draft',
        }

class SaleOrderLinePrevisional(orm.Model):
    ''' Previsional production
    '''

    _name = 'sale.order.line.previsional'

    def set_updated(self, cr, uid, ids, context=None):
        ''' Check the updated boolean (for speed up)
        '''
        self.write(cr, uid, ids, {'updated': True}, context=context)
        return True

    _columns = {
        'partner_id':fields.many2one(
            'res.partner', 'Customer', required=False),
        'product_id': fields.many2one(
            'product.product', 'Product', required=False),
        'product_tmpl_id': fields.many2one(
            'product.template', 'Product', required=False),
        'deadline': fields.date('Deadline'),
        'note': fields.text('Note'),
        'product_uom_qty': fields.float('Quantity', digits=(16, 2),
            required=True),

        'updated': fields.boolean('Updated',
            help='Manually updated on accounting program'),
        'mrp_id': fields.many2one(
            'mrp.production', 'Production', ondelete='cascade', ),
        }

class MrpProduction(orm.Model):
    ''' Add extra field to manage connection with accounting
    '''

    _inherit = 'mrp.production'

    # -------------
    # Button event:
    # -------------
    def close_all_production(self, cr, uid, ids, context=None):
        ''' Close all production
        '''
        line_proxy = self.browse(cr, uid, ids, context=context).order_line_ids

        # Loop for close all (use original button event):
        for line in line_proxy:
           if not line.is_produced:
               self.pool.get(
                   'sale.order.line').close_production(
                       cr, uid, [line.id], context=context)
        return True

    def force_production_sequence(self, cr, uid, ids, context=None):
        ''' Set current order depend on default code
            Note: currently is forced for particular customization
            maybe if order is not your you could override this procedure
        '''
        mrp_proxy = self.browse(cr, uid, ids, context=context)
        order = []
        for line in mrp_proxy.order_line_ids:
            order.append((line.default_code, line.id))
            #order.append((
            #    '%3s%2s' % (
            #        line.default_code[:3],
            #        line.default_code[5:7],
            #        ),
            #    line.id))
        line_pool = self.pool.get('sale.order.line')
        i = 0
        for code, item_id in sorted(order):
            i += 1
            line_pool.write(cr, uid, item_id, {
                'mrp_sequence': i,
                }, context=context)
        return True

    def free_line(self, cr, uid, ids, context=None):
        ''' Free the line from production order
        '''
        return self.write(cr, uid, ids, {
            'used_by_mrp_id': False, }, context=context)

    def _get_totals(self, cr, uid, ids, fields=None, args=None, context=None):
        ''' Calculate all totals
            oc_qty = sum (qty for all line)
            extra_qty = total production - oc_qty
        '''
        res = {}
        for order in self.browse(cr, uid, ids, context=context):
            res[order.id] = {}
            res[order.id]['oc_qty'] = 0.0
            res[order.id]['previsional_qty'] = 0.0
            res[order.id]['use_extra_qty'] = 0.0

            for line in order.order_line_ids:
                res[order.id]['oc_qty'] += line.product_uom_qty # TODO UM?

            for line in order.previsional_line_ids:
                res[order.id]['previsional_qty'] += line.product_uom_qty

            for line in order.use_mrp_ids: # TO correct (for recursion)
                res[order.id]['use_extra_qty'] += self.browse(cr, uid, line.id).extra_qty

                #res[order.id]['use_extra_qty'] += line.extra_qty

            res[order.id]['extra_qty'] = (        # Extra =
                order.product_qty +               # Production
                res[order.id]['use_extra_qty'] -  # + extra qty used
                res[order.id]['oc_qty'] -         # - Ordered
                res[order.id]['previsional_qty']) # - Previsional
            res[order.id]['error_qty'] = res[order.id]['extra_qty'] < 0.0
            res[order.id]['has_extra_qty'] = res[order.id]['extra_qty'] > 0.0
        return res

    # Fields function:
    def _get_mandatory_delivery(
            self, cr, uid, ids, fields, args, context=None):
        ''' Number of fix delivery
        '''
        res = {}
        for mo in self.browse(cr, uid, ids, context=context):
            res[mo.id] = {
                'has_mandatory_delivery': '',
                'mandatory_delivery': 0,
                }
            for so in mo.order_line_ids:
                if so.has_mandatory_delivery:
                    res[mo.id]['mandatory_delivery'] += 1
                    res[mo.id]['has_mandatory_delivery'] += "*"
        return res

    _columns = {
        'oc_qty': fields.function(
            _get_totals, method=True, type='float',
            string='OC qty', store=False, readonly=True, multi=True),
        'previsional_qty': fields.function(
            _get_totals, method=True, type='float',
            string='Previsional qty', store=False, readonly=True, multi=True),
        'use_extra_qty': fields.function(
            _get_totals, method=True, type='float',
            string='Use extra qty', store=False, readonly=True, multi=True),
        'extra_qty': fields.function(
            _get_totals, method=True, type='float',
            string='Extra qty', store=False, readonly=True, multi=True),
        'has_extra_qty': fields.function(
            _get_totals, method=True, type='boolean', string='Has extra',
            store=True,
            #{'sale.order.line':(_get_extra_production,['product_uom_qty'],10)},
            readonly=True, multi=True),
        'error_qty': fields.function(
            _get_totals, method=True, type='boolean',
            string='Total error', store=False, readonly=True, multi=True),

        'used_by_mrp_id': fields.many2one('mrp.production', 'Used by'),

        'use_mrp_ids': fields.one2many(
            'mrp.production', 'used_by_mrp_id', 'Use mrp'),
        'order_line_ids': fields.one2many(
            'sale.order.line', 'mrp_id', 'Order line'),
        'previsional_line_ids': fields.one2many(
            'sale.order.line.previsional', 'mrp_id', 'Previsional order'),
        'updated':fields.boolean('Label', required=False),
        'has_mandatory_delivery': fields.function(_get_mandatory_delivery,
            method=True, type='char', size=1, string='Has fix delivery',
            store=False, multi=True),
        'mandatory_delivery': fields.function(_get_mandatory_delivery,
            method=True, type='integer', string='Fix delivery',
            store=False, multi=True),
        }

class MrpProductionWorkcenterLine(orm.Model):
    ''' Accounting external fields
    '''

    _inherit = 'mrp.production.workcenter.line'

    _columns = {
        #'product_id': fields.related('mrp_id', 'product_id',
        #    type='many2one', relation='product.product', string='Product'),
        'has_mandatory_delivery': fields.related('production_id',
            'has_mandatory_delivery', type='char', size=1,
            string='Has fix delivery'),
        'mandatory_delivery': fields.related('production_id',
            'mandatory_delivery', type='integer', string='Fix delivery'),
        }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>manage colls for calculate correct total of units<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
from openerp import netsvc
import logging
from openerp.osv import osv, orm, fields
from datetime import datetime, timedelta
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT, DATETIME_FORMATS_MAP, float_compare)
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _

_logger = logging.getLogger(__name__)

class ProductTemplateAccounting(orm.Model):
    ''' Accounting external fields
    '''

    _inherit = 'product.template'

    _columns = {
        'minimum_qty': fields.float('Min. quantity', digits=(10, 2),
            help="Minimum value for stock"),
        'maximum_qty': fields.float('Max. quantity', digits=(10, 2),
            help="Maximum value for stock"),
        'accounting_qty': fields.float('Accounting quantity', digits=(10, 2),
            help="Accounting existence updated today"),
    }

class SaleOrderLine(orm.Model):
    ''' Add extra field to manage connection with accounting
    '''

    _inherit = 'sale.order.line'

    # Force new order for production
    _order = 'mrp_sequence,order_id,sequence'

    # -------------
    # Button event:
    # -------------
    """def force_fast_creation(self, cr, uid, ids, context=None):
        ''' Force fast creation of wizard passing default elements
        '''
        line_proxy = self.browse(cr, uid, ids, context=context)[0]

        # Create a wizard record (setting default) and simulate button pressure
        wiz_pool = self.pool.get('mrp.production.create.wizard')
        wiz_id = wiz_pool.create(cr, uid, {
            'total': wiz_pool.wiz_total,
            'product_tmpl_id': wiz_proxy.product_id.product_tmpl_id.id,
            #'bom_id' << default (hoping is present!)
            'schedule_from_date': wiz_proxy.wiz_date,
            #'workhour_id': wiz_proxy.wiz_workhour_id.id
            'operation': 'lavoration',
            }, context=context)
        return True"""

    def free_line(self, cr, uid, ids, context=None):
        ''' Free the line from production order
        '''
        return self.write(cr, uid, ids, {
            'mrp_id': False,
            'mrp_sequence': False, # reset order
            }, context=context)

    def close_production(self, cr, uid, ids, context=None):
        ''' Close production
        '''
        # TODO (interact with accounting)
        line_proxy = self.browse(cr, uid, ids, context=context)[0]

        #if line_proxy.product_uom_maked_qty: # partial
        #    pass # TODO close (partial)
        #else: # TODO manage well (now not correct)
        return self.write(cr, uid, ids, {
            'product_uom_maked_qty':
                line_proxy.product_uom_qty,
            'sync_state': 'closed',
            }, context=context)

    def accounting_sync(self, cr, uid, ids, context=None):
        ''' Read all line to sync in accounting and produce it for
            XMLRPC call
        '''
        # Read all line to close
        sol_ids = self.search(cr, uid, [
            ('sync_state', 'in', ('partial', 'closed'))
            ],
            order='order_id', # TODO line sequence?
            context=context, )

        # Write in file:
        temp_file = 'close.txt'
        out = open(temp_file, 'w')
        for line in self.browse(cr, uid, sol_ids, context=context):
            out.write("%10s" % ( # TODO
                line.order_id.name,
                ))
        out.close()

        # XMLRPC call for import the file

        return True

    _columns = {
        'mrp_id': fields.many2one(
            'mrp.production', 'Production', ondelete='set null', ),
        'product_uom_maked_qty': fields.float(
            'Maked', digits=(16, 2), ),
        'production_note': fields.char('Note', size=100),

        #'default_code': fields.related('product_id','default_code',
        #    type='char', string='Code'),
        # TODO remove with state?
        #'is_produced': fields.boolean('Is produced', required=False),
        'mrp_sequence': fields.integer('MRP order'),

        'sync_state': fields.selection([
            ('draft', 'Draft'),
            ('partial', 'Partial'),
            ('closed', 'Closed'),
            ('sync', 'Sync'), ],'Sync state', select=True),
        }

    _defaults = {
        'sync_state': lambda *x: 'draft',
        }

class SaleOrderLinePrevisional(orm.Model):
    ''' Previsional production
    '''

    _name = 'sale.order.line.previsional'

    def set_updated(self, cr, uid, ids, context=None):
        ''' Check the updated boolean (for speed up)
        '''
        self.write(cr, uid, ids, {'updated': True}, context=context)
        return True

    _columns = {
        'partner_id':fields.many2one(
            'res.partner', 'Customer', required=False),
        'product_id': fields.many2one(
            'product.product', 'Product', required=False),
        'product_tmpl_id': fields.many2one(
            'product.template', 'Product', required=False),
        'deadline': fields.date('Deadline'),
        'note': fields.text('Note'),
        'product_uom_qty': fields.float('Quantity', digits=(16, 2),
            required=True),

        'updated': fields.boolean('Updated',
            help='Manually updated on accounting program'),
        'mrp_id': fields.many2one(
            'mrp.production', 'Production', ondelete='cascade', ),
        }

class MrpProduction(orm.Model):
    ''' Add extra field to manage connection with accounting
    '''

    _inherit = 'mrp.production'

    # -------------
    # Button event:
    # -------------
    def close_all_production(self, cr, uid, ids, context=None):
        ''' Close all production
        '''
        line_proxy = self.browse(cr, uid, ids, context=context).order_line_ids

        # Loop for close all (use original button event):
        for line in line_proxy:
           if not line.is_produced:
               self.pool.get(
                   'sale.order.line').close_production(
                       cr, uid, [line.id], context=context)
        return True

    def force_production_sequence(self, cr, uid, ids, context=None):
        ''' Set current order depend on default code
            Note: currently is forced for particular customization
            maybe if order is not your you could override this procedure
        '''
        mrp_proxy = self.browse(cr, uid, ids, context=context)
        order = []
        for line in mrp_proxy.order_line_ids:
            order.append((line.default_code, line.id))
            #order.append((
            #    '%3s%2s' % (
            #        line.default_code[:3],
            #        line.default_code[5:7],
            #        ),
            #    line.id))
        line_pool = self.pool.get('sale.order.line')
        i = 0
        for code, item_id in sorted(order):
            i += 1
            line_pool.write(cr, uid, item_id, {
                'mrp_sequence': i,
                }, context=context)
        return True

    def free_line(self, cr, uid, ids, context=None):
        ''' Free the line from production order
        '''
        return self.write(cr, uid, ids, {
            'used_by_mrp_id': False, }, context=context)

    def _get_totals(self, cr, uid, ids, fields=None, args=None, context=None):
        ''' Calculate all totals
            oc_qty = sum (qty for all line)
            extra_qty = total production - oc_qty
        '''
        res = {}
        for order in self.browse(cr, uid, ids, context=context):
            res[order.id] = {}
            res[order.id]['oc_qty'] = 0.0
            res[order.id]['previsional_qty'] = 0.0
            res[order.id]['use_extra_qty'] = 0.0

            for line in order.order_line_ids:
                res[order.id]['oc_qty'] += line.product_uom_qty # TODO UM?

            for line in order.previsional_line_ids:
                res[order.id]['previsional_qty'] += line.product_uom_qty

            for line in order.use_mrp_ids: # TO correct (for recursion)
                res[order.id]['use_extra_qty'] += self.browse(cr, uid, line.id).extra_qty

                #res[order.id]['use_extra_qty'] += line.extra_qty

            res[order.id]['extra_qty'] = (        # Extra =
                order.product_qty +               # Production
                res[order.id]['use_extra_qty'] -  # + extra qty used
                res[order.id]['oc_qty'] -         # - Ordered
                res[order.id]['previsional_qty']) # - Previsional
            res[order.id]['error_qty'] = res[order.id]['extra_qty'] < 0.0
            res[order.id]['has_extra_qty'] = res[order.id]['extra_qty'] > 0.0
        return res

    # Fields function:
    def _get_mandatory_delivery(
            self, cr, uid, ids, fields, args, context=None):
        ''' Number of fix delivery
        '''
        res = {}
        for mo in self.browse(cr, uid, ids, context=context):
            res[mo.id] = {
                'has_mandatory_delivery': '',
                'mandatory_delivery': 0,
                }
            for so in mo.order_line_ids:
                if so.has_mandatory_delivery:
                    res[mo.id]['mandatory_delivery'] += 1
                    res[mo.id]['has_mandatory_delivery'] += "*"
        return res

    _columns = {
        'oc_qty': fields.function(
            _get_totals, method=True, type='float',
            string='OC qty', store=False, readonly=True, multi=True),
        'previsional_qty': fields.function(
            _get_totals, method=True, type='float',
            string='Previsional qty', store=False, readonly=True, multi=True),
        'use_extra_qty': fields.function(
            _get_totals, method=True, type='float',
            string='Use extra qty', store=False, readonly=True, multi=True),
        'extra_qty': fields.function(
            _get_totals, method=True, type='float',
            string='Extra qty', store=False, readonly=True, multi=True),
        'has_extra_qty': fields.function(
            _get_totals, method=True, type='boolean', string='Has extra',
            store=True,
            #{'sale.order.line':(_get_extra_production,['product_uom_qty'],10)},
            readonly=True, multi=True),
        'error_qty': fields.function(
            _get_totals, method=True, type='boolean',
            string='Total error', store=False, readonly=True, multi=True),

        'used_by_mrp_id': fields.many2one('mrp.production', 'Used by'),

        'use_mrp_ids': fields.one2many(
            'mrp.production', 'used_by_mrp_id', 'Use mrp'),
        'order_line_ids': fields.one2many(
            'sale.order.line', 'mrp_id', 'Order line'),
        'previsional_line_ids': fields.one2many(
            'sale.order.line.previsional', 'mrp_id', 'Previsional order'),
        'updated':fields.boolean('Label', required=False),
        'has_mandatory_delivery': fields.function(_get_mandatory_delivery,
            method=True, type='char', size=1, string='Has fix delivery',
            store=False, multi=True),
        'mandatory_delivery': fields.function(_get_mandatory_delivery,
            method=True, type='integer', string='Fix delivery',
            store=False, multi=True),
        }

class MrpProductionWorkcenterLine(orm.Model):
    ''' Accounting external fields
    '''

    _inherit = 'mrp.production.workcenter.line'

    _columns = {
        #'product_id': fields.related('mrp_id', 'product_id',
        #    type='many2one', relation='product.product', string='Product'),
        'has_mandatory_delivery': fields.related('production_id',
            'has_mandatory_delivery', type='char', size=1,
            string='Has fix delivery'),
        'mandatory_delivery': fields.related('production_id',
            'mandatory_delivery', type='integer', string='Fix delivery'),
        }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>import ck.kernel as ck
import copy
import re

#######################################
#    Description
#
#
#    KERNEL = Xgemm
#    INPUT = M,N,K SIZE. PRECISION
#    OUTPUT = CONFIGURATION
#######################################

'''
Set interval or Resolution 25%
Set resolution 0 //powersafe mode take min
Set resolution 1// performance mode take max
Set resolution x[0 and 1]
           Convert x in percent
Start from lower Freq. The next freq is the first freq with freq_min + x % of power.
Set resolution 2
            Try all frequencies

Example of Set resolution x
Freq availables 100 MHz, 200MHz, 400MHz, 800Mhz, 2000Mhz
Set interval 50%.
Lvl 0: 100Mhz
Lvl 1: 150Mhz // value not allowed
Lvl 2: 200Mhz OK
Lvl 3: 250Mhz
'''

# clock_resolution
# 0 min freq
# 1 max freq [default]
# (0,1) called resolution convert. Create intervals starting from min. Interval 0 = frequencies between min and min+(min*resolution)
#otherwise takes min and mix and divide per a fixed number
# 2 run all the frequencies
clock_resolution = 1.0
kernel = ['xgemm_direct-fp32']
title =  "My Experiment"
# size of the matrix m,n,k
size_m = ["512", "256", "128", "1024" ]
size_n = ["256", "512", "128", "1024" ]
size_k = ["128","256",  "1024","128" ]

precision=32 # default
run = 1 # default

VERBOSE=0
VERBOSE_STR="[VERBOSE] "
DEBUG = 0
DEBUG_STR="[DEBUG] "

def do(i):
    if VERBOSE or DEBUG:
	    print '[Experiment] ', title
	    print '[Preparing pipeline] Clock resolution: ', clock_resolution
	    print '[Preparing pipeline] Matrix sizes (m,n,k): ', size_m, size_n, size_k
	    print '[Preparing pipeline] Precision: ', precision
	    print '[Preparing pipeline] Run for configuration: ', run
	    print '[Preparing pipeline] More parms... '
    # Detect basic platform info.
    ii={'action':'detect',
        'module_uoa':'platform',
        'out':'out'}
    r=ck.access(ii)
    if DEBUG: print DEBUG_STR, r
    if r['return']>0: return r

    # Host and target OS params.
    hos=r['host_os_uoa']
    hosd=r['host_os_dict']
    tos=r['os_uoa']
    tosd=r['os_dict']
    tdid=r['device_id']

    if DEBUG: print DEBUG_STR, hos, hosd
    if DEBUG: print DEBUG_STR, tos, tosd, tdid

    # Load CLBLAST program meta and desc to check deps.
    ii={'action':'load',
        'module_uoa':'program',
        'data_uoa':'clblast-tune'}
    rx=ck.access(ii)
    if DEBUG: print DEBUG_STR, rx
    if rx['return']>0: return rx
    meta= rx['dict']
    if VERBOSE: print VERBOSE_STR, meta

    # Get compile-time and run-time deps.
    cdeps=meta.get('compile_deps',{})
    rdeps=meta.get('run_deps',{})

    # Merge rdeps with cdeps for setting up the pipeline (which uses
    # common deps), but tag them as "for_run_time".
    for k in rdeps:
        cdeps[k]=rdeps[k]
        cdeps[k]['for_run_time']='yes'
    # CLblast libs.
    depl=copy.deepcopy(cdeps['lib-clblast'])
    if VERBOSE: print VERBOSE_STR, depl
    #ON LOCAL MACHINE
    ii={'action':'resolve',
    'module_uoa':'env',
    'host_os':hos,
    'target_os':tos,
    'device_id':tdid,
    'deps':{'lib-clblast':copy.deepcopy(depl)}
    }
    r=ck.access(ii)
    if DEBUG: print DEBUG_STR, r
    if r['return']>0: return r
    udepl=r['deps']['lib-clblast'].get('choices',[])
    if len(udepl)==0: return {'return':1, 'error':'no installed CLBlast libs'}

    #prepare pipeline
    '''
    ii={'action':'pipeline',
        'module_uoa':'program',
        'data_uoa':'clblast-tune',
        'prepare':'yes',
        'dependencies': cdeps,
        'no_compiler_description':'yes',
        'cmd_key':kernel[0],
        'cpu_freq':'max',
        'gpu_freq':'max',
        'flags':'-O3',
    }
    '''
    ii={'action':'pipeline',
        'module_uoa':'program',
        'data_uoa':'clblast-tune',
        'prepare':'yes',
        'dependencies': cdeps,
        'no_compiler_description':'yes',
        'cmd_key':kernel[0],
        "target_os":"android21-arm-v7a",
        "device_id":"192.168.0.18:5566",
        'flags':'-O3',
    }
    r=ck.access(ii)
    if DEBUG: print DEBUG_STR, r
    if r['return']>0: return r
    fail=r.get('fail','')
    if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}

    ready=r.get('ready','')
    if ready!='yes': return {'return':11, 'error':'pipeline not ready'}

    state=r['state']
    tmp_dir=state['tmp_dir']
    xcdeps=r.get('dependencies',{})
    # Clean pipeline.
    if 'ready' in r: del(r['ready'])
    if 'fail' in r: del(r['fail'])
    if 'return' in r: del(r['return'])
    pipeline=copy.deepcopy(r)

    record_repo='local'
    record_uoa='explore-matrix-size-'+kernel[0]
    ck.out('---------------------------------------------------------------------------------------')
    ck.out('Experiment - %s:%s' % (record_repo, record_uoa))

    cpipeline=copy.deepcopy(pipeline)
    ii={
        'action':'autotune',
        'module_uoa':'pipeline',
        'data_uoa':'program',
        'choices_order':[
            [
	     '##env#CK_CLBLAST_MSIZE'
	    ],
	    [
	     '##env#CK_CLBLAST_NSIZE',
	    ],
	    [
	     '##env#CK_CLBLAST_KSIZE'
	    ]
        ],
        'choices_selection':[
            {"type":"loop-with-next", "choice":size_m, "default":"256"},
            {"type":"loop-with-next", "choice":size_n, "default":"256"},
            {"type":"loop-with-next", "choice":size_k, "default":"256"}

        ],
        'features_keys_to_process':['##choices#*'],

        'iterations':-1,
        'repetitions':1,
        'record':'yes',
        'record_failed':'yes',
        'record_params':{
            'search_point_by_features':'yes'
        },
        'record_repo':record_repo,
        'record_uoa':record_uoa,
        'tags':['explore-clblast-matrix-size', kernel[0]],
        'pipeline': cpipeline,
        'out':'no'

    }
    r=ck.access(ii)
    if DEBUG > 0: print DEBUG_STR, r
    if r['return']>0: return r
    fail=r.get('fail','')
    if fail=='yes':
       return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}

    return  {'return':0}

r=do({})
if r['return']>0: ck.err(r)
<commit_msg>Fix CLBlast-tune pipeline<commit_after>import ck.kernel as ck
import copy
import re

#######################################
#    Description
#
#
#    KERNEL = Xgemm
#    INPUT = M,N,K SIZE. PRECISION
#    OUTPUT = CONFIGURATION
#######################################

'''
Set interval or Resolution 25%
Set resolution 0 //powersafe mode take min
Set resolution 1// performance mode take max
Set resolution x[0 and 1]
           Convert x in percent
Start from lower Freq. The next freq is the first freq with freq_min + x % of power.
Set resolution 2
            Try all frequencies

Example of Set resolution x
Freq availables 100 MHz, 200MHz, 400MHz, 800Mhz, 2000Mhz
Set interval 50%.
Lvl 0: 100Mhz
Lvl 1: 150Mhz // value not allowed
Lvl 2: 200Mhz OK
Lvl 3: 250Mhz
'''

# clock_resolution
# 0 min freq
# 1 max freq [default]
# (0,1) called resolution convert. Create intervals starting from min. Interval 0 = frequencies between min and min+(min*resolution)
#otherwise takes min and mix and divide per a fixed number
# 2 run all the frequencies
clock_resolution = 1.0
kernel = ['xgemm_direct-fp32']
title =  "My Experiment"
# size of the matrix m,n,k
size_m = ["512", "256", "128", "1024" ]
size_n = ["256", "512", "128", "1024" ]
size_k = ["128","256",  "1024","128" ]

precision=32 # default
run = 1 # default

VERBOSE=0
VERBOSE_STR="[VERBOSE] "
DEBUG = 0
DEBUG_STR="[DEBUG] "

def do(i):
    if VERBOSE or DEBUG:
	    print '[Experiment] ', title
	    print '[Preparing pipeline] Clock resolution: ', clock_resolution
	    print '[Preparing pipeline] Matrix sizes (m,n,k): ', size_m, size_n, size_k
	    print '[Preparing pipeline] Precision: ', precision
	    print '[Preparing pipeline] Run for configuration: ', run
	    print '[Preparing pipeline] More parms... '
    # Detect basic platform info.
    ii={'action':'detect',
        'module_uoa':'platform',
        'con':'con'}
    r=ck.access(ii)
    if DEBUG: print DEBUG_STR, r
    if r['return']>0: return r

    # Host and target OS params.
    hos=r['host_os_uoa']
    hosd=r['host_os_dict']
    tos=r['os_uoa']
    tosd=r['os_dict']
    tdid=r['device_id']

    if DEBUG: print DEBUG_STR, hos, hosd
    if DEBUG: print DEBUG_STR, tos, tosd, tdid

    # Load CLBLAST program meta and desc to check deps.
    ii={'action':'load',
        'module_uoa':'program',
        'data_uoa':'clblast-tune'}
    rx=ck.access(ii)
    if DEBUG: print DEBUG_STR, rx
    if rx['return']>0: return rx
    meta= rx['dict']
    if VERBOSE: print VERBOSE_STR, meta

    # Get compile-time and run-time deps.
    cdeps=meta.get('compile_deps',{})
    rdeps=meta.get('run_deps',{})

    # Merge rdeps with cdeps for setting up the pipeline (which uses
    # common deps), but tag them as "for_run_time".
    for k in rdeps:
        cdeps[k]=rdeps[k]
        cdeps[k]['for_run_time']='yes'
    # CLblast libs.
    depl=copy.deepcopy(cdeps['lib-clblast'])
    if VERBOSE: print VERBOSE_STR, depl
    #ON LOCAL MACHINE
    tdid='192.168.0.18:5566'
    tos='android21-arm-v7a'
    ii={'action':'resolve',
    'module_uoa':'env',
    'host_os':hos,
    'target_os':tos,
    'device_id':tdid,
    'out':'con',
    'deps':{'lib-clblast':copy.deepcopy(depl)}
    }
    r=ck.access(ii)
    if DEBUG: print DEBUG_STR, r
    if r['return']>0: return r
    udepl=r['deps']['lib-clblast'].get('choices',[])
    if len(udepl)==0: return {'return':1, 'error':'no installed CLBlast libs'}

    #prepare pipeline
    ii={'action':'pipeline',
        'module_uoa':'program',
        'data_uoa':'clblast-tune',
        'prepare':'yes',
        'dependencies': cdeps,
        'no_compiler_description':'yes',
        'cmd_key':kernel[0],
        "target_os":tos,
        "device_id":tdid,
        "out":'con',
        'flags':'-O3',
    }
    r=ck.access(ii)
    if DEBUG: print DEBUG_STR, r
    if r['return']>0: return r
    fail=r.get('fail','')
    if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}

    ready=r.get('ready','')
    if ready!='yes': return {'return':11, 'error':'pipeline not ready'}

    state=r['state']
    tmp_dir=state['tmp_dir']
    xcdeps=r.get('dependencies',{})
    # Clean pipeline.
    if 'ready' in r: del(r['ready'])
    if 'fail' in r: del(r['fail'])
    if 'return' in r: del(r['return'])
    pipeline=copy.deepcopy(r)

    record_repo='local'
    record_uoa='explore-matrix-size-'+kernel[0]
    ck.out('---------------------------------------------------------------------------------------')
    ck.out('Experiment - %s:%s' % (record_repo, record_uoa))

    cpipeline=copy.deepcopy(pipeline)
    ii={
        'action':'autotune',
        'module_uoa':'pipeline',
        'data_uoa':'program',
        'choices_order':[
            [
	     '##env#CK_CLBLAST_MSIZE'
	    ],
	    [
	     '##env#CK_CLBLAST_NSIZE',
	    ],
	    [
	     '##env#CK_CLBLAST_KSIZE'
	    ]
        ],
        'choices_selection':[
            {"type":"loop-with-next", "choice":size_m, "default":"256"},
            {"type":"loop-with-next", "choice":size_n, "default":"256"},
            {"type":"loop-with-next", "choice":size_k, "default":"256"}

        ],
        'features_keys_to_process':['##choices#*'],

        'iterations':-1,
        'repetitions':1,
        'record':'yes',
        'record_failed':'yes',
        'record_params':{
            'search_point_by_features':'yes'
        },
        'record_repo':record_repo,
        'record_uoa':record_uoa,
        'tags':['explore-clblast-matrix-size', kernel[0]],
        'pipeline': cpipeline,
        'out':'con'

    }
    r=ck.access(ii)
    if DEBUG > 0: print DEBUG_STR, r
    if r['return']>0: return r
    fail=r.get('fail','')
    if fail=='yes':
       return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}

    return  {'return':0}

r=do({})
if r['return']>0: ck.err(r)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

import socket
import json
import time
import copy
import os
import hashlib

import certifi
import urllib3

from . dq import DiskQueue

import logging
log = logging.getLogger(__name__)

__version__ = '1.0'

_max_content_bytes = 100000
http_event_collector_debug = False

# the list of collector URLs given to the HEC object
# are hashed into an md5 string that identifies the URL set
# these maximums are per URL set, not for the entire disk cache
max_diskqueue_items = 200
max_diskqueue_size  = 10 * (1024 ** 2)

class Payload(object):
    ''' formatters for final payload stringification
        and a convenient place to store retry counter information

        note that formatting a payload is different from formatting an event
        say our event is a line like
          "Sat Aug 18 ProcessName [3828282] DEBUG: oops"
        or perhaps some data like
          {'blah': 'happened"}
        Then the payload would look like this:
          {'host': 'something', 'sourcetype': 'something', 'event': 'Sat Aug 18...'}
          {'host': 'something', 'sourcetype': 'something', 'event': {'blah': 'happened'}}

        For reasons regarding the above, we provide a classmethod to format events:

        p = Payload.format_event({'blah': 'happened'}, sourcetype='blah')
    '''
    host = None

    @classmethod
    def format_event(cls, event, **payload):
        if 'host' not in payload:
            payload['host'] = cls.host
        if 'sourcetype' not in payload:
            payload['sourcetype'] = 'hubble'
        payload['event'] = event
        return cls(payload)

    @classmethod
    def promote(cls, payload, eventtime='', no_queue=False):
        if isinstance(payload, cls):
            return payload
        return Payload(payload, eventtime=eventtime, no_queue=no_queue)

    def __init__(self, dat, eventtime='', no_queue=False):
        if self.host is None:
            self.__class__.host = socket.gethostname()

        self.requeues = 0
        self.no_queue = no_queue or dat.pop('_no_queue', False)

        if 'host' not in dat or dat['host'] is None:
            dat['host'] = self.host

        if eventtime:
            dat['time'] = eventtime
        elif 'time' not in dat:
            dat['time'] = str(int(time.time()))

        self.strip_empty_dict_entries(dat)
        self.dat = json.dumps(dat)

    def __repr__(self):
        return 'Payload({0})'.format(self)

    def __str__(self):
        return self.dat

    def __len__(self):
        return len(self.dat)

    @classmethod
    def strip_empty_dict_entries(cls, dat):
        todo = [dat]
        while todo:
            dat = todo.pop(0)
            if isinstance(dat, dict):
                remove_keys = [k for k in dat if not dat[k] and dat[k] not in (0,False) ]
                for k in remove_keys:
                    del dat[k]
                for v in dat.values():
                    if isinstance(v, (list,tuple,dict)):
                        if v not in todo:
                            todo.append(v)
            else:
                for item in dat:
                    if isinstance(item, (list,tuple,dict)):
                        if item not in todo:
                            todo.append(item)

# Thanks to George Starcher for the http_event_collector class (https://github.com/georgestarcher/)
# Default batch max size to match splunk's default limits for max byte
# See http_input stanza in limits.conf; note in testing I had to limit to
# 100,000 to avoid http event collector breaking connection Auto flush will
# occur if next event payload will exceed limit

class HEC(object):
    flushing_queue = False

    class Server(object):
        bad = False
        def __init__(self, host, port=8080, proto='https'):
            if '://' in host:
                proto,host = host.split('://')
            if ':' in host:
                host,port = host.split(':')
            self.uri = '{proto}://{host}:{port}/services/collector/event'.format(
                proto=proto, host=host, port=port)
            self.fails = 0

        def __str__(self):
            r = self.uri
            if self.fails:
                r += ' (fails: {0})'.format(self.fails)
            return r

    def __init__(self, token, http_event_server, host='', http_event_port='8088',
                 http_event_server_ssl=True, http_event_collector_ssl_verify=True,
                 max_bytes=_max_content_bytes, proxy=None, timeout=9.05,
                 disk_queue='/var/cache/hubble/dq'):

        self.max_requeues =  5
        self.queue_overflow_msg_delay = 10
        self.retry_diskqueue_interval = 60

        self.timeout = timeout
        self.token = token
        self.batchEvents = []
        self.maxByteLength = max_bytes
        self.currentByteLength = 0
        self.server_uri = []

        if proxy and http_event_server_ssl:
            self.proxy = {'https': 'https://{0}'.format(proxy)}
        elif proxy:
            self.proxy = {'http': 'http://{0}'.format(proxy)}
        else:
            self.proxy = {}

        # Set host to specified value or default to localhostname if no value provided
        if host:
            self.host = host
        else:
            self.host = socket.gethostname()

        Payload.host = self.host

        # Build and set server_uri for http event collector
        # Defaults to SSL if flag not passed
        # Defaults to port 8088 if port not passed

        servers = http_event_server
        if not isinstance(servers, list):
            servers = [servers]
        for server in servers:
            if http_event_server_ssl:
                self.server_uri.append(self.Server(server, http_event_port, proto='https'))
            else:
                self.server_uri.append(self.Server(server, http_event_port, proto='http'))

        # build headers once
        self.headers = urllib3.make_headers( keep_alive=True,
            user_agent='hubble-hec/{0}'.format(__version__),
            accept_encoding=True)
        self.headers.update({ 'Content-Type': 'application/json',
            'Authorization': 'Splunk {0}'.format(self.token) })

        # retries can be made much more flexible than shown here theoretically,
        # it could take the load off overloaded servers through the backoff and
        # improve overall throughput at those (usually transient) bottlenecks
        # -- the number 3 was chosen essentially at random
        pm_kw = {
            'timeout': self.timeout,
            'retries': urllib3.util.retry.Retry(
                total=3, redirect=10, backoff_factor=3,
                connect=self.timeout, read=self.timeout,
                respect_retry_after_header=True)
        }

        if http_event_collector_ssl_verify:
            pm_kw.update({'cert_reqs': 'CERT_REQUIRED', 'ca_certs': certifi.where()})
        else:
            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

        if self.proxy:
            self.pool_manager = urllib3.ProxyManager(self.proxy, **pm_kw)
        else:
            self.pool_manager = urllib3.PoolManager(**pm_kw)

        md5 = hashlib.md5()
        uril = sorted([ x.uri for x in self.server_uri ])
        for u in uril:
            md5.update(u)
        actual_disk_queue = os.path.join(disk_queue, md5.hexdigest())
        log.debug("disk_queue for %s: %s", uril, actual_disk_queue)
        self.queue = DiskQueue(actual_disk_queue, restrict_to=Payload,
            max_items=max_diskqueue_items, max_size=max_diskqueue_size)

    def queueEvent(self, dat, eventtime=''):
        if not isinstance(dat, Payload):
            payload = Payload(dat, eventtime, no_queue=no_queue)
        if payload.no_queue: # here you silly hec, queue this no_queue payload...
            return
        self.queue.put(payload)

    def flushQueue(self):
        if self.flushing_queue:
            log.debug('already flushing queue')
            return
        if self.queue.eventcount < 1:
            log.debug('nothing in queue')
            return
        dt = time.time() - self.queue.last_get
        if dt >= self.retry_diskqueue_interval and self.queue.eventcount:
            log.debug('flushing queue eventscount=%d', self.queue.eventcount)
        self.flushing_queue = True
        while self.flushing_queue:
            x = self.queue.pull()
            if not x:
                break
            self.batchEvent(x)
        self.flushBatch()
        self.flushing_queue = False

    def _requeue(self, payloads):
        log.debug("_requeue")
        if self.flushing_queue:
            log.debug("aborting queue flush due to requeue")
            self.flushing_queue = False
        if not isinstance(payloads, (list,tuple)):
            payloads = [ payloads ]
        for pload in payloads:
            if pload.no_queue:
                continue
            if pload.requeues < self.max_requeues:
                log.debug("requeueing payload (requeues so far: %d)", pload.requeues)
                pload.requeues += 1
                self.queue.put(pload)

    def _send(self, *payload):
        data = ' '.join([ str(x) for x in payload ])

        servers = [ x for x in self.server_uri if not x.bad ]
        if not servers:
            log.error("all servers are marked 'bad', aborting send")
            return

        # This logic is overly complicated originally the plan was to have the
        # HEC() object manage multiple servers and choose the successful one to
        # send to. Mostly if the hubble configs contain multiple endpoints,
        # hubble sends to all of them, not just the first successful one.  and
        # mostly that logic is handled outside of the HEC class (see
        # splunklogging.py)
        for server in sorted(servers, key=lambda u: u.fails):
            try:
                r = self.pool_manager.request('POST', server.uri, body=data, headers=self.headers)
                server.fails = 0
            except urllib3.exceptions.LocationParseError as e:
                if self.log_other_exceptions:
                    log.error('server uri parse error "{0}": {1}'.format(server.uri, e))
                server.bad = True
                continue
            except Exception as e:
                server.fails += 1
                continue
            if r.status < 400:
                return r

        log.error('failed to send payload, queueing for later delivery')
        self._requeue(payload)

    def _finish_send(self, r):
        if r is not None and hasattr(r, 'text'):
            log.debug(r.text)
            self.flushQueue()

    def sendEvent(self, payload, eventtime='', no_queue=False):
        r = self._send( Payload.promote(payload, eventtime=eventtime, no_queue=no_queue) )
        self._finish_send(r)

    def batchEvent(self, dat, eventtime='', no_queue=False):
        payload = Payload.promote(dat, eventtime, no_queue=False)

        if (self.currentByteLength + len(payload)) > self.maxByteLength:
            self.flushBatch()
            if http_event_collector_debug:
                log.debug('auto flushing')
        else:
            self.currentByteLength = self.currentByteLength + len(payload)
        self.batchEvents.append(payload)

    def flushBatch(self):
        if self.batchEvents:
            r = self._send( *self.batchEvents )
            self.batchEvents = []
            self.currentByteLength = 0
            self._finish_send(r)

http_event_collector = HEC
<commit_msg>this is where the filter goes... I need to prove it's useful though.<commit_after># -*- encoding: utf-8 -*-

import socket
import json
import time
import copy
import os
import hashlib

import certifi
import urllib3

from . dq import DiskQueue

import logging
log = logging.getLogger(__name__)

__version__ = '1.0'

_max_content_bytes = 100000
http_event_collector_debug = False

# the list of collector URLs given to the HEC object
# are hashed into an md5 string that identifies the URL set
# these maximums are per URL set, not for the entire disk cache
max_diskqueue_items = 200
max_diskqueue_size  = 10 * (1024 ** 2)

class Payload(object):
    ''' formatters for final payload stringification
        and a convenient place to store retry counter information

        note that formatting a payload is different from formatting an event
        say our event is a line like
          "Sat Aug 18 ProcessName [3828282] DEBUG: oops"
        or perhaps some data like
          {'blah': 'happened"}
        Then the payload would look like this:
          {'host': 'something', 'sourcetype': 'something', 'event': 'Sat Aug 18...'}
          {'host': 'something', 'sourcetype': 'something', 'event': {'blah': 'happened'}}

        For reasons regarding the above, we provide a classmethod to format events:

        p = Payload.format_event({'blah': 'happened'}, sourcetype='blah')
    '''
    host = None

    @classmethod
    def format_event(cls, event, **payload):
        if 'host' not in payload:
            payload['host'] = cls.host
        if 'sourcetype' not in payload:
            payload['sourcetype'] = 'hubble'
        payload['event'] = event
        return cls(payload)

    @classmethod
    def promote(cls, payload, eventtime='', no_queue=False):
        if isinstance(payload, cls):
            return payload
        return Payload(payload, eventtime=eventtime, no_queue=no_queue)

    def __init__(self, dat, eventtime='', no_queue=False):
        if self.host is None:
            self.__class__.host = socket.gethostname()

        self.requeues = 0
        self.no_queue = no_queue or dat.pop('_no_queue', False)

        if 'host' not in dat or dat['host'] is None:
            dat['host'] = self.host

        if eventtime:
            dat['time'] = eventtime
        elif 'time' not in dat:
            dat['time'] = str(int(time.time()))

        self.strip_empty_dict_entries(dat)
        self.rename_event_fields_in_payload(dat)
        self.dat = json.dumps(dat)

    def __repr__(self):
        return 'Payload({0})'.format(self)

    def __str__(self):
        return self.dat

    def __len__(self):
        return len(self.dat)

    @classmethod
    def rename_event_fields_in_payload(cls, dat):
        pass

    @classmethod
    def strip_empty_dict_entries(cls, dat):
        todo = [dat]
        while todo:
            dat = todo.pop(0)
            if isinstance(dat, dict):
                remove_keys = [k for k in dat if not dat[k] and dat[k] not in (0,False) ]
                for k in remove_keys:
                    del dat[k]
                for v in dat.values():
                    if isinstance(v, (list,tuple,dict)):
                        if v not in todo:
                            todo.append(v)
            else:
                for item in dat:
                    if isinstance(item, (list,tuple,dict)):
                        if item not in todo:
                            todo.append(item)

# Thanks to George Starcher for the http_event_collector class (https://github.com/georgestarcher/)
# Default batch max size to match splunk's default limits for max byte
# See http_input stanza in limits.conf; note in testing I had to limit to
# 100,000 to avoid http event collector breaking connection Auto flush will
# occur if next event payload will exceed limit

class HEC(object):
    flushing_queue = False

    class Server(object):
        bad = False
        def __init__(self, host, port=8080, proto='https'):
            if '://' in host:
                proto,host = host.split('://')
            if ':' in host:
                host,port = host.split(':')
            self.uri = '{proto}://{host}:{port}/services/collector/event'.format(
                proto=proto, host=host, port=port)
            self.fails = 0

        def __str__(self):
            r = self.uri
            if self.fails:
                r += ' (fails: {0})'.format(self.fails)
            return r

    def __init__(self, token, http_event_server, host='', http_event_port='8088',
                 http_event_server_ssl=True, http_event_collector_ssl_verify=True,
                 max_bytes=_max_content_bytes, proxy=None, timeout=9.05,
                 disk_queue='/var/cache/hubble/dq'):

        self.max_requeues =  5
        self.queue_overflow_msg_delay = 10
        self.retry_diskqueue_interval = 60

        self.timeout = timeout
        self.token = token
        self.batchEvents = []
        self.maxByteLength = max_bytes
        self.currentByteLength = 0
        self.server_uri = []

        if proxy and http_event_server_ssl:
            self.proxy = {'https': 'https://{0}'.format(proxy)}
        elif proxy:
            self.proxy = {'http': 'http://{0}'.format(proxy)}
        else:
            self.proxy = {}

        # Set host to specified value or default to localhostname if no value provided
        if host:
            self.host = host
        else:
            self.host = socket.gethostname()

        Payload.host = self.host

        # Build and set server_uri for http event collector
        # Defaults to SSL if flag not passed
        # Defaults to port 8088 if port not passed

        servers = http_event_server
        if not isinstance(servers, list):
            servers = [servers]
        for server in servers:
            if http_event_server_ssl:
                self.server_uri.append(self.Server(server, http_event_port, proto='https'))
            else:
                self.server_uri.append(self.Server(server, http_event_port, proto='http'))

        # build headers once
        self.headers = urllib3.make_headers( keep_alive=True,
            user_agent='hubble-hec/{0}'.format(__version__),
            accept_encoding=True)
        self.headers.update({ 'Content-Type': 'application/json',
            'Authorization': 'Splunk {0}'.format(self.token) })

        # retries can be made much more flexible than shown here theoretically,
        # it could take the load off overloaded servers through the backoff and
        # improve overall throughput at those (usually transient) bottlenecks
        # -- the number 3 was chosen essentially at random
        pm_kw = {
            'timeout': self.timeout,
            'retries': urllib3.util.retry.Retry(
                total=3, redirect=10, backoff_factor=3,
                connect=self.timeout, read=self.timeout,
                respect_retry_after_header=True)
        }

        if http_event_collector_ssl_verify:
            pm_kw.update({'cert_reqs': 'CERT_REQUIRED', 'ca_certs': certifi.where()})
        else:
            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

        if self.proxy:
            self.pool_manager = urllib3.ProxyManager(self.proxy, **pm_kw)
        else:
            self.pool_manager = urllib3.PoolManager(**pm_kw)

        md5 = hashlib.md5()
        uril = sorted([ x.uri for x in self.server_uri ])
        for u in uril:
            md5.update(u)
        actual_disk_queue = os.path.join(disk_queue, md5.hexdigest())
        log.debug("disk_queue for %s: %s", uril, actual_disk_queue)
        self.queue = DiskQueue(actual_disk_queue, restrict_to=Payload,
            max_items=max_diskqueue_items, max_size=max_diskqueue_size)

    def queueEvent(self, dat, eventtime=''):
        if not isinstance(dat, Payload):
            payload = Payload(dat, eventtime, no_queue=no_queue)
        if payload.no_queue: # here you silly hec, queue this no_queue payload...
            return
        self.queue.put(payload)

    def flushQueue(self):
        if self.flushing_queue:
            log.debug('already flushing queue')
            return
        if self.queue.eventcount < 1:
            log.debug('nothing in queue')
            return
        dt = time.time() - self.queue.last_get
        if dt >= self.retry_diskqueue_interval and self.queue.eventcount:
            log.debug('flushing queue eventscount=%d', self.queue.eventcount)
        self.flushing_queue = True
        while self.flushing_queue:
            x = self.queue.pull()
            if not x:
                break
            self.batchEvent(x)
        self.flushBatch()
        self.flushing_queue = False

    def _requeue(self, payloads):
        log.debug("_requeue")
        if self.flushing_queue:
            log.debug("aborting queue flush due to requeue")
            self.flushing_queue = False
        if not isinstance(payloads, (list,tuple)):
            payloads = [ payloads ]
        for pload in payloads:
            if pload.no_queue:
                continue
            if pload.requeues < self.max_requeues:
                log.debug("requeueing payload (requeues so far: %d)", pload.requeues)
                pload.requeues += 1
                self.queue.put(pload)

    def _send(self, *payload):
        data = ' '.join([ str(x) for x in payload ])

        servers = [ x for x in self.server_uri if not x.bad ]
        if not servers:
            log.error("all servers are marked 'bad', aborting send")
            return

        # This logic is overly complicated originally the plan was to have the
        # HEC() object manage multiple servers and choose the successful one to
        # send to. Mostly if the hubble configs contain multiple endpoints,
        # hubble sends to all of them, not just the first successful one.  and
        # mostly that logic is handled outside of the HEC class (see
        # splunklogging.py)
        for server in sorted(servers, key=lambda u: u.fails):
            try:
                r = self.pool_manager.request('POST', server.uri, body=data, headers=self.headers)
                server.fails = 0
            except urllib3.exceptions.LocationParseError as e:
                if self.log_other_exceptions:
                    log.error('server uri parse error "{0}": {1}'.format(server.uri, e))
                server.bad = True
                continue
            except Exception as e:
                server.fails += 1
                continue
            if r.status < 400:
                return r

        log.error('failed to send payload, queueing for later delivery')
        self._requeue(payload)

    def _finish_send(self, r):
        if r is not None and hasattr(r, 'text'):
            log.debug(r.text)
            self.flushQueue()

    def sendEvent(self, payload, eventtime='', no_queue=False):
        r = self._send( Payload.promote(payload, eventtime=eventtime, no_queue=no_queue) )
        self._finish_send(r)

    def batchEvent(self, dat, eventtime='', no_queue=False):
        payload = Payload.promote(dat, eventtime, no_queue=False)

        if (self.currentByteLength + len(payload)) > self.maxByteLength:
            self.flushBatch()
            if http_event_collector_debug:
                log.debug('auto flushing')
        else:
            self.currentByteLength = self.currentByteLength + len(payload)
        self.batchEvents.append(payload)

    def flushBatch(self):
        if self.batchEvents:
            r = self._send( *self.batchEvents )
            self.batchEvents = []
            self.currentByteLength = 0
            self._finish_send(r)

http_event_collector = HEC
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
hyper/http20/stream
~~~~~~~~~~~~~~~~~~~

Objects that make up the stream-level abstraction of hyper's HTTP/2 support.

These objects are not expected to be part of the public HTTP/2 API: they're
intended purely for use inside hyper's HTTP/2 abstraction.

Conceptually, a single HTTP/2 connection is made up of many streams: each
stream is an independent, bi-directional sequence of HTTP headers and data.
Each stream is identified by a monotonically increasing integer, assigned to
the stream by the endpoint that initiated the stream.
"""
from .exceptions import ProtocolError
from .frame import (
    FRAME_MAX_LEN, FRAMES, HeadersFrame, DataFrame, PushPromiseFrame,
    WindowUpdateFrame, ContinuationFrame, BlockedFrame
)
from .util import get_from_key_value_set
import collections
import logging
import zlib

log = logging.getLogger(__name__)

# Define a set of states for a HTTP/2 stream.
STATE_IDLE               = 0
STATE_OPEN               = 1
STATE_HALF_CLOSED_LOCAL  = 2
STATE_HALF_CLOSED_REMOTE = 3
STATE_CLOSED             = 4

# Define the largest chunk of data we'll send in one go. Realistically, we
# should take the MSS into account but that's pretty dull, so let's just say
# 1kB and call it a day.
MAX_CHUNK = 1024

class Stream(object):
    """
    A single HTTP/2 stream.

    A stream is an independent, bi-directional sequence of HTTP headers and
    data. Each stream is identified by a single integer. From a HTTP
    perspective, a stream _approximately_ matches a single request-response
    pair.
    """
    def __init__(self,
                 stream_id,
                 data_cb,
                 recv_cb,
                 close_cb,
                 header_encoder,
                 header_decoder,
                 window_manager,
                 local_closed=False):
        self.stream_id = stream_id
        self.state = STATE_HALF_CLOSED_LOCAL if local_closed else STATE_IDLE
        self.headers = []

        # Set to a key-value set of the response headers once their
        # HEADERS..CONTINUATION frame sequence finishes.
        self.response_headers = None

        # Set to a key-value set of the response trailers once their
        # HEADERS..CONTINUATION frame sequence finishes.
        self.response_trailers = None

        # A dict mapping the promised stream ID of a pushed resource to a
        # key-value set of its request headers. Entries are added once their
        # PUSH_PROMISE..CONTINUATION frame sequence finishes.
        self.promised_headers = {}

        # Chunks of encoded header data from the current
        # (HEADERS|PUSH_PROMISE)..CONTINUATION frame sequence. Since sending any
        # frame other than a CONTINUATION is disallowed while a header block is
        # being transmitted, this and ``promised_stream_id`` are the only pieces
        # of state we have to track.
        self.header_data = []
        self.promised_stream_id = None

        # Unconsumed response data chunks. Empties after every call to _read().
        self.data = []

        # There are two flow control windows: one for data we're sending,
        # one for data being sent to us.
        self._in_window_manager = window_manager
        self._out_flow_control_window = 65535

        # This is the callback handed to the stream by its parent connection.
        # It is called when the stream wants to send data. It expects to
        # receive a list of frames that will be automatically serialized.
        self._data_cb = data_cb

        # Similarly, this is a callback that reads one frame off the
        # connection.
        self._recv_cb = recv_cb

        # This is the callback to be called when the stream is closed.
        self._close_cb = close_cb

        # A reference to the header encoder and decoder objects belonging to
        # the parent connection.
        self._encoder = header_encoder
        self._decoder = header_decoder

    def add_header(self, name, value):
        """
        Adds a single HTTP header to the headers to be sent on the request.
        """
        self.headers.append((name.lower(), value))

    def send_data(self, data, final):
        """
        Send some data on the stream. If this is the end of the data to be
        sent, the ``final`` flag _must_ be set to True. If no data is to be
        sent, set ``data`` to ``None``.
        """
        # Define a utility iterator for file objects.
        def file_iterator(fobj):
            while True:
                data = fobj.read(MAX_CHUNK)
                yield data
                if len(data) < MAX_CHUNK:
                    break

        # Build the appropriate iterator for the data, in chunks of CHUNK_SIZE.
        if hasattr(data, 'read'):
            chunks = file_iterator(data)
        else:
            chunks = (data[i:i+MAX_CHUNK]
                      for i in range(0, len(data), MAX_CHUNK))

        for chunk in chunks:
            self._send_chunk(chunk, final)

    @property
    def _local_closed(self):
        return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL)

    @property
    def _remote_closed(self):
        return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE)

    @property
    def _local_open(self):
        return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE)

    def _close_local(self):
        self.state = (
            STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN
            else STATE_CLOSED
        )

    def _close_remote(self):
        self.state = (
            STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN
            else STATE_CLOSED
        )

    def _read(self, amt=None):
        """
        Read data from the stream. Unlike a normal read behaviour, this
        function returns _at least_ ``amt`` data, but may return more.
        """
        def listlen(list):
            return sum(map(len, list))

        # Keep reading until the stream is closed or we get enough data.
        while not self._remote_closed and (amt is None or listlen(self.data) < amt):
            self._recv_cb()

        result = b''.join(self.data)
        self.data = []
        return result

    def receive_frame(self, frame):
        """
        Handle a frame received on this stream.
        """
        if 'END_STREAM' in frame.flags:
            self._close_remote()

        if frame.type == WindowUpdateFrame.type:
            self._out_flow_control_window += frame.window_increment
        elif frame.type == HeadersFrame.type:
            # Begin the header block for the response headers.
            self.promised_stream_id = None
            self.header_data = [frame.data]
        elif frame.type == PushPromiseFrame.type:
            # Begin a header block for the request headers of a pushed resource.
            self.promised_stream_id = frame.promised_stream_id
            self.header_data = [frame.data]
        elif frame.type == ContinuationFrame.type:
            # Continue a header block begun with either HEADERS or PUSH_PROMISE.
            self.header_data.append(frame.data)
        elif frame.type == DataFrame.type:
            # Increase the window size. Only do this if the data frame contains
            # actual data.
            size = frame.flow_controlled_length
            increment = self._in_window_manager._handle_frame(size)

            # Append the data to the buffer.
            self.data.append(frame.data)

            if increment and not self._remote_closed:
                w = WindowUpdateFrame(self.stream_id)
                w.window_increment = increment
                self._data_cb(w, True)
        elif frame.type == BlockedFrame.type:
            # If we've been blocked we may want to fixup the window.
            increment = self._in_window_manager._blocked()
            if increment:
                w = WindowUpdateFrame(self.stream_id)
                w.window_increment = increment
                self._data_cb(w, True)
        elif frame.type in FRAMES:
            # This frame isn't valid at this point.
            raise ValueError("Unexpected frame %s." % frame)
        else:  # pragma: no cover
            # Unknown frames belong to extensions. Just drop it on the
            # floor, but log so that users know that something happened.
            log.warning("Received unknown frame, type %d", frame.type)
            pass

        if 'END_HEADERS' in frame.flags:
            # Begin by decoding the header block. If this fails, we need to
            # tear down the entire connection. TODO: actually do that.
            headers = self._decoder.decode(b''.join(self.header_data))

            # If we're involved in a PUSH_PROMISE sequence, this header block
            # is the proposed request headers. Save it off. Otherwise, handle
            # it as an in-stream HEADERS block.
            if (self.promised_stream_id is None):
                self._handle_header_block(headers)
            else:
                self.promised_headers[self.promised_stream_id] = headers

            # We've handled the headers, zero them out.
            self.header_data = None

    def open(self, end):
        """
        Open the stream. Does this by encoding and sending the headers: no more
        calls to ``add_header`` are allowed after this method is called.
        The `end` flag controls whether this will be the end of the stream, or
        whether data will follow.
        """
        # Encode the headers.
        encoded_headers = self._encoder.encode(self.headers)

        # It's possible that there is a substantial amount of data here. The
        # data needs to go into one HEADERS frame, followed by a number of
        # CONTINUATION frames. For now, for ease of implementation, let's just
        # assume that's never going to happen (16kB of headers is lots!).
        # Additionally, since this is so unlikely, there's no point writing a
        # test for this: it's just so simple.
        if len(encoded_headers) > FRAME_MAX_LEN:  # pragma: no cover
            raise ValueError("Header block too large.")

        header_frame = HeadersFrame(self.stream_id)
        header_frame.data = encoded_headers

        # If no data has been provided, this is the end of the stream. Either
        # way, due to the restriction above it's definitely the end of the
        # headers.
        header_frame.flags.add('END_HEADERS')

        if end:
            header_frame.flags.add('END_STREAM')

        # Send the header frame.
        self._data_cb(header_frame)

        # Transition the stream state appropriately.
        self.state = STATE_HALF_CLOSED_LOCAL if end else STATE_OPEN

        return

    def getheaders(self):
        """
        Once all data has been sent on this connection, returns a key-value set
        of the headers of the response to the original request.
        """
        assert self._local_closed

        # Keep reading until all headers are received.
        while self.response_headers is None:
            self._recv_cb()

        # Find the Content-Length header if present.
        self._in_window_manager.document_size = (
            int(get_from_key_value_set(self.response_headers, 'content-length', 0))
        )

        return self.response_headers

    def getpushes(self, capture_all=False):
        """
        Returns a generator that yields push promises from the server. Note that
        this method is not idempotent; promises returned in one call will not be
        returned in subsequent calls. Iterating through generators returned by
        multiple calls to this method simultaneously results in undefined
        behavior.

        :param capture_all: If ``False``, the generator will yield all buffered
            push promises without blocking. If ``True``, the generator will
            first yield all buffered push promises, then yield additional ones
            as they arrive, and terminate when the original stream closes.
        """
        while True:
            for pair in self.promised_headers.items():
                yield pair
            self.promised_headers = {}
            if not capture_all or self._remote_closed:
                break
            self._recv_cb()

    def close(self, error_code=None):
        """
        Closes the stream. If the stream is currently open, attempts to close
        it as gracefully as possible.

        :param error_code: (optional) The error code to reset the stream with.
        :returns: Nothing.
        """
        # Right now let's not bother with grace, let's just call close on the
        # connection.
        self._close_cb(self.stream_id, error_code)

    def _handle_header_block(self, headers):
        """
        Handles the logic for receiving a completed headers block.

        A headers block is an uninterrupted sequence of one HEADERS frame
        followed by zero or more CONTINUATION frames, and is terminated by a
        frame bearing the END_HEADERS flag.

        HTTP/2 allows receipt of up to three such blocks on a stream. The first
        is optional, and contains a 1XX response. The second is mandatory, and
        must contain a final response (200 or higher). The third is optional,
        and may contain 'trailers', headers that are sent after a chunk-encoded
        body is sent. This method enforces the logic associated with this,
        storing the headers in the appropriate places.
        """

        # At this stage we should check for a provisional response (1XX). As
        # hyper doesn't currently support such responses, we'll leave that as a
        # TODO.
        # TODO: Handle 1XX responses here.

        # The header block may be for trailers or headers. If we've already
        # received headers these _must_ be for trailers.
        if self.response_headers is None:
            self.response_headers = headers
        elif self.response_trailers is None:
            self.response_trailers = headers
        else:
            # Received too many headers blocks.
            raise ProtocolError("Too many header blocks.")

        self.header_data = None

    def _send_chunk(self, data, final):
        """
        Implements most of the sending logic.

        Takes a single chunk of size at most MAX_CHUNK, wraps it in a frame and
        sends it. Optionally sets the END_STREAM flag if this is the last chunk
        (determined by being of size less than MAX_CHUNK) and no more data is
        to be sent.
        """
        assert self._local_open

        f = DataFrame(self.stream_id)
        f.data = data

        # If the length of the data is less than MAX_CHUNK, we're probably
        # at the end of the file. If this is the end of the data, mark it
        # as END_STREAM.
        if len(data) < MAX_CHUNK and final:
            f.flags.add('END_STREAM')

        # If we don't fit in the connection window, try popping frames off the
        # connection in hope that one might be a Window Update frame.
        while len(data) > self._out_flow_control_window:
            self._recv_cb()

        # Send the frame and decrement the flow control window.
        self._data_cb(f)
        self._out_flow_control_window -= len(data)

        # If no more data is to be sent on this stream, transition our state.
        if len(data) < MAX_CHUNK and final:
            self._close_local()
<commit_msg>Remove duplicate statement.<commit_after># -*- coding: utf-8 -*-
"""
hyper/http20/stream
~~~~~~~~~~~~~~~~~~~

Objects that make up the stream-level abstraction of hyper's HTTP/2 support.

These objects are not expected to be part of the public HTTP/2 API: they're
intended purely for use inside hyper's HTTP/2 abstraction.

Conceptually, a single HTTP/2 connection is made up of many streams: each
stream is an independent, bi-directional sequence of HTTP headers and data.
Each stream is identified by a monotonically increasing integer, assigned to
the stream by the endpoint that initiated the stream.
"""
from .exceptions import ProtocolError
from .frame import (
    FRAME_MAX_LEN, FRAMES, HeadersFrame, DataFrame, PushPromiseFrame,
    WindowUpdateFrame, ContinuationFrame, BlockedFrame
)
from .util import get_from_key_value_set
import collections
import logging
import zlib

log = logging.getLogger(__name__)

# Define a set of states for a HTTP/2 stream.
STATE_IDLE               = 0
STATE_OPEN               = 1
STATE_HALF_CLOSED_LOCAL  = 2
STATE_HALF_CLOSED_REMOTE = 3
STATE_CLOSED             = 4

# Define the largest chunk of data we'll send in one go. Realistically, we
# should take the MSS into account but that's pretty dull, so let's just say
# 1kB and call it a day.
MAX_CHUNK = 1024

class Stream(object):
    """
    A single HTTP/2 stream.

    A stream is an independent, bi-directional sequence of HTTP headers and
    data. Each stream is identified by a single integer. From a HTTP
    perspective, a stream _approximately_ matches a single request-response
    pair.
    """
    def __init__(self,
                 stream_id,
                 data_cb,
                 recv_cb,
                 close_cb,
                 header_encoder,
                 header_decoder,
                 window_manager,
                 local_closed=False):
        self.stream_id = stream_id
        self.state = STATE_HALF_CLOSED_LOCAL if local_closed else STATE_IDLE
        self.headers = []

        # Set to a key-value set of the response headers once their
        # HEADERS..CONTINUATION frame sequence finishes.
        self.response_headers = None

        # Set to a key-value set of the response trailers once their
        # HEADERS..CONTINUATION frame sequence finishes.
        self.response_trailers = None

        # A dict mapping the promised stream ID of a pushed resource to a
        # key-value set of its request headers. Entries are added once their
        # PUSH_PROMISE..CONTINUATION frame sequence finishes.
        self.promised_headers = {}

        # Chunks of encoded header data from the current
        # (HEADERS|PUSH_PROMISE)..CONTINUATION frame sequence. Since sending any
        # frame other than a CONTINUATION is disallowed while a header block is
        # being transmitted, this and ``promised_stream_id`` are the only pieces
        # of state we have to track.
        self.header_data = []
        self.promised_stream_id = None

        # Unconsumed response data chunks. Empties after every call to _read().
        self.data = []

        # There are two flow control windows: one for data we're sending,
        # one for data being sent to us.
        self._in_window_manager = window_manager
        self._out_flow_control_window = 65535

        # This is the callback handed to the stream by its parent connection.
        # It is called when the stream wants to send data. It expects to
        # receive a list of frames that will be automatically serialized.
        self._data_cb = data_cb

        # Similarly, this is a callback that reads one frame off the
        # connection.
        self._recv_cb = recv_cb

        # This is the callback to be called when the stream is closed.
        self._close_cb = close_cb

        # A reference to the header encoder and decoder objects belonging to
        # the parent connection.
        self._encoder = header_encoder
        self._decoder = header_decoder

    def add_header(self, name, value):
        """
        Adds a single HTTP header to the headers to be sent on the request.
        """
        self.headers.append((name.lower(), value))

    def send_data(self, data, final):
        """
        Send some data on the stream. If this is the end of the data to be
        sent, the ``final`` flag _must_ be set to True. If no data is to be
        sent, set ``data`` to ``None``.
        """
        # Define a utility iterator for file objects.
        def file_iterator(fobj):
            while True:
                data = fobj.read(MAX_CHUNK)
                yield data
                if len(data) < MAX_CHUNK:
                    break

        # Build the appropriate iterator for the data, in chunks of CHUNK_SIZE.
        if hasattr(data, 'read'):
            chunks = file_iterator(data)
        else:
            chunks = (data[i:i+MAX_CHUNK]
                      for i in range(0, len(data), MAX_CHUNK))

        for chunk in chunks:
            self._send_chunk(chunk, final)

    @property
    def _local_closed(self):
        return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL)

    @property
    def _remote_closed(self):
        return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE)

    @property
    def _local_open(self):
        return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE)

    def _close_local(self):
        self.state = (
            STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN
            else STATE_CLOSED
        )

    def _close_remote(self):
        self.state = (
            STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN
            else STATE_CLOSED
        )

    def _read(self, amt=None):
        """
        Read data from the stream. Unlike a normal read behaviour, this
        function returns _at least_ ``amt`` data, but may return more.
        """
        def listlen(list):
            return sum(map(len, list))

        # Keep reading until the stream is closed or we get enough data.
        while not self._remote_closed and (amt is None or listlen(self.data) < amt):
            self._recv_cb()

        result = b''.join(self.data)
        self.data = []
        return result

    def receive_frame(self, frame):
        """
        Handle a frame received on this stream.
        """
        if 'END_STREAM' in frame.flags:
            self._close_remote()

        if frame.type == WindowUpdateFrame.type:
            self._out_flow_control_window += frame.window_increment
        elif frame.type == HeadersFrame.type:
            # Begin the header block for the response headers.
            self.promised_stream_id = None
            self.header_data = [frame.data]
        elif frame.type == PushPromiseFrame.type:
            # Begin a header block for the request headers of a pushed resource.
            self.promised_stream_id = frame.promised_stream_id
            self.header_data = [frame.data]
        elif frame.type == ContinuationFrame.type:
            # Continue a header block begun with either HEADERS or PUSH_PROMISE.
            self.header_data.append(frame.data)
        elif frame.type == DataFrame.type:
            # Increase the window size. Only do this if the data frame contains
            # actual data.
            size = frame.flow_controlled_length
            increment = self._in_window_manager._handle_frame(size)

            # Append the data to the buffer.
            self.data.append(frame.data)

            if increment and not self._remote_closed:
                w = WindowUpdateFrame(self.stream_id)
                w.window_increment = increment
                self._data_cb(w, True)
        elif frame.type == BlockedFrame.type:
            # If we've been blocked we may want to fixup the window.
            increment = self._in_window_manager._blocked()
            if increment:
                w = WindowUpdateFrame(self.stream_id)
                w.window_increment = increment
                self._data_cb(w, True)
        elif frame.type in FRAMES:
            # This frame isn't valid at this point.
            raise ValueError("Unexpected frame %s." % frame)
        else:  # pragma: no cover
            # Unknown frames belong to extensions. Just drop it on the
            # floor, but log so that users know that something happened.
            log.warning("Received unknown frame, type %d", frame.type)
            pass

        if 'END_HEADERS' in frame.flags:
            # Begin by decoding the header block. If this fails, we need to
            # tear down the entire connection. TODO: actually do that.
            headers = self._decoder.decode(b''.join(self.header_data))

            # If we're involved in a PUSH_PROMISE sequence, this header block
            # is the proposed request headers. Save it off. Otherwise, handle
            # it as an in-stream HEADERS block.
            if (self.promised_stream_id is None):
                self._handle_header_block(headers)
            else:
                self.promised_headers[self.promised_stream_id] = headers

            # We've handled the headers, zero them out.
            self.header_data = None

    def open(self, end):
        """
        Open the stream. Does this by encoding and sending the headers: no more
        calls to ``add_header`` are allowed after this method is called.
        The `end` flag controls whether this will be the end of the stream, or
        whether data will follow.
        """
        # Encode the headers.
        encoded_headers = self._encoder.encode(self.headers)

        # It's possible that there is a substantial amount of data here. The
        # data needs to go into one HEADERS frame, followed by a number of
        # CONTINUATION frames. For now, for ease of implementation, let's just
        # assume that's never going to happen (16kB of headers is lots!).
        # Additionally, since this is so unlikely, there's no point writing a
        # test for this: it's just so simple.
        if len(encoded_headers) > FRAME_MAX_LEN:  # pragma: no cover
            raise ValueError("Header block too large.")

        header_frame = HeadersFrame(self.stream_id)
        header_frame.data = encoded_headers

        # If no data has been provided, this is the end of the stream. Either
        # way, due to the restriction above it's definitely the end of the
        # headers.
        header_frame.flags.add('END_HEADERS')

        if end:
            header_frame.flags.add('END_STREAM')

        # Send the header frame.
        self._data_cb(header_frame)

        # Transition the stream state appropriately.
        self.state = STATE_HALF_CLOSED_LOCAL if end else STATE_OPEN

        return

    def getheaders(self):
        """
        Once all data has been sent on this connection, returns a key-value set
        of the headers of the response to the original request.
        """
        assert self._local_closed

        # Keep reading until all headers are received.
        while self.response_headers is None:
            self._recv_cb()

        # Find the Content-Length header if present.
        self._in_window_manager.document_size = (
            int(get_from_key_value_set(self.response_headers, 'content-length', 0))
        )

        return self.response_headers

    def getpushes(self, capture_all=False):
        """
        Returns a generator that yields push promises from the server. Note that
        this method is not idempotent; promises returned in one call will not be
        returned in subsequent calls. Iterating through generators returned by
        multiple calls to this method simultaneously results in undefined
        behavior.

        :param capture_all: If ``False``, the generator will yield all buffered
            push promises without blocking. If ``True``, the generator will
            first yield all buffered push promises, then yield additional ones
            as they arrive, and terminate when the original stream closes.
        """
        while True:
            for pair in self.promised_headers.items():
                yield pair
            self.promised_headers = {}
            if not capture_all or self._remote_closed:
                break
            self._recv_cb()

    def close(self, error_code=None):
        """
        Closes the stream. If the stream is currently open, attempts to close
        it as gracefully as possible.

        :param error_code: (optional) The error code to reset the stream with.
        :returns: Nothing.
        """
        # Right now let's not bother with grace, let's just call close on the
        # connection.
        self._close_cb(self.stream_id, error_code)

    def _handle_header_block(self, headers):
        """
        Handles the logic for receiving a completed headers block.

        A headers block is an uninterrupted sequence of one HEADERS frame
        followed by zero or more CONTINUATION frames, and is terminated by a
        frame bearing the END_HEADERS flag.

        HTTP/2 allows receipt of up to three such blocks on a stream. The first
        is optional, and contains a 1XX response. The second is mandatory, and
        must contain a final response (200 or higher). The third is optional,
        and may contain 'trailers', headers that are sent after a chunk-encoded
        body is sent. This method enforces the logic associated with this,
        storing the headers in the appropriate places.
        """
        # At this stage we should check for a provisional response (1XX). As
        # hyper doesn't currently support such responses, we'll leave that as a
        # TODO.
        # TODO: Handle 1XX responses here.

        # The header block may be for trailers or headers. If we've already
        # received headers these _must_ be for trailers.
        if self.response_headers is None:
            self.response_headers = headers
        elif self.response_trailers is None:
            self.response_trailers = headers
        else:
            # Received too many headers blocks.
            raise ProtocolError("Too many header blocks.")

        return

    def _send_chunk(self, data, final):
        """
        Implements most of the sending logic.

        Takes a single chunk of size at most MAX_CHUNK, wraps it in a frame and
        sends it. Optionally sets the END_STREAM flag if this is the last chunk
        (determined by being of size less than MAX_CHUNK) and no more data is
        to be sent.
        """
        assert self._local_open

        f = DataFrame(self.stream_id)
        f.data = data

        # If the length of the data is less than MAX_CHUNK, we're probably
        # at the end of the file. If this is the end of the data, mark it
        # as END_STREAM.
        if len(data) < MAX_CHUNK and final:
            f.flags.add('END_STREAM')

        # If we don't fit in the connection window, try popping frames off the
        # connection in hope that one might be a Window Update frame.
        while len(data) > self._out_flow_control_window:
            self._recv_cb()

        # Send the frame and decrement the flow control window.
        self._data_cb(f)
        self._out_flow_control_window -= len(data)

        # If no more data is to be sent on this stream, transition our state.
        if len(data) < MAX_CHUNK and final:
            self._close_local()
<|endoftext|>


<commit_before># Copyright 2016 IBM All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
The Conversation v1 service
(https://www.ibm.com/watson/developercloud/conversation.html)
"""

from .watson_developer_cloud_service import WatsonDeveloperCloudService

class ConversationV1(WatsonDeveloperCloudService):
    """Client for the Conversation service"""

    default_url = 'https://gateway.watsonplatform.net/conversation/api'
    latest_version = '2016-07-11'

    def __init__(self, version, url=default_url, **kwargs):
        WatsonDeveloperCloudService.__init__(self, 'conversation', url, **kwargs)
        self.version = version

    def message(self, workspace_id, message_input=None, context=None, entities=None, intents=None, output=None, alternate_intents=None):
        """
        Retrieves information about a specific classifier.
        :param workspace_id: The workspace to use
        :param message_input: The input, usually containing a text field
        :param context: The optional context object
        :param entities: The optional entities
        :param intents: The optional intents
        :param alternate_intents: Whether to return more than one intent.
        :param output: The optional output object
        """

        params = {'version': self.version}
        data = {'input': message_input,
                'context': context,
                'entities': entities,
                'intents': intents,
                'alternate_intents': alternate_intents,
                'output': output}
        return self.request(method='POST', url='/v1/workspaces/{0}/message'.format(workspace_id), params=params,
                            json=data, accept_json=True)
<commit_msg>tweaking conversation alternate_intents<commit_after># Copyright 2016 IBM All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
The Conversation v1 service
(https://www.ibm.com/watson/developercloud/conversation.html)
"""

from .watson_developer_cloud_service import WatsonDeveloperCloudService

class ConversationV1(WatsonDeveloperCloudService):
    """Client for the Conversation service"""

    default_url = 'https://gateway.watsonplatform.net/conversation/api'
    latest_version = '2016-07-11'

    def __init__(self, version, url=default_url, **kwargs):
        WatsonDeveloperCloudService.__init__(self, 'conversation', url, **kwargs)
        self.version = version

    def message(self, workspace_id, message_input=None, context=None, entities=None, intents=None, output=None,
                alternate_intents=False):
        """
        Retrieves information about a specific classifier.
        :param workspace_id: The workspace to use
        :param message_input: The input, usually containing a text field
        :param context: The optional context object
        :param entities: The optional entities
        :param intents: The optional intents
        :param alternate_intents: Whether to return more than one intent.
        :param output: The optional output object
        """

        params = {'version': self.version}
        data = {'input': message_input,
                'context': context,
                'entities': entities,
                'intents': intents,
                'alternate_intents': alternate_intents,
                'output': output}
        return self.request(method='POST', url='/v1/workspaces/{0}/message'.format(workspace_id), params=params,
                            json=data, accept_json=True)
<|endoftext|>


<commit_before>from django.test import TestCase
import pytest

from ..models import JSONReformTaxCalculator
from ..forms import TaxBrainForm
from ...test_assets.utils import get_taxbrain_model, stringify_fields
from ...test_assets.test_models import TaxBrainTableResults, TaxBrainModelsTest

class TaxBrainJSONReformModelTest(TestCase):
    """Test taxbrain JSONReformTaxCalculator."""

    def setUp(self):
        # Every test needs a client.
        self.test_string = "".join(["1" for x in range(100000)])

    def test_create_reforms(self):
        self.reforms = JSONReformTaxCalculator.objects.create(
            reform_text=self.test_string,
            raw_reform_text=self.test_string,
            assumption_text=self.test_string,
            raw_assumption_text=self.test_string
        )

class TaxBrainStaticResultsTest(TaxBrainTableResults, TestCase):

    def test_static_tc_lt_0130(self):
        self.tc_lt_0130(self.test_coverage_fields)

    def test_static_tc_gt_0130(self):
        self.tc_gt_0130(self.test_coverage_fields)

class TaxBrainStaticFieldsTest(TaxBrainModelsTest, TestCase):

    def test_set_fields(self):
        start_year = 2017
        fields = self.test_coverage_gui_fields.copy()
        fields['first_year'] = start_year

        self.parse_fields(start_year, fields, Form=TaxBrainForm)
<commit_msg>Use correct base class in taxbrain/test_models<commit_after>from django.test import TestCase
import pytest

from ..models import JSONReformTaxCalculator
from ..forms import TaxBrainForm
from ...test_assets.utils import get_taxbrain_model, stringify_fields
from ...test_assets.test_models import TaxBrainTableResults, TaxBrainFieldsTest

class TaxBrainJSONReformModelTest(TestCase):
    """Test taxbrain JSONReformTaxCalculator."""

    def setUp(self):
        # Every test needs a client.
        self.test_string = "".join(["1" for x in range(100000)])

    def test_create_reforms(self):
        self.reforms = JSONReformTaxCalculator.objects.create(
            reform_text=self.test_string,
            raw_reform_text=self.test_string,
            assumption_text=self.test_string,
            raw_assumption_text=self.test_string
        )

class TaxBrainStaticResultsTest(TaxBrainTableResults, TestCase):

    def test_static_tc_lt_0130(self):
        self.tc_lt_0130(self.test_coverage_fields)

    def test_static_tc_gt_0130(self):
        self.tc_gt_0130(self.test_coverage_fields)

class TaxBrainStaticFieldsTest(TaxBrainFieldsTest, TestCase):

    def test_set_fields(self):
        start_year = 2017
        fields = self.test_coverage_gui_fields.copy()
        fields['first_year'] = start_year

        self.parse_fields(start_year, fields, Form=TaxBrainForm)
<|endoftext|>