<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-TODAY OpenERP SA (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
from osv import fields, osv
from tools.misc import DEFAULT_SERVER_DATETIME_FORMAT
import decimal_precision as dp
from lxml import etree
from tools.translate import _

class stock_partial_picking_line(osv.TransientModel):

    def _tracking(self, cursor, user, ids, name, arg, context=None):
        res = {}
        for tracklot in self.browse(cursor, user, ids, context=context):
            tracking = False
            if (tracklot.move_id.picking_id.type == 'in' and tracklot.product_id.track_incoming == True) or \
                (tracklot.move_id.picking_id.type == 'out' and tracklot.product_id.track_outgoing == True):
                tracking = True
            res[tracklot.id] = tracking
        return res

    _name = "stock.partial.picking.line"
    _rec_name = 'product_id'
    _columns = {
        'product_id' : fields.many2one('product.product', string="Product", required=True, ondelete='CASCADE'),
        'quantity' : fields.float("Quantity", digits_compute=dp.get_precision('Product UoM'), required=True),
        'product_uom': fields.many2one('product.uom', 'Unit of Measure', required=True, ondelete='CASCADE'),
        'prodlot_id' : fields.many2one('stock.production.lot', 'Production Lot', ondelete='CASCADE'),
        'location_id': fields.many2one('stock.location', 'Location', required=True, ondelete='CASCADE', domain = [('usage','<>','view')]),
        'location_dest_id': fields.many2one('stock.location', 'Dest. Location', required=True, ondelete='CASCADE',domain = [('usage','<>','view')]),
        'move_id' : fields.many2one('stock.move', "Move", ondelete='CASCADE'),
        'wizard_id' : fields.many2one('stock.partial.picking', string="Wizard", ondelete='CASCADE'),
        'update_cost': fields.boolean('Need cost update'),
        'cost' : fields.float("Cost", help="Unit Cost for this product line"),
        'currency' : fields.many2one('res.currency', string="Currency", help="Currency in which Unit cost is expressed", ondelete='CASCADE'),
        'tracking': fields.function(_tracking, string='Tracking', type='boolean'),
    }

class stock_partial_picking(osv.osv_memory):
    _name = "stock.partial.picking"
    _description = "Partial Picking Processing Wizard"

    def _hide_tracking(self, cursor, user, ids, name, arg, context=None):
        res = {}
        for wizard in self.browse(cursor, user, ids, context=context):
            res[wizard.id] = any([not(x.tracking) for x in wizard.move_ids])
        return res

    _columns = {
        'date': fields.datetime('Date', required=True),
        'move_ids' : fields.one2many('stock.partial.picking.line', 'wizard_id', 'Product Moves'),
        'picking_id': fields.many2one('stock.picking', 'Picking', required=True, ondelete='CASCADE'),
        'hide_tracking': fields.function(_hide_tracking, string='Tracking', type='boolean', help='This field is for internal purpose. It is used to decide if the column prodlot has to be shown on the move_ids field or not'),
     }

    def fields_view_get(self, cr, uid, view_id=None, view_type='form',
                        context=None, toolbar=False, submenu=False):
        if context is None:
            context={}
        res = super(stock_partial_picking, self).fields_view_get(cr, uid, view_id=view_id, view_type=view_type, context=context, toolbar=toolbar,submenu=False)
        type = context.get('default_type', False)
        doc = etree.XML(res['arch'])
        nodes = doc.xpath("//group/button[@string='_Validate']")
        for field in res['fields']:
            if type == 'in':
                for node in nodes:
                    node.set('string', '_Receive')
            if type == 'internal':
                for node in nodes:
                    node.set('string', '_Move')
            if type == 'out':
                for node in nodes:
                    node.set('string', '_Deliver')

        res['arch'] = etree.tostring(doc)
        return res

    def default_get(self, cr, uid, fields, context=None):
        if context is None: context = {}
        res = super(stock_partial_picking, self).default_get(cr, uid, fields, context=context)
        picking_ids = context.get('active_ids', [])
        if not picking_ids or (not context.get('active_model') == 'stock.picking') \
            or len(picking_ids) != 1:
            # Partial Picking Processing may only be done for one picking at a time
            return res
        picking_id, = picking_ids
        if 'picking_id' in fields:
            res.update(picking_id=picking_id)
        if 'move_ids' in fields:
            picking = self.pool.get('stock.picking').browse(cr, uid, picking_id, context=context)
            moves = [self._partial_move_for(cr, uid, m) for m in picking.move_lines if m.state not in ('done','cancel')]
            res.update(move_ids=moves)
        if 'date' in fields:
            res.update(date=time.strftime(DEFAULT_SERVER_DATETIME_FORMAT))
        return res

    def _product_cost_for_average_update(self, cr, uid, move):
        """Returns product cost and currency ID for the given move, suited for re-computing
           the average product cost.

           :return: map of the form::

                {'cost': 123.34,
                 'currency': 42}
        """
        # Currently, the cost on the product form is supposed to be expressed in the currency
        # of the company owning the product. If not set, we fall back to the picking's company,
        # which should work in simple cases.
        return {'cost': move.product_id.standard_price,
                'currency': move.product_id.company_id.currency_id.id \
                                or move.picking_id.company_id.currency_id.id \
                                or False}

    def _partial_move_for(self, cr, uid, move):
        partial_move = {
            'product_id' : move.product_id.id,
            'quantity' : move.state in ('assigned','new') and move.product_qty or 0,
            'product_uom' : move.product_uom.id,
            'prodlot_id' : move.prodlot_id.id,
            'move_id' : move.id,
            'location_id' : move.location_id.id,
            'location_dest_id' : move.location_dest_id.id,
        }
        if move.picking_id.type == 'in' and move.product_id.cost_method == 'average':
            partial_move.update(update_cost=True, **self._product_cost_for_average_update(cr, uid, move))
        return partial_move

    def do_partial(self, cr, uid, ids, context=None):
        assert len(ids) == 1, 'Partial picking processing may only be done one at a time'
        stock_picking = self.pool.get('stock.picking')
        stock_move = self.pool.get('stock.move')
        uom_obj = self.pool.get('product.uom')
        partial = self.browse(cr, uid, ids[0], context=context)
        partial_data = {
            'delivery_date' : partial.date
        }
        picking_type = partial.picking_id.type
        for wizard_line in partial.move_ids:
            line_uom = wizard_line.product_uom
            move_id = wizard_line.move_id.id

            #Quantiny must be Positive
            if wizard_line.quantity < 0:
                raise osv.except_osv(_('Warning!'), _('Please provide Proper Quantity !'))

            #Compute the quantity for respective wizard_line in the line uom (this jsut do the rounding if necessary)
            qty_in_line_uom = uom_obj._compute_qty(cr, uid, line_uom.id, wizard_line.quantity, line_uom.id)

            if line_uom.factor and line_uom.factor <> 0:
                if qty_in_line_uom <> wizard_line.quantity:
                    raise osv.except_osv(_('Warning'), _('The uom rounding does not allow you to ship "%s %s", only roundings of "%s %s" is accepted by the uom.') % (wizard_line.quantity, line_uom.name, line_uom.rounding, line_uom.name))
            if move_id:
                #Check rounding Quantity.ex.
                #picking: 1kg, uom kg rounding = 0.01 (rounding to 10g),
                #partial delivery: 253g
                #=> result= refused, as the qty left on picking would be 0.747kg and only 0.75 is accepted by the uom.
                initial_uom = wizard_line.move_id.product_uom
                #Compute the quantity for respective wizard_line in the initial uom
                qty_in_initial_uom = uom_obj._compute_qty(cr, uid, line_uom.id, wizard_line.quantity, initial_uom.id)
                without_rounding_qty = (wizard_line.quantity / line_uom.factor) * initial_uom.factor
                if qty_in_initial_uom <> without_rounding_qty:
                    raise osv.except_osv(_('Warning'), _('The rounding of the initial uom does not allow you to ship "%s %s", as it would let a quantity of "%s %s" to ship and only roundings of "%s %s" is accepted by the uom.') % (wizard_line.quantity, line_uom.name, wizard_line.move_id.product_qty - without_rounding_qty, initial_uom.name, initial_uom.rounding, initial_uom.name))
            else:
                seq_obj_name =  'stock.picking.' + picking_type
                move_id = stock_move.create(cr,uid,{'name' : self.pool.get('ir.sequence').get(cr, uid, seq_obj_name),
                                                    'product_id': wizard_line.product_id.id,
                                                    'product_qty': wizard_line.quantity,
                                                    'product_uom': wizard_line.product_uom.id,
                                                    'prodlot_id': wizard_line.prodlot_id.id,
                                                    'location_id' : wizard_line.location_id.id,
                                                    'location_dest_id' : wizard_line.location_dest_id.id,
                                                    'picking_id': partial.picking_id.id
                                                    },context=context)
                stock_move.action_confirm(cr, uid, [move_id], context)
            partial_data['move%s' % (move_id)] = {
                'product_id': wizard_line.product_id.id,
                'product_qty': wizard_line.quantity,
                'product_uom': wizard_line.product_uom.id,
                'prodlot_id': wizard_line.prodlot_id.id,
            }
            if (picking_type == 'in') and (wizard_line.product_id.cost_method == 'average'):
                partial_data['move%s' % (wizard_line.move_id.id)].update(product_price=wizard_line.cost,
                                                                  product_currency=wizard_line.currency.id)
        stock_picking.do_partial(cr, uid, [partial.picking_id.id], partial_data, context=context)
        return {'type': 'ir.actions.act_window_close'}

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP]stock: added a separator via field_view_get for related view in partial wizard<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-TODAY OpenERP SA (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
from osv import fields, osv
from tools.misc import DEFAULT_SERVER_DATETIME_FORMAT
import decimal_precision as dp
from lxml import etree
from tools.translate import _

class stock_partial_picking_line(osv.TransientModel):

    def _tracking(self, cursor, user, ids, name, arg, context=None):
        res = {}
        for tracklot in self.browse(cursor, user, ids, context=context):
            tracking = False
            if (tracklot.move_id.picking_id.type == 'in' and tracklot.product_id.track_incoming == True) or \
                (tracklot.move_id.picking_id.type == 'out' and tracklot.product_id.track_outgoing == True):
                tracking = True
            res[tracklot.id] = tracking
        return res

    _name = "stock.partial.picking.line"
    _rec_name = 'product_id'
    _columns = {
        'product_id' : fields.many2one('product.product', string="Product", required=True, ondelete='CASCADE'),
        'quantity' : fields.float("Quantity", digits_compute=dp.get_precision('Product UoM'), required=True),
        'product_uom': fields.many2one('product.uom', 'Unit of Measure', required=True, ondelete='CASCADE'),
        'prodlot_id' : fields.many2one('stock.production.lot', 'Production Lot', ondelete='CASCADE'),
        'location_id': fields.many2one('stock.location', 'Location', required=True, ondelete='CASCADE', domain = [('usage','<>','view')]),
        'location_dest_id': fields.many2one('stock.location', 'Dest. Location', required=True, ondelete='CASCADE',domain = [('usage','<>','view')]),
        'move_id' : fields.many2one('stock.move', "Move", ondelete='CASCADE'),
        'wizard_id' : fields.many2one('stock.partial.picking', string="Wizard", ondelete='CASCADE'),
        'update_cost': fields.boolean('Need cost update'),
        'cost' : fields.float("Cost", help="Unit Cost for this product line"),
        'currency' : fields.many2one('res.currency', string="Currency", help="Currency in which Unit cost is expressed", ondelete='CASCADE'),
        'tracking': fields.function(_tracking, string='Tracking', type='boolean'),
    }

class stock_partial_picking(osv.osv_memory):
    _name = "stock.partial.picking"
    _description = "Partial Picking Processing Wizard"

    def _hide_tracking(self, cursor, user, ids, name, arg, context=None):
        res = {}
        for wizard in self.browse(cursor, user, ids, context=context):
            res[wizard.id] = any([not(x.tracking) for x in wizard.move_ids])
        return res

    _columns = {
        'date': fields.datetime('Date', required=True),
        'move_ids' : fields.one2many('stock.partial.picking.line', 'wizard_id', 'Product Moves'),
        'picking_id': fields.many2one('stock.picking', 'Picking', required=True, ondelete='CASCADE'),
        'hide_tracking': fields.function(_hide_tracking, string='Tracking', type='boolean', help='This field is for internal purpose. It is used to decide if the column prodlot has to be shown on the move_ids field or not'),
     }

    def fields_view_get(self, cr, uid, view_id=None, view_type='form',
                        context=None, toolbar=False, submenu=False):
        if context is None:
            context={}
        res = super(stock_partial_picking, self).fields_view_get(cr, uid, view_id=view_id, view_type=view_type, context=context, toolbar=toolbar,submenu=False)
        type = context.get('default_type', False)
        doc = etree.XML(res['arch'])
        nodes = doc.xpath("//group/button[@string='_Validate']")
        separator_extended = ''
        for field in res['fields']:
            if type == 'in':
                for node in nodes:
                    separator_extended = '''<separator colspan="4" string="Receive Products"/>'''
                    node.set('string', '_Receive')
            if type == 'internal':
                for node in nodes:
                    separator_extended = '''<separator colspan="4" string="Move Products"/>'''
                    node.set('string', '_Move')
            if type == 'out':
                for node in nodes:
                    separator_extended = '''<separator colspan="4" string="Deliver Products"/>'''
                    node.set('string', '_Deliver')

        res['arch'] = etree.tostring(doc)
        res['arch'] = res['arch'].replace('<separator colspan="4" string="Products"/>', separator_extended)
        return res

    def default_get(self, cr, uid, fields, context=None):
        if context is None: context = {}
        res = super(stock_partial_picking, self).default_get(cr, uid, fields, context=context)
        picking_ids = context.get('active_ids', [])
        if not picking_ids or (not context.get('active_model') == 'stock.picking') \
            or len(picking_ids) != 1:
            # Partial Picking Processing may only be done for one picking at a time
            return res
        picking_id, = picking_ids
        if 'picking_id' in fields:
            res.update(picking_id=picking_id)
        if 'move_ids' in fields:
            picking = self.pool.get('stock.picking').browse(cr, uid, picking_id, context=context)
            moves = [self._partial_move_for(cr, uid, m) for m in picking.move_lines if m.state not in ('done','cancel')]
            res.update(move_ids=moves)
        if 'date' in fields:
            res.update(date=time.strftime(DEFAULT_SERVER_DATETIME_FORMAT))
        return res

    def _product_cost_for_average_update(self, cr, uid, move):
        """Returns product cost and currency ID for the given move, suited for re-computing
           the average product cost.

           :return: map of the form::

                {'cost': 123.34,
                 'currency': 42}
        """
        # Currently, the cost on the product form is supposed to be expressed in the currency
        # of the company owning the product. If not set, we fall back to the picking's company,
        # which should work in simple cases.
        return {'cost': move.product_id.standard_price,
                'currency': move.product_id.company_id.currency_id.id \
                                or move.picking_id.company_id.currency_id.id \
                                or False}

    def _partial_move_for(self, cr, uid, move):
        partial_move = {
            'product_id' : move.product_id.id,
            'quantity' : move.state in ('assigned','new') and move.product_qty or 0,
            'product_uom' : move.product_uom.id,
            'prodlot_id' : move.prodlot_id.id,
            'move_id' : move.id,
            'location_id' : move.location_id.id,
            'location_dest_id' : move.location_dest_id.id,
        }
        if move.picking_id.type == 'in' and move.product_id.cost_method == 'average':
            partial_move.update(update_cost=True, **self._product_cost_for_average_update(cr, uid, move))
        return partial_move

    def do_partial(self, cr, uid, ids, context=None):
        assert len(ids) == 1, 'Partial picking processing may only be done one at a time'
        stock_picking = self.pool.get('stock.picking')
        stock_move = self.pool.get('stock.move')
        uom_obj = self.pool.get('product.uom')
        partial = self.browse(cr, uid, ids[0], context=context)
        partial_data = {
            'delivery_date' : partial.date
        }
        picking_type = partial.picking_id.type
        for wizard_line in partial.move_ids:
            line_uom = wizard_line.product_uom
            move_id = wizard_line.move_id.id

            #Quantiny must be Positive
            if wizard_line.quantity < 0:
                raise osv.except_osv(_('Warning!'), _('Please provide Proper Quantity !'))

            #Compute the quantity for respective wizard_line in the line uom (this jsut do the rounding if necessary)
            qty_in_line_uom = uom_obj._compute_qty(cr, uid, line_uom.id, wizard_line.quantity, line_uom.id)

            if line_uom.factor and line_uom.factor <> 0:
                if qty_in_line_uom <> wizard_line.quantity:
                    raise osv.except_osv(_('Warning'), _('The uom rounding does not allow you to ship "%s %s", only roundings of "%s %s" is accepted by the uom.') % (wizard_line.quantity, line_uom.name, line_uom.rounding, line_uom.name))
            if move_id:
                #Check rounding Quantity.ex.
                #picking: 1kg, uom kg rounding = 0.01 (rounding to 10g),
                #partial delivery: 253g
                #=> result= refused, as the qty left on picking would be 0.747kg and only 0.75 is accepted by the uom.
                initial_uom = wizard_line.move_id.product_uom
                #Compute the quantity for respective wizard_line in the initial uom
                qty_in_initial_uom = uom_obj._compute_qty(cr, uid, line_uom.id, wizard_line.quantity, initial_uom.id)
                without_rounding_qty = (wizard_line.quantity / line_uom.factor) * initial_uom.factor
                if qty_in_initial_uom <> without_rounding_qty:
                    raise osv.except_osv(_('Warning'), _('The rounding of the initial uom does not allow you to ship "%s %s", as it would let a quantity of "%s %s" to ship and only roundings of "%s %s" is accepted by the uom.') % (wizard_line.quantity, line_uom.name, wizard_line.move_id.product_qty - without_rounding_qty, initial_uom.name, initial_uom.rounding, initial_uom.name))
            else:
                seq_obj_name =  'stock.picking.' + picking_type
                move_id = stock_move.create(cr,uid,{'name' : self.pool.get('ir.sequence').get(cr, uid, seq_obj_name),
                                                    'product_id': wizard_line.product_id.id,
                                                    'product_qty': wizard_line.quantity,
                                                    'product_uom': wizard_line.product_uom.id,
                                                    'prodlot_id': wizard_line.prodlot_id.id,
                                                    'location_id' : wizard_line.location_id.id,
                                                    'location_dest_id' : wizard_line.location_dest_id.id,
                                                    'picking_id': partial.picking_id.id
                                                    },context=context)
                stock_move.action_confirm(cr, uid, [move_id], context)
            partial_data['move%s' % (move_id)] = {
                'product_id': wizard_line.product_id.id,
                'product_qty': wizard_line.quantity,
                'product_uom': wizard_line.product_uom.id,
                'prodlot_id': wizard_line.prodlot_id.id,
            }
            if (picking_type == 'in') and (wizard_line.product_id.cost_method == 'average'):
                partial_data['move%s' % (wizard_line.move_id.id)].update(product_price=wizard_line.cost,
                                                                  product_currency=wizard_line.currency.id)
        stock_picking.do_partial(cr, uid, [partial.picking_id.id], partial_data, context=context)
        return {'type': 'ir.actions.act_window_close'}

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>"""
Functions for reading Magritek Spinsolve binary (dx/1d) files and
parameter (acqu.par/proc.par) files.
"""

from warnings import warn

import os
import numpy as np

from . import fileiobase
from . import jcampdx

__developer_info__ = """
Spinsolve is the software used on the Magritek benchtop NMR devices.

A spectrum is saved in a folder with several files. The spectral data is
stored in these files: 'data.1d' (FID), 'spectrum.1d' (Fourier transformed)
and 'spectrum_processed.1d' (FT + processed by spinsolve)
Optional spectral data (System->Prefs->Setup->Global data storage):
'nmr_fid.dx' (FID stored in `JCAMP-DX standard <http://www.jcamp-dx.org/>`),
'spectrum.csv' and 'spectrum_processed.csv' (FT + processed by Spinsolve with ppm for each
point and intensity delimited by ';')
Other files:
'acqu.par' - all parameters that are used for acquisition
'protocol.par' - text file used to reload data back into the Spinsolve software
'processing.script' - text file to transfer Spinsolve software protocol settings
into MNOVA

The Spinsolve Expert software has a slightly different output:
[Needs to be double checked as I do not have access to this software -LCageman]
- Output into JCAMP-DX is not possible
- 'spectrum_processed.1d' is not generated
- (new) 'fid.1d' - seems to be the same as 'data.1d'
- (new) 'proc.par' - contains processing parameters in the same style as 'acqu.par'
- (new) .pt1 files - seem to be plot files specific for the expert software, cannot
be read by NMRglue
"""

def parse_spinsolve_acqu_line(line):
    """
    Parse lines in acqu.par and return a tuple (paramter name, parameter value)
    """
    line = line.strip()  # Drop newline
    name, value = line.split("=")  # Split at equal sign

    # remove spaces
    name = name.strip()
    value = value.strip()

    # Detect value type
    if value[0] == value[-1] == '"':  # String
        return name, str(value[1:-1])  # Drop quote marks
    if "." in value:  # Float
        return name, float(value)
    else:
        return name, int(value)

def read(dir='.', specfile=None, acqupar="acqu.par", procpar="proc.par"):
    """
    Reads spinsolve files from a directory
    When no spectrum filename is given (specfile), the following list is tried, in
    that specific order
    ["nmr_fid.dx", "data.1d", "fid.1d", "spectrum.1d", "spectrum_processed.1d"]
    To use the resolution enhanced spectrum use the './Enhanced' folder as input.
    Note that spectrum.1d and spectrum_processed.1d contain only data in the
    frequency domain, so no Fourier transformation is needed. Also, use
    dic["spectrum"]["xaxis"] to plot the x-axis

    Parameters
    ----------
    dir : str
        Directory to read from
    specfile : str, optional
        Filename to import spectral data from. None uses standard filename from:
        ["nmr_fid.dx", "data.1d", "fid.1d", "spectrum.1d", "spectrum_processed.1d"]
    acqupar : str, optional
        Filename for acquisition parameters. None uses standard name.
    procpar : str, optional
        Filename for processing parameters. None uses standard name.

    Returns
    -------
    dic : dict
        All parameters that can be present in the data folder:
        dic["spectrum"] - First bytes of spectrum(_processed).1d
        dic["acqu"] - Parameters present in acqu.par
        dic["proc"] - Parameters present in proc.par
        dic["dx"] - - Parameters present in the header of nmr_fid.dx
    data : ndarray
        Array of NMR data

    """

    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # Create empty dic
    dic = {
        "spectrum": {},
        "acqu": {},
        "proc": {},
        "dx": {}
    }

    # Read in acqu.par and write to dic
    acqupar = os.path.join(dir, acqupar)
    if os.path.isfile(acqupar):
        with open(acqupar, "r") as f:
            info = f.readlines()
        for line in info:
            par_name, par_value = parse_spinsolve_acqu_line(line)

            if par_name is not None:
                dic["acqu"][par_name] = par_value

    # Read in proc.par and write to dic
    procpar = os.path.join(dir, procpar)
    if os.path.isfile(procpar):
        with open(procpar, "r") as f:
            info = f.readlines()
        for line in info:
            line = line.replace("\n", "")
            k, v = line.split("=")
            dic["proc"][k.strip()] = v.strip()

    # Define which spectrumfile to take, using 'specfile' when defined, otherwise
    # the files in 'priority_list' are tried, in that particular order
    priority_list = ["nmr_fid.dx", "data.1d", "fid.1d", "spectrum.1d", "spectrum_processed.1d"]
    inputfile = None
    if specfile:
        inputfile = os.path.join(dir, specfile)
        if not os.path.isfile(inputfile):
            raise IOError("File %s does not exist" % (inputfile))
    else:
        for priority in priority_list:
            inputfile = os.path.join(dir, priority)
            if os.path.isfile(inputfile):
                break
    if inputfile is None:
        raise IOError("directory %s does not contain spectral data" % (dir))

    # Detect which file we are dealing with from the extension and read in the spectral data

    # Reading .dx file using existing nmrglue.fileio.jcampdx module
    if inputfile.split('.')[-1] == "dx":
        dic["dx"], raw_data = jcampdx.read(inputfile)
        data = raw_data[0][:] + 1j * raw_data[1][:]

    # Reading .1d files
    elif inputfile.split('.')[-1] == ".1d":
        with open(inputfile, "rb") as f:
            raw_data = f.read()

        # Write out parameters from the first 32 bytes into dic["spectrum"]
        keys = ["owner", "format", "version", "dataType", "xDim", "yDim", "zDim", "qDim"]
        for i, k in enumerate(keys):
            start = i * 4
            end = start + 4
            value = int.from_bytes(raw_data[start:end], "little")
            dic["spectrum"][k] = value
        data = np.frombuffer(raw_data[32:], "<f")

        # The first 1/3 of the file is xaxis data (s or ppm)
        split = data.shape[-1] // 3
        xscale = data[0: split]
        dic["spectrum"]["xaxis"] = xscale

        # The rest is real and imaginary data points interleaved
        data = data[split:: 2] + 1j * data[split + 1:: 2]

    else:
        raise IOError("File %s cannot be interpreted, use .dx or .1d instead" % (inputfile))

    return dic, data

def get_udic_from_acqu_dict(param: dict):
    """
    Returns an udic from the parameters in acqu dictionary provided.
    """
    return_dict = dict()

    # Spectral Width (sw, in Hz)
    return_dict['sw'] = float(param.get('bandwidth')) * 1000

    # Observation Frequency (obs, in MHz)
    return_dict['obs'] = float(param.get('b1Freq'))

    # Carrier frequency (car, in Hz)
    return_dict['car'] = float(param.get('lowestFrequency')) + (return_dict['sw'] / 2)

    # Label (str describing the axis name)
    return_dict['label'] = param['rxChannel']

    return return_dict

def get_udic_from_jcamp_dict(param):
    """
    Returns an udic from the parameters in the JCAMP-DX dict provided.
    """
    return_dict = dict()

    # Observation Frequency (obs, in MHz)
    try:
        return_dict['obs'] = float(param['$BF1'][0])
    except KeyError:
        warn("Cannot set observation frequency - set manually using: 'udic[0]['obs'] = x' "
             "where x is magnetic field in MHz")

    # Spectral Width (sw, in Hz)
    try:
        return_dict['sw'] = float(param['$SW'][0]) * return_dict['obs']
    except KeyError:
        warn("Cannot set spectral width - set manually using: 'udic[0]['sw'] = x' "
             "where x is the spectral width in Hz")

    # Carrier frequency (car, in Hz)
    try:
        return_dict['car'] = -float(param['$REFERENCEPOINT'][0]) + (return_dict['sw'] / 2)
    except KeyError:
        pass
    try:
        return_dict['car'] = (return_dict['obs'] - float(param['$SF'][0])) * 1e6
    except KeyError:
        warn("Cannot set carrier - try: 'udic[0]['car'] = x * udic[0]['obs']' "
             "where x is the center of the spectrum in ppm")

    # Label (str describing the axis name)
    try:
        return_dict['label'] = param[".OBSERVENUCLEUS"][0].replace("^", "")
    except KeyError:
        warn("Cannot set observed nucleus label")

    return return_dict

def guess_udic(dic, data):
    """
    Guess parameters of universal dictionary from dic, data pair.

    Parameters
    ----------
    dic : dict
        Dictionary of JCAMP-DX, acqu, proc and spectrum parameters.
    data : ndarray
        Array of NMR data.

    Returns
    -------
    udic : dict
        Universal dictionary of spectral parameters.
    """

    # Create an empty universal dictionary
    udic = fileiobase.create_blank_udic(1)

    # Size can be calculated from data len
    if data is not None:
        udic[0]["size"] = len(data)
    else:
        warn('No data, cannot set udic size')

    # For the other parameters, first try to use info from acqu.par (dic['acqu']), then JCAMP-DX header parameters
    if dic["acqu"]:
        udic[0].update(get_udic_from_acqu_dict(dic["acqu"]))
    else:
        udic[0].update(get_udic_from_jcamp_dict(dic["dx"]))

    # keys left to default
    # udic[0]['complex']
    # udic[0]['encoding']
    # udic[0]['time'] = True
    # udic[0]['freq'] = False
    return udic
<commit_msg>Fix regression<commit_after>"""
Functions for reading Magritek Spinsolve binary (dx/1d) files and
parameter (acqu.par/proc.par) files.
"""

from warnings import warn

import os
import numpy as np

from . import fileiobase
from . import jcampdx

__developer_info__ = """
Spinsolve is the software used on the Magritek benchtop NMR devices.

A spectrum is saved in a folder with several files. The spectral data is
stored in these files: 'data.1d' (FID), 'spectrum.1d' (Fourier transformed)
and 'spectrum_processed.1d' (FT + processed by spinsolve)
Optional spectral data (System->Prefs->Setup->Global data storage):
'nmr_fid.dx' (FID stored in `JCAMP-DX standard <http://www.jcamp-dx.org/>`),
'spectrum.csv' and 'spectrum_processed.csv' (FT + processed by Spinsolve with ppm for each
point and intensity delimited by ';')
Other files:
'acqu.par' - all parameters that are used for acquisition
'protocol.par' - text file used to reload data back into the Spinsolve software
'processing.script' - text file to transfer Spinsolve software protocol settings
into MNOVA

The Spinsolve Expert software has a slightly different output:
[Needs to be double checked as I do not have access to this software -LCageman]
- Output into JCAMP-DX is not possible
- 'spectrum_processed.1d' is not generated
- (new) 'fid.1d' - seems to be the same as 'data.1d'
- (new) 'proc.par' - contains processing parameters in the same style as 'acqu.par'
- (new) .pt1 files - seem to be plot files specific for the expert software, cannot
be read by NMRglue
"""

def parse_spinsolve_acqu_line(line):
    """
    Parse lines in acqu.par and return a tuple (paramter name, parameter value)
    """
    line = line.strip()  # Drop newline
    name, value = line.split("=")  # Split at equal sign

    # remove spaces
    name = name.strip()
    value = value.strip()

    # Detect value type
    if value[0] == value[-1] == '"':  # String
        return name, str(value[1:-1])  # Drop quote marks
    if "." in value:  # Float
        return name, float(value)
    else:
        return name, int(value)

def read(dir='.', specfile=None, acqupar="acqu.par", procpar="proc.par"):
    """
    Reads spinsolve files from a directory
    When no spectrum filename is given (specfile), the following list is tried, in
    that specific order
    ["nmr_fid.dx", "data.1d", "fid.1d", "spectrum.1d", "spectrum_processed.1d"]
    To use the resolution enhanced spectrum use the './Enhanced' folder as input.
    Note that spectrum.1d and spectrum_processed.1d contain only data in the
    frequency domain, so no Fourier transformation is needed. Also, use
    dic["spectrum"]["xaxis"] to plot the x-axis

    Parameters
    ----------
    dir : str
        Directory to read from
    specfile : str, optional
        Filename to import spectral data from. None uses standard filename from:
        ["nmr_fid.dx", "data.1d", "fid.1d", "spectrum.1d", "spectrum_processed.1d"]
    acqupar : str, optional
        Filename for acquisition parameters. None uses standard name.
    procpar : str, optional
        Filename for processing parameters. None uses standard name.

    Returns
    -------
    dic : dict
        All parameters that can be present in the data folder:
        dic["spectrum"] - First bytes of spectrum(_processed).1d
        dic["acqu"] - Parameters present in acqu.par
        dic["proc"] - Parameters present in proc.par
        dic["dx"] - - Parameters present in the header of nmr_fid.dx
    data : ndarray
        Array of NMR data

    """

    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # Create empty dic
    dic = {
        "spectrum": {},
        "acqu": {},
        "proc": {},
        "dx": {}
    }

    # Read in acqu.par and write to dic
    acqupar = os.path.join(dir, acqupar)
    if os.path.isfile(acqupar):
        with open(acqupar, "r") as f:
            info = f.readlines()
        for line in info:
            par_name, par_value = parse_spinsolve_acqu_line(line)

            if par_name is not None:
                dic["acqu"][par_name] = par_value

    # Read in proc.par and write to dic
    procpar = os.path.join(dir, procpar)
    if os.path.isfile(procpar):
        with open(procpar, "r") as f:
            info = f.readlines()
        for line in info:
            line = line.replace("\n", "")
            k, v = line.split("=")
            dic["proc"][k.strip()] = v.strip()

    # Define which spectrumfile to take, using 'specfile' when defined, otherwise
    # the files in 'priority_list' are tried, in that particular order
    priority_list = ["nmr_fid.dx", "data.1d", "fid.1d", "spectrum.1d", "spectrum_processed.1d"]
    inputfile = None
    if specfile:
        inputfile = os.path.join(dir, specfile)
        if not os.path.isfile(inputfile):
            raise IOError("File %s does not exist" % (inputfile))
    else:
        for priority in priority_list:
            inputfile = os.path.join(dir, priority)
            if os.path.isfile(inputfile):
                break
    if inputfile is None:
        raise IOError("directory %s does not contain spectral data" % (dir))

    # Detect which file we are dealing with from the extension and read in the spectral data

    # Reading .dx file using existing nmrglue.fileio.jcampdx module
    if inputfile.split('.')[-1] == "dx":
        dic["dx"], raw_data = jcampdx.read(inputfile)
        data = raw_data[0][:] + 1j * raw_data[1][:]

    # Reading .1d files
    elif inputfile.split('.')[-1] == "1d":
        with open(inputfile, "rb") as f:
            raw_data = f.read()

        # Write out parameters from the first 32 bytes into dic["spectrum"]
        keys = ["owner", "format", "version", "dataType", "xDim", "yDim", "zDim", "qDim"]
        for i, k in enumerate(keys):
            start = i * 4
            end = start + 4
            value = int.from_bytes(raw_data[start:end], "little")
            dic["spectrum"][k] = value
        data = np.frombuffer(raw_data[32:], "<f")

        # The first 1/3 of the file is xaxis data (s or ppm)
        split = data.shape[-1] // 3
        xscale = data[0: split]
        dic["spectrum"]["xaxis"] = xscale

        # The rest is real and imaginary data points interleaved
        data = data[split:: 2] + 1j * data[split + 1:: 2]

    else:
        raise IOError("File %s cannot be interpreted, use .dx or .1d instead" % (inputfile))

    return dic, data

def get_udic_from_acqu_dict(param: dict):
    """
    Returns an udic from the parameters in acqu dictionary provided.
    """
    return_dict = dict()

    # Spectral Width (sw, in Hz)
    return_dict['sw'] = float(param.get('bandwidth')) * 1000

    # Observation Frequency (obs, in MHz)
    return_dict['obs'] = float(param.get('b1Freq'))

    # Carrier frequency (car, in Hz)
    return_dict['car'] = float(param.get('lowestFrequency')) + (return_dict['sw'] / 2)

    # Label (str describing the axis name)
    return_dict['label'] = param['rxChannel']

    return return_dict

def get_udic_from_jcamp_dict(param):
    """
    Returns an udic from the parameters in the JCAMP-DX dict provided.
    """
    return_dict = dict()

    # Observation Frequency (obs, in MHz)
    try:
        return_dict['obs'] = float(param['$BF1'][0])
    except KeyError:
        warn("Cannot set observation frequency - set manually using: 'udic[0]['obs'] = x' "
             "where x is magnetic field in MHz")

    # Spectral Width (sw, in Hz)
    try:
        return_dict['sw'] = float(param['$SW'][0]) * return_dict['obs']
    except KeyError:
        warn("Cannot set spectral width - set manually using: 'udic[0]['sw'] = x' "
             "where x is the spectral width in Hz")

    # Carrier frequency (car, in Hz)
    try:
        return_dict['car'] = -float(param['$REFERENCEPOINT'][0]) + (return_dict['sw'] / 2)
    except KeyError:
        pass
    try:
        return_dict['car'] = (return_dict['obs'] - float(param['$SF'][0])) * 1e6
    except KeyError:
        warn("Cannot set carrier - try: 'udic[0]['car'] = x * udic[0]['obs']' "
             "where x is the center of the spectrum in ppm")

    # Label (str describing the axis name)
    try:
        return_dict['label'] = param[".OBSERVENUCLEUS"][0].replace("^", "")
    except KeyError:
        warn("Cannot set observed nucleus label")

    return return_dict

def guess_udic(dic, data):
    """
    Guess parameters of universal dictionary from dic, data pair.

    Parameters
    ----------
    dic : dict
        Dictionary of JCAMP-DX, acqu, proc and spectrum parameters.
    data : ndarray
        Array of NMR data.

    Returns
    -------
    udic : dict
        Universal dictionary of spectral parameters.
    """

    # Create an empty universal dictionary
    udic = fileiobase.create_blank_udic(1)

    # Size can be calculated from data len
    if data is not None:
        udic[0]["size"] = len(data)
    else:
        warn('No data, cannot set udic size')

    # For the other parameters, first try to use info from acqu.par (dic['acqu']), then JCAMP-DX header parameters
    if dic["acqu"]:
        udic[0].update(get_udic_from_acqu_dict(dic["acqu"]))
    else:
        udic[0].update(get_udic_from_jcamp_dict(dic["dx"]))

    # keys left to default
    # udic[0]['complex']
    # udic[0]['encoding']
    # udic[0]['time'] = True
    # udic[0]['freq'] = False
    return udic
<|endoftext|>


<commit_before>import collections
import inspect
import sys

def make_decorator(test_case_generator):
    # `test_case_generator` is a callable that receives the source test class
    # (typically a subclass of unittest.TestCase) and returns an iterable of
    # generated test cases.
    # Each element of the iterable is a 3-element tuple:
    # [0] Generated class name
    # [1] Dict of members
    # [2] Method generator
    # The method generator is also a callable that receives an original test
    # method and returns a new test method.

    def f(cls):
        # The input is a bare test case
        module_name = cls.__module__

        # Generate parameterized test cases out of the input test case.
        module = sys.modules[module_name]
        for cls_name, members, method_generator in test_case_generator(cls):
            _generate_case(
                cls, module, cls_name, members, method_generator)

        # Remove original base class
        return None
    return f

def _generate_case(base, module, cls_name, mb, method_generator):
    members = mb.copy()
    base_methods = inspect.getmembers(base, predicate=inspect.isfunction)
    for name, value in base_methods:
        if not name.startswith('test_'):
            continue
        value = method_generator(value)
        # If the return value of method_generator is None, None is assigned
        # as the value of the test method and it is ignored by pytest.
        members[name] = value

    cls = type(cls_name, (base,), members)

    # Add new test class to module
    setattr(module, cls_name, cls)
<commit_msg>flake8: remove unused import<commit_after>import inspect
import sys

def make_decorator(test_case_generator):
    # `test_case_generator` is a callable that receives the source test class
    # (typically a subclass of unittest.TestCase) and returns an iterable of
    # generated test cases.
    # Each element of the iterable is a 3-element tuple:
    # [0] Generated class name
    # [1] Dict of members
    # [2] Method generator
    # The method generator is also a callable that receives an original test
    # method and returns a new test method.

    def f(cls):
        # The input is a bare test case
        module_name = cls.__module__

        # Generate parameterized test cases out of the input test case.
        module = sys.modules[module_name]
        for cls_name, members, method_generator in test_case_generator(cls):
            _generate_case(
                cls, module, cls_name, members, method_generator)

        # Remove original base class
        return None
    return f

def _generate_case(base, module, cls_name, mb, method_generator):
    members = mb.copy()
    base_methods = inspect.getmembers(base, predicate=inspect.isfunction)
    for name, value in base_methods:
        if not name.startswith('test_'):
            continue
        value = method_generator(value)
        # If the return value of method_generator is None, None is assigned
        # as the value of the test method and it is ignored by pytest.
        members[name] = value

    cls = type(cls_name, (base,), members)

    # Add new test class to module
    setattr(module, cls_name, cls)
<|endoftext|>


<commit_before>import json, warnings
from six.moves import urllib
try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False

from wepay.exceptions import *

class Post(object):
    """This is a helper class that uses either `urllib` or `requests` library to
    perform POST requests.

    """

    def __init__(self, use_requests=None, silent=None):
        self._use_requests = HAS_REQUESTS and (
            use_requests is None or use_requests)
        if not silent and use_requests and not self._use_requests:
            message = "Using requests library was specified, but there was a problem " \
                      "importing it. Falling back to urllib."
            if silent is not None:
                raise WePayWarning(message)
            warnings.warn(message, WePayWarning)

    def __call__(self, url, params, headers, timeout):
        if self._use_requests:
            return self._post_requests(url, params, headers, timeout)
        return self._post_urllib(url, params, headers, timeout)

    def _post_urllib(self, url, params, headers, timeout):
        data = urllib.parse.urlencode(params).encode('utf-8')
        request = urllib.request.Request(url, data=data, headers=headers)
        try:
            response = urllib.request.urlopen(request, timeout=timeout)
        except urllib.error.HTTPError as exc:
            try:
                kwargs = json.loads(exc.read().decode('utf-8'))
            except ValueError:
                kwargs = {}
            self._raise_error(exc, exc.code, **kwargs)
        except urllib.error.URLError as exc:
            raise WePayConnectionError(exc)
        return json.loads(response.read().decode('utf-8'))

    def _post_requests(self, url, params, headers, timeout):
        data = json.dumps(params)
        try:
            response = requests.post(
                url, data=data, headers=headers, timeout=timeout)
            response.raise_for_status()
        except requests.exceptions.HTTPError as exc:
            try:
                kwargs = exc.response.json()
            except ValueError: # JSONDecodeError is a subclass of ValueError
                kwargs = {}
            self._raise_error(exc, exc.response.status_code, **kwargs)
        except requests.exceptions.RequestException as exc:
            raise WePayConnectionError(exc)
        return response.json()

    def _raise_error(self, exc, status_code, **kwargs):
        if status_code >= 500:
            raise WePayServerError(exc, status_code, **kwargs)
        if status_code >= 400:
            raise WePayClientError(exc, status_code, **kwargs)
        # Should never really happen
        raise WePayHTTPError(exc, status_code, **kwargs)

class cached_property(object):

    def __init__(self, fget):
        self.fget = fget
        self.__doc__ = fget.__doc__
        self.__name__ = fget.__name__
        self.__module__ = fget.__module__
        self._key = "_%s" % fget.__name__

    def __get__(self, obj, cls):
        if obj is None:
            return self
        value = getattr(obj, self._key, None)
        if value is None:
            value = self.fget(obj)
            setattr(obj, self._key, value)
        return value

    def __set__(self, obj, value):
        setattr(obj, self._key, value)
<commit_msg>removed unnecessary line and imports<commit_after>import json, warnings
from six.moves import urllib
try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False

from wepay.exceptions import WePayClientError, WePayServerError

class Post(object):
    """This is a helper class that uses either `urllib` or `requests` library to
    perform POST requests.

    """

    def __init__(self, use_requests=None, silent=None):
        self._use_requests = HAS_REQUESTS and (
            use_requests is None or use_requests)
        if not silent and use_requests and not self._use_requests:
            message = "Using requests library was specified, but there was a problem " \
                      "importing it. Falling back to urllib."
            if silent is not None:
                raise WePayWarning(message)
            warnings.warn(message, WePayWarning)

    def __call__(self, url, params, headers, timeout):
        if self._use_requests:
            return self._post_requests(url, params, headers, timeout)
        return self._post_urllib(url, params, headers, timeout)

    def _post_urllib(self, url, params, headers, timeout):
        data = urllib.parse.urlencode(params).encode('utf-8')
        request = urllib.request.Request(url, data=data, headers=headers)
        try:
            response = urllib.request.urlopen(request, timeout=timeout)
        except urllib.error.HTTPError as exc:
            try:
                kwargs = json.loads(exc.read().decode('utf-8'))
            except ValueError:
                kwargs = {}
            self._raise_error(exc, exc.code, **kwargs)
        except urllib.error.URLError as exc:
            raise WePayConnectionError(exc)
        return json.loads(response.read().decode('utf-8'))

    def _post_requests(self, url, params, headers, timeout):
        data = json.dumps(params)
        try:
            response = requests.post(
                url, data=data, headers=headers, timeout=timeout)
            response.raise_for_status()
        except requests.exceptions.HTTPError as exc:
            try:
                kwargs = exc.response.json()
            except ValueError: # JSONDecodeError is a subclass of ValueError
                kwargs = {}
            self._raise_error(exc, exc.response.status_code, **kwargs)
        except requests.exceptions.RequestException as exc:
            raise WePayConnectionError(exc)
        return response.json()

    def _raise_error(self, exc, status_code, **kwargs):
        if status_code >= 500:
            raise WePayServerError(exc, status_code, **kwargs)
        if status_code >= 400:
            raise WePayClientError(exc, status_code, **kwargs)

class cached_property(object):

    def __init__(self, fget):
        self.fget = fget
        self.__doc__ = fget.__doc__
        self.__name__ = fget.__name__
        self.__module__ = fget.__module__
        self._key = "_%s" % fget.__name__

    def __get__(self, obj, cls):
        if obj is None:
            return self
        value = getattr(obj, self._key, None)
        if value is None:
            value = self.fget(obj)
            setattr(obj, self._key, value)
        return value

    def __set__(self, obj, value):
        setattr(obj, self._key, value)
<|endoftext|>


<commit_before>#----------------------------------------------------------------------
# Copyright (c) 2012 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------

from omnilib.frameworks.framework_pg import Framework as pg_framework
from geni.util.urn_util import is_valid_urn, URN, string_to_urn_format
from urlparse import urlparse

class Framework(pg_framework):
    """The a ProtoGENI CH shim for Omni.
    """

    def __init__(self, config, opts):
        pg_framework.__init__(self,config, opts)
        self.opts = opts

    def slice_name_to_urn(self, name):
        """Convert a slice name and project name to a slice urn."""
        #
        # Sample URNs:
        #   urn:publicid:IDN+portal:myproject+slice+myexperiment
        #

        if name is None or name.strip() == '':
            raise Exception('Empty slice name')

        if not self.config.has_key('sa'):
            raise Exception("Invalid configuration: no slice authority (sa) defined")

        if (not self.opts.project) and (not self.config.has_key('default_project')):
            raise Exception("Invalid configuration: no default project defined")

        if self.opts.project:
            # use the command line option --project
            project = self.opts.project
        else:
            # otherwise, default to 'default_project' in 'omni_config'
            project = self.config['default_project']

        # if config has an authority, make sure it matches
        if self.config.has_key('sa'):
            url = urlparse(self.config['sa'])
        sa_host = url.hostname
        try:
            sa_hostname, sa_domain = sa_host.split(".",1)
            auth = sa_hostname
        except:
            # Funny SA
            self.logger.debug("Found no . in sa hostname. Using whole hostname")
            auth = sa_host

        # Authority is of form: host:project
        auth = auth+":"+project

        # Could use is_valid_urn_bytype here, or just let the SA/AM do the check
        if is_valid_urn(name):
            urn = URN(None, None, None, urn=name)
            if not string_to_urn_format(urn.getType()) == "slice":
                raise Exception("Invalid Slice name: got a non Slice URN %s"% name)
            # if config has an authority, make sure it matches
            urn_auth = string_to_urn_format(urn.getAuthority())
            if urn_auth != auth:
                self.logger.warn("CAREFUL: slice' authority (%s) doesn't match current configured authority (%s)" % (urn_auth, auth))
                self.logger.info("This may be OK though if you are using delegated slice credentials...")
            return name

        return URN(auth, "slice", name).urn_string()
<commit_msg>clean up comments. Add own logger.<commit_after>#----------------------------------------------------------------------
# Copyright (c) 2012 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------

from omnilib.frameworks.framework_pg import Framework as pg_framework
from geni.util.urn_util import is_valid_urn, URN, string_to_urn_format

import logging
from urlparse import urlparse

class Framework(pg_framework):
    """Framework to talk to the GENI Clearinghouse (or similar) using PG CH APIs.
    The difference here is that there is a project name which is appended to the
    SA hostname to construct the authority field in the slice URN.
    """

    def __init__(self, config, opts):
        pg_framework.__init__(self,config, opts)
        self.opts = opts
        self.logger = logging.getLogger("omni.pgch")

    def slice_name_to_urn(self, name):
        """Convert a slice name and project name to a slice urn."""
        #
        # Sample URN:
        #   urn:publicid:IDN+portal:myproject+slice+myexperiment
        #

        if name is None or name.strip() == '':
            raise Exception('Empty slice name')

        if not self.config.has_key('sa'):
            raise Exception("Invalid configuration: no slice authority (sa) defined")

        if (not self.opts.project) and (not self.config.has_key('default_project')):
            raise Exception("Invalid configuration: no default project defined")

        if self.opts.project:
            # use the command line option --project
            project = self.opts.project
        else:
            # otherwise, default to 'default_project' in 'omni_config'
            project = self.config['default_project']

        # Get the authority from the SA hostname
        url = urlparse(self.config['sa'])
        sa_host = url.hostname
        try:
            sa_hostname, sa_domain = sa_host.split(".",1)
            auth = sa_hostname
        except:
            # Funny SA
            self.logger.debug("Found no . in sa hostname. Using whole hostname")
            auth = sa_host

        # Authority is of form: host:project
        auth = auth+":"+project

        # Could use is_valid_urn_bytype here, or just let the SA/AM do the check
        if is_valid_urn(name):
            urn = URN(None, None, None, urn=name)
            if not string_to_urn_format(urn.getType()) == "slice":
                raise Exception("Invalid Slice name: got a non Slice URN %s"% name)
            # if config has an authority, make sure it matches
            urn_auth = string_to_urn_format(urn.getAuthority())
            if urn_auth != auth:
                self.logger.warn("CAREFUL: slice' authority (%s) doesn't match current configured authority (%s)" % (urn_auth, auth))
                self.logger.info("This may be OK though if you are using delegated slice credentials...")
            return name

        return URN(auth, "slice", name).urn_string()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys

from sasila.slow_system.core.request_spider import RequestSpider
from sasila.slow_system.pipeline.kafa_pipeline import KafkaPipeline
from base_processor import BaseProcessor
from sasila.slow_system.downloader.http.spider_request import Request
from bs4 import BeautifulSoup as bs
import time

import traceback

reload(sys)
sys.setdefaultencoding('utf-8')

class QccProcessor(BaseProcessor):
    spider_id = 'qcc'
    spider_name = 'qcc'
    allowed_domains = ['qichacha.com']

    start_requests = [
        Request(url='http://www.qichacha.com/search?key=%E5%B0%8F%E9%A2%9D%E8%B4%B7%E6%AC%BE')]

    def process(self, response):
        soup = bs(response.m_response.content, "lxml")
        province_list = soup.select_one("dl#provinceOld").select("div.pull-left")[1].select("dd a")
        for province in province_list:
            province_name = province.string.strip()
            province_id = province["data-value"].strip()
            request = Request(
                    url="http://www.qichacha.com/search_getCityListHtml?province=" + province_id + "&q_type=1",
                    callback="get_city", priority=0)
            request.meta["province_name"] = province_name
            request.meta["province_id"] = province_id
            yield request

            # province_name = "北京"
            # province_id = "BJ"
            # request = Request(
            #         url="http://www.qichacha.com/search_getCityListHtml?province=" + province_id + "&q_type=1",
            #         callback="get_city", priority=0)
            # request.meta["province_name"] = province_name
            # request.meta["province_id"] = province_id
            # yield request

    def get_city(self, response):
        if response.m_response.content == "":
            request = Request(
                    url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                        response.request.meta["province_id"] + "&",
                    callback="get_all_page", priority=1)
            request.meta["city_name"] = ""
            request.meta["city_id"] = ""
            request.meta["province_name"] = response.request.meta["province_name"]
            request.meta["province_id"] = response.request.meta["province_id"]
            request.meta["only_province_sign"] = "0"
            yield request
        else:
            soup = bs(response.m_response.content, "lxml")
            city_list = soup.select("a")
            for city in city_list:
                city_name = city.string.strip()
                city_id = city["data-value"].strip()
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&city=" + city_id + "&",
                        callback="get_all_page", priority=1)
                request.meta["city_name"] = city_name
                request.meta["city_id"] = city_id
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request

    def get_all_page(self, response):
        soup = bs(response.m_response.content, "lxml")
        try:
            temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findNextSibling()
            if temp_page:
                page = temp_page.select_one("a")
                if page:
                    total_page = int(page.string.strip().replace("...", ""))
                else:
                    total_page = 1
            else:
                temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findPreviousSibling()
                if temp_page:
                    page = temp_page.select_one("a")
                    if page:
                        total_page = int(page.string.strip().replace("...", ""))
                    else:
                        total_page = 1
                else:
                    total_page = 1
        except:
            total_page = 1

        now_page = 1
        while now_page <= total_page:
            if "only_province_sign" in response.request.meta.keys():
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&p=" + str(now_page) + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            else:
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&&p=" + str(
                                now_page) + "&province=" +
                            response.request.meta["province_id"] + "&city=" + response.request.meta["city_id"] + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            now_page += 1

    def get_content(self, response):
        soup = bs(response.m_response.content, "lxml")
        content_list = soup.select("table.m_srchList tbody tr")
        for content in content_list:
            try:
                result_item = dict()
                result_item["province"] = response.request.meta["province_name"]
                result_item["city"] = response.request.meta["city_name"]
                result_item["company_name"] = content.select("td")[1].text.split('\n')[0].strip()
                result_item["company_man"] = content.select("td")[1].text.split('\n')[1].strip().replace("企业法人：", "")
                result_item["company_telephone"] = content.select("td")[1].text.split('\n')[2].strip().replace("联系方式：",
                                                                                "")
                result_item["company_address"] = content.select("td")[1].text.split('\n')[3].strip().replace("地址：", "")
                result_item["company_registered_capital"] = content.select("td")[2].text.strip()
                result_item["company_registered_time"] = content.select("td")[3].text.strip()
                result_item["company_status"] = content.select("td")[4].text.strip()
                result_item["source"] = "企查查"
                result_item["update_time"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
                yield result_item
            except Exception:
                print traceback.format_exc()

qcc_spider = RequestSpider(QccProcessor(), time_sleep=1).set_pipeline(KafkaPipeline())
if __name__ == '__main__':
    qcc_spider.start()
<commit_msg>sasila<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys

from sasila.slow_system.core.request_spider import RequestSpider
from sasila.slow_system.pipeline.kafa_pipeline import KafkaPipeline
from base_processor import BaseProcessor
from sasila.slow_system.downloader.http.spider_request import Request
from bs4 import BeautifulSoup as bs
import time

import traceback

reload(sys)
sys.setdefaultencoding('utf-8')

class QccProcessor(BaseProcessor):
    spider_id = 'qcc'
    spider_name = 'qcc'
    allowed_domains = ['qichacha.com']

    start_requests = [
        Request(url='http://www.qichacha.com/search?key=%E5%B0%8F%E9%A2%9D%E8%B4%B7%E6%AC%BE')]

    def process(self, response):
        soup = bs(response.m_response.content, "lxml")
        province_list = soup.select_one("dl#provinceOld").select("div.pull-left")[1].select("dd a")
        for province in province_list:
            province_name = province.string.strip()
            province_id = province["data-value"].strip()
            request = Request(
                    url="http://www.qichacha.com/search_getCityListHtml?province=" + province_id + "&q_type=1",
                    callback="get_city", priority=0)
            request.meta["province_name"] = province_name
            request.meta["province_id"] = province_id
            yield request

            # province_name = "北京"
            # province_id = "BJ"
            # request = Request(
            #         url="http://www.qichacha.com/search_getCityListHtml?province=" + province_id + "&q_type=1",
            #         callback="get_city", priority=0)
            # request.meta["province_name"] = province_name
            # request.meta["province_id"] = province_id
            # yield request

    def get_city(self, response):
        if response.m_response.content == "":
            request = Request(
                    url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                        response.request.meta["province_id"] + "&",
                    callback="get_all_page", priority=1)
            request.meta["city_name"] = ""
            request.meta["city_id"] = ""
            request.meta["province_name"] = response.request.meta["province_name"]
            request.meta["province_id"] = response.request.meta["province_id"]
            request.meta["only_province_sign"] = "0"
            yield request
        else:
            soup = bs(response.m_response.content, "lxml")
            city_list = soup.select("a")
            for city in city_list:
                city_name = city.string.strip()
                city_id = city["data-value"].strip()
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&city=" + city_id + "&",
                        callback="get_all_page", priority=1)
                request.meta["city_name"] = city_name
                request.meta["city_id"] = city_id
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request

    def get_all_page(self, response):
        soup = bs(response.m_response.content, "lxml")
        try:
            temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findNextSibling()
            if temp_page:
                page = temp_page.select_one("a")
                if page:
                    total_page = int(page.string.strip().replace("...", ""))
                else:
                    total_page = 1
            else:
                temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findPreviousSibling()
                if temp_page:
                    page = temp_page.select_one("a")
                    if page:
                        total_page = int(page.string.strip().replace("...", ""))
                    else:
                        total_page = 1
                else:
                    total_page = 1
        except:
            total_page = 1

        now_page = 1
        while now_page <= total_page:
            if "only_province_sign" in response.request.meta.keys():
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&p=" + str(now_page) + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            else:
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&&p=" + str(
                                now_page) + "&province=" +
                            response.request.meta["province_id"] + "&city=" + response.request.meta["city_id"] + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            now_page += 1

    def get_content(self, response):
        soup = bs(response.m_response.content, "lxml")
        content_list = soup.select("table.m_srchList tbody tr")
        for content in content_list:
            try:
                result_item = dict()
                result_item["province"] = response.request.meta["province_name"]
                result_item["city"] = response.request.meta["city_name"]
                result_item["company_name"] = content.select("td")[1].text.split('\n')[0].strip()
                result_item["company_man"] = content.select("td")[1].text.split('\n')[1].strip().replace("企业法人：", "")
                result_item["company_telephone"] = content.select("td")[1].text.split('\n')[2].strip().replace("联系方式：",
                                                                                "")
                result_item["company_address"] = content.select("td")[1].text.split('\n')[3].strip()
                if "地址：" in result_item["company_address"]:
                    result_item["company_address"] = result_item["company_address"].replace("地址：", "")
                else:
                    result_item["company_address"] = ""
                result_item["company_registered_capital"] = content.select("td")[2].text.strip()
                result_item["company_registered_time"] = content.select("td")[3].text.strip()
                result_item["company_status"] = content.select("td")[4].text.strip()
                result_item["source"] = "企查查"
                result_item["update_time"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
                yield result_item
            except Exception:
                print traceback.format_exc()

qcc_spider = RequestSpider(QccProcessor(), time_sleep=1).set_pipeline(KafkaPipeline())
if __name__ == '__main__':
    qcc_spider.start()
<|endoftext|>


<commit_before>POSTCODES = [70173, 70174, 70176, 70178, 70180, 70182, 70184, 70186, 70188,
             70190, 70191, 70192, 70193, 70195, 70197, 70199, 70327, 70329,
             70372, 70374, 70378, 70435, 70437, 70439, 70469, 70476, 70499,
             70563, 70565, 70567, 70569, 70597, 70599, 70619, 70629]

SCRAPE_SLEEP = 5 # pause between each GET request, in seconds

OUTPUT_DIR = '/home/kopf/www/gsack-output'

RETRIES = 5

<commit_msg>Remove non-existant PLZs<commit_after>POSTCODES = [70173, 70174, 70176, 70178, 70180, 70182, 70184, 70186, 70188,
             70190, 70191, 70192, 70193, 70195, 70197, 70199, 70327, 70329,
             70372, 70374, 70378, 70435, 70437, 70439, 70469, 70499, 70563,
             70565, 70567, 70569, 70597, 70599, 70619]

SCRAPE_SLEEP = 5 # pause between each GET request, in seconds

OUTPUT_DIR = '/home/kopf/www/gsack-output'

RETRIES = 5

<|endoftext|>


<commit_before>#!/usr/bin/python2
import os
import sys
import shutil
from datetime import datetime
from optparse import OptionParser, OptionGroup
import logging

import yaml

import wok
from wok import page
from wok import renderers
from wok import util
from wok import devserver

class Engine(object):
    """
    The main engine of wok. Upon initialization, it generates a site from the
    source files.
    """
    default_options = {
        'content_dir' : 'content',
        'template_dir': 'templates',
        'output_dir'  : 'output',
        'media_dir'   : 'media',
        'site_title'  : 'Some random Wok site',
        'url_pattern' : '/{category}/{slug}{page}.{type}',
    }

    def __init__(self, output_lvl = 1):

        parser = OptionParser(version='%prog v{0}'.format(wok.version))

        # Options for noisiness level and logging
        logging_grp = OptionGroup(parser, "Logging")
        parser.set_defaults(loglevel=logging.WARNING)
        logging_grp.add_option('-q', '--quiet', action='store_const',
                const=logging.ERROR, dest='loglevel',
                help="be completely quiet, log nothing")
        logging_grp.add_option('--warnings', action='store_const',
                const=logging.WARNING, dest='loglevel',
                help="log warnings in addition to errors")
        logging_grp.add_option('-v', '--verbose', action='store_const',
                const=logging.INFO, dest='loglevel',
                help="log ALL the things!")
        logging_grp.add_option('--debug', action='store_const',
                const=logging.DEBUG, dest='loglevel',
                help="log debugging info in addition to warnings and errors")
        logging_grp.add_option('--log', '-l', dest='logfile',
                help="log to the specified LOGFILE instead of standard out")
        parser.add_option_group(logging_grp)

        # Add option to to run the development server after generating pages
        devserver_grp = OptionGroup(parser, "Development server")
        devserver_grp.add_option('--server', action='store_true',
                dest='runserver',
                help="run a development server after generating the site")
        devserver_grp.add_option('--address', action='store', dest='address',
                help="specify ADDRESS on which to run development server")
        devserver_grp.add_option('--port', action='store', dest='port',
                type='int',
                help="specify PORT on which to run development server")
        parser.add_option_group(devserver_grp)

        cli_options, args = parser.parse_args()
        logging_options = {
            'format': '%(levelname)s: %(message)s',
            'level': cli_options.loglevel,
        }
        if cli_options.logfile:
            logging_options['filename'] = cli_options.logfile
        else:
            logging_options['stream'] = sys.stdout

        logging.basicConfig(**logging_options)

        self.all_pages = []

        self.read_options()
        self.sanity_check()
        self.prepare_output()
        self.load_pages()
        self.make_tree()
        self.render_site()

        # Run the dev server after generating pages if the user said to
        if cli_options.runserver:
            devserver.run(cli_options.address, cli_options.port,
                    serv_dir=os.path.join(self.options['output_dir']))

    def read_options(self):
        """Load options from the config file."""
        self.options = Engine.default_options.copy()

        if os.path.isfile('config'):
            with open('config') as f:
                yaml_config = yaml.load(f)

            if yaml_config:
                self.options.update(yaml_config)

        authors = self.options.get('authors', self.options.get('author', None))
        if isinstance(authors, list):
            self.options['authors'] = [page.Author.parse(a) for a in authors]
        elif isinstance(authors, str):
            self.options['authors'] = [page.Author.parse(a) for a in authors.split(',')]

    def sanity_check(self):
        """Basic sanity checks."""
        # Make sure that this is (probabably) a wok source directory.
        if not (os.path.isdir('templates') or os.path.isdir('content')):
            logging.critical("This doesn't look like a wok site. Aborting.")
            sys.exit(1)

    def prepare_output(self):
        """
        Prepare the output directory. Remove any contents there already, and
        then copy over the media files, if they exist.
        """
        if os.path.isdir(self.options['output_dir']):
            shutil.rmtree(self.options['output_dir'])
        os.mkdir(self.options['output_dir'])

        # Copy the media directory to the output folder
        try:
            for name in os.listdir(self.options['media_dir']):
                path = os.path.join(self.options['media_dir'], name)
                if os.path.isdir(path):
                    shutil.copytree(
                            path,
                            os.path.join(self.options['output_dir'], name),
                            symlinks=True
                    )
                else:
                    shutil.copy(path, self.options['output_dir'])

        # Do nothing if the media directory doesn't exist
        except OSError:
            # XXX: We should verify that the problem was the media dir
            pass

    def load_pages(self):
        """Load all the content files."""
        for root, dirs, files in os.walk(self.options['content_dir']):
            # Grab all the parsable files
            for f in files:
                # As long as the current file is not hidden, append it to the
                # page list
                if not f.startswith('.'):
                    ext = f.split('.')[-1]
                    renderer = renderers.Plain

                    for r in renderers.all:
                        if ext in r.extensions:
                            renderer = r
                            break
                    else:
                        logging.warning('No parser found '
                                'for {0}. Using default renderer.'.format(f))
                        renderer = renderers.Renderer

                    self.all_pages.append(page.Page(os.path.join(root,f),
                            self.options, renderer))

    def make_tree(self):
        """
        Make the category pseduo-tree.

        In this structure, each node is a page. Pages with sub pages are
        interior nodes, and leaf nodes have no sub pages. It is not truely a
        tree, because the root node doesn't exist.
        """
        self.categories = {}
        site_tree = []
        # We want to parse these in a approximately breadth first order
        self.all_pages.sort(key=lambda p: len(p.meta['category']))

        for p in [p for p in self.all_pages]:
            if len(p.meta['category']) > 0:
                top_cat = p.meta['category'][0]
                if not top_cat in self.categories:
                    self.categories[top_cat] = []

                self.categories[top_cat].append(p.meta)

            try:
                siblings = site_tree
                for cat in p.meta['category']:
                    # This line will fail if the page is an orphan
                    parent = [subpage for subpage in siblings
                                 if subpage['slug']== cat][0]
                    siblings = parent['subpages']
                siblings.append(p.meta)
            except IndexError:
                logging.error('It looks like the page "{0}" is an orphan! '
                        'This will probably cause problems.'.format(p.path))

    def render_site(self):
        """Render every page and write the output files."""
        # Gather tags
        tag_set = set()
        for p in self.all_pages:
            tag_set = tag_set.union(p.meta['tags'])
        tag_dict = dict()
        for tag in tag_set:
            tag_dict[tag] = [p.meta for p in self.all_pages if tag in p.meta['tags']]

        for p in self.all_pages:
            if p.meta['published']:
                # Construct this every time, to avoid sharing one instance
                # between page objects.
                templ_vars = {
                    'site': {
                        'title': self.options.get('site_title', 'Untitled'),
                        'datetime': datetime.now(),
                        'tags': tag_dict,
                        'pages': self.all_pages[:],
                        'categories': self.categories,
                    },
                }

                for k, v in self.options.iteritems():
                    if k not in ('site_title', 'output_dir', 'content_dir',
                            'templates_dir', 'media_dir', 'url_pattern'):

                        templ_vars['site'][k] = v

                if 'author' in self.options:
                    templ_vars['site']['author'] = self.options['author']

                # Rendering the page might give us back more pages to render.
                new_pages = p.render(templ_vars)
                p.write()
                if new_pages:
                    logging.debug('found new_pages')
                    self.all_pages += new_pages

if __name__ == '__main__':
    Engine()
<commit_msg>Swapping order of dev server and log options so the dev server options appear before the logging options. (Looks prettier in --help)<commit_after>#!/usr/bin/python2
import os
import sys
import shutil
from datetime import datetime
from optparse import OptionParser, OptionGroup
import logging

import yaml

import wok
from wok import page
from wok import renderers
from wok import util
from wok import devserver

class Engine(object):
    """
    The main engine of wok. Upon initialization, it generates a site from the
    source files.
    """
    default_options = {
        'content_dir' : 'content',
        'template_dir': 'templates',
        'output_dir'  : 'output',
        'media_dir'   : 'media',
        'site_title'  : 'Some random Wok site',
        'url_pattern' : '/{category}/{slug}{page}.{type}',
    }

    def __init__(self, output_lvl = 1):

        parser = OptionParser(version='%prog v{0}'.format(wok.version))

        # Add option to to run the development server after generating pages
        devserver_grp = OptionGroup(parser, "Development server")
        devserver_grp.add_option('--server', action='store_true',
                dest='runserver',
                help="run a development server after generating the site")
        devserver_grp.add_option('--address', action='store', dest='address',
                help="specify ADDRESS on which to run development server")
        devserver_grp.add_option('--port', action='store', dest='port',
                type='int',
                help="specify PORT on which to run development server")
        parser.add_option_group(devserver_grp)

        # Options for noisiness level and logging
        logging_grp = OptionGroup(parser, "Logging")
        parser.set_defaults(loglevel=logging.WARNING)
        logging_grp.add_option('-q', '--quiet', action='store_const',
                const=logging.ERROR, dest='loglevel',
                help="be completely quiet, log nothing")
        logging_grp.add_option('--warnings', action='store_const',
                const=logging.WARNING, dest='loglevel',
                help="log warnings in addition to errors")
        logging_grp.add_option('-v', '--verbose', action='store_const',
                const=logging.INFO, dest='loglevel',
                help="log ALL the things!")
        logging_grp.add_option('--debug', action='store_const',
                const=logging.DEBUG, dest='loglevel',
                help="log debugging info in addition to warnings and errors")
        logging_grp.add_option('--log', '-l', dest='logfile',
                help="log to the specified LOGFILE instead of standard out")
        parser.add_option_group(logging_grp)

        cli_options, args = parser.parse_args()
        logging_options = {
            'format': '%(levelname)s: %(message)s',
            'level': cli_options.loglevel,
        }
        if cli_options.logfile:
            logging_options['filename'] = cli_options.logfile
        else:
            logging_options['stream'] = sys.stdout

        logging.basicConfig(**logging_options)

        self.all_pages = []

        self.read_options()
        self.sanity_check()
        self.prepare_output()
        self.load_pages()
        self.make_tree()
        self.render_site()

        # Run the dev server after generating pages if the user said to
        if cli_options.runserver:
            devserver.run(cli_options.address, cli_options.port,
                    serv_dir=os.path.join(self.options['output_dir']))

    def read_options(self):
        """Load options from the config file."""
        self.options = Engine.default_options.copy()

        if os.path.isfile('config'):
            with open('config') as f:
                yaml_config = yaml.load(f)

            if yaml_config:
                self.options.update(yaml_config)

        authors = self.options.get('authors', self.options.get('author', None))
        if isinstance(authors, list):
            self.options['authors'] = [page.Author.parse(a) for a in authors]
        elif isinstance(authors, str):
            self.options['authors'] = [page.Author.parse(a) for a in authors.split(',')]

    def sanity_check(self):
        """Basic sanity checks."""
        # Make sure that this is (probabably) a wok source directory.
        if not (os.path.isdir('templates') or os.path.isdir('content')):
            logging.critical("This doesn't look like a wok site. Aborting.")
            sys.exit(1)

    def prepare_output(self):
        """
        Prepare the output directory. Remove any contents there already, and
        then copy over the media files, if they exist.
        """
        if os.path.isdir(self.options['output_dir']):
            shutil.rmtree(self.options['output_dir'])
        os.mkdir(self.options['output_dir'])

        # Copy the media directory to the output folder
        try:
            for name in os.listdir(self.options['media_dir']):
                path = os.path.join(self.options['media_dir'], name)
                if os.path.isdir(path):
                    shutil.copytree(
                            path,
                            os.path.join(self.options['output_dir'], name),
                            symlinks=True
                    )
                else:
                    shutil.copy(path, self.options['output_dir'])

        # Do nothing if the media directory doesn't exist
        except OSError:
            # XXX: We should verify that the problem was the media dir
            pass

    def load_pages(self):
        """Load all the content files."""
        for root, dirs, files in os.walk(self.options['content_dir']):
            # Grab all the parsable files
            for f in files:
                # As long as the current file is not hidden, append it to the
                # page list
                if not f.startswith('.'):
                    ext = f.split('.')[-1]
                    renderer = renderers.Plain

                    for r in renderers.all:
                        if ext in r.extensions:
                            renderer = r
                            break
                    else:
                        logging.warning('No parser found '
                                'for {0}. Using default renderer.'.format(f))
                        renderer = renderers.Renderer

                    self.all_pages.append(page.Page(os.path.join(root,f),
                            self.options, renderer))

    def make_tree(self):
        """
        Make the category pseduo-tree.

        In this structure, each node is a page. Pages with sub pages are
        interior nodes, and leaf nodes have no sub pages. It is not truely a
        tree, because the root node doesn't exist.
        """
        self.categories = {}
        site_tree = []
        # We want to parse these in a approximately breadth first order
        self.all_pages.sort(key=lambda p: len(p.meta['category']))

        for p in [p for p in self.all_pages]:
            if len(p.meta['category']) > 0:
                top_cat = p.meta['category'][0]
                if not top_cat in self.categories:
                    self.categories[top_cat] = []

                self.categories[top_cat].append(p.meta)

            try:
                siblings = site_tree
                for cat in p.meta['category']:
                    # This line will fail if the page is an orphan
                    parent = [subpage for subpage in siblings
                                 if subpage['slug']== cat][0]
                    siblings = parent['subpages']
                siblings.append(p.meta)
            except IndexError:
                logging.error('It looks like the page "{0}" is an orphan! '
                        'This will probably cause problems.'.format(p.path))

    def render_site(self):
        """Render every page and write the output files."""
        # Gather tags
        tag_set = set()
        for p in self.all_pages:
            tag_set = tag_set.union(p.meta['tags'])
        tag_dict = dict()
        for tag in tag_set:
            tag_dict[tag] = [p.meta for p in self.all_pages if tag in p.meta['tags']]

        for p in self.all_pages:
            if p.meta['published']:
                # Construct this every time, to avoid sharing one instance
                # between page objects.
                templ_vars = {
                    'site': {
                        'title': self.options.get('site_title', 'Untitled'),
                        'datetime': datetime.now(),
                        'tags': tag_dict,
                        'pages': self.all_pages[:],
                        'categories': self.categories,
                    },
                }

                for k, v in self.options.iteritems():
                    if k not in ('site_title', 'output_dir', 'content_dir',
                            'templates_dir', 'media_dir', 'url_pattern'):

                        templ_vars['site'][k] = v

                if 'author' in self.options:
                    templ_vars['site']['author'] = self.options['author']

                # Rendering the page might give us back more pages to render.
                new_pages = p.render(templ_vars)
                p.write()
                if new_pages:
                    logging.debug('found new_pages')
                    self.all_pages += new_pages

if __name__ == '__main__':
    Engine()
<|endoftext|>