<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import api, fields, models, _
from odoo.tools.float_utils import float_compare

class AccountInvoice(models.Model):
    _inherit = 'account.invoice'

    purchase_id = fields.Many2one('purchase.order', string='Add Purchase Order',
        help='Encoding help. When selected, the associated purchase order lines are added to the vendor bill. Several PO can be selected.')

    @api.onchange('state', 'partner_id', 'invoice_line_ids')
    def _onchange_allowed_purchase_ids(self):
        '''
        The purpose of the method is to define a domain for the available
        purchase orders.
        '''
        result = {}

        # A PO can be selected only if at least one PO line is not already in the invoice
        purchase_line_ids = self.invoice_line_ids.mapped('purchase_line_id')
        purchase_ids = self.invoice_line_ids.mapped('purchase_id').filtered(lambda r: r.order_line <= purchase_line_ids)

        result['domain'] = {'purchase_id': [
            ('invoice_status', '=', 'to invoice'),
            ('partner_id', 'child_of', self.partner_id.id),
            ('id', 'not in', purchase_ids.ids),
            ]}
        return result

    def _prepare_invoice_line_from_po_line(self, line):
        if line.product_id.purchase_method == 'purchase':
            qty = line.product_qty - line.qty_invoiced
        else:
            qty = line.qty_received - line.qty_invoiced
        if float_compare(qty, 0.0, precision_rounding=line.product_uom.rounding) <= 0:
            qty = 0.0
        taxes = line.taxes_id
        invoice_line_tax_ids = self.purchase_id.fiscal_position_id.map_tax(taxes)
        invoice_line = self.env['account.invoice.line']
        data = {
            'purchase_line_id': line.id,
            'name': self.purchase_id.name+': '+line.name,
            'origin': self.purchase_id.origin,
            'uom_id': line.product_uom.id,
            'product_id': line.product_id.id,
            'account_id': invoice_line.with_context({'journal_id': self.journal_id.id, 'type': 'in_invoice'})._default_account(),
            'price_unit': line.order_id.currency_id.compute(line.price_unit, self.currency_id, round=False),
            'quantity': qty,
            'discount': 0.0,
            'account_analytic_id': line.account_analytic_id.id,
            'analytic_tag_ids': line.analytic_tag_ids.ids,
            'invoice_line_tax_ids': invoice_line_tax_ids.ids
        }
        account = invoice_line.get_invoice_line_account('in_invoice', line.product_id, self.purchase_id.fiscal_position_id, self.env.user.company_id)
        if account:
            data['account_id'] = account.id
        return data

    # Load all unsold PO lines
    @api.onchange('purchase_id')
    def purchase_order_change(self):
        if not self.purchase_id:
            return {}
        if not self.partner_id:
            self.partner_id = self.purchase_id.partner_id.id

        new_lines = self.env['account.invoice.line']
        for line in self.purchase_id.order_line:
            # Load a PO line only once
            if line in self.invoice_line_ids.mapped('purchase_line_id'):
                continue
            data = self._prepare_invoice_line_from_po_line(line)
            new_line = new_lines.new(data)
            new_line._set_additional_fields(self)
            new_lines += new_line

        self.invoice_line_ids += new_lines
        self.purchase_id = False
        return {}

    @api.onchange('currency_id')
    def _onchange_currency_id(self):
        if self.currency_id:
            for line in self.invoice_line_ids.filtered(lambda r: r.purchase_line_id):
                line.price_unit = line.purchase_id.currency_id.compute(line.purchase_line_id.price_unit, self.currency_id, round=False)

    @api.onchange('invoice_line_ids')
    def _onchange_origin(self):
        purchase_ids = self.invoice_line_ids.mapped('purchase_id')
        if purchase_ids:
            self.origin = ', '.join(purchase_ids.mapped('name'))

    @api.onchange('partner_id', 'company_id')
    def _onchange_partner_id(self):
        res = super(AccountInvoice, self)._onchange_partner_id()
        if not self.env.context.get('default_journal_id') and self.partner_id and self.currency_id and\
                self.type in ['in_invoice', 'in_refund'] and\
                self.currency_id != self.partner_id.property_purchase_currency_id:
            journal_domain = [
                ('type', '=', 'purchase'),
                ('company_id', '=', self.company_id.id),
                ('currency_id', '=', self.partner_id.property_purchase_currency_id.id),
            ]
            default_journal_id = self.env['account.journal'].search(journal_domain, limit=1)
            if default_journal_id:
                self.journal_id = default_journal_id
        return res

    @api.model
    def invoice_line_move_line_get(self):
        res = super(AccountInvoice, self).invoice_line_move_line_get()

        if self.env.user.company_id.anglo_saxon_accounting:
            if self.type in ['in_invoice', 'in_refund']:
                for i_line in self.invoice_line_ids:
                    res.extend(self._anglo_saxon_purchase_move_lines(i_line, res))
        return res

    @api.model
    def _anglo_saxon_purchase_move_lines(self, i_line, res):
        """Return the additional move lines for purchase invoices and refunds.

        i_line: An account.invoice.line object.
        res: The move line entries produced so far by the parent move_line_get.
        """
        inv = i_line.invoice_id
        company_currency = inv.company_id.currency_id
        if i_line.product_id and i_line.product_id.valuation == 'real_time' and i_line.product_id.type == 'product':
            # get the fiscal position
            fpos = i_line.invoice_id.fiscal_position_id
            # get the price difference account at the product
            acc = i_line.product_id.property_account_creditor_price_difference
            if not acc:
                # if not found on the product get the price difference account at the category
                acc = i_line.product_id.categ_id.property_account_creditor_price_difference_categ
            acc = fpos.map_account(acc).id
            # reference_account_id is the stock input account
            reference_account_id = i_line.product_id.product_tmpl_id.get_product_accounts(fiscal_pos=fpos)['stock_input'].id
            diff_res = []
            account_prec = inv.company_id.currency_id.decimal_places
            # calculate and write down the possible price difference between invoice price and product price
            for line in res:
                if line.get('invl_id', 0) == i_line.id and reference_account_id == line['account_id']:
                    valuation_price_unit = i_line.product_id.uom_id._compute_price(i_line.product_id.standard_price, i_line.uom_id)
                    if i_line.product_id.cost_method != 'standard' and i_line.purchase_line_id:
                        #for average/fifo/lifo costing method, fetch real cost price from incomming moves
                        stock_move_obj = self.env['stock.move']
                        valuation_stock_move = stock_move_obj.search([('purchase_line_id', '=', i_line.purchase_line_id.id), ('state', '=', 'done')])
                        if valuation_stock_move:
                            valuation_price_unit_total = 0
                            valuation_total_qty = 0
                            for val_stock_move in valuation_stock_move:
                                valuation_price_unit_total += val_stock_move.price_unit * val_stock_move.product_qty
                                valuation_total_qty += val_stock_move.product_qty
                            valuation_price_unit = valuation_price_unit_total / valuation_total_qty
                            valuation_price_unit = i_line.product_id.uom_id._compute_price(valuation_price_unit, i_line.uom_id)
                    if inv.currency_id.id != company_currency.id:
                            valuation_price_unit = company_currency.with_context(date=inv.date_invoice).compute(valuation_price_unit, inv.currency_id)
                    if valuation_price_unit != i_line.price_unit and line['price_unit'] == i_line.price_unit and acc:
                        # price with discount and without tax included
                        price_unit = i_line.price_unit * (1 - (i_line.discount or 0.0) / 100.0)
                        if line['tax_ids']:
                            #line['tax_ids'] is like [(4, tax_id, None), (4, tax_id2, None)...]
                            taxes = self.env['account.tax'].browse([x[1] for x in line['tax_ids']])
                            price_unit = taxes.compute_all(price_unit, currency=inv.currency_id, quantity=1.0)['total_excluded']
                        price_before = line.get('price', 0.0)
                        line.update({'price': round(valuation_price_unit * line['quantity'], account_prec)})
                        diff_res.append({
                            'type': 'src',
                            'name': i_line.name[:64],
                            'price_unit': round(price_unit - valuation_price_unit, account_prec),
                            'quantity': line['quantity'],
                            'price': round(price_before - line.get('price', 0.0), account_prec),
                            'account_id': acc,
                            'product_id': line['product_id'],
                            'uom_id': line['uom_id'],
                            'account_analytic_id': line['account_analytic_id'],
                            })
            return diff_res
        return []

    @api.model
    def create(self, vals):
        invoice = super(AccountInvoice, self).create(vals)
        purchase = invoice.invoice_line_ids.mapped('purchase_line_id.order_id')
        if purchase and not invoice.refund_invoice_id:
            message = _("This vendor bill has been created from: %s") % (",".join([""+order.name+"" for order in purchase]))
            invoice.message_post(body=message)
        return invoice

    @api.multi
    def write(self, vals):
        result = True
        for invoice in self:
            purchase_old = invoice.invoice_line_ids.mapped('purchase_line_id.order_id')
            result = result and super(AccountInvoice, invoice).write(vals)
            purchase_new = invoice.invoice_line_ids.mapped('purchase_line_id.order_id')
            #To get all po reference when updating invoice line or adding purchase order reference from vendor bill.
            purchase = (purchase_old | purchase_new) - (purchase_old & purchase_new)
            if purchase:
                message = _("This vendor bill has been modified from: %s") % (",".join([""+order.name+"" for order in purchase]))
                invoice.message_post(body=message)
        return result

class AccountInvoiceLine(models.Model):
    """ Override AccountInvoice_line to add the link to the purchase order line it is related to"""
    _inherit = 'account.invoice.line'

    purchase_line_id = fields.Many2one('purchase.order.line', 'Purchase Order Line', ondelete='set null', index=True, readonly=True)
    purchase_id = fields.Many2one('purchase.order', related='purchase_line_id.order_id', string='Purchase Order', store=False, readonly=True,
        help='Associated Purchase Order. Filled in automatically when a PO is chosen on the vendor bill.')
<commit_msg>[FIX] To make function _prepare_invoice_line_from_po_line more generic<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import api, fields, models, _
from odoo.tools.float_utils import float_compare

class AccountInvoice(models.Model):
    _inherit = 'account.invoice'

    purchase_id = fields.Many2one('purchase.order', string='Add Purchase Order',
        help='Encoding help. When selected, the associated purchase order lines are added to the vendor bill. Several PO can be selected.')

    @api.onchange('state', 'partner_id', 'invoice_line_ids')
    def _onchange_allowed_purchase_ids(self):
        '''
        The purpose of the method is to define a domain for the available
        purchase orders.
        '''
        result = {}

        # A PO can be selected only if at least one PO line is not already in the invoice
        purchase_line_ids = self.invoice_line_ids.mapped('purchase_line_id')
        purchase_ids = self.invoice_line_ids.mapped('purchase_id').filtered(lambda r: r.order_line <= purchase_line_ids)

        result['domain'] = {'purchase_id': [
            ('invoice_status', '=', 'to invoice'),
            ('partner_id', 'child_of', self.partner_id.id),
            ('id', 'not in', purchase_ids.ids),
            ]}
        return result

    def _prepare_invoice_line_from_po_line(self, line):
        if line.product_id.purchase_method == 'purchase':
            qty = line.product_qty - line.qty_invoiced
        else:
            qty = line.qty_received - line.qty_invoiced
        if float_compare(qty, 0.0, precision_rounding=line.product_uom.rounding) <= 0:
            qty = 0.0
        taxes = line.taxes_id
        invoice_line_tax_ids = line.order_id.fiscal_position_id.map_tax(taxes)
        invoice_line = self.env['account.invoice.line']
        data = {
            'purchase_line_id': line.id,
            'name': line.order_id.name+': '+line.name,
            'origin': line.order_id.origin,
            'uom_id': line.product_uom.id,
            'product_id': line.product_id.id,
            'account_id': invoice_line.with_context({'journal_id': self.journal_id.id, 'type': 'in_invoice'})._default_account(),
            'price_unit': line.order_id.currency_id.compute(line.price_unit, self.currency_id, round=False),
            'quantity': qty,
            'discount': 0.0,
            'account_analytic_id': line.account_analytic_id.id,
            'analytic_tag_ids': line.analytic_tag_ids.ids,
            'invoice_line_tax_ids': invoice_line_tax_ids.ids
        }
        account = invoice_line.get_invoice_line_account('in_invoice', line.product_id, line.order_id.fiscal_position_id, self.env.user.company_id)
        if account:
            data['account_id'] = account.id
        return data

    # Load all unsold PO lines
    @api.onchange('purchase_id')
    def purchase_order_change(self):
        if not self.purchase_id:
            return {}
        if not self.partner_id:
            self.partner_id = self.purchase_id.partner_id.id

        new_lines = self.env['account.invoice.line']
        for line in self.purchase_id.order_line:
            # Load a PO line only once
            if line in self.invoice_line_ids.mapped('purchase_line_id'):
                continue
            data = self._prepare_invoice_line_from_po_line(line)
            new_line = new_lines.new(data)
            new_line._set_additional_fields(self)
            new_lines += new_line

        self.invoice_line_ids += new_lines
        self.purchase_id = False
        return {}

    @api.onchange('currency_id')
    def _onchange_currency_id(self):
        if self.currency_id:
            for line in self.invoice_line_ids.filtered(lambda r: r.purchase_line_id):
                line.price_unit = line.purchase_id.currency_id.compute(line.purchase_line_id.price_unit, self.currency_id, round=False)

    @api.onchange('invoice_line_ids')
    def _onchange_origin(self):
        purchase_ids = self.invoice_line_ids.mapped('purchase_id')
        if purchase_ids:
            self.origin = ', '.join(purchase_ids.mapped('name'))

    @api.onchange('partner_id', 'company_id')
    def _onchange_partner_id(self):
        res = super(AccountInvoice, self)._onchange_partner_id()
        if not self.env.context.get('default_journal_id') and self.partner_id and self.currency_id and\
                self.type in ['in_invoice', 'in_refund'] and\
                self.currency_id != self.partner_id.property_purchase_currency_id:
            journal_domain = [
                ('type', '=', 'purchase'),
                ('company_id', '=', self.company_id.id),
                ('currency_id', '=', self.partner_id.property_purchase_currency_id.id),
            ]
            default_journal_id = self.env['account.journal'].search(journal_domain, limit=1)
            if default_journal_id:
                self.journal_id = default_journal_id
        return res

    @api.model
    def invoice_line_move_line_get(self):
        res = super(AccountInvoice, self).invoice_line_move_line_get()

        if self.env.user.company_id.anglo_saxon_accounting:
            if self.type in ['in_invoice', 'in_refund']:
                for i_line in self.invoice_line_ids:
                    res.extend(self._anglo_saxon_purchase_move_lines(i_line, res))
        return res

    @api.model
    def _anglo_saxon_purchase_move_lines(self, i_line, res):
        """Return the additional move lines for purchase invoices and refunds.

        i_line: An account.invoice.line object.
        res: The move line entries produced so far by the parent move_line_get.
        """
        inv = i_line.invoice_id
        company_currency = inv.company_id.currency_id
        if i_line.product_id and i_line.product_id.valuation == 'real_time' and i_line.product_id.type == 'product':
            # get the fiscal position
            fpos = i_line.invoice_id.fiscal_position_id
            # get the price difference account at the product
            acc = i_line.product_id.property_account_creditor_price_difference
            if not acc:
                # if not found on the product get the price difference account at the category
                acc = i_line.product_id.categ_id.property_account_creditor_price_difference_categ
            acc = fpos.map_account(acc).id
            # reference_account_id is the stock input account
            reference_account_id = i_line.product_id.product_tmpl_id.get_product_accounts(fiscal_pos=fpos)['stock_input'].id
            diff_res = []
            account_prec = inv.company_id.currency_id.decimal_places
            # calculate and write down the possible price difference between invoice price and product price
            for line in res:
                if line.get('invl_id', 0) == i_line.id and reference_account_id == line['account_id']:
                    valuation_price_unit = i_line.product_id.uom_id._compute_price(i_line.product_id.standard_price, i_line.uom_id)
                    if i_line.product_id.cost_method != 'standard' and i_line.purchase_line_id:
                        #for average/fifo/lifo costing method, fetch real cost price from incomming moves
                        stock_move_obj = self.env['stock.move']
                        valuation_stock_move = stock_move_obj.search([('purchase_line_id', '=', i_line.purchase_line_id.id), ('state', '=', 'done')])
                        if valuation_stock_move:
                            valuation_price_unit_total = 0
                            valuation_total_qty = 0
                            for val_stock_move in valuation_stock_move:
                                valuation_price_unit_total += val_stock_move.price_unit * val_stock_move.product_qty
                                valuation_total_qty += val_stock_move.product_qty
                            valuation_price_unit = valuation_price_unit_total / valuation_total_qty
                            valuation_price_unit = i_line.product_id.uom_id._compute_price(valuation_price_unit, i_line.uom_id)
                    if inv.currency_id.id != company_currency.id:
                            valuation_price_unit = company_currency.with_context(date=inv.date_invoice).compute(valuation_price_unit, inv.currency_id)
                    if valuation_price_unit != i_line.price_unit and line['price_unit'] == i_line.price_unit and acc:
                        # price with discount and without tax included
                        price_unit = i_line.price_unit * (1 - (i_line.discount or 0.0) / 100.0)
                        if line['tax_ids']:
                            #line['tax_ids'] is like [(4, tax_id, None), (4, tax_id2, None)...]
                            taxes = self.env['account.tax'].browse([x[1] for x in line['tax_ids']])
                            price_unit = taxes.compute_all(price_unit, currency=inv.currency_id, quantity=1.0)['total_excluded']
                        price_before = line.get('price', 0.0)
                        line.update({'price': round(valuation_price_unit * line['quantity'], account_prec)})
                        diff_res.append({
                            'type': 'src',
                            'name': i_line.name[:64],
                            'price_unit': round(price_unit - valuation_price_unit, account_prec),
                            'quantity': line['quantity'],
                            'price': round(price_before - line.get('price', 0.0), account_prec),
                            'account_id': acc,
                            'product_id': line['product_id'],
                            'uom_id': line['uom_id'],
                            'account_analytic_id': line['account_analytic_id'],
                            })
            return diff_res
        return []

    @api.model
    def create(self, vals):
        invoice = super(AccountInvoice, self).create(vals)
        purchase = invoice.invoice_line_ids.mapped('purchase_line_id.order_id')
        if purchase and not invoice.refund_invoice_id:
            message = _("This vendor bill has been created from: %s") % (",".join([""+order.name+"" for order in purchase]))
            invoice.message_post(body=message)
        return invoice

    @api.multi
    def write(self, vals):
        result = True
        for invoice in self:
            purchase_old = invoice.invoice_line_ids.mapped('purchase_line_id.order_id')
            result = result and super(AccountInvoice, invoice).write(vals)
            purchase_new = invoice.invoice_line_ids.mapped('purchase_line_id.order_id')
            #To get all po reference when updating invoice line or adding purchase order reference from vendor bill.
            purchase = (purchase_old | purchase_new) - (purchase_old & purchase_new)
            if purchase:
                message = _("This vendor bill has been modified from: %s") % (",".join([""+order.name+"" for order in purchase]))
                invoice.message_post(body=message)
        return result

class AccountInvoiceLine(models.Model):
    """ Override AccountInvoice_line to add the link to the purchase order line it is related to"""
    _inherit = 'account.invoice.line'

    purchase_line_id = fields.Many2one('purchase.order.line', 'Purchase Order Line', ondelete='set null', index=True, readonly=True)
    purchase_id = fields.Many2one('purchase.order', related='purchase_line_id.order_id', string='Purchase Order', store=False, readonly=True,
        help='Associated Purchase Order. Filled in automatically when a PO is chosen on the vendor bill.')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import http
from odoo.http import request
from odoo import tools
from odoo.tools.translate import _

from odoo.fields import Date

class website_account(http.Controller):

    MANDATORY_BILLING_FIELDS = ["name", "phone", "email", "street2", "city", "country_id"]
    OPTIONAL_BILLING_FIELDS = ["zipcode", "state_id", "vat", "street"]

    _items_per_page = 10

    def _prepare_portal_layout_values(self):
        """ prepare the values to render portal layout """
        partner = request.env.user.partner_id
        # get customer sales rep
        if partner.user_id:
            sales_rep = partner.user_id
        else:
            sales_rep = False
        values = {
            'sales_rep': sales_rep,
            'company': request.website.company_id,
            'user': request.env.user
        }
        return values

    def _get_archive_groups(self, model, domain=None, fields=None, groupby="create_date", order="create_date desc"):
        if not model:
            return []
        if domain is None:
            domain = []
        if fields is None:
            fields = ['name', 'create_date']
        groups = []
        for group in request.env[model]._read_group_raw(domain, fields=fields, groupby=groupby, orderby=order):
            dates, label = group[groupby]
            date_begin, date_end = dates.split('/')
            groups.append({
                'date_begin': Date.to_string(Date.from_string(date_begin)),
                'date_end': Date.to_string(Date.from_string(date_end)),
                'name': label,
                'item_count': group[groupby + '_count']
            })
        return groups

    @http.route(['/my', '/my/home'], type='http', auth="public", website=True)
    def account(self, **kw):
        values = self._prepare_portal_layout_values()
        return request.website.render("website_portal.portal_my_home", values)

    @http.route(['/my/account'], type='http', auth='user', website=True)
    def details(self, redirect=None, **post):
        partner = request.env['res.users'].browse(request.uid).partner_id
        values = {
            'error': {},
            'error_message': []
        }

        if post:
            error, error_message = self.details_form_validate(post)
            values.update({'error': error, 'error_message': error_message})
            values.update(post)
            if not error:
                values = {key: post[key] for key in self.MANDATORY_BILLING_FIELDS + self.OPTIONAL_BILLING_FIELDS}
                values.update({'zip': post.pop('zipcode', '')})
                partner.sudo().write(values)
                if redirect:
                    return request.redirect(redirect)
                return request.redirect('/my/home')

        countries = request.env['res.country'].sudo().search([])
        states = request.env['res.country.state'].sudo().search([])

        values.update({
            'partner': partner,
            'countries': countries,
            'states': states,
            'has_check_vat': hasattr(request.env['res.partner'], 'check_vat'),
            'redirect': redirect,
        })

        return request.website.render("website_portal.details", values)

    def details_form_validate(self, data):
        error = dict()
        error_message = []

        # Validation
        for field_name in self.MANDATORY_BILLING_FIELDS:
            if not data.get(field_name):
                error[field_name] = 'missing'

        # email validation
        if data.get('email') and not tools.single_email_re.match(data.get('email')):
            error["email"] = 'error'
            error_message.append(_('Invalid Email! Please enter a valid email address.'))

        # vat validation
        if data.get("vat") and hasattr(request.env["res.partner"], "check_vat"):
            if request.website.company_id.vat_check_vies:
                # force full VIES online check
                check_func = request.env["res.partner"].vies_vat_check
            else:
                # quick and partial off-line checksum validation
                check_func = request.env["res.partner"].simple_vat_check
            vat_country, vat_number = request.env["res.partner"]._split_vat(data.get("vat"))
            if not check_func(vat_country, vat_number):  # simple_vat_check
                error["vat"] = 'error'

        # error message for empty required fields
        if [err for err in error.values() if err == 'missing']:
            error_message.append(_('Some required fields are empty.'))

        unknown = [k for k in data.iterkeys() if k not in self.MANDATORY_BILLING_FIELDS + self.OPTIONAL_BILLING_FIELDS]
        if unknown:
            error['common'] = 'Unknown field'
            error_message.append("Unknown field '%s'" % ','.join(unknown))

        return error, error_message
<commit_msg>[FIX] website_portal: fix typo<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import http
from odoo.http import request
from odoo import tools
from odoo.tools.translate import _

from odoo.fields import Date

class website_account(http.Controller):

    MANDATORY_BILLING_FIELDS = ["name", "phone", "email", "street2", "city", "country_id"]
    OPTIONAL_BILLING_FIELDS = ["zipcode", "state_id", "vat", "street"]

    _items_per_page = 10

    def _prepare_portal_layout_values(self):
        """ prepare the values to render portal layout """
        partner = request.env.user.partner_id
        # get customer sales rep
        if partner.user_id:
            sales_rep = partner.user_id
        else:
            sales_rep = False
        values = {
            'sales_rep': sales_rep,
            'company': request.website.company_id,
            'user': request.env.user
        }
        return values

    def _get_archive_groups(self, model, domain=None, fields=None, groupby="create_date", order="create_date desc"):
        if not model:
            return []
        if domain is None:
            domain = []
        if fields is None:
            fields = ['name', 'create_date']
        groups = []
        for group in request.env[model]._read_group_raw(domain, fields=fields, groupby=groupby, orderby=order):
            dates, label = group[groupby]
            date_begin, date_end = dates.split('/')
            groups.append({
                'date_begin': Date.to_string(Date.from_string(date_begin)),
                'date_end': Date.to_string(Date.from_string(date_end)),
                'name': label,
                'item_count': group[groupby + '_count']
            })
        return groups

    @http.route(['/my', '/my/home'], type='http', auth="public", website=True)
    def account(self, **kw):
        values = self._prepare_portal_layout_values()
        return request.website.render("website_portal.portal_my_home", values)

    @http.route(['/my/account'], type='http', auth='user', website=True)
    def details(self, redirect=None, **post):
        partner = request.env['res.users'].browse(request.uid).partner_id
        values = {
            'error': {},
            'error_message': []
        }

        if post:
            error, error_message = self.details_form_validate(post)
            values.update({'error': error, 'error_message': error_message})
            values.update(post)
            if not error:
                values = {key: post[key] for key in self.MANDATORY_BILLING_FIELDS + self.OPTIONAL_BILLING_FIELDS}
                values.update({'zip': values.pop('zipcode', '')})
                partner.sudo().write(values)
                if redirect:
                    return request.redirect(redirect)
                return request.redirect('/my/home')

        countries = request.env['res.country'].sudo().search([])
        states = request.env['res.country.state'].sudo().search([])

        values.update({
            'partner': partner,
            'countries': countries,
            'states': states,
            'has_check_vat': hasattr(request.env['res.partner'], 'check_vat'),
            'redirect': redirect,
        })

        return request.website.render("website_portal.details", values)

    def details_form_validate(self, data):
        error = dict()
        error_message = []

        # Validation
        for field_name in self.MANDATORY_BILLING_FIELDS:
            if not data.get(field_name):
                error[field_name] = 'missing'

        # email validation
        if data.get('email') and not tools.single_email_re.match(data.get('email')):
            error["email"] = 'error'
            error_message.append(_('Invalid Email! Please enter a valid email address.'))

        # vat validation
        if data.get("vat") and hasattr(request.env["res.partner"], "check_vat"):
            if request.website.company_id.vat_check_vies:
                # force full VIES online check
                check_func = request.env["res.partner"].vies_vat_check
            else:
                # quick and partial off-line checksum validation
                check_func = request.env["res.partner"].simple_vat_check
            vat_country, vat_number = request.env["res.partner"]._split_vat(data.get("vat"))
            if not check_func(vat_country, vat_number):  # simple_vat_check
                error["vat"] = 'error'

        # error message for empty required fields
        if [err for err in error.values() if err == 'missing']:
            error_message.append(_('Some required fields are empty.'))

        unknown = [k for k in data.iterkeys() if k not in self.MANDATORY_BILLING_FIELDS + self.OPTIONAL_BILLING_FIELDS]
        if unknown:
            error['common'] = 'Unknown field'
            error_message.append("Unknown field '%s'" % ','.join(unknown))

        return error, error_message
<|endoftext|>


<commit_before>NEW_LINE = b'\r\n'

class HeaderParser(object):
    __slots__ = ['_buffer', 'complete']

    RETAIN_KEYS = set(['CONTENT_TYPE', 'CONTENT_LENGTH'])

    def __init__(self):
        self._buffer = b''
        self.complete = False

    def _make_key(self, key):
        key = key.strip().upper().replace('-', '_')
        if key in self.RETAIN_KEYS:
            return key
        return 'HTTP_' + key

    def feed(self, data, env):
        initial_size = len(self._buffer)
        consumed_size = 0
        self._buffer += data

        while True:
            if self._buffer[:2] == NEW_LINE:
                self.complete = True
                self._buffer = self._buffer[2:]
                consumed_size += 2
                break

            try:
                i = self._buffer.index(NEW_LINE)
                line = self._buffer[:i].decode('utf8')
                self._buffer = self._buffer[i+2:]
                k, v = line.split(':', 1)
                k = self._make_key(k)
                v = v.strip()
                env[k] = v
                consumed_size += i + 2
            except Exception as _:
                return len(data)

        assert consumed_size > initial_size  # Safety check
        return consumed_size - initial_size
<commit_msg>Fixing header encoding to support python 2.<commit_after>import six

NEW_LINE = b'\r\n'

class HeaderParser(object):
    __slots__ = ['_buffer', 'complete']

    RETAIN_KEYS = set(['CONTENT_TYPE', 'CONTENT_LENGTH'])

    def __init__(self):
        self._buffer = b''
        self.complete = False

    def _make_key(self, key):
        key = key.strip().upper().replace('-', '_')
        if key in self.RETAIN_KEYS:
            return key
        return 'HTTP_' + key

    def feed(self, data, env):
        initial_size = len(self._buffer)
        consumed_size = 0
        self._buffer += data

        while True:
            if self._buffer[:2] == NEW_LINE:
                self.complete = True
                self._buffer = self._buffer[2:]
                consumed_size += 2
                break

            try:
                i = self._buffer.index(NEW_LINE)
                line = self._buffer[:i]
                if six.PY3:
                    line = line.decode('ascii')

                self._buffer = self._buffer[i+2:]
                k, v = line.split(':', 1)
                k = self._make_key(k)
                v = v.strip()
                env[k] = v
                consumed_size += i + 2
            except Exception as _:
                return len(data)

        assert consumed_size > initial_size  # Safety check
        return consumed_size - initial_size
<|endoftext|>


<commit_before>"""
Django settings for budget_proj project.

Generated by 'django-admin startproject' using Django 1.10.5.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

DEBUG = False

# Note: the 192.168.99.100 address is necessary to enable testing with Docker Toolbox for Mac and Windows
ALLOWED_HOSTS = ['127.0.0.1', 'localhost', '192.168.99.100']

# Application definition
INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'whitenoise.runserver_nostatic',
    'django.contrib.staticfiles',
    # *******************************
    # non-default 3rd party packages
    'rest_framework',
    'rest_framework_swagger',
    'corsheaders',
    # project-specific code
    'budget_app'
    # *******************************
]

MIDDLEWARE = [
    # should be placed as high as possible and before CommonMiddleware response generator
    'corsheaders.middleware.CorsMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'budget_proj.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'budget_proj.wsgi.application'

# DRF Settings:
REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
    'PAGE_SIZE': 100,
    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',),
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

STATIC_URL = "/budget/static/"

# This seems to be necessary to enable the Django app to correctly style
# the Swagger wrapper when the Django app runs inside a Docker container
STATIC_ROOT = 'staticfiles'

# This allows any site to call these API endpoints, which is entirely the point of making them available
CORS_ORIGIN_ALLOW_ALL = True

STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

# LOGGING = {
#     'version': 1,
#     'disable_existing_loggers': False,
#     'formatters': {
#         'verbose': {
#             'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
#         },
#     },
#     'handlers': {
#         'console': {
#             'class': 'logging.StreamHandler',
#             'formatter': 'verbose',
#         },
#         'file': {
#             'level': 'DEBUG',
#             'class': 'logging.FileHandler',
#             'filename': '/var/log/django_debug.log',
#         },
#     },
#     'loggers': {
#         'django': {
#             'handlers': ['console', 'file'],
#             'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
#             'propagate': True,
#         },
#     },
# }
<commit_msg>Experimenting with page size.<commit_after>"""
Django settings for budget_proj project.

Generated by 'django-admin startproject' using Django 1.10.5.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

DEBUG = False

# Note: the 192.168.99.100 address is necessary to enable testing with Docker Toolbox for Mac and Windows
ALLOWED_HOSTS = ['127.0.0.1', 'localhost', '192.168.99.100']

# Application definition
INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'whitenoise.runserver_nostatic',
    'django.contrib.staticfiles',
    # *******************************
    # non-default 3rd party packages
    'rest_framework',
    'rest_framework_swagger',
    'corsheaders',
    # project-specific code
    'budget_app'
    # *******************************
]

MIDDLEWARE = [
    # should be placed as high as possible and before CommonMiddleware response generator
    'corsheaders.middleware.CorsMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'budget_proj.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'budget_proj.wsgi.application'

# DRF Settings:
REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
    'PAGE_SIZE': 2500,
    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',),
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

STATIC_URL = "/budget/static/"

# This seems to be necessary to enable the Django app to correctly style
# the Swagger wrapper when the Django app runs inside a Docker container
STATIC_ROOT = 'staticfiles'

# This allows any site to call these API endpoints, which is entirely the point of making them available
CORS_ORIGIN_ALLOW_ALL = True

STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

# LOGGING = {
#     'version': 1,
#     'disable_existing_loggers': False,
#     'formatters': {
#         'verbose': {
#             'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
#         },
#     },
#     'handlers': {
#         'console': {
#             'class': 'logging.StreamHandler',
#             'formatter': 'verbose',
#         },
#         'file': {
#             'level': 'DEBUG',
#             'class': 'logging.FileHandler',
#             'filename': '/var/log/django_debug.log',
#         },
#     },
#     'loggers': {
#         'django': {
#             'handlers': ['console', 'file'],
#             'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
#             'propagate': True,
#         },
#     },
# }
<|endoftext|>


<commit_before>

class BaseContentType(object):

    def __str__(self):
        raise NotImplementedError("Please implement this method in subclass")

    def __repr__(self):
        return "<{self.__class__.__name__}: {self}>".format(self=self)

    @classmethod
    def from_string(cls, value):
        raise NotImplementedError("This method should return an instance of the content type")

class DeviceAccess(BaseContentType):
    TYPE_ALL = "all"
    TYPE_CHAR = "c"
    TYPE_BLOCK = "b"

    ACCESS_UNSPEC = 0
    ACCESS_READ = 1
    ACCESS_WRITE = 2
    ACCESS_MKNOD = 4

    def __init__(self, dev_type=None, major=None, minor=None, access=None):
        self.dev_type = dev_type or self.TYPE_ALL

        # the default behaviour of device access cgroups if unspecified is as follows
        self.major = major or "*"
        self.minor = minor or "*"
        self.access = access or (self.ACCESS_READ | self.ACCESS_WRITE | self.ACCESS_MKNOD)

    def _check_access_bit(self, offset):
        mask = 1 << offset
        return self.access & mask

    @property
    def can_read(self):
        return self._check_access_bit(0) == self.ACCESS_READ

    @property
    def can_write(self):
        return self._check_access_bit(1) == self.ACCESS_WRITE

    @property
    def can_mknod(self):
        return self._check_access_bit(2) == self.ACCESS_MKNOD

    @property
    def access_string(self):
        accstr = ""
        if self.can_read:
            accstr += "r"
        if self.can_write:
            accstr += "w"
        if self.can_mknod:
            accstr += "m"
        return accstr

    def __str__(self):
        return "{self.dev_type} {self.major}:{self.minor} {self.access_string}".format(self=self)

    @classmethod
    def from_string(cls, value):
        dev_type, major_minor, access_string = value.split()
        major, minor = major_minor.split(":")
        major = int(major) if major != "*" else None
        minor == int(minor) if minor != "*" else None

        access_mode = 0
        for idx, char in enumerate("rwm"):
            if char in access_string:
                access_mode |= (1 << idx)
        return cls(dev_type, major, minor, access_mode)

class DeviceThrottle(BaseContentType):

    def __init__(self, limit, major=None, minor=None, ):
        self.limit = limit
        self.major = major or "*"
        self.minor = minor or "*"

    def __str__(self):
        return "{self.major}:{self.minor} {self.limit}".format(self=self)

    @classmethod
    def from_string(cls, value):
        if not value:
            return None

        try:
            major_minor, limit = value.split()
            major, minor = major_minor.split(":")
            return cls(int(limit), major, minor)
        except Exception:
            raise RuntimeError("Value {} cannot be converted to a string that matches the pattern: "
                               "[device major]:[device minor] [throttle limit in bytes]".format(value))
<commit_msg>Improve the content types - better __init__ methods, add __eq__ methods, add license info, fix == to be =<commit_after>"""
Copyright (c) 2014, CloudSigma AG
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the CloudSigma AG nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL CLOUDSIGMA AG BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

class BaseContentType(object):

    def __str__(self):
        raise NotImplementedError("Please implement this method in subclass")

    def __repr__(self):
        return "<{self.__class__.__name__}: {self}>".format(self=self)

    @classmethod
    def from_string(cls, value):
        raise NotImplementedError("This method should return an instance of the content type")

class DeviceAccess(BaseContentType):
    TYPE_ALL = "all"
    TYPE_CHAR = "c"
    TYPE_BLOCK = "b"

    ACCESS_UNSPEC = 0
    ACCESS_READ = 1
    ACCESS_WRITE = 2
    ACCESS_MKNOD = 4

    def __init__(self, dev_type=None, major=None, minor=None, access=None):
        self.dev_type = dev_type or self.TYPE_ALL

        # the default behaviour of device access cgroups if unspecified is as follows
        if major is not None:
            self.major = int(major)
        else:
            self.major = "*"

        if minor is not None:
            self.minor = int(minor)
        else:
            self.minor = "*"

        if isinstance(access, str):
            value = 0
            if 'r' in access:
                value |= self.ACCESS_READ
            if 'w' in access:
                value |= self.ACCESS_WRITE
            if 'm' in access:
                value |= self.ACCESS_MKNOD
            self.access = value
        else:
            self.access = access or (self.ACCESS_READ | self.ACCESS_WRITE | self.ACCESS_MKNOD)

    def _check_access_bit(self, offset):
        mask = 1 << offset
        return self.access & mask

    @property
    def can_read(self):
        return self._check_access_bit(0) == self.ACCESS_READ

    @property
    def can_write(self):
        return self._check_access_bit(1) == self.ACCESS_WRITE

    @property
    def can_mknod(self):
        return self._check_access_bit(2) == self.ACCESS_MKNOD

    @property
    def access_string(self):
        accstr = ""
        if self.can_read:
            accstr += "r"
        if self.can_write:
            accstr += "w"
        if self.can_mknod:
            accstr += "m"
        return accstr

    def __str__(self):
        return "{self.dev_type} {self.major}:{self.minor} {self.access_string}".format(self=self)

    def __eq__(self, other):
        return self.dev_type == other.dev_type and self.major == other.major \
               and self.minor == other.minor and self.access_string == other.access_string

    @classmethod
    def from_string(cls, value):
        dev_type, major_minor, access_string = value.split()
        major, minor = major_minor.split(":")
        major = int(major) if major != "*" else None
        minor = int(minor) if minor != "*" else None

        access_mode = 0
        for idx, char in enumerate("rwm"):
            if char in access_string:
                access_mode |= (1 << idx)
        return cls(dev_type, major, minor, access_mode)

class DeviceThrottle(BaseContentType):

    def __init__(self, limit, major=None, minor=None, ):
        self.limit = limit
        if major is not None and major != '*':
            self.major = int(major)
        else:
            self.major = '*'

        if minor is not None and minor != '*':
            self.minor = int(minor)
        else:
            self.minor = '*'

    def __eq__(self, other):
        return self.limit == other.limit and self.major == other.major and self.minor == other.minor

    def __str__(self):
        return "{self.major}:{self.minor} {self.limit}".format(self=self)

    @classmethod
    def from_string(cls, value):
        if not value:
            return None

        try:
            major_minor, limit = value.split()
            major, minor = major_minor.split(":")
            return cls(int(limit), major, minor)
        except Exception:
            raise RuntimeError("Value {} cannot be converted to a string that matches the pattern: "
                               "[device major]:[device minor] [throttle limit in bytes]".format(value))
<|endoftext|>


<commit_before>#! python3
# bulletPointAdder.py - Adds Wikipedia bullet points to the start
# of each line of text on the clipboard.

import pyperclip
text = pyperclip.paste()

# TODO: Separate lines and add starts.
# Separate lines and add stars.

lines = text.split('\n')
for i in range(len(lines)): # loop through all indexes in the "lines" list
    lines[i] = '* ' + lines[i] # add star to each string in "lines" list

pyperclip.copy(text)

<commit_msg>Added line<commit_after>#! python3
# bulletPointAdder.py - Adds Wikipedia bullet points to the start
# of each line of text on the clipboard.

import pyperclip
text = pyperclip.paste()

# TODO: Separate lines and add starts.
# Separate lines and add stars.

lines = text.split('\n')
for i in range(len(lines)): # loop through all indexes in the "lines" list
    lines[i] = '* ' + lines[i] # add star to each string in "lines" list
text = '\n'.join(lines)

pyperclip.copy(text)

<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as plt
from pylab import *
from view.visualizations.xy_plot import XYPlot

def class_name():
    return "DogePlot"

class DogePlot(XYPlot):
    """DogePlot
    """

    def name(self):
        return "Doge Plot"

    def __init__(self, sim_manager, main_controller, **kwargs):
        super(DogePlot, self).__init__(sim_manager, main_controller, **kwargs)
        self.text = []

    def display_name(self):
        return "so science"

    def update(self, data, start_time):
        super(DogePlot, self).update(data, start_time)
        if(not self.text and len(data) > 10):
            self.text.append(plt.text(0.27, 0.77, "such line", fontsize=12, color='orange'))
            self.text.append(plt.text(0.7, 0.57, "very neuron", fontsize=12, color='green'))
            self.text.append(plt.text(0.77, 0.2, "wow", fontsize=12, color='purple'))
            self.text.append(plt.text(0.07, 0.32, "so science", fontsize=12, color='cyan'))
        elif (self.text and len(data) < 275):
            for txt in self.text:
                txt.set_x((txt.get_position()[0]*100000 + len(data))/100000 % 1 )

<commit_msg>dogeplot fix<commit_after>import numpy as np
import matplotlib.pyplot as plt
from view.visualizations.xy_plot import XYPlot

def class_name():
    return "DogePlot"

class DogePlot(XYPlot):
    """DogePlot
    """

    def name(self):
        return "Doge Plot"

    def __init__(self, sim_manager, main_controller, **kwargs):
        super(DogePlot, self).__init__(sim_manager, main_controller, **kwargs)
        self.text = []
        self.text.append(plt.text(0.27, 0.77, "such line", fontsize=12, color='orange'))
        self.text.append(plt.text(0.7, 0.57, "very neuron", fontsize=12, color='green'))
        self.text.append(plt.text(0.77, 0.2, "wow", fontsize=12, color='purple'))
        self.text.append(plt.text(0.07, 0.32, "so science", fontsize=12, color='cyan'))

    def display_name(self):
        return "so science"

    def update(self, data, start_time):
        super(DogePlot, self).update(data, start_time)
        if (self.text and len(data) < 275):
            for txt in self.text:
                txt.set_x((txt.get_position()[0]*100000 + len(data))/100000 % 1 )

<|endoftext|>


<commit_before>import datetime
import mailman
import mailbox
import numpy as np
from bigbang.thread import Thread
from bigbang.thread import Node
import bigbang.process as process
import pandas as pd
import pytz
import utils
import logging

def load(path):
    data = pd.read_csv(path)
    return Archive(data)

class Archive(object):

    """
    A representation of a mailing list archive.
    """

    data = None
    activity = None
    threads = None
    entities = None

    def __init__(self, data, archive_dir="archives", mbox=False):
        """
        Initializes an Archive object.

        The behavior of the constructor depends on the type
        of its first argument, data.

        If data is a Pandas DataFrame, it is treated as a representation of
        email messages with columns for Message-ID, From, Date, In-Reply-To,
        References, and Body. The created Archive becomes a wrapper around a
        copy of the input DataFrame.

        If data is a string, then it is interpreted as a path to either a
        single .mbox file (if the optional argument single_file is True) or
        else to a directory of .mbox files (also in .mbox format). Note that
        the file extensions need not be .mbox; frequently they will be .txt.

        Upon initialization, the Archive object drops duplicate entries
        and sorts its member variable *data* by Date.
        """

        if isinstance(data, pd.core.frame.DataFrame):
            self.data = data.copy()
        elif isinstance(data, str):
            self.data = mailman.load_data(data,archive_dir=archive_dir,mbox=mbox)

        try:
            self.data['Date'] = pd.to_datetime(self.data['Date'], errors='coerce', infer_datetime_format=True, utc=True)
        except:
            out_path = 'datetime-exception.csv'

            with open(out_path, 'w') as f:
                self.data.to_csv(f, encoding='utf-8')

            logging.error('Error while converting to datetime, despite coerce mode.')
            raise

        try:
            self.data.drop_duplicates(inplace=True)
        except:
            logging.error('Error while removing duplicate messages, maybe timezone issues?', exc_info=True)

        # Drops any entries with no Date field.
        # It may be wiser to optionally
        # do interpolation here.
        if self.data['Date'].isnull().any():
            #self.data.dropna(subset=['Date'], inplace=True)
            self.data = self.data[self.data['Date'].notnull()]
            # workaround for https://github.com/pandas-dev/pandas/issues/13407

        #convert any null fields to None -- csv saves these as nan sometimes
        self.data = self.data.where(pd.notnull(self.data),None)

        try:
            #set the index to be the Message-ID column
            self.data.set_index('Message-ID',inplace=True)
        except KeyError:
            #will get KeyError if Message-ID is already index
            pass

        self.data.sort_values(by='Date', inplace=True)

    def resolve_entities(self,inplace=True):
        if self.entities is None:
            if self.activity is None:
                self.get_activity()

            self.entities = process.resolve_sender_entities(self.activity)

        to_replace = []

        value = []

        for e, names in self.entities.items():
            for n in names:
                to_replace.append(n)
                value.append(e)

        data = self.data.replace(to_replace=to_replace,value=value,inplace=inplace)

        # clear and replace activity with resolved activity
        self.activity = None
        self.get_activity()

        if inplace:
            return self.data
        else:
            return data

    def get_activity(self,resolved=False):
        """
        Get the activity matrix of an Archive.
        Columns of the returned DataFrame are the Senders of emails.
        Rows are indexed by ordinal date.
        Cells are the number of emails sent by each sender on each data.

        If *resolved* is true, then default entity resolution is run on the
        activity matrix before it is returned.
        """
        if self.activity is None:
            self.activity = self.compute_activity(self)

        if resolved:
            self.entities = process.resolve_sender_entities(self.activity)
            eact = utils.repartition_dataframe(self.activity,self.entities)

            return eact

        return self.activity

    def compute_activity(self, clean=True):
        mdf = self.data

        if clean:
            # unnecessary?
            if mdf['Date'].isnull().any():
                mdf = mdf.dropna(subset=['Date'])

            mdf = mdf[
                mdf['Date'] < datetime.datetime.now(
                    pytz.utc)]  # drop messages apparently in the future

        mdf2 = mdf.reindex(columns = ['From', 'Date'])
        mdf2['Date'] = mdf['Date'].apply(lambda x: x.toordinal())

        activity = mdf2.groupby(
            ['From', 'Date']).size().unstack('From').fillna(0)
        new_date_range = np.arange(mdf2['Date'].min(), mdf2['Date'].max())
        # activity.set_index('Date')
        activity = activity.reindex(new_date_range, fill_value=0)

        return activity

    def get_threads(self, verbose=False):

        if self.threads is not None:
            return self.threads

        df = self.data

        threads = list()
        visited = dict()

        total = df.shape[0]
        c = 0

        for i in df.iterrows():

            if verbose:
                c += 1
                if c % 1000 == 0:
                    print "Processed %d of %d" %(c,total)

            if(i[1]['In-Reply-To'] is None):
                root = Node(i[0], i[1])
                visited[i[0]] = root
                threads.append(Thread(root))
            elif(i[1]['In-Reply-To'] not in visited.keys()):
                root = Node(i[1]['In-Reply-To'])
                succ = Node(i[0],i[1], root)
                root.add_successor(succ)
                visited[i[1]['In-Reply-To']] = root
                visited[i[0]] = succ
                threads.append(Thread(root, known_root=False))
            else:
                parent = visited[i[1]['In-Reply-To']]
                node = Node(i[0],i[1], parent)
                parent.add_successor(node)
                visited[i[0]] = node

        self.threads = threads

        return threads

    def save(self, path,encoding='utf-8'):
        self.data.to_csv(path, ",",encoding=encoding)

def find_footer(messages,number=1):
    '''
    Returns the footer of a DataFrame of emails.
    A footer is a string occurring at the tail of most messages.
    Messages can be a DataFrame or a Series
    '''
    if isinstance(messages,pd.DataFrame):
        messages = messages['Body']

    # sort in lexical order of reverse strings to maximize foot length
    srb = messages.apply(lambda x: None if x is None else x[::-1]).order()
    #srb = df.apply(lambda x: None if x['Body'] is None else x['Body'][::-1],
    #              axis=1).order()
    # begin walking down the series looking for maximal overlap
    counts = {}

    last = None

    def clean_footer(foot):
        return foot.strip()

    for b in srb:
        if last is None:
            last = b
            continue
        elif b is None:
            continue
        else:
            head,i = utils.get_common_head(b,last,delimiter='\n')
            head = clean_footer(head[::-1])
            last = b

            if head in counts:
                counts[head] = counts[head] + 1
            else:
                counts[head] = 1

        last = b

    # reduce candidates that are strictly longer and less frequent
    # than most promising footer candidates
    for n,foot1 in sorted([(v,k) for k,v in counts.items()],reverse=True):
        for foot2, m in counts.items():
            if n > m and foot1 in foot2 and len(foot1) > 0:
                counts[foot1] = counts[foot1] + counts[foot2]
                del counts[foot2]

    candidates = sorted([(v,k) for k,v in counts.items()],reverse=True)

    return candidates[0:number]
<commit_msg>better error handling on tzinfo in particular<commit_after>import datetime
import mailman
import mailbox
import numpy as np
from bigbang.thread import Thread
from bigbang.thread import Node
import bigbang.process as process
import pandas as pd
import pytz
import utils
import logging

def load(path):
    data = pd.read_csv(path)
    return Archive(data)

class Archive(object):

    """
    A representation of a mailing list archive.
    """

    data = None
    activity = None
    threads = None
    entities = None

    def __init__(self, data, archive_dir="archives", mbox=False):
        """
        Initializes an Archive object.

        The behavior of the constructor depends on the type
        of its first argument, data.

        If data is a Pandas DataFrame, it is treated as a representation of
        email messages with columns for Message-ID, From, Date, In-Reply-To,
        References, and Body. The created Archive becomes a wrapper around a
        copy of the input DataFrame.

        If data is a string, then it is interpreted as a path to either a
        single .mbox file (if the optional argument single_file is True) or
        else to a directory of .mbox files (also in .mbox format). Note that
        the file extensions need not be .mbox; frequently they will be .txt.

        Upon initialization, the Archive object drops duplicate entries
        and sorts its member variable *data* by Date.
        """

        if isinstance(data, pd.core.frame.DataFrame):
            self.data = data.copy()
        elif isinstance(data, str):
            self.data = mailman.load_data(data,archive_dir=archive_dir,mbox=mbox)

        try:
            self.data['Date'] = pd.to_datetime(self.data['Date'], errors='coerce', infer_datetime_format=True, utc=True)
        except:
            out_path = 'datetime-exception.csv'

            with open(out_path, 'w') as f:
                self.data.to_csv(f, encoding='utf-8')

            logging.error('Error while converting to datetime, despite coerce mode.')
            raise

        try:
            self.data.drop_duplicates(inplace=True)
        except:
            logging.error('Error while removing duplicate messages, maybe timezone issues?', exc_info=True)

        # Drops any entries with no Date field.
        # It may be wiser to optionally
        # do interpolation here.
        if self.data['Date'].isnull().any():
            #self.data.dropna(subset=['Date'], inplace=True)
            self.data = self.data[self.data['Date'].notnull()]
            # workaround for https://github.com/pandas-dev/pandas/issues/13407

        #convert any null fields to None -- csv saves these as nan sometimes
        self.data = self.data.where(pd.notnull(self.data),None)

        try:
            #set the index to be the Message-ID column
            self.data.set_index('Message-ID',inplace=True)
        except KeyError:
            #will get KeyError if Message-ID is already index
            pass

        # let's do a pass to try to find bad tzinfo's
        bad_indices = []
        for i, row in self.data.iterrows():
            try:
                future_comparison = row['Date'] < datetime.datetime.now(pytz.utc)
            except:
                logging.error('Error timezone issues while detecting bad rows', exc_info=True)
                bad_indices.append(i)
                logging.info('Bad timezone on %s', row['Date'])
        if len(bad_indices) > 0:
            # drop those rows that threw an error
            self.data = self.data.drop(bad_indices)
            logging.info('Dropped %d rows', len(bad_indices))

        try:
            self.data.sort_values(by='Date', inplace=True)
        except:
            logging.error('Error while sorting, maybe timezone issues?', exc_info=True)

    def resolve_entities(self,inplace=True):
        if self.entities is None:
            if self.activity is None:
                self.get_activity()

            self.entities = process.resolve_sender_entities(self.activity)

        to_replace = []

        value = []

        for e, names in self.entities.items():
            for n in names:
                to_replace.append(n)
                value.append(e)

        data = self.data.replace(to_replace=to_replace,value=value,inplace=inplace)

        # clear and replace activity with resolved activity
        self.activity = None
        self.get_activity()

        if inplace:
            return self.data
        else:
            return data

    def get_activity(self,resolved=False):
        """
        Get the activity matrix of an Archive.
        Columns of the returned DataFrame are the Senders of emails.
        Rows are indexed by ordinal date.
        Cells are the number of emails sent by each sender on each data.

        If *resolved* is true, then default entity resolution is run on the
        activity matrix before it is returned.
        """
        if self.activity is None:
            self.activity = self.compute_activity(self)

        if resolved:
            self.entities = process.resolve_sender_entities(self.activity)
            eact = utils.repartition_dataframe(self.activity,self.entities)

            return eact

        return self.activity

    def compute_activity(self, clean=True):
        mdf = self.data

        if clean:
            # unnecessary?
            if mdf['Date'].isnull().any():
                mdf = mdf.dropna(subset=['Date'])

            mdf = mdf[
                mdf['Date'] < datetime.datetime.now(
                    pytz.utc)]  # drop messages apparently in the future

        mdf2 = mdf.reindex(columns = ['From', 'Date'])
        mdf2['Date'] = mdf['Date'].apply(lambda x: x.toordinal())

        activity = mdf2.groupby(
            ['From', 'Date']).size().unstack('From').fillna(0)
        new_date_range = np.arange(mdf2['Date'].min(), mdf2['Date'].max())
        # activity.set_index('Date')
        activity = activity.reindex(new_date_range, fill_value=0)

        return activity

    def get_threads(self, verbose=False):

        if self.threads is not None:
            return self.threads

        df = self.data

        threads = list()
        visited = dict()

        total = df.shape[0]
        c = 0

        for i in df.iterrows():

            if verbose:
                c += 1
                if c % 1000 == 0:
                    print "Processed %d of %d" %(c,total)

            if(i[1]['In-Reply-To'] is None):
                root = Node(i[0], i[1])
                visited[i[0]] = root
                threads.append(Thread(root))
            elif(i[1]['In-Reply-To'] not in visited.keys()):
                root = Node(i[1]['In-Reply-To'])
                succ = Node(i[0],i[1], root)
                root.add_successor(succ)
                visited[i[1]['In-Reply-To']] = root
                visited[i[0]] = succ
                threads.append(Thread(root, known_root=False))
            else:
                parent = visited[i[1]['In-Reply-To']]
                node = Node(i[0],i[1], parent)
                parent.add_successor(node)
                visited[i[0]] = node

        self.threads = threads

        return threads

    def save(self, path,encoding='utf-8'):
        self.data.to_csv(path, ",",encoding=encoding)

def find_footer(messages,number=1):
    '''
    Returns the footer of a DataFrame of emails.
    A footer is a string occurring at the tail of most messages.
    Messages can be a DataFrame or a Series
    '''
    if isinstance(messages,pd.DataFrame):
        messages = messages['Body']

    # sort in lexical order of reverse strings to maximize foot length
    srb = messages.apply(lambda x: None if x is None else x[::-1]).order()
    #srb = df.apply(lambda x: None if x['Body'] is None else x['Body'][::-1],
    #              axis=1).order()
    # begin walking down the series looking for maximal overlap
    counts = {}

    last = None

    def clean_footer(foot):
        return foot.strip()

    for b in srb:
        if last is None:
            last = b
            continue
        elif b is None:
            continue
        else:
            head,i = utils.get_common_head(b,last,delimiter='\n')
            head = clean_footer(head[::-1])
            last = b

            if head in counts:
                counts[head] = counts[head] + 1
            else:
                counts[head] = 1

        last = b

    # reduce candidates that are strictly longer and less frequent
    # than most promising footer candidates
    for n,foot1 in sorted([(v,k) for k,v in counts.items()],reverse=True):
        for foot2, m in counts.items():
            if n > m and foot1 in foot2 and len(foot1) > 0:
                counts[foot1] = counts[foot1] + counts[foot2]
                del counts[foot2]

    candidates = sorted([(v,k) for k,v in counts.items()],reverse=True)

    return candidates[0:number]
<|endoftext|>


<commit_before>import urllib2
import urllib
import gzip
import re
import os
import mailbox
import parse
from pprint import pprint as pp
import w3crawl

ml_exp = re.compile('/([\w-]*)/$')

gz_exp = re.compile('href="(\d\d\d\d-\w*\.txt\.gz)"')
ietf_ml_exp = re.compile('href="([\d-]+.mail)"')
w3c_archives_exp = re.compile('lists\.w3\.org')

mailing_list_path_expressions = [gz_exp, ietf_ml_exp]

def collect_from_url(url):
    url = url.rstrip()
    collect_archive_from_url(url)
    unzip_archive(url)

def collect_from_file(urls_file):
    for url in open(urls_file):
        collect_archive_from_url(url)

def get_list_name(url):
    url = url.rstrip()

    return ml_exp.search(url).groups()[0]

def archive_directory(base_dir,list_name):
    arc_dir = os.path.join(base_dir,list_name)
    if not os.path.exists(arc_dir):
        os.makedirs(arc_dir)
    return arc_dir

def collect_archive_from_url(url,base_arch_dir="archives"):
    list_name = get_list_name(url)
    pp("Getting archive page for %s" % list_name)

    if w3c_archives_exp.search(url):
      return w3crawl.collect_from_url(url, base_arch_dir)

    response = urllib2.urlopen(url)
    html = response.read()

    results = []
    for exp in mailing_list_path_expressions:
      results.extend(exp.findall(html))

    pp(results)

    # directory for downloaded files
    arc_dir = archive_directory(base_arch_dir,list_name)

    # download monthly archives
    for res in results:
        result_path = os.path.join(arc_dir,res)
        #this check is redundant with urlretrieve
        if not os.path.isfile(result_path):
            gz_url = url + res
            pp('retrieving %s' % gz_url)
            info = urllib.urlretrieve(gz_url,result_path)
            print info

def unzip_archive(url,base_arc_dir="archives"):
    arc_dir = archive_directory(base_arc_dir,get_list_name(url))

    gzs = [os.path.join(arc_dir,fn) for fn
           in os.listdir(arc_dir)
           if fn.endswith('.txt.gz')]

    print 'unzipping %d archive files' % (len(gzs))

    for gz in gzs:
        try:
            f = gzip.open(gz,'rb')
            content = f.read()
            f.close()

            txt_fn = str(gz[:-3])

            f2 = open(txt_fn,'w')
            f2.write(content)
            f2.close()
        except Exception as e:
            print e

# This works for the names of the files. Order them.
# datetime.datetime.strptime('2000-November',"%Y-%B")

# This doesn't yet work for parsing the dates. Because of %z Bullshit
#datetime.datetime.strptime(arch[0][0].get('Date'),"%a, %d %b %Y %H:%M:%S %z")

def open_list_archives(url,base_arc_dir="archives"):
    list_name = get_list_name(url)
    arc_dir = archive_directory(base_arc_dir,list_name)

    file_extensions = [".txt", ".mail", ".mbox"]

    txts = [os.path.join(arc_dir,fn) for fn
            in os.listdir(arc_dir)
            if any([fn.endswith(extension) for extension in file_extensions])]

    print 'Opening %d archive files' % (len(txts))
    arch = [mailbox.mbox(txt, create=False).values() for txt in txts]

    messages = [item for sublist in arch for item in sublist]
    return messages

<commit_msg>comment on open_list_archives<commit_after>import urllib2
import urllib
import gzip
import re
import os
import mailbox
import parse
from pprint import pprint as pp
import w3crawl

ml_exp = re.compile('/([\w-]*)/$')

gz_exp = re.compile('href="(\d\d\d\d-\w*\.txt\.gz)"')
ietf_ml_exp = re.compile('href="([\d-]+.mail)"')
w3c_archives_exp = re.compile('lists\.w3\.org')

mailing_list_path_expressions = [gz_exp, ietf_ml_exp]

def collect_from_url(url):
    url = url.rstrip()
    collect_archive_from_url(url)
    unzip_archive(url)

def collect_from_file(urls_file):
    for url in open(urls_file):
        collect_archive_from_url(url)

def get_list_name(url):
    url = url.rstrip()

    return ml_exp.search(url).groups()[0]

def archive_directory(base_dir,list_name):
    arc_dir = os.path.join(base_dir,list_name)
    if not os.path.exists(arc_dir):
        os.makedirs(arc_dir)
    return arc_dir

def collect_archive_from_url(url,base_arch_dir="archives"):
    list_name = get_list_name(url)
    pp("Getting archive page for %s" % list_name)

    if w3c_archives_exp.search(url):
      return w3crawl.collect_from_url(url, base_arch_dir)

    response = urllib2.urlopen(url)
    html = response.read()

    results = []
    for exp in mailing_list_path_expressions:
      results.extend(exp.findall(html))

    pp(results)

    # directory for downloaded files
    arc_dir = archive_directory(base_arch_dir,list_name)

    # download monthly archives
    for res in results:
        result_path = os.path.join(arc_dir,res)
        #this check is redundant with urlretrieve
        if not os.path.isfile(result_path):
            gz_url = url + res
            pp('retrieving %s' % gz_url)
            info = urllib.urlretrieve(gz_url,result_path)
            print info

def unzip_archive(url,base_arc_dir="archives"):
    arc_dir = archive_directory(base_arc_dir,get_list_name(url))

    gzs = [os.path.join(arc_dir,fn) for fn
           in os.listdir(arc_dir)
           if fn.endswith('.txt.gz')]

    print 'unzipping %d archive files' % (len(gzs))

    for gz in gzs:
        try:
            f = gzip.open(gz,'rb')
            content = f.read()
            f.close()

            txt_fn = str(gz[:-3])

            f2 = open(txt_fn,'w')
            f2.write(content)
            f2.close()
        except Exception as e:
            print e

# This works for the names of the files. Order them.
# datetime.datetime.strptime('2000-November',"%Y-%B")

# This doesn't yet work for parsing the dates. Because of %z Bullshit
#datetime.datetime.strptime(arch[0][0].get('Date'),"%a, %d %b %Y %H:%M:%S %z")

def open_list_archives(url,base_arc_dir="archives"):
    """
    Returns a list of all email messages contained in the specified directory.

    The argument *url* here is taken to be the name of a subdirectory
    of the directory specified in argument *base_arc_dir*.

    This directory is expected to contain files with extensions .txt,
    .mail, or .mbox. These files are all expected to be in mbox format--
    i.e. a series of blocks of text starting with headers (colon-separated
    key-value pairs) followed by an email body.
    """
    list_name = get_list_name(url)
    arc_dir = archive_directory(base_arc_dir,list_name)

    file_extensions = [".txt", ".mail", ".mbox"]

    txts = [os.path.join(arc_dir,fn) for fn
            in os.listdir(arc_dir)
            if any([fn.endswith(extension) for extension in file_extensions])]

    print 'Opening %d archive files' % (len(txts))
    arch = [mailbox.mbox(txt, create=False).values() for txt in txts]

    messages = [item for sublist in arch for item in sublist]
    return messages

<|endoftext|>