<commit_before>from pathlib import Path
from num2words import num2words

import subprocess
from django.template.loader import get_template
from django.utils.safestring import mark_safe
from markdown import markdown
from markdown.extensions.toc import TocExtension

from agir.lib.display import display_price

def display_place_of_birth(contract_information):
    if contract_information["country_of_birth"] == "FR":
        return f'{contract_information["city_of_birth"]} ({contract_information["departement_of_birth"]})'
    else:
        return f'{contract_information["city_of_birth"]} ({contract_information["country_of_birth"]}'

def display_full_address(contract_information):
    return (
        f'{contract_information["location_address1"]} {contract_information["location_address2"]} '
        f'{contract_information["location_zip"]} {contract_information["location_city"]} '
        f'{contract_information["location_country"]} '
    )

SUBSTITUTIONS = {
    "cher_preteur": {
        "M": "Cher prêteur",
        "F": "Chère prêteuse",
        "O": "Cher⋅e prêteur⋅se",
    },
    "final_e": {"M": "", "F": "e", "O": "⋅e"},
    "lender": {"M": "prêteur", "F": "prêteuse", "O": "prêteur⋅euse"},
    "article": {"M": "le", "F": "la", "O": "le-la"},
    "pronoun": {"M": "il", "F": "elle", "O": "il-elle"},
    "determinant": {"M": "du", "F": "de la", "O": "du/de la"},
    "payment": {
        "check_afce": "chèque bancaire tiré de son compte personnel",
        "system_pay_afce_pret": "paiement par carte bancaire depuis son compte personnel",
    },
}

def generate_html_contract(contract_information, baselevel=1):
    gender = contract_information["gender"]
    signed = "signature_datetime" in contract_information

    signature_image_path = (
        Path(__file__)
        .parent.joinpath("static", "europeennes", "signature_manon_aubry.png")
        .absolute()
        .as_uri()
    )

    contract_markdown = get_template("europeennes/loans/contract.md").render(
        context={
            "lender_date_of_birth": "22/12/1989",
            "lender_place_of_birth": "Fréjus (Var)",
            "name": f'{contract_information["first_name"]} {contract_information["last_name"]}',
            "address": "personal address",
            "date_of_birth": contract_information["date_of_birth"],
            "place_of_birth": display_place_of_birth(contract_information),
            "full_address": display_full_address(contract_information),
            "amount_letters": num2words(contract_information["amount"] / 100, lang="fr")
            + " euros",
            "amount_figure": display_price(contract_information["amount"]),
            "signature_date": contract_information.get(
                "signature_datetime", "XX/XX/XXXX"
            ),
            "e": SUBSTITUTIONS["final_e"][gender],
            "preteur": SUBSTITUTIONS["lender"][gender],
            "le": SUBSTITUTIONS["article"][gender],
            "Le": SUBSTITUTIONS["article"][gender].capitalize(),
            "du": SUBSTITUTIONS["determinant"][gender],
            "il": SUBSTITUTIONS["pronoun"][gender],
            "mode_paiement": SUBSTITUTIONS["payment"][
                contract_information["payment_mode"]
            ],
            "signature": f"Accepté en ligne le {contract_information['acceptance_datetime']}"
            if signed
            else "",
            "signature_emprunteuse": mark_safe(
                f''
            )
            if signed
            else "",
        }
    )

    return mark_safe(
        markdown(
            contract_markdown, extensions=["extra", TocExtension(baselevel=baselevel)]
        )
    )

def save_pdf_contract(contract_information, dest_path):
    dest_dir = Path(dest_path).parent
    dest_dir.mkdir(parents=True, exist_ok=True)

    html_contract = generate_html_contract(contract_information)
    contract_with_layout = get_template(
        "europeennes/loans/contract_layout.html"
    ).render(context={"contract_body": mark_safe(html_contract)})

    proc = subprocess.Popen(
        ["wkhtmltopdf", "--encoding", "utf-8", "-", str(dest_path)],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )

    try:
        out, errs = proc.communicate(input=contract_with_layout.encode(), timeout=10)
        return_code = proc.wait(timeout=10)
    except subprocess.TimeoutExpired:
        proc.kill()
        proc.communicate()
        raise

    if return_code != 0:
        raise RuntimeError(f"PDF conversion failed\nOUT: {out}\nERR: {errs}")
<commit_msg>Meilleur mise en page des noms de pays et des adresses dans le contrat<commit_after>from pathlib import Path

from django_countries import countries
from num2words import num2words

import subprocess
from django.template.loader import get_template
from django.utils.safestring import mark_safe
from markdown import markdown
from markdown.extensions.toc import TocExtension

from agir.lib.display import display_price

def display_place_of_birth(contract_information):
    country_of_birth = countries.name(contract_information["country_of_birth"])

    if contract_information["country_of_birth"] == "FR":
        return f'{contract_information["city_of_birth"]} ({contract_information["departement_of_birth"]})'
    else:
        return f'{contract_information["city_of_birth"]} ({country_of_birth})'

def display_full_address(contract_information):
    street_address = contract_information["location_address1"]
    if contract_information["location_address2"]:
        street_address += ", " + contract_information["location_address2"]

    if contract_information["location_zip"]:
        city_address = (
            contract_information["location_zip"]
            + " "
            + contract_information["location_city"]
        )
    else:
        city_address = contract_information["location_city"]

    country_address = countries.name(contract_information["location_country"])

    return f"{street_address}, {city_address}, {country_address}"

SUBSTITUTIONS = {
    "cher_preteur": {
        "M": "Cher prêteur",
        "F": "Chère prêteuse",
        "O": "Cher⋅e prêteur⋅se",
    },
    "final_e": {"M": "", "F": "e", "O": "⋅e"},
    "lender": {"M": "prêteur", "F": "prêteuse", "O": "prêteur⋅euse"},
    "article": {"M": "le", "F": "la", "O": "le-la"},
    "pronoun": {"M": "il", "F": "elle", "O": "il-elle"},
    "determinant": {"M": "du", "F": "de la", "O": "du/de la"},
    "payment": {
        "check_afce": "chèque bancaire tiré de son compte personnel",
        "system_pay_afce_pret": "paiement par carte bancaire depuis son compte personnel",
    },
}

def generate_html_contract(contract_information, baselevel=1):
    gender = contract_information["gender"]
    signed = "signature_datetime" in contract_information

    signature_image_path = (
        Path(__file__)
        .parent.joinpath("static", "europeennes", "signature_manon_aubry.png")
        .absolute()
        .as_uri()
    )

    contract_markdown = get_template("europeennes/loans/contract.md").render(
        context={
            "lender_date_of_birth": "22/12/1989",
            "lender_place_of_birth": "Fréjus (Var)",
            "name": f'{contract_information["first_name"]} {contract_information["last_name"]}',
            "address": "personal address",
            "date_of_birth": contract_information["date_of_birth"],
            "place_of_birth": display_place_of_birth(contract_information),
            "full_address": display_full_address(contract_information),
            "amount_letters": num2words(contract_information["amount"] / 100, lang="fr")
            + " euros",
            "amount_figure": display_price(contract_information["amount"]),
            "signature_date": contract_information.get(
                "signature_datetime", "XX/XX/XXXX"
            ),
            "e": SUBSTITUTIONS["final_e"][gender],
            "preteur": SUBSTITUTIONS["lender"][gender],
            "le": SUBSTITUTIONS["article"][gender],
            "Le": SUBSTITUTIONS["article"][gender].capitalize(),
            "du": SUBSTITUTIONS["determinant"][gender],
            "il": SUBSTITUTIONS["pronoun"][gender],
            "mode_paiement": SUBSTITUTIONS["payment"][
                contract_information["payment_mode"]
            ],
            "signature": f"Accepté en ligne le {contract_information['acceptance_datetime']}"
            if signed
            else "",
            "signature_emprunteuse": mark_safe(
                f''
            )
            if signed
            else "",
        }
    )

    return mark_safe(
        markdown(
            contract_markdown, extensions=["extra", TocExtension(baselevel=baselevel)]
        )
    )

def save_pdf_contract(contract_information, dest_path):
    dest_dir = Path(dest_path).parent
    dest_dir.mkdir(parents=True, exist_ok=True)

    html_contract = generate_html_contract(contract_information)
    contract_with_layout = get_template(
        "europeennes/loans/contract_layout.html"
    ).render(context={"contract_body": mark_safe(html_contract)})

    proc = subprocess.Popen(
        ["wkhtmltopdf", "--encoding", "utf-8", "-", str(dest_path)],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )

    try:
        out, errs = proc.communicate(input=contract_with_layout.encode(), timeout=10)
        return_code = proc.wait(timeout=10)
    except subprocess.TimeoutExpired:
        proc.kill()
        proc.communicate()
        raise

    if return_code != 0:
        raise RuntimeError(f"PDF conversion failed\nOUT: {out}\nERR: {errs}")
<|endoftext|>


<commit_before>##############################################################################
#
#    Copyright (C) 2016 Comunitea Servicios Tecnológicos S.L.
#    $Omar Castiñeira Saavedra$ <omar@comunitea.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import math
from odoo import models, fields, api
import odoo.addons.decimal_precision as dp

class ProductTemplate(models.Model):
    _inherit = 'product.template'

    @api.multi
    def _get_no_wh_internal_stock(self):
        for product in self:
            locs = []
            locs.append(self.env.ref('location_moves.stock_location_kitchen').id)
            locs.append(self.env.ref('location_moves.stock_location_pantry').id)
            qty = product.with_context(location=locs).qty_available
            product.qty_available_wo_wh = qty

    @api.multi
    def _get_input_loc_stock(self):
        for product in self:
            locs = []
            qty = 0.0
            for wh in self.env["stock.warehouse"].search([]):
                locs.append(wh.wh_input_stock_loc_id.id)
                qty += product.with_context(location=locs).qty_available
            product.qty_available_input_loc = qty

    @api.multi
    def _get_in_production_stock(self):
        for product in self:
            if product.product_variant_ids:
                moves = self.env["stock.move"].search([('product_id', 'in', product.product_variant_ids.ids),
                                                       ('purchase_line_id', '!=', False),
                                                       ('picking_id', '=', False),
                                                       ('state', '!=', 'cancel')])
                product.qty_in_production = sum(moves.mapped('product_uom_qty')) or 0.0
            else:
                product.qty_in_production = 0.0

    @api.multi
    def _stock_conservative(self):
        for product in self:
            pack_stock = 0
            first_subproduct = True
            if product.bom_ids:
                for bom in product.bom_ids:
                    if bom.type == 'phantom':
                        for subproduct in bom.bom_line_ids:
                            subproduct_quantity_next = subproduct.product_qty
                            if subproduct_quantity_next:
                                subproduct_stock_next = \
                                    subproduct.product_id.qty_available - subproduct.product_id.outgoing_picking_reserved_qty - \
                                    subproduct.product_id.reservation_count
                                pack_stock_next = math.\
                                    floor(subproduct_stock_next /
                                          subproduct_quantity_next)
                                if first_subproduct:
                                    pack_stock = pack_stock_next
                                    first_subproduct = False
                                else:
                                    if pack_stock_next < pack_stock:
                                        pack_stock = pack_stock_next
                        product.virtual_stock_conservative = pack_stock
                    else:
                        product.virtual_stock_conservative = \
                            product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count
            else:
                product.virtual_stock_conservative = \
                    product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count

    @api.multi
    def _get_avail_conservative(self):
        for product in self:
            pack_stock = 0
            first_subproduct = True
            if product.bom_ids:
                for bom in product.bom_ids:
                    if bom.type == 'phantom':
                        for subproduct in bom.bom_line_ids:
                            subproduct_quantity_next = subproduct.product_qty
                            if subproduct_quantity_next:
                                subproduct_stock_next = \
                                    subproduct.product_id.qty_available - subproduct.product_id.outgoing_picking_reserved_qty - \
                                    subproduct.product_id.reservation_count - \
                                    subproduct.product_id.qty_available_wo_wh - \
                                    subproduct.product_id.qty_available_input_loc
                                pack_stock_next = math.\
                                    floor(subproduct_stock_next /
                                          subproduct_quantity_next)
                                if first_subproduct:
                                    pack_stock = pack_stock_next
                                    first_subproduct = False
                                else:
                                    if pack_stock_next < pack_stock:
                                        pack_stock = pack_stock_next
                        product.virtual_available_wo_incoming = pack_stock
                    else:
                        product.virtual_available_wo_incoming = \
                            product.virtual_stock_conservative - \
                            product.qty_available_wo_wh - \
                            product.qty_available_input_loc
            else:
                product.virtual_available_wo_incoming = \
                    product.virtual_stock_conservative - \
                    product.qty_available_wo_wh - \
                    product.qty_available_input_loc

    qty_available_wo_wh = fields.\
        Float(string="Qty. on kitchen", compute="_get_no_wh_internal_stock",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))

    outgoing_picking_reserved_qty = fields.Float(
        compute='_get_outgoing_picking_qty', readonly=True,
        digits=dp.get_precision('Product Unit of Measure'))
    qty_available_input_loc = fields.\
        Float(string="Qty. on input", compute="_get_input_loc_stock",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))
    qty_in_production = fields.\
        Float("Qty. in production", compute="_get_in_production_stock",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))
    virtual_available_wo_incoming = fields.\
        Float("Virtual avail. conservative",
              compute="_get_avail_conservative",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))

    @api.multi
    def _get_outgoing_picking_qty(self):
        for product in self:
            domain = [('product_id', 'in', product.product_variant_ids.ids),
                      ('state', 'in', ('confirmed', 'assigned', 'partially_available', 'waiting')),
                      ('picking_id', '!=', False),
                      ('location_id', '=', 12)]
            product.outgoing_picking_reserved_qty = sum(item['product_uom_qty'] for item in self.env['stock.move'].search_read(domain, ['product_uom_qty']))

    @api.multi
    def _get_stock_italy(self):
        location_id = self.env['stock.location'].search([('name', '=', 'Depósito Visiotech Italia')]).id
        for product in self:
            qty = product.with_context(location=location_id).qty_available
            product.qty_available_italy = qty

    qty_available_italy = fields.Float(string="Qty. available Italy", compute="_get_stock_italy",
                             readonly=True,
                             digits=dp.get_precision('Product Unit of Measure'))

class ProductProduct(models.Model):

    _inherit = 'product.product'

    @api.multi
    def _stock_conservative(self):
        for product in self:
            pack_stock = 0
            first_subproduct = True
            if product.bom_ids:
                for bom in product.bom_ids:
                    if bom.type == 'phantom':
                        for subproduct in bom.bom_line_ids:
                            subproduct_quantity_next = subproduct.product_qty
                            if subproduct_quantity_next:
                                product_id = subproduct.product_id
                                subproduct_stock_next = \
                                    product_id.qty_available - \
                                    product_id.outgoing_picking_reserved_qty - \
                                    product_id.reservation_count
                                pack_stock_next = math.\
                                    floor(subproduct_stock_next /
                                          subproduct_quantity_next)
                                if first_subproduct:
                                    pack_stock = pack_stock_next
                                    first_subproduct = False
                                else:
                                    if pack_stock_next < pack_stock:
                                        pack_stock = pack_stock_next
                        product.virtual_stock_conservative = pack_stock
                    else:
                        product.virtual_stock_conservative = \
                            product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count
            else:
                product.virtual_stock_conservative = \
                    product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count

    @api.multi
    def _compute_reservation_count(self):
        super()._compute_reservation_count()
        for product in self:
            domain = [('product_id', 'in', product.product_variant_ids.ids),
                      ('state', 'in', ('confirmed', 'assigned', 'partially_available', 'waiting')),
                      ('raw_material_production_id', '!=', False),
                      ('picking_id', '=', False)]
            product.reservation_count += sum(
                item['product_uom_qty'] for item in self.env['stock.move'].search_read(domain, ['product_uom_qty']))
<commit_msg>[FIX] sale_display_stock: cambiado id de existencia por el external_id<commit_after>##############################################################################
#
#    Copyright (C) 2016 Comunitea Servicios Tecnológicos S.L.
#    $Omar Castiñeira Saavedra$ <omar@comunitea.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import math
from odoo import models, fields, api
import odoo.addons.decimal_precision as dp

class ProductTemplate(models.Model):
    _inherit = 'product.template'

    @api.multi
    def _get_no_wh_internal_stock(self):
        for product in self:
            locs = []
            locs.append(self.env.ref('location_moves.stock_location_kitchen').id)
            locs.append(self.env.ref('location_moves.stock_location_pantry').id)
            qty = product.with_context(location=locs).qty_available
            product.qty_available_wo_wh = qty

    @api.multi
    def _get_input_loc_stock(self):
        for product in self:
            locs = []
            qty = 0.0
            for wh in self.env["stock.warehouse"].search([]):
                locs.append(wh.wh_input_stock_loc_id.id)
                qty += product.with_context(location=locs).qty_available
            product.qty_available_input_loc = qty

    @api.multi
    def _get_in_production_stock(self):
        for product in self:
            if product.product_variant_ids:
                moves = self.env["stock.move"].search([('product_id', 'in', product.product_variant_ids.ids),
                                                       ('purchase_line_id', '!=', False),
                                                       ('picking_id', '=', False),
                                                       ('state', '!=', 'cancel')])
                product.qty_in_production = sum(moves.mapped('product_uom_qty')) or 0.0
            else:
                product.qty_in_production = 0.0

    @api.multi
    def _stock_conservative(self):
        for product in self:
            pack_stock = 0
            first_subproduct = True
            if product.bom_ids:
                for bom in product.bom_ids:
                    if bom.type == 'phantom':
                        for subproduct in bom.bom_line_ids:
                            subproduct_quantity_next = subproduct.product_qty
                            if subproduct_quantity_next:
                                subproduct_stock_next = \
                                    subproduct.product_id.qty_available - subproduct.product_id.outgoing_picking_reserved_qty - \
                                    subproduct.product_id.reservation_count
                                pack_stock_next = math.\
                                    floor(subproduct_stock_next /
                                          subproduct_quantity_next)
                                if first_subproduct:
                                    pack_stock = pack_stock_next
                                    first_subproduct = False
                                else:
                                    if pack_stock_next < pack_stock:
                                        pack_stock = pack_stock_next
                        product.virtual_stock_conservative = pack_stock
                    else:
                        product.virtual_stock_conservative = \
                            product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count
            else:
                product.virtual_stock_conservative = \
                    product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count

    @api.multi
    def _get_avail_conservative(self):
        for product in self:
            pack_stock = 0
            first_subproduct = True
            if product.bom_ids:
                for bom in product.bom_ids:
                    if bom.type == 'phantom':
                        for subproduct in bom.bom_line_ids:
                            subproduct_quantity_next = subproduct.product_qty
                            if subproduct_quantity_next:
                                subproduct_stock_next = \
                                    subproduct.product_id.qty_available - subproduct.product_id.outgoing_picking_reserved_qty - \
                                    subproduct.product_id.reservation_count - \
                                    subproduct.product_id.qty_available_wo_wh - \
                                    subproduct.product_id.qty_available_input_loc
                                pack_stock_next = math.\
                                    floor(subproduct_stock_next /
                                          subproduct_quantity_next)
                                if first_subproduct:
                                    pack_stock = pack_stock_next
                                    first_subproduct = False
                                else:
                                    if pack_stock_next < pack_stock:
                                        pack_stock = pack_stock_next
                        product.virtual_available_wo_incoming = pack_stock
                    else:
                        product.virtual_available_wo_incoming = \
                            product.virtual_stock_conservative - \
                            product.qty_available_wo_wh - \
                            product.qty_available_input_loc
            else:
                product.virtual_available_wo_incoming = \
                    product.virtual_stock_conservative - \
                    product.qty_available_wo_wh - \
                    product.qty_available_input_loc

    qty_available_wo_wh = fields.\
        Float(string="Qty. on kitchen", compute="_get_no_wh_internal_stock",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))

    outgoing_picking_reserved_qty = fields.Float(
        compute='_get_outgoing_picking_qty', readonly=True,
        digits=dp.get_precision('Product Unit of Measure'))
    qty_available_input_loc = fields.\
        Float(string="Qty. on input", compute="_get_input_loc_stock",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))
    qty_in_production = fields.\
        Float("Qty. in production", compute="_get_in_production_stock",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))
    virtual_available_wo_incoming = fields.\
        Float("Virtual avail. conservative",
              compute="_get_avail_conservative",
              readonly=True,
              digits=dp.get_precision('Product Unit of Measure'))

    @api.multi
    def _get_outgoing_picking_qty(self):
        for product in self:
            domain = [('product_id', 'in', product.product_variant_ids.ids),
                      ('state', 'in', ('confirmed', 'assigned', 'partially_available', 'waiting')),
                      ('picking_id', '!=', False),
                      ('location_id', '=', self.env.ref('stock.stock_location_stock').id)]
            product.outgoing_picking_reserved_qty = sum(item['product_uom_qty'] for item in self.env['stock.move'].search_read(domain, ['product_uom_qty']))

    @api.multi
    def _get_stock_italy(self):
        location_id = self.env['stock.location'].search([('name', '=', 'Depósito Visiotech Italia')]).id
        for product in self:
            qty = product.with_context(location=location_id).qty_available
            product.qty_available_italy = qty

    qty_available_italy = fields.Float(string="Qty. available Italy", compute="_get_stock_italy",
                             readonly=True,
                             digits=dp.get_precision('Product Unit of Measure'))

class ProductProduct(models.Model):

    _inherit = 'product.product'

    @api.multi
    def _stock_conservative(self):
        for product in self:
            pack_stock = 0
            first_subproduct = True
            if product.bom_ids:
                for bom in product.bom_ids:
                    if bom.type == 'phantom':
                        for subproduct in bom.bom_line_ids:
                            subproduct_quantity_next = subproduct.product_qty
                            if subproduct_quantity_next:
                                product_id = subproduct.product_id
                                subproduct_stock_next = \
                                    product_id.qty_available - \
                                    product_id.outgoing_picking_reserved_qty - \
                                    product_id.reservation_count
                                pack_stock_next = math.\
                                    floor(subproduct_stock_next /
                                          subproduct_quantity_next)
                                if first_subproduct:
                                    pack_stock = pack_stock_next
                                    first_subproduct = False
                                else:
                                    if pack_stock_next < pack_stock:
                                        pack_stock = pack_stock_next
                        product.virtual_stock_conservative = pack_stock
                    else:
                        product.virtual_stock_conservative = \
                            product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count
            else:
                product.virtual_stock_conservative = \
                    product.qty_available - product.outgoing_picking_reserved_qty - product.reservation_count

    @api.multi
    def _compute_reservation_count(self):
        super()._compute_reservation_count()
        for product in self:
            domain = [('product_id', 'in', product.product_variant_ids.ids),
                      ('state', 'in', ('confirmed', 'assigned', 'partially_available', 'waiting')),
                      ('raw_material_production_id', '!=', False),
                      ('picking_id', '=', False)]
            product.reservation_count += sum(
                item['product_uom_qty'] for item in self.env['stock.move'].search_read(domain, ['product_uom_qty']))
<|endoftext|>


<commit_before>from pelican import signals
from pelican.contents import Content, Article
from bs4 import BeautifulSoup

def images_extraction(instance):
    representativeImage = None
    if type(instance) == Article:
        if 'image' in instance.metadata:
            representativeImage = instance.metadata['image']

        # Process Summary:
        # If summary contains images, extract one to be the representativeImage and remove images from summary
        soup = BeautifulSoup(instance.summary, 'html.parser')
        images = soup.find_all('img')
        for i in images:
            if not representativeImage:
                representativeImage = i['src']
            i.extract()
        if len(images) > 0:
            # set _summary field which is based on metadata. summary field is only based on article's content and not settable
            instance._summary = unicode(soup)

        # If there are no image in summary, look for it in the content body
        if not representativeImage:
            soup = BeautifulSoup(instance.content, 'html.parser')
            imageTag = soup.find('img')
            if imageTag:
                representativeImage = imageTag['src']

        # Set the attribute to content instance
        instance.featured_image = representativeImage

def register():
    signals.content_object_init.connect(images_extraction)
<commit_msg>Featured images could be used among all kinds of pages.<commit_after>from pelican import signals
from pelican.contents import Article, Draft, Page
from bs4 import BeautifulSoup

def images_extraction(instance):
    representativeImage = None
    if type(instance) in (Article, Draft, Page):
        if 'image' in instance.metadata:
            representativeImage = instance.metadata['image']

        # Process Summary:
        # If summary contains images, extract one to be the representativeImage and remove images from summary
        soup = BeautifulSoup(instance.summary, 'html.parser')
        images = soup.find_all('img')
        for i in images:
            if not representativeImage:
                representativeImage = i['src']
            i.extract()
        if len(images) > 0:
            # set _summary field which is based on metadata. summary field is only based on article's content and not settable
            instance._summary = unicode(soup)

        # If there are no image in summary, look for it in the content body
        if not representativeImage:
            soup = BeautifulSoup(instance.content, 'html.parser')
            imageTag = soup.find('img')
            if imageTag:
                representativeImage = imageTag['src']

        # Set the attribute to content instance
        instance.featured_image = representativeImage

def register():
    signals.content_object_init.connect(images_extraction)
<|endoftext|>


<commit_before>#  corda.py
#
#  Copyright 2016 Christian Diener <mail[at]cdiener.com>
#
#  MIT license. See LICENSE for more information.

from cobra.solvers import solver_dict, get_solver_name
from cobra.manipulation.modify import convert_to_irreversible
from cobra import Model, Metabolite, Reaction
import numpy as np
from collections import Counter
from .util import safe_revert_reversible
import sys
from os import devnull

TOL = 1e-5  # Tolerance to judge whether a flux is non-zero
UPPER = 1e6 # default upper bound

class CORDA(object):

    def __init__(self, model, confidence, met_prod=None, flux_target=1, n=10,
        penalty_factor=100, support=5, solver=None, **solver_kwargs):
        """Initialize parameters and model"""
        self.model = model.copy()
        self.objective = model.objective.copy()
        convert_to_irreversible(self.model)

        # Map confidences from forward to backward reactions
        self.conf = {}
        for r in self.model.reactions:
            r.objective_coefficient = 0
            r.upper_bound = UPPER
            if r.id in confidence: self.conf[r.id] = confidence[r.id]
            elif "reflection" in r.notes:
                rev = self.model.reactions.get_by_id(r.notes["reflection"])
                self.conf[r.id] = confidence[rev.id]
            else:
                raise ValueError("{} missing from confidence!".format(r.id))
        self.__conf_old = self.conf.copy()

        self.built = False
        self.tflux = flux_target
        self.impossible = []
        self.n = n
        self.noise = 0.5
        self.support = support
        self.pf = penalty_factor
        self.solver = solver_dict[get_solver_name() if solver is None else solver]
        self.sargs = solver_kwargs

        if met_prod:
            for mid in met_prod:
                r = Reaction("EX_CORDA_" + mid)
                r.notes["mock"] = mid
                r.upper_bound = UPPER
                self.model.add_reaction(r)
                if type(mid) == str:
                    r.add_metabolites({mid: -1})
                elif type(mid) == dict:
                    r.add_metabolites(mid)
                else:
                    raise ValueError("metabolite test not string or dictionary")
                self.conf[r.id] = 3

        self.m_pen = Metabolite("penalty")
        self.r_pen = Reaction("EX_penalty")
        self.r_pen.notes["mock"] = self.m_pen
        self.r_pen.add_metabolites({self.m_pen: -1})
        self.r_pen.objective_coefficient = 1.0
        self.r_pen.upper_bound = len(self.model.reactions)*self.pf*UPPER+1

    def __perturb(self, lp, m):
        noise = np.random.uniform(high=self.noise, size=len(m.reactions))
        pen_id = m.metabolites.index("penalty")

        for i, r in enumerate(m.reactions):
            if self.m_pen in r.metabolites and r.id != "EX_penalty":
                coef = r.metabolites[self.m_pen]
                self.solver.change_coefficient(lp, pen_id,
                    i, coef + noise[i])

        del noise, pen_id

    def __quiet_solve(self, lp, os):
        old = sys.stdout
        f = open(devnull, 'w')
        sys.stdout = f
        try:
            sol = self.solver.solve_problem(lp, objective_sense=os,
                **self.sargs)
        finally:
            sys.stdout = old
            f.close()
        return sol

    def associated(self, targets, conf=None):
        """Gets the associated reactions for the target reactions"""

        if conf is None: conf = self.conf

        m = self.model.copy()
        m.add_reaction(self.r_pen)

        for r in m.reactions:
            if r.id == "EX_penalty": continue
            if conf[r.id] == 2 or conf[r.id] == 1:
                r.add_metabolites({self.m_pen: 1})
            elif conf[r.id] == -1:
                r.add_metabolites({self.m_pen: self.pf})

        lp = self.solver.create_problem(m)

        needed = {}
        for rid in targets:
            ti = m.reactions.index(rid)

            upper = m.reactions.get_by_id(rid).upper_bound
            self.solver.change_variable_bounds(lp, ti, self.tflux, upper)
            needed[rid] = np.array([])
            for _ in range(self.n):
                self.__perturb(lp, m)
                sol = self.__quiet_solve(lp, "minimize")
                if(sol != "optimal"):
                    self.impossible.append(rid)
                    needed[rid] = np.array([])
                    self.conf[rid] = -1
                    break
                sol = self.solver.format_solution(lp, m)
                need = [r for r in sol.x_dict if sol.x_dict[r] > TOL \
                    and r != "EX_penalty" and conf[r] in [-1, 1, 2]]
                need = np.hstack([needed[rid], need])
                needed[rid] = np.unique(need)

            self.solver.change_variable_bounds(lp, ti, 0.0, upper)

        return needed

    def build(self):
        """Constructs a tissue-specific model"""
        if self.built: raise ValueError("Model has been constructed already!")

        # First iteration - find reactions required for high confidence
        include = [r.id for r in self.model.reactions if self.conf[r.id] == 3]
        need = self.associated(include)
        add = np.unique([x for v in need.values() for x in v])
        for a in add: self.conf[a] = 3

        # Second iteration - add the best no confidence and independent medium
        # confidence
        include = [r.id for r in self.model.reactions if self.conf[r.id] == 1 \
            or self.conf[r.id] == 2]
        need = self.associated(include)
        add = [x for v in need.values() for x in v if self.conf[x] == -1 \
            and x not in self.impossible]
        count = Counter(add)
        add = [k for k in count if count[k] >= self.support]
        for a in add: self.conf[a] = 3

        not_included = [rid for rid in self.conf if self.conf[rid] == -1]
        for rid in not_included:
            self.model.reactions.get_by_id(rid).upper_bound = 0.0
        lp = self.solver.create_problem(self.model)
        for i, r in enumerate(self.model.reactions):
            if self.conf[r.id] == 1 or self.conf[r.id] == 2:
                self.solver.change_variable_objective(lp, i, 1.0)
                sol = self.__quiet_solve(lp, "maximize")
                sol = self.solver.format_solution(lp, self.model)
                if sol.f > TOL: self.conf[r.id] = 3
            self.solver.change_variable_objective(lp, i, 0.0)

        # Third iteration block all non-included N+M add free reactions
        for rid, co in self.conf.items():
            if co == 1 or co == 2:
                self.model.reactions.get_by_id(rid).upper_bound = 0.0
            if co == 0:
                self.conf[rid] = -1
        need = self.associated([k for k in self.conf if self.conf[k] == 3])
        add = np.unique([x for v in need.values() for x in v \
            if x not in self.impossible])
        for a in add: self.conf[a] = 3

        self.built = True

    def __str__(self):
        old_counts = Counter(self.__conf_old.values())
        if not self.built:
            out = "build status: not built\n" + \
                "#reactions (including mock): {}\n".\
                    format(len(self.__conf_old)) + \
                "Reaction confidence:\n" + \
                " - unclear: {}\n".format(old_counts[0]) + \
                " - exclude: {}\n".format(old_counts[-1]) + \
                " - low and medium: {}\n".format(old_counts[1] + \
                    old_counts[2]) + \
                " - high: {}\n".format(old_counts[3])
        else:
            rids = self.conf.keys()
            old = np.array([self.__conf_old[k] for k in rids])
            new = np.array([self.conf[k] for k in rids])
            med_inc = np.sum(((old == 1) | (old == 2)) & (new == 3))
            noc_inc = np.sum((old == -1) & (new == 3))
            free_inc = np.sum((old == 0) & (new == 3))
            high_inc = np.sum((old == 3) & (new == 3))
            out = "build status: reconstruction complete\n" + \
                "Inc. reactions: {}/{}\n".format(np.sum(new == 3), len(old)) +\
                " - unclear: {}/{}\n".format(free_inc, old_counts[0]) + \
                " - exclude: {}/{}\n".format(noc_inc, old_counts[-1]) + \
                " - low and medium: {}/{}\n".format(med_inc, old_counts[1] + \
                    old_counts[2]) + \
                " - high: {}/{}\n".format(high_inc, old_counts[3])
        return out

    def cobra_model(self, name, reversible=True, bound=1000):
        new_mod = Model(name)
        for rid in self.conf:
            r = self.model.reactions.get_by_id(rid)
            if self.conf[rid] == 3 and "mock" not in r.notes:
                r.upper_bound = bound
                new_mod.add_reaction(r)

        if reversible: safe_revert_reversible(new_mod)
        still_valid = True
        for r in self.objective:
            still_valid &= (self.conf[r.id] == 3)
        if still_valid: new_mod.change_objective(self.objective)
        return new_mod
<commit_msg>Tried to improve numerical stability<commit_after>#  corda.py
#
#  Copyright 2016 Christian Diener <mail[at]cdiener.com>
#
#  MIT license. See LICENSE for more information.

from cobra.solvers import solver_dict, get_solver_name
from cobra.manipulation.modify import convert_to_irreversible
from cobra import Model, Metabolite, Reaction
import numpy as np
from collections import Counter
from .util import safe_revert_reversible
import sys
from os import devnull

TOL = 1e-5  # Tolerance to judge whether a flux is non-zero
UPPER = 1e16 # default upper bound

class CORDA(object):

    def __init__(self, model, confidence, met_prod=None, n=10,
        penalty_factor=100, support=5, solver=None, **solver_kwargs):
        """Initialize parameters and model"""
        self.model = model.copy()
        self.objective = model.objective.copy()
        convert_to_irreversible(self.model)

        # Map confidences from forward to backward reactions
        self.conf = {}
        for r in self.model.reactions:
            r.objective_coefficient = 0
            r.upper_bound = UPPER
            if r.id in confidence: self.conf[r.id] = confidence[r.id]
            elif "reflection" in r.notes:
                rev = self.model.reactions.get_by_id(r.notes["reflection"])
                self.conf[r.id] = confidence[rev.id]
            else:
                raise ValueError("{} missing from confidence!".format(r.id))
        self.__conf_old = self.conf.copy()

        self.built = False
        self.tflux = 1
        self.impossible = []
        self.unstable = []
        self.n = n
        self.noise = 0.5
        self.support = support
        self.pf = penalty_factor
        self.solver = solver_dict[get_solver_name() if solver is None else solver]
        self.sargs = solver_kwargs

        if met_prod:
            for mid in met_prod:
                r = Reaction("EX_CORDA_" + mid)
                r.notes["mock"] = mid
                r.upper_bound = UPPER
                self.model.add_reaction(r)
                if type(mid) == str:
                    r.add_metabolites({mid: -1})
                elif type(mid) == dict:
                    r.add_metabolites(mid)
                else:
                    raise ValueError("metabolite test not string or dictionary")
                self.conf[r.id] = 3

    def __perturb(self, lp, pen):
        noise = np.random.uniform(high=self.noise, size=len(pen))

        for i, p in enumerate(pen):
            if p > 0.0:
                self.solver.change_variable_objective(lp, i, p + noise[i])

    def __quiet_solve(self, lp, os):
        old = sys.stdout
        f = open(devnull, 'w')
        sys.stdout = f
        try:
            sol = self.solver.solve_problem(lp, objective_sense=os,
                **self.sargs)
        finally:
            sys.stdout = old
            f.close()
        return sol

    def __zero_objective(self, lp, m):
        for i, r in enumerate(m.reactions):
            self.solver.change_variable_objective(lp, i, 0.0)

    def associated(self, targets, conf=None):
        """Gets the associated reactions for the target reactions"""

        if conf is None: conf = self.conf

        m = self.model

        penalties = []
        for r in m.reactions:
            if conf[r.id] == 2 or conf[r.id] == 1:
                penalties.append(1)
            elif conf[r.id] == -1:
                penalties.append(self.pf)
            else: penalties.append(0)

        lp = self.solver.create_problem(self.model)

        needed = {}
        for rid in targets:
            ti = m.reactions.index(rid)
            needed[rid] = np.array([])

            self.__zero_objective(lp, m)
            self.solver.change_variable_objective(lp, ti, 1.0)
            sol = self.__quiet_solve(lp, "maximize")
            if sol != "optimal": self.unstable.append(rid)
            else:
                sol = self.solver.format_solution(lp, m)
                if sol.f < self.tflux:
                    self.impossible.append(rid)
                    self.conf[rid] = 0
                    continue

            self.solver.change_variable_bounds(lp, ti, self.tflux, UPPER)
            for _ in range(self.n):
                self.__perturb(lp, penalties)
                sol = self.__quiet_solve(lp, "minimize")
                if(sol != "optimal"):
                    self.unstable.append(rid)
                    continue
                sol = self.solver.format_solution(lp, m)
                need = [r for r in sol.x_dict if sol.x_dict[r] > TOL \
                    and r != "EX_penalty" and conf[r] in [-1, 1, 2]]
                need = np.hstack([needed[rid], need])
                needed[rid] = np.unique(need)

            self.solver.change_variable_bounds(lp, ti, 0.0, UPPER)

        return needed

    def build(self):
        """Constructs a tissue-specific model"""
        if self.built: raise ValueError("Model has been constructed already!")

        # First iteration - find reactions required for high confidence
        include = [r.id for r in self.model.reactions if self.conf[r.id] == 3]
        need = self.associated(include)
        add = np.unique([x for v in need.values() for x in v])
        for a in add: self.conf[a] = 3

        # Second iteration - add the best no confidence and independent medium
        # confidence
        include = [r.id for r in self.model.reactions if self.conf[r.id] == 1 \
            or self.conf[r.id] == 2]
        need = self.associated(include)
        add = [x for v in need.values() for x in v if self.conf[x] == -1 \
            and x not in self.impossible]
        count = Counter(add)
        add = [k for k in count if count[k] >= self.support]
        for a in add: self.conf[a] = 3

        not_included = [rid for rid in self.conf if self.conf[rid] == -1]
        for rid in not_included:
            self.model.reactions.get_by_id(rid).upper_bound = 0.0
        lp = self.solver.create_problem(self.model)
        for i, r in enumerate(self.model.reactions):
            if self.conf[r.id] == 1 or self.conf[r.id] == 2:
                self.solver.change_variable_objective(lp, i, 1.0)
                sol = self.__quiet_solve(lp, "maximize")
                sol = self.solver.format_solution(lp, self.model)
                if sol.f > TOL: self.conf[r.id] = 3
            self.solver.change_variable_objective(lp, i, 0.0)

        # Third iteration block all non-included N+M add free reactions
        for rid, co in self.conf.items():
            if co == 1 or co == 2:
                self.model.reactions.get_by_id(rid).upper_bound = 0.0
            if co == 0:
                self.conf[rid] = -1
        need = self.associated([k for k in self.conf if self.conf[k] == 3])
        add = np.unique([x for v in need.values() for x in v \
            if x not in self.impossible])
        for a in add: self.conf[a] = 3

        self.impossible = np.unique(self.impossible)
        self.unstable = np.unique(self.unstable)
        self.built = True

    def __str__(self):
        old_counts = Counter(self.__conf_old.values())
        if not self.built:
            out = "build status: not built\n" + \
                "#reactions (including mock): {}\n".\
                    format(len(self.__conf_old)) + \
                "Reaction confidence:\n" + \
                " - unclear: {}\n".format(old_counts[0]) + \
                " - exclude: {}\n".format(old_counts[-1]) + \
                " - low and medium: {}\n".format(old_counts[1] + \
                    old_counts[2]) + \
                " - high: {}\n".format(old_counts[3])
        else:
            rids = self.conf.keys()
            old = np.array([self.__conf_old[k] for k in rids])
            new = np.array([self.conf[k] for k in rids])
            med_inc = np.sum(((old == 1) | (old == 2)) & (new == 3))
            noc_inc = np.sum((old == -1) & (new == 3))
            free_inc = np.sum((old == 0) & (new == 3))
            high_inc = np.sum((old == 3) & (new == 3))
            out = "build status: reconstruction complete\n" + \
                "Inc. reactions: {}/{}\n".format(np.sum(new == 3), len(old)) +\
                " - unclear: {}/{}\n".format(free_inc, old_counts[0]) + \
                " - exclude: {}/{}\n".format(noc_inc, old_counts[-1]) + \
                " - low and medium: {}/{}\n".format(med_inc, old_counts[1] + \
                    old_counts[2]) + \
                " - high: {}/{}\n".format(high_inc, old_counts[3])
        return out

    def cobra_model(self, name, reversible=True, bound=1000):
        new_mod = Model(name)
        for rid in self.conf:
            r = self.model.reactions.get_by_id(rid)
            if self.conf[rid] == 3 and "mock" not in r.notes:
                r.upper_bound = bound
                new_mod.add_reaction(r)

        if reversible: safe_revert_reversible(new_mod)
        still_valid = True
        for r in self.objective:
            still_valid &= (self.conf[r.id] == 3)
        if still_valid: new_mod.change_objective(self.objective)
        return new_mod
<|endoftext|>


<commit_before>"""Connoisseur Utils.

Author: Lucas David -- <ld492@drexel.edu>
Licence: MIT License 2016 (c)

"""
import argparse
import gc
from jsmin import jsmin
import logging
from datetime import datetime

from .timer import Timer

logger = logging.getLogger('artificial')

class Constants:
    """Container for constants of an experiment, as well as an automatic
    loader for these.

    Parameters:
        data   -- the dict containing info regarding an execution.

    """

    def __init__(self, data):
        self._data = data

    def __getattr__(self, item):
        return self[item]

    def __getitem__(self, item):
        if item not in self._data:
            raise AttributeError('%s not an attribute of constants: %s'
                                 % (item, list(self._data.keys())))
        return self._data[item]

    def __str__(self):
        return str(self._data)

    def __contains__(self, item):
        return item in self._data

class Experiment(object):
    """Base Class for Experiments.

    Notes:
        the `run` method should be overridden for the experiment to be
        actually performed.

    Usage:

    >>> class ToyExperiment(Experiment):
    ...     def run(self):
    ...         print('Hello World!')
    ...
    >>> with ToyExperiment() as t:
    ...     t.run()
    Hello World
    >>> print(t.started_at)
    2016-10-11 14:40:22.454985
    >>> print(t.ended_at)
    2016-10-11 14:40:22.455061

    """

    def __init__(self, consts=None):
        self.consts = consts
        self.started_at = self.ended_at = None

    def setup(self):
        """Setup experiment to run.

        This event is always called right before the execution of the
        `Experiment.run` method.

        """

    def teardown(self):
        """Teardown experiment.

        This event is called after `Experiment.run` method has executed.

        """

    def run(self):
        raise NotImplementedError

    def __enter__(self):
        self.started_at = datetime.now()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.ended_at = datetime.now()

class ExperimentSet(object):
    """Container for experiments.

    Useful when executing multiple experiments or executing from multiple
    different environments.

    Usage:

    >>> class ToyExperiment(Experiment):
    ...     def run(self):
    ...         print('Hello World!')
    ...
    >>> experiments = ExperimentSet(ToyExperiment)
    >>> experiments.load_from_json('toy-experiments.json')
    >>> # Run all experiments!
    >>> experiments.run()
    >>> # Find out more about when these experiments started and ended.
    >>> for e in experiments:
    ...     print('experiment %s started at %s and ended at %s'
    ...           % (e.consts.code_name, e.started_at, e.ended_at))
    ...
    experiment julia started at 2016-10-11 14:52:12.216573
    and ended at 2016-10-11 14:52:14.688444

    """

    def __init__(self, experiment_cls, data=None):
        self.experiment_cls = experiment_cls
        self._data = None
        self._experiment_constants = None
        self.current_experiment_ = -1

        if data: self.load_from_object(data)

    def load_from_json(self, filename='./constants.json',
                       raise_on_not_found=False):
        data = {}

        try:
            with open(filename) as f:
                data = jsmin(f)
        except IOError:
            if raise_on_not_found:
                raise

        return self.load_from_object(data)

    def load_from_object(self, data):
        self._data = data = data.copy()

        if isinstance(data, dict):
            base_params = (data['base_parameters']
                           if 'base_parameters' in data
                           else data)

            experiments_params_lst = (data['executions']
                                      if 'executions' in data
                                      else [{'code_name': 'julia'}])
        else:
            base_params = []
            experiments_params_lst = data

        self._experiment_constants = []

        for experiment_params in experiments_params_lst:
            params = base_params.copy()
            params.update(experiment_params)

            self._experiment_constants.append(Constants(params))

        return self

    def __iter__(self):
        self.current_experiment_ = -1
        return self

    def __next__(self):
        return self.next()

    def __len__(self):
        return len(self._experiment_constants)

    def next(self):
        self.current_experiment_ += 1

        if self.current_experiment_ >= len(self._experiment_constants):
            self.current_experiment_ = -1
            raise StopIteration

        consts = self._experiment_constants[self.current_experiment_]
        return self.experiment_cls(consts)

    def run(self):
        t = Timer()

        try:
            logger.info('%i experiments in set', len(self))

            for i, e in enumerate(self):
                try:
                    with e:
                        e.setup()

                        logger.info('Experiment #%i: %s (%s)',
                                    i, e.consts.code_name, e.started_at)
                        logger.info('constants: %s', e.consts)

                        e.run()
                        e.teardown()
                    logger.info('experiment completed (%s)\n'
                                '-------------------------' % e.ended_at)

                    del e
                    gc.collect()
                except KeyboardInterrupt:
                    logger.warning('experiment interrupted by user')
                except Exception as e:
                    logger.error('an error was raised (%s): %s',
                                 e.__class__.__name__, e)
        except KeyboardInterrupt:
            logger.warning('experiment set interrupted by user')
        finally:
            logger.info('time elapsed for all experiments: %s\n', t)

arg_parser = argparse.ArgumentParser(
    description='Experiments on Art Connoisseurship')

arg_parser.add_argument('--constants', type=str, default='./constants.json',
                        help='JSON file containing definitions for the '
                             'experiment.')
<commit_msg>update jsmin<commit_after>"""Connoisseur Utils.

Author: Lucas David -- <ld492@drexel.edu>
Licence: MIT License 2016 (c)

"""
import argparse
import gc
import json

from jsmin import jsmin
import logging
from datetime import datetime

from .timer import Timer

logger = logging.getLogger('artificial')

class Constants:
    """Container for constants of an experiment, as well as an automatic
    loader for these.

    Parameters:
        data   -- the dict containing info regarding an execution.

    """

    def __init__(self, data):
        self._data = data

    def __getattr__(self, item):
        return self[item]

    def __getitem__(self, item):
        if item not in self._data:
            raise AttributeError('%s not an attribute of constants: %s'
                                 % (item, list(self._data.keys())))
        return self._data[item]

    def __str__(self):
        return str(self._data)

    def __contains__(self, item):
        return item in self._data

class Experiment(object):
    """Base Class for Experiments.

    Notes:
        the `run` method should be overridden for the experiment to be
        actually performed.

    Usage:

    >>> class ToyExperiment(Experiment):
    ...     def run(self):
    ...         print('Hello World!')
    ...
    >>> with ToyExperiment() as t:
    ...     t.run()
    Hello World
    >>> print(t.started_at)
    2016-10-11 14:40:22.454985
    >>> print(t.ended_at)
    2016-10-11 14:40:22.455061

    """

    def __init__(self, consts=None):
        self.consts = consts
        self.started_at = self.ended_at = None

    def setup(self):
        """Setup experiment to run.

        This event is always called right before the execution of the
        `Experiment.run` method.

        """

    def teardown(self):
        """Teardown experiment.

        This event is called after `Experiment.run` method has executed.

        """

    def run(self):
        raise NotImplementedError

    def __enter__(self):
        self.started_at = datetime.now()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.ended_at = datetime.now()

class ExperimentSet(object):
    """Container for experiments.

    Useful when executing multiple experiments or executing from multiple
    different environments.

    Usage:

    >>> class ToyExperiment(Experiment):
    ...     def run(self):
    ...         print('Hello World!')
    ...
    >>> experiments = ExperimentSet(ToyExperiment)
    >>> experiments.load_from_json('toy-experiments.json')
    >>> # Run all experiments!
    >>> experiments.run()
    >>> # Find out more about when these experiments started and ended.
    >>> for e in experiments:
    ...     print('experiment %s started at %s and ended at %s'
    ...           % (e.consts.code_name, e.started_at, e.ended_at))
    ...
    experiment julia started at 2016-10-11 14:52:12.216573
    and ended at 2016-10-11 14:52:14.688444

    """

    def __init__(self, experiment_cls, data=None):
        self.experiment_cls = experiment_cls
        self._data = None
        self._experiment_constants = None
        self.current_experiment_ = -1

        if data: self.load_from_object(data)

    def load_from_json(self, filename='./constants.json',
                       raise_on_not_found=False):
        data = {}

        try:
            with open(filename) as f:
                data = json.loads(jsmin(f.read()))
        except IOError:
            if raise_on_not_found:
                raise

        return self.load_from_object(data)

    def load_from_object(self, data):
        self._data = data = data.copy()

        if isinstance(data, dict):
            base_params = (data['base_parameters']
                           if 'base_parameters' in data
                           else data)

            experiments_params_lst = (data['executions']
                                      if 'executions' in data
                                      else [{'code_name': 'julia'}])
        else:
            base_params = []
            experiments_params_lst = data

        self._experiment_constants = []

        for experiment_params in experiments_params_lst:
            params = base_params.copy()
            params.update(experiment_params)

            self._experiment_constants.append(Constants(params))

        return self

    def __iter__(self):
        self.current_experiment_ = -1
        return self

    def __next__(self):
        return self.next()

    def __len__(self):
        return len(self._experiment_constants)

    def next(self):
        self.current_experiment_ += 1

        if self.current_experiment_ >= len(self._experiment_constants):
            self.current_experiment_ = -1
            raise StopIteration

        consts = self._experiment_constants[self.current_experiment_]
        return self.experiment_cls(consts)

    def run(self):
        t = Timer()

        try:
            logger.info('%i experiments in set', len(self))

            for i, e in enumerate(self):
                try:
                    with e:
                        e.setup()

                        logger.info('Experiment #%i: %s (%s)',
                                    i, e.consts.code_name, e.started_at)
                        logger.info('constants: %s', e.consts)

                        e.run()
                        e.teardown()
                    logger.info('experiment completed (%s)\n'
                                '-------------------------' % e.ended_at)

                    del e
                    gc.collect()
                except KeyboardInterrupt:
                    logger.warning('experiment interrupted by user')
                except Exception as e:
                    logger.error('an error was raised (%s): %s',
                                 e.__class__.__name__, e)
        except KeyboardInterrupt:
            logger.warning('experiment set interrupted by user')
        finally:
            logger.info('time elapsed for all experiments: %s\n', t)

arg_parser = argparse.ArgumentParser(
    description='Experiments on Art Connoisseurship')

arg_parser.add_argument('--constants', type=str, default='./constants.json',
                        help='JSON file containing definitions for the '
                             'experiment.')
<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

'''

Function for calculating statistics for sensitivity analysis

'''

import numpy as np
import sys
import os
import copy
from datetime import datetime
from astropy.io.fits import getdata

from turbustat.statistics import stats_wrapper
from turbustat.data_reduction import property_arrays

keywords = {"centroid", "centroid_error", "integrated_intensity",
            "integrated_intensity_error", "linewidth",
            "linewidth_error", "moment0", "moment0_error", "cube"}

def timestep_wrapper(fiducial_timestep, testing_timestep, statistics):

    # Derive the property arrays assuming uniform noise (for sims)
    fiducial_dataset = load_and_reduce(fiducial_timestep)
    testing_dataset = load_and_reduce(testing_timestep)

    distances = stats_wrapper(fiducial_dataset, testing_dataset,
                              statistics=statistics, multicore=True)
    return distances

def single_input(a):
    return timestep_wrapper(*a)

def run_all(fiducial, simulation_runs, face, statistics, savename,
            multicore=True, ncores=10, verbose=False, comp_face=None,
            multi_timesteps=False):
    '''
    Given a fiducial set and a series of sets to compare to, loop
    through and compare all sets and their timesteps. Return an array of
    the distances.

    Parameters
    ----------
    verbose : bool, optional
        Prints out the time when completing a set.
    comp_face : int or None, optional
        Face to compare to. If None, this is set to the fiducial face.
    multi_timesteps : bool, optional
        If multiple timesteps are given for each simulation run, parallelize
        over the timesteps. If only one is given, parallelize over the
        simulation runs.
    '''
    if comp_face is None:
        comp_face = face

    fiducial_timesteps = fiducial[face]

    if verbose:
        # print "Simulation runs to be analyzed: %s" % (simulation_runs)
        print "Started at "+str(datetime.now())

    if multi_timesteps:
        # Distances will be stored in an array of dimensions
        # # statistics x # sim runs x # timesteps
        distances_storage = np.zeros((len(statistics),
                                      len(simulation_runs),
                                      len(fiducial_timesteps)))

        print distances_storage.shape

        for i, key in enumerate(simulation_runs.keys()):
            timesteps = simulation_runs[key][comp_face]

            if verbose:
                print "On Simulation %s/%s" % (i+1, len(simulation_runs))
                print str(datetime.now())
            if multicore:
                pool = Pool(processes=ncores)
                distances = pool.map(single_input, izip(fiducial_timesteps,
                                                        timesteps,
                                                        repeat(statistics)))
                pool.close()
                pool.join()
                distances_storage[:, i, :] = \
                    sort_distances(statistics, distances).T

            else:
                for ii, timestep in enumerate(timesteps):
                    fiducial_dataset = load_and_reduce(fiducial_timesteps[ii])
                    testing_dataset = load_and_reduce(timestep)
                    if i == 0:
                        distances, fiducial_models = \
                            stats_wrapper(fiducial_dataset, testing_dataset,
                                          statistics=statistics)
                        all_fiducial_models = fiducial_models
                    else:
                        distances = \
                            stats_wrapper(fiducial_dataset, testing_dataset,
                                          fiducial_models=all_fiducial_models,
                                          statistics=statistics)
                    distances = [distances]
                    distances_storage[:, i, ii:ii+1] = \
                        sort_distances(statistics, distances).T

    else:
        distances_storage = np.zeros((len(statistics),
                                      len(simulation_runs)))

        if multicore:
            pool = Pool(processes=ncores)
            distances = pool.map(single_input, izip(repeat(fiducial),
                                                    simulation_runs,
                                                    repeat(statistics)))
            pool.close()
            pool.join()
            distances_storage = sort_distances(statistics, distances).T

        else:
            for i, key in enumerate(simulation_runs.keys()):
                fiducial_dataset = load_and_reduce(fiducial[face])
                testing_dataset = \
                    load_and_reduce(simulation_runs[key][comp_face])
                if i == 0:
                    distances, fiducial_models = \
                        stats_wrapper(fiducial_dataset, testing_dataset,
                                      statistics=statistics)
                    all_fiducial_models = fiducial_models
                else:
                    distances = \
                        stats_wrapper(fiducial_dataset, testing_dataset,
                                      fiducial_models=all_fiducial_models,
                                      statistics=statistics)
                distances = [distances]
                distances_storage[:, i:i+1] = \
                    sort_distances(statistics, distances).T

    return distances_storage

def sort_distances(statistics, distances):
    if len(statistics) > 1:
        distance_array = np.empty((len(distances), len(statistics)))
    elif len(statistics) == 1:
        distance_array = np.empty((len(distances), 1))

    for j, dist in enumerate(distances):
        distance_array[j, :] = [distances[j][stat] for stat in statistics]

    return distance_array

def files_sorter(folder, fiducial_labels=np.arange(0, 5, 1),
                 design_labels=np.arange(0, 32, 1), timesteps='last',
                 faces=[0, 1, 2], suffix="fits", append_prefix=False):
    '''
    If the entire simulation suite is in one directory, this function
    will spit out appropriate groupings.

    Parameters
    ----------
    folder : str
        Folder where data is.
    fiducial_labels : list or numpy.ndarray, optional
        List of the fiducial numbers.
    design_labels : list or numpy.ndarray, optional
        List of the design numbers.
    timesteps : 'last' or list or numpy.ndarray, optional
        List of timesteps to analyze. If 'last', the last timestep
        found for each simulation is used.
    faces : list
        Faces of the simulations to use.
    suffix : str, optional
        File suffix.
    '''

    # Get the files and remove any sub-directories.
    files = [f for f in os.listdir(folder) if not os.path.isdir(f) and
             f[-len(suffix):] == suffix]

    # Set up the dictionaries.
    fiducials = dict.fromkeys(fiducial_labels)
    for lab in fiducial_labels:
        fiducials[lab] = dict((face, []) for face in faces)
    designs = dict.fromkeys(design_labels)
    for lab in design_labels:
        designs[lab] = dict((face, []) for face in faces)
    timestep_labels = dict.fromkeys(design_labels)
    for lab in design_labels:
        timestep_labels[lab] = dict((face, []) for face in faces)

    # Sort the files
    for f in files:
        if "Fiducial" in f:
            for lab in fiducial_labels:
                if not "Fiducial"+str(lab)+"_" in f:
                    continue
                for face in faces:
                    if "_0"+str(face)+"_" in f:
                        if append_prefix:
                            fiducials[lab][face].append(folder+f)
                        else:
                            fiducials[lab][face].append(f)

        elif "Design" in f:
            for lab in design_labels:
                if not "Design"+str(lab)+"_" in f:
                    continue
                for face in faces:
                    if "_0"+str(face)+"_" in f:
                        if append_prefix:
                            designs[lab][face].append(folder+f)
                        else:
                            designs[lab][face].append(f)

        else:
            print "Could not find a category for " + f

    # Sort and keep only the specified timesteps
    _timestep_sort(fiducials, timesteps)
    _timestep_sort(designs, timesteps, labels=timestep_labels)

    return fiducials, designs, timestep_labels

def _timestep_sort(d, timesteps, labels=None):
    '''
    Helper function for segmenting by timesteps.
    '''
    for lab in d.keys():
        for face in d[lab].keys():
            # Check for empty lists.
            if d[lab][face] == []:
                continue
            d[lab][face].sort()
            if timesteps == 'last':  # Grab the last one
                if labels is not None:
                    labels[lab][face].append(d[lab][face][-1][-16:-14])
                d[lab][face] = d[lab][face][-1]
            elif timesteps == 'max':  # Keep all available
                # Reverse the order so the comparisons are between the highest
                # time steps.
                d[lab][face] = d[lab][face][::-1]
            elif isinstance(timesteps, int):  # Slice out a certain section
                d[lab][face] = d[lab][face][timesteps:]
                if labels is None:
                    continue
                for val in d[lab][face]:
                    labels[lab][face].append(val[-16:-14])
            else:  # Make a copy and loop through the steps
                good_files = copy.copy(d[lab][face])
                for f in d[lab][face]:
                    match = ["_00"+str(step)+"_" in f for step in timesteps]
                    if not any(match):
                        good_files.remove(f)
                    if labels is not None:
                        labels[lab][face].append(f[-16:-14])
                d[lab][face] = good_files

def load_and_reduce(filename):
    '''
    Load the cube in and derive the property arrays.
    '''

    cube = getdata(filename, header=True)

    reduction = property_arrays(cube, rms_noise=0.001)
    reduction.return_all(save=False)

    return reduction.dataset

if __name__ == "__main__":

    # Call as:
    # python output.py path/to/folder/ 0 0 1 max fiducial0 T 10
    # The args correspond to: directory, fiducial number, face,
    # comparison face, time steps to use, output file prefix,
    # use multiple cores?, how many cores?

    from multiprocessing import Pool
    from itertools import izip, repeat

    statistics = ["Wavelet", "MVC", "PSpec", "Bispectrum", "DeltaVariance",
                  "Genus", "VCS", "VCA", "Tsallis", "PCA", "SCF", "Cramer",
                  "Skewness", "Kurtosis", "VCS_Density", "VCS_Velocity",
                  "Dendrogram_Hist", "Dendrogram_Num", "PDF"]

    print "Statistics to run: %s" % (statistics)
    num_statistics = len(statistics)

    # Read in cmd line args
    try:
        fiducial_num = int(sys.argv[2])
    except ValueError:
        fiducial_num = str(sys.argv[2])
    face = int(sys.argv[3])
    comp_face = int(sys.argv[4])
    try:
        timesteps = int(sys.argv[5])
    except ValueError:
        timesteps = str(sys.argv[5])
    save_name = str(sys.argv[6])
    MULTICORE = str(sys.argv[7])
    if MULTICORE == "T":
        MULTICORE = True
    else:
        MULTICORE = False
        NCORES = 1  # Placeholder to pass into run_all
    if MULTICORE:
        NCORES = int(sys.argv[8])

    # Read in all files in the given directory
    PREFIX = str(sys.argv[1])

    fiducials, designs, timesteps_labels = \
        files_sorter(PREFIX, timesteps="max",
                     append_prefix=True, design_labels=[0])

    if fiducial_num == "fid_comp":  # Run all the comparisons of fiducials

        print "Fiducials to compare %s" % (fiducials.keys())
        fiducial_index = []
        fiducial_col = []

        # number of comparisons b/w all fiducials
        num_comp = (len(fiducials)**2. - len(fiducials))/2
        # Change dim 2 to match number of time steps
        distances_storage = np.zeros((num_statistics, num_comp, 10))
        posn = 0
        prev = 0
        # no need to loop over the last one
        for fid_num, i in zip(fiducials.keys()[:-1],
                              np.arange(len(fiducials), 0, -1)):
            posn += i
            comparisons = fiducials.copy()
            for key in range(fid_num):
                del comparisons[key]
            partial_distances = \
                run_all(fiducials[fid_num], comparisons,
                        face, statistics, save_name, multicore=MULTICORE,
                        ncores=NCORES, comp_face=comp_face,
                        multi_timesteps=True, verbose=True)
            distances_storage[:, prev:posn, :] = partial_distances
            prev += i

            fiducial_index.extend(fiducials.keys()[fid_num:])

            fiducial_col.extend([posn-prev] * len(fiducials.keys()[fid_num:]))

        # consistent naming with non-fiducial case
        simulation_runs = fiducial_index
        face = comp_face
    else:  # Normal case of comparing to single fiducial

        distances_storage = \
            run_all(fiducials[fiducial_num], designs,
                    face, statistics, save_name,
                    multicore=MULTICORE, ncores=NCORES, comp_face=comp_face,
                    multi_timesteps=True)

        simulation_runs = designs.keys()
        fiducial_index = [fiducial_num] * len(designs.keys())

    # If using timesteps 'max', some comparisons will remain zero
    # To distinguish a bit better, set the non-comparisons to zero
    distances_storage[np.where(distances_storage == 0)] = np.NaN

    filename = save_name + str(face) + "_" + str(comp_face) + \
        "_distance_results.h5"
    from pandas import DataFrame, HDFStore, concat, Series

    ## Save data for each statistic in a dataframe.
    ## Each dataframe is saved in a single hdf5 file

    store = HDFStore("lustre/home/ekoch/results/"+filename)

    for i in range(num_statistics):
        # If timesteps is 'max', there will be different number of labels
        # in this case, don't bother specifying column names.
        if not 'max' in timesteps:
            df = DataFrame(distances_storage[i, :, :], index=simulation_runs,
                           columns=timesteps_labels[0][face])
        else:
            df = DataFrame(distances_storage[i, :, :], index=simulation_runs)

        if not "Fiducial" in df.columns:
            df["Fiducial"] = Series(fiducial_index, index=df.index)
        if statistics[i] in store:
            existing_df = store[statistics[i]]
            if len(existing_df.index) == len(df.index):
                store[statistics[i]] = df
            else:  # Append on
                for ind in df.index:
                    if ind in list(existing_df.index):
                        existing_df.ix[ind] = df.ix[ind]
                    else:
                        existing_df = concat([existing_df, df])
                    store[statistics[i]] = existing_df
        else:
            store[statistics[i]] = df

    store.close()

    print "Done at " + str(datetime.now())
<commit_msg>Enable printing.<commit_after># Licensed under an MIT open source license - see LICENSE

'''

Function for calculating statistics for sensitivity analysis

'''

import numpy as np
import sys
import os
import copy
from datetime import datetime
from astropy.io.fits import getdata

from turbustat.statistics import stats_wrapper
from turbustat.data_reduction import property_arrays

keywords = {"centroid", "centroid_error", "integrated_intensity",
            "integrated_intensity_error", "linewidth",
            "linewidth_error", "moment0", "moment0_error", "cube"}

def timestep_wrapper(fiducial_timestep, testing_timestep, statistics):

    # Derive the property arrays assuming uniform noise (for sims)
    fiducial_dataset = load_and_reduce(fiducial_timestep)
    testing_dataset = load_and_reduce(testing_timestep)

    distances = stats_wrapper(fiducial_dataset, testing_dataset,
                              statistics=statistics, multicore=True)
    return distances

def single_input(a):
    return timestep_wrapper(*a)

def run_all(fiducial, simulation_runs, face, statistics, savename,
            multicore=True, ncores=10, verbose=True, comp_face=None,
            multi_timesteps=False):
    '''
    Given a fiducial set and a series of sets to compare to, loop
    through and compare all sets and their timesteps. Return an array of
    the distances.

    Parameters
    ----------
    verbose : bool, optional
        Prints out the time when completing a set.
    comp_face : int or None, optional
        Face to compare to. If None, this is set to the fiducial face.
    multi_timesteps : bool, optional
        If multiple timesteps are given for each simulation run, parallelize
        over the timesteps. If only one is given, parallelize over the
        simulation runs.
    '''
    if comp_face is None:
        comp_face = face

    fiducial_timesteps = fiducial[face]

    if verbose:
        # print "Simulation runs to be analyzed: %s" % (simulation_runs)
        print "Started at "+str(datetime.now())

    if multi_timesteps:
        # Distances will be stored in an array of dimensions
        # # statistics x # sim runs x # timesteps
        distances_storage = np.zeros((len(statistics),
                                      len(simulation_runs),
                                      len(fiducial_timesteps)))

        print distances_storage.shape

        for i, key in enumerate(simulation_runs.keys()):
            timesteps = simulation_runs[key][comp_face]

            if verbose:
                print "On Simulation %s/%s" % (i+1, len(simulation_runs))
                print str(datetime.now())
            if multicore:
                pool = Pool(processes=ncores)
                distances = pool.map(single_input, izip(fiducial_timesteps,
                                                        timesteps,
                                                        repeat(statistics)))
                pool.close()
                pool.join()
                distances_storage[:, i, :] = \
                    sort_distances(statistics, distances).T

            else:
                for ii, timestep in enumerate(timesteps):
                    fiducial_dataset = load_and_reduce(fiducial_timesteps[ii])
                    testing_dataset = load_and_reduce(timestep)
                    if i == 0:
                        distances, fiducial_models = \
                            stats_wrapper(fiducial_dataset, testing_dataset,
                                          statistics=statistics)
                        all_fiducial_models = fiducial_models
                    else:
                        distances = \
                            stats_wrapper(fiducial_dataset, testing_dataset,
                                          fiducial_models=all_fiducial_models,
                                          statistics=statistics)
                    distances = [distances]
                    distances_storage[:, i, ii:ii+1] = \
                        sort_distances(statistics, distances).T

    else:
        distances_storage = np.zeros((len(statistics),
                                      len(simulation_runs)))

        if multicore:
            pool = Pool(processes=ncores)
            distances = pool.map(single_input, izip(repeat(fiducial),
                                                    simulation_runs,
                                                    repeat(statistics)))
            pool.close()
            pool.join()
            distances_storage = sort_distances(statistics, distances).T

        else:
            for i, key in enumerate(simulation_runs.keys()):
                fiducial_dataset = load_and_reduce(fiducial[face])
                testing_dataset = \
                    load_and_reduce(simulation_runs[key][comp_face])
                if i == 0:
                    distances, fiducial_models = \
                        stats_wrapper(fiducial_dataset, testing_dataset,
                                      statistics=statistics)
                    all_fiducial_models = fiducial_models
                else:
                    distances = \
                        stats_wrapper(fiducial_dataset, testing_dataset,
                                      fiducial_models=all_fiducial_models,
                                      statistics=statistics)
                distances = [distances]
                distances_storage[:, i:i+1] = \
                    sort_distances(statistics, distances).T

    return distances_storage

def sort_distances(statistics, distances):
    if len(statistics) > 1:
        distance_array = np.empty((len(distances), len(statistics)))
    elif len(statistics) == 1:
        distance_array = np.empty((len(distances), 1))

    for j, dist in enumerate(distances):
        distance_array[j, :] = [distances[j][stat] for stat in statistics]

    return distance_array

def files_sorter(folder, fiducial_labels=np.arange(0, 5, 1),
                 design_labels=np.arange(0, 32, 1), timesteps='last',
                 faces=[0, 1, 2], suffix="fits", append_prefix=False):
    '''
    If the entire simulation suite is in one directory, this function
    will spit out appropriate groupings.

    Parameters
    ----------
    folder : str
        Folder where data is.
    fiducial_labels : list or numpy.ndarray, optional
        List of the fiducial numbers.
    design_labels : list or numpy.ndarray, optional
        List of the design numbers.
    timesteps : 'last' or list or numpy.ndarray, optional
        List of timesteps to analyze. If 'last', the last timestep
        found for each simulation is used.
    faces : list
        Faces of the simulations to use.
    suffix : str, optional
        File suffix.
    '''

    # Get the files and remove any sub-directories.
    files = [f for f in os.listdir(folder) if not os.path.isdir(f) and
             f[-len(suffix):] == suffix]

    # Set up the dictionaries.
    fiducials = dict.fromkeys(fiducial_labels)
    for lab in fiducial_labels:
        fiducials[lab] = dict((face, []) for face in faces)
    designs = dict.fromkeys(design_labels)
    for lab in design_labels:
        designs[lab] = dict((face, []) for face in faces)
    timestep_labels = dict.fromkeys(design_labels)
    for lab in design_labels:
        timestep_labels[lab] = dict((face, []) for face in faces)

    # Sort the files
    for f in files:
        if "Fiducial" in f:
            for lab in fiducial_labels:
                if not "Fiducial"+str(lab)+"_" in f:
                    continue
                for face in faces:
                    if "_0"+str(face)+"_" in f:
                        if append_prefix:
                            fiducials[lab][face].append(folder+f)
                        else:
                            fiducials[lab][face].append(f)

        elif "Design" in f:
            for lab in design_labels:
                if not "Design"+str(lab)+"_" in f:
                    continue
                for face in faces:
                    if "_0"+str(face)+"_" in f:
                        if append_prefix:
                            designs[lab][face].append(folder+f)
                        else:
                            designs[lab][face].append(f)

        else:
            print "Could not find a category for " + f

    # Sort and keep only the specified timesteps
    _timestep_sort(fiducials, timesteps)
    _timestep_sort(designs, timesteps, labels=timestep_labels)

    return fiducials, designs, timestep_labels

def _timestep_sort(d, timesteps, labels=None):
    '''
    Helper function for segmenting by timesteps.
    '''
    for lab in d.keys():
        for face in d[lab].keys():
            # Check for empty lists.
            if d[lab][face] == []:
                continue
            d[lab][face].sort()
            if timesteps == 'last':  # Grab the last one
                if labels is not None:
                    labels[lab][face].append(d[lab][face][-1][-16:-14])
                d[lab][face] = d[lab][face][-1]
            elif timesteps == 'max':  # Keep all available
                # Reverse the order so the comparisons are between the highest
                # time steps.
                d[lab][face] = d[lab][face][::-1]
            elif isinstance(timesteps, int):  # Slice out a certain section
                d[lab][face] = d[lab][face][timesteps:]
                if labels is None:
                    continue
                for val in d[lab][face]:
                    labels[lab][face].append(val[-16:-14])
            else:  # Make a copy and loop through the steps
                good_files = copy.copy(d[lab][face])
                for f in d[lab][face]:
                    match = ["_00"+str(step)+"_" in f for step in timesteps]
                    if not any(match):
                        good_files.remove(f)
                    if labels is not None:
                        labels[lab][face].append(f[-16:-14])
                d[lab][face] = good_files

def load_and_reduce(filename):
    '''
    Load the cube in and derive the property arrays.
    '''

    cube = getdata(filename, header=True)

    reduction = property_arrays(cube, rms_noise=0.001)
    reduction.return_all(save=False)

    return reduction.dataset

if __name__ == "__main__":

    # Call as:
    # python output.py path/to/folder/ 0 0 1 max fiducial0 T 10
    # The args correspond to: directory, fiducial number, face,
    # comparison face, time steps to use, output file prefix,
    # use multiple cores?, how many cores?

    from multiprocessing import Pool
    from itertools import izip, repeat

    statistics = ["Wavelet", "MVC", "PSpec", "Bispectrum", "DeltaVariance",
                  "Genus", "VCS", "VCA", "Tsallis", "PCA", "SCF", "Cramer",
                  "Skewness", "Kurtosis", "VCS_Density", "VCS_Velocity",
                  "Dendrogram_Hist", "Dendrogram_Num", "PDF"]

    print "Statistics to run: %s" % (statistics)
    num_statistics = len(statistics)

    # Read in cmd line args
    try:
        fiducial_num = int(sys.argv[2])
    except ValueError:
        fiducial_num = str(sys.argv[2])
    face = int(sys.argv[3])
    comp_face = int(sys.argv[4])
    try:
        timesteps = int(sys.argv[5])
    except ValueError:
        timesteps = str(sys.argv[5])
    save_name = str(sys.argv[6])
    MULTICORE = str(sys.argv[7])
    if MULTICORE == "T":
        MULTICORE = True
    else:
        MULTICORE = False
        NCORES = 1  # Placeholder to pass into run_all
    if MULTICORE:
        NCORES = int(sys.argv[8])

    # Read in all files in the given directory
    PREFIX = str(sys.argv[1])

    fiducials, designs, timesteps_labels = \
        files_sorter(PREFIX, timesteps="max",
                     append_prefix=True, design_labels=[0])

    if fiducial_num == "fid_comp":  # Run all the comparisons of fiducials

        print "Fiducials to compare %s" % (fiducials.keys())
        fiducial_index = []
        fiducial_col = []

        # number of comparisons b/w all fiducials
        num_comp = (len(fiducials)**2. - len(fiducials))/2
        # Change dim 2 to match number of time steps
        distances_storage = np.zeros((num_statistics, num_comp, 10))
        posn = 0
        prev = 0
        # no need to loop over the last one
        for fid_num, i in zip(fiducials.keys()[:-1],
                              np.arange(len(fiducials), 0, -1)):
            posn += i
            comparisons = fiducials.copy()
            for key in range(fid_num):
                del comparisons[key]
            partial_distances = \
                run_all(fiducials[fid_num], comparisons,
                        face, statistics, save_name, multicore=MULTICORE,
                        ncores=NCORES, comp_face=comp_face,
                        multi_timesteps=True, verbose=True)
            distances_storage[:, prev:posn, :] = partial_distances
            prev += i

            fiducial_index.extend(fiducials.keys()[fid_num:])

            fiducial_col.extend([posn-prev] * len(fiducials.keys()[fid_num:]))

        # consistent naming with non-fiducial case
        simulation_runs = fiducial_index
        face = comp_face
    else:  # Normal case of comparing to single fiducial

        distances_storage = \
            run_all(fiducials[fiducial_num], designs,
                    face, statistics, save_name,
                    multicore=MULTICORE, ncores=NCORES, comp_face=comp_face,
                    multi_timesteps=True)

        simulation_runs = designs.keys()
        fiducial_index = [fiducial_num] * len(designs.keys())

    # If using timesteps 'max', some comparisons will remain zero
    # To distinguish a bit better, set the non-comparisons to zero
    distances_storage[np.where(distances_storage == 0)] = np.NaN

    filename = save_name + str(face) + "_" + str(comp_face) + \
        "_distance_results.h5"
    from pandas import DataFrame, HDFStore, concat, Series

    ## Save data for each statistic in a dataframe.
    ## Each dataframe is saved in a single hdf5 file

    store = HDFStore("lustre/home/ekoch/results/"+filename)

    for i in range(num_statistics):
        # If timesteps is 'max', there will be different number of labels
        # in this case, don't bother specifying column names.
        if not 'max' in timesteps:
            df = DataFrame(distances_storage[i, :, :], index=simulation_runs,
                           columns=timesteps_labels[0][face])
        else:
            df = DataFrame(distances_storage[i, :, :], index=simulation_runs)

        if not "Fiducial" in df.columns:
            df["Fiducial"] = Series(fiducial_index, index=df.index)
        if statistics[i] in store:
            existing_df = store[statistics[i]]
            if len(existing_df.index) == len(df.index):
                store[statistics[i]] = df
            else:  # Append on
                for ind in df.index:
                    if ind in list(existing_df.index):
                        existing_df.ix[ind] = df.ix[ind]
                    else:
                        existing_df = concat([existing_df, df])
                    store[statistics[i]] = existing_df
        else:
            store[statistics[i]] = df

    store.close()

    print "Done at " + str(datetime.now())
<|endoftext|>