<commit_before># coding: utf-8
# Copyright (C) 2015 - Today: GRAP (http://www.grap.coop)
# @author: Sylvain LE GAL (https://twitter.com/legalsylvain)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    'name': 'Intercompany Trade - Purchase',
    'version': '8.0.1.0.0',
    'category': 'Intercompany Trade',
    'author': 'GRAP',
    'website': 'http://www.grap.coop',
    'license': 'AGPL-3',
    'depends': [
        'intercompany_trade_base',
        'intercompany_trade_product',
        'intercompany_trade_stock',
        'purchase',
    ],
    'demo': [
        'demo/res_groups.xml',
        'demo/product_pricelist.xml',
    ],
    'auto_install': False,
    'installable': True,
}
<commit_msg>[ADD] dependencies<commit_after># coding: utf-8
# Copyright (C) 2015 - Today: GRAP (http://www.grap.coop)
# @author: Sylvain LE GAL (https://twitter.com/legalsylvain)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    'name': 'Intercompany Trade - Purchase',
    'version': '8.0.1.0.0',
    'category': 'Intercompany Trade',
    'author': 'GRAP',
    'website': 'http://www.grap.coop',
    'license': 'AGPL-3',
    'depends': [
        'intercompany_trade_base',
        'intercompany_trade_product',
        'intercompany_trade_stock',
        'intercompany_trade_account',
        'purchase',
    ],
    'demo': [
        'demo/res_groups.xml',
        'demo/product_pricelist.xml',
    ],
    'auto_install': False,
    'installable': True,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class ProductProduct(orm.Model):
    """ Model name: ProductProduct
    """

    _inherit = 'product.product'

    def get_purchase_cost_value(self, cr, uid, context=None):
        ''' Check price from purchase order after from
        '''
        _logger.info('Check price from purchase')

        # ---------------------------------------------------------------------
        # Start with supplier order price:
        # ---------------------------------------------------------------------
        of_cost = {}
        po_pool = self.pool.get('purchase.order')
        po_ids = po_pool.search(cr, uid, [
            ('state', 'in', ('approved', )),
            #('date_order', '>=', '%s-01-01' % year),
            #('date_order', '<=', '%s-12-31' % year),
            ], order='date_order desc', context=context)

        for po in po_pool.browse(cr, uid, po_ids, context=context):
            for line in po.order_line:
                if not line.price_unit:
                    continue # jump no price line
                product = line.product_id
                default_code = product.default_code
                if default_code and default_code not in of_cost:
                    of_cost[default_code] = line.price_unit

        # ---------------------------------------------------------------------
        # Load cost check costs:
        # ---------------------------------------------------------------------
        costs = {}
        product_pool = self.pool.get('product.product')
        _logger.info('Check price from check costs')

        log_file = os.path.join(os.path.expanduser('~'), 'price.csv')
        log_f = open(log_file, 'w')

        product_ids = product_pool.search(cr, uid, [
            ('defualt_code', '!=', False),
            ], context=context)
        for product in product_pool.browse(cr, uid, product_ids,
                context=context):
            default_code = product.default_code
            for seller in product.seller_ids:
                if default_code not in costs:
                    break

                for pl in sorted(
                        seller.pricelist_ids,
                        lambda x: x.date_quotation,
                        reverse=True):
                    costs[default_code] = pl.price
                    log_f.write('%s|%s|%s\n' % (
                        default_code, pl.price, pl.date_quotation))
                    break

            # -----------------------------------------------------------------
            # Use anagraphic if not present
            # -----------------------------------------------------------------
            if not costs.get(default_code, False):
                costs[default_code] = \
                    of_cost.get(default_code, False) or product.standard_price
        return costs

    _columns = {
        'inv_cost_account': fields.char('Inv. cost account', size=8),
        'inv_revenue_account': fields.char('Inv. cost account', size=8),
        'inv_first_supplier': fields.char('Inv. First supplier', size=8),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>fast git<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class ProductProduct(orm.Model):
    """ Model name: ProductProduct
    """

    _inherit = 'product.product'

    def get_purchase_cost_value(self, cr, uid, context=None):
        ''' Check price from purchase order after from
        '''
        _logger.info('Check price from purchase')

        # ---------------------------------------------------------------------
        # Start with supplier order price:
        # ---------------------------------------------------------------------
        of_cost = {}
        po_pool = self.pool.get('purchase.order')
        po_ids = po_pool.search(cr, uid, [
            ('state', 'in', ('approved', )),
            #('date_order', '>=', '%s-01-01' % year),
            #('date_order', '<=', '%s-12-31' % year),
            ], order='date_order desc', context=context)

        for po in po_pool.browse(cr, uid, po_ids, context=context):
            for line in po.order_line:
                if not line.price_unit:
                    continue # jump no price line
                product = line.product_id
                default_code = product.default_code
                if default_code and default_code not in of_cost:
                    of_cost[default_code] = line.price_unit

        # ---------------------------------------------------------------------
        # Load cost check costs:
        # ---------------------------------------------------------------------
        costs = {}
        product_pool = self.pool.get('product.product')
        _logger.info('Check price from check costs')

        log_file = os.path.join(os.path.expanduser('~'), 'price.csv')
        log_f = open(log_file, 'w')

        product_ids = product_pool.search(cr, uid, [
            ('default_code', '!=', False),
            ], context=context)
        for product in product_pool.browse(cr, uid, product_ids,
                context=context):
            default_code = product.default_code
            for seller in product.seller_ids:
                if default_code not in costs:
                    break

                for pl in sorted(
                        seller.pricelist_ids,
                        lambda x: x.date_quotation,
                        reverse=True):
                    costs[default_code] = pl.price
                    log_f.write('%s|%s|%s\n' % (
                        default_code, pl.price, pl.date_quotation))
                    break

            # -----------------------------------------------------------------
            # Use anagraphic if not present
            # -----------------------------------------------------------------
            if not costs.get(default_code, False):
                costs[default_code] = \
                    of_cost.get(default_code, False) or product.standard_price
        return costs

    _columns = {
        'inv_cost_account': fields.char('Inv. cost account', size=8),
        'inv_revenue_account': fields.char('Inv. cost account', size=8),
        'inv_first_supplier': fields.char('Inv. First supplier', size=8),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>"""
Copy static files to nginx location

.. moduleauthor:: Infoxchange Development Team <development@infoxchange.net.au>

"""
import os
from shutil import copy2, copystat

from django.conf import settings
from django.core.management.base import NoArgsCommand

class Command(NoArgsCommand):
    """
    Copy static files to nginx location
    """

    def handle_noargs(self, **options):
        try:
            static_dir = settings.NGINX_STATIC_DIR
        except AttributeError:
            static_dir = None

        if not static_dir:
            self.stdout.write("Static dir not defined, "
                              "copy static content skipped")
            return

        self.stdout.write("Copying static content to %s", static_dir)

        self.copytree(settings.STATIC_ROOT, static_dir)

    def copytree(self, src, dst):
        """
        A version of copytree I don't hate
        """

        if not (os.path.exists(dst) and os.path.isdir(dst)):
            self.stdout.write("Creating directory %s", dst)
            os.makedirs(dst)
            copystat(src, dst)

        for name in os.listdir(src):
            srcname = os.path.join(src, name)
            dstname = os.path.join(dst, name)

            if os.path.isdir(srcname):
                self.stdout.write("Copying directory %s", name)
                self.copytree(srcname, dstname)
            else:
                self.stdout.write("Copying file %s", name)
                copy2(srcname, dstname)
<commit_msg>syntax<commit_after>"""
Copy static files to nginx location

.. moduleauthor:: Infoxchange Development Team <development@infoxchange.net.au>

"""
import os
from shutil import copy2, copystat

from django.conf import settings
from django.core.management.base import NoArgsCommand

class Command(NoArgsCommand):
    """
    Copy static files to nginx location
    """

    def handle_noargs(self, **options):
        try:
            static_dir = settings.NGINX_STATIC_DIR
        except AttributeError:
            static_dir = None

        if not static_dir:
            self.stdout.write("Static dir not defined, "
                              "copy static content skipped")
            return

        self.stdout.write("Copying static content to %s" % static_dir)

        self.copytree(settings.STATIC_ROOT, static_dir)

    def copytree(self, src, dst):
        """
        A version of copytree I don't hate
        """

        if not (os.path.exists(dst) and os.path.isdir(dst)):
            self.stdout.write("Creating directory %s" % dst)
            os.makedirs(dst)
            copystat(src, dst)

        for name in os.listdir(src):
            srcname = os.path.join(src, name)
            dstname = os.path.join(dst, name)

            if os.path.isdir(srcname):
                self.stdout.write("Copying directory %s" % name)
                self.copytree(srcname, dstname)
            else:
                self.stdout.write("Copying file %s" % name)
                copy2(srcname, dstname)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
__author__ = 'luissaguas'

#from jasper_erpnext_report import jasperserverlib
import jasper_erpnext_report as jr
import copy

try:
	import jasperserver.core as jasper
	from jasperserver.core.reportingService import ReportingService
	from jasperserver.core.ReportExecutionRequest import ReportExecutionRequest
	from jasperserver.resource_details import Details
	from jasperserver.repo_search import Search
	from jasperserver.resource_download import DownloadBinary
	from jasperserver.core.exceptions import Unauthorized, NotFound
	jr.jasperserverlib = True
except:
	jr.jasperserverlib = False

from frappe.utils import pprint_dict

from frappe import _
import frappe

import json
from io import BytesIO
import inspect

from jasper_erpnext_report.utils.file import JasperXmlReport
import jasper_erpnext_report.utils.utils as utils

import JasperBase as Jb

_logger = frappe.get_logger("jasper_erpnext_report")

def _jasperserver(fn):
	"""
	decorator for jasperserver functions
	"""
	def innerfn(*args, **kwargs):
		newargs = {}
		me = args[0]
		try:
			fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
			for a in fnargs:
				if a in kwargs:
					newargs[a] = kwargs.get(a)
			fn_result = fn(*args, **newargs)
			me.update()
			return fn_result

		except Unauthorized as e:
			me._timeout()
			fn_result = fn(*args, **newargs)
			me.update()
			utils.jaspersession_set_value("last_jasper_session_timeout", frappe.utils.now())
			return fn_result
		except Exception as e:
			print "Problems: {}\n".format(e)
			_logger.error(_("Problems {}".format(e)))

	return innerfn

class JasperServer(Jb.JasperBase):
	def __init__(self, doc=None):
		self.is_login = False
		self.session = frappe.local.jasper_session = frappe._dict({'session': frappe._dict({})})
		super(JasperServer, self).__init__(doc)
		self.check_session()

	def check_session(self):
		if not jr.jasperserverlib:
			return
		if self.data['data'] and self.data['data'].get('cookie', None):
			self.resume_connection()
		else:
			self.login()

	def login(self):
		self.connect()
		if not self.in_jasper_session() and self.user == "Administrator":
			sessionId = "login_error"
			msg = _("Jasper Server is down. Please check Jasper Server or change to local report only (you will need pyhton module pyjnius).")
			title = _("Jasper Server, login error")
			self.send_mail_and_logger(sessionId, msg, title)
			#last_login = utils.jaspersession_get_value("login_error")
			#if not last_login:
			#	last_login = utils.add_to_time_str(hours=-5)
			#time_diff = frappe.utils.time_diff_in_hours(frappe.utils.now(), last_login)
			#if time_diff >= 4:
			#	self.send_email(_("Jasper Server is down. Please check Jasper Server or change to local report only (you will need pyhton module pyjnius)."), _("Jasper Server, login error"))
			#	utils.jaspersession_set_value("login_error", frappe.utils.now())
			#	_logger.error(_("Jasper Server is down. Please check Jasper Server or change to local report only (you will need pyhton module pyjnius)."))

	def in_jasper_session(self):
		try:
			session = bool(self.session.session.getSessionId())
		except:
			session = False

		return session

	def resume_connection(self):
		self.session = frappe.local.jasper_session = jasper.session(self.doc.get("jasper_server_url"), resume=True)
		if self.session:
			self.session.resume(self.data['data']['cookie'])
			self.is_login = True

	def connect(self):
		if self.user=="Guest":
			return

		try:
			if not self.doc:
				self.get_jasperconfig_from_db()

			self.session = frappe.local.jasper_session = jasper.session(self.doc.get("jasper_server_url"), self.doc.get("jasper_username"), self.doc.get("jasper_server_password"))

			self.update_cookie()
			self.is_login = True

		except Exception as e:
			self.is_login = False
			sessionId = "connect_error"
			msg = _("Jasper Server, login error. Reason: {}".format(e))
			title = _("Jasper Server, login error")
			self.send_mail_and_logger(sessionId, msg, title)
			#cur_user = "no_reply@gmail.com" if self.user == "Administrator" else self.user
			#last_conn = utils.jaspersession_get_value("connect_error")
			#if not last_conn:
			#	last_conn = utils.add_to_time_str(hours=-5)
			#time_diff = frappe.utils.time_diff_in_hours(frappe.utils.now(), last_conn)
			#if time_diff >= 4:
			#	self.send_email(_("Jasper Server, login error. Reason: {}").format(e), _("Jasper Server, login error"), user=cur_user)
			#	utils.jaspersession_set_value("connect_error", frappe.utils.now())
			#	_logger.error(_("Jasper Server, login error. Reason: {}").format(e))

	def send_mail_and_logger(self, sessionId, msg, title, log=True):
		cur_user = "no_reply@gmail.com" if self.user == "Administrator" else self.user
		last_err = utils.jaspersession_get_value(sessionId)
		if not last_err:
			last_err = utils.add_to_time_str(hours=-5)
		time_diff = frappe.utils.time_diff_in_hours(frappe.utils.now(), last_err)
		print "last_err 4 {} time_diff {} site {}".format(last_err, time_diff, getattr(frappe.local, 'site', None))
		if time_diff >= 4:
			self.send_email(msg, title, user=cur_user)
			utils.jaspersession_set_value(sessionId, frappe.utils.now())
		if log:
			_logger.error(msg)

	def logout(self):
		if self.session:
			self.session.logout()

	def _timeout(self):
		try:
			self.logout()
			self.login()
		except:
			_logger.error(_("_login: JasperServerSession Error while timeout and login"))

		_logger.info("_timeout JasperServerSession login successfuly {0}".format(self.doc))

	def update_cookie(self):
		self.data['data']['cookie'] = frappe.local.jasper_session.session.getSessionId() if frappe.local.jasper_session.session else {}
		self.update(force_cache=False, force_db=True)

	def get_server_info(self):
		details = Details(self.session)
		serverInfo = details.serverInfo()
		return serverInfo

	def get_jrxml_from_server(self, uri):
		f = DownloadBinary(self.session, uri)
		f.downloadBinary(file=True)
		return f.getFileContent()

	def import_all_jasper_reports(self, data, force=True):
		if self.is_login:
			reports = self.get_reports_list_from_server(force=force)
			docs = utils.do_doctype_from_jasper(data, reports, force=True)
			utils.import_all_jasper_remote_reports(docs, force)

	@_jasperserver
	def get_reports_list_from_server(self, force=False):
		ret = {}
		s = Search(self.session)
		result = s.search(path=self.doc.get("jasper_report_root_path"), type="reportUnit")
		reports = result.getDescriptor().json_descriptor()
		for report in reports:
			ics = []
			d = Details(self.session, report.get("uri"))
			r_details = d.details(expanded=False)
			r_param = r_details.getDescriptor().json_descriptor()
			uri = r_param[0].get('jrxml').get('jrxmlFileReference').get('uri')
			file_content = self.get_jrxml_from_server(uri)
			xmldoc = JasperXmlReport(BytesIO(file_content))
			params = xmldoc.get_params()
			query = self.get_query_jrxmlFile_from_server(file_content)
			for param in params:
				pname = param.xpath('./@name')
				pclass = param.xpath('./@class')
				ptype = pclass[0].split(".")
				c = len(ptype) - 1
				ics.append({"label":pname[0], "type":ptype[c].lower()})
			updateDate = report.get("updateDate", None)
			if not updateDate:
				updateDate = report.get("creationDate", frappe.utils.now())

			ret[report.get("label")] = {"uri":report.get("uri"), "inputControls": ics, "updateDate": updateDate, "queryString": query}

		return ret

	def run_remote_report_async(self, path, doc, data=None, params=None, pformat="pdf", ncopies=1):
		"""
		resps = []
		data = self.run_report_async(doc, data=data, params=params)
		if data.get('jasper_report_type', None) == "Form" or doc.jasper_report_type == "Form":
			ids = data.get('ids')
			for id in ids:
				data['ids'] = [id]
				resps.append(self._run_report_async(path, doc, data=data, params=params, pformat=pformat, ncopies=ncopies))
		else:
			resps.append(self._run_report_async(path, doc, data=data, params=params, pformat=pformat, ncopies=ncopies))
		cresp = self.prepareCollectResponse(resps)
		cresp["origin"] = "server"
		return [cresp]
		"""
		cresp = self.prepare_report_async(path, doc, data=data, params=params, pformat=pformat, ncopies=ncopies, for_all_sites=0)
		cresp["origin"] = "server"
		return [cresp]

	#run reports with http POST and run async and sync
	def _run_report_async(self, path, doc, data=None, params=None, pformat="pdf", ncopies=1, for_all_sites=0):
		data = data or {}
		pram, pram_server, copies = self.do_params(data, params, pformat)
		pram_copy_index = copies.get("pram_copy_index", -1)
		pram_copy_page_index = copies.get("pram_copy_page_index", -1)
		resp = []

		pram.extend(self.get_param_hook(doc, data, pram_server))

		copies = [_("Original"), _("Duplicated"), _("Triplicate")]

		for m in range(ncopies):
			npram = copy.deepcopy(pram)
			if pram_copy_index != -1:
				values = pram[pram_copy_index].get("value","")
				print "values in range 3 {}".format(values)
				if not values or not values[0]:
					npram[pram_copy_index]['value'] = [copies[m]]
				else:
					npram[pram_copy_index]['value'] = [frappe.utils.strip(values[m], ' \t\n\r')]

			if pram_copy_page_index != -1:
				npram[pram_copy_page_index]['value'] = [str(m) + _(" of ") + str(ncopies)]
			result = self.run_async(path, npram, data.get("report_name"), pformat=pformat)
			if result:
				requestId = result.get('requestId')
				reqDbObj = {"data":{"result": result, "report_name": data.get("report_name"), "last_updated": frappe.utils.now(),'session_expiry': utils.get_expiry_period()}}
				self.insert_jasper_reqid_record(requestId, reqDbObj)
				resp.append({"requestId":requestId, "report_name": data.get("report_name"), "status": result.get('status')})
			else:
				frappe.msgprint(_("There was an error in report request."),raise_exception=True)
		#TODO get reqId for check report state
		return resp

	@_jasperserver
	def run_async(self, path, pram, report_name, pformat="pdf"):
		rs = ReportingService(self.session)
		rr = ReportExecutionRequest()
		rr.setReportUnitUri(path)
		rr.setOutputFormat(pformat)
		rr.setParameters(pram)
		rr.setAsync(True)
		if pformat == "html":
			rr.setAttachmentsPrefix("./images/")
			#rr.setIgnorePagination(True)
		try:
			req = rs.newReportExecutionRequest(rr)
			res = req.run().getResult("content")
			res = json.loads(res)
			resp = self.prepareResponse(res, res.get("requestId"))
			resp["status"] = res.get("status")
			return resp
		except Exception as e:
			if isinstance(e, Unauthorized):

				raise Unauthorized
			else:
				frappe.throw(_("Error in report %s, server error is: %s." % (self.doc.jasper_report_name, e)))

	@_jasperserver
	def polling(self, reqId):
		res = []
		try:
			rs = ReportingService(self.session)
			rexecreq = rs.reportExecutionRequest(reqId)
			status = rexecreq.status().content
			status = json.loads(status)
			if status.get("value") == "ready":
				detail = self.reportExecDetail(rexecreq)
				res = self.prepareResponse(detail, reqId)
				if res.get('status', "") == "not ready":
					res = self.prepareResponse({}, reqId)
				else:
					res["status"] = "ready"
			elif status.get("value") == "failed":
				frappe.throw(_("Error in report %s, server error is: %s." % (self.doc.jasper_report_name, status['errorDescriptor'].get('message'))))
			else:
				res = self.prepareResponse({}, reqId)
		except NotFound:
			frappe.throw(_("Not Found."))
		return res

	#rexecreq : ReportExecutionRequestBuilder instance
	#check if the report is ready and get the expids for ready reports
	def reportExecDetail(self, rexecreq):
		return rexecreq.executionDetails()

	#rs: ReportingService instance
	#get the report with reqId for the given expId
	@_jasperserver
	def getReport(self, reqId, expId):
		try:
			rs = ReportingService(self.session)
			rexecreq = rs.reportExecutionRequest(reqId, expId)
			report = rexecreq.export().outputResource().getResult("content")
			return report
		except NotFound:
			frappe.throw(_("Not Found."))

	def getAttachment(self, reqId, expId, attachmentId):
		try:
			rs = ReportingService(self.session)
			rexecreq = rs.reportExecutionRequest(reqId, expId)
			content = rexecreq.export().attachment(attachmentId)
			return content
		except NotFound:
			frappe.throw(_("Not Found."))
<commit_msg>Add logger.<commit_after>from __future__ import unicode_literals
__author__ = 'luissaguas'

#from jasper_erpnext_report import jasperserverlib
import jasper_erpnext_report as jr
import copy

try:
	import jasperserver.core as jasper
	from jasperserver.core.reportingService import ReportingService
	from jasperserver.core.ReportExecutionRequest import ReportExecutionRequest
	from jasperserver.resource_details import Details
	from jasperserver.repo_search import Search
	from jasperserver.resource_download import DownloadBinary
	from jasperserver.core.exceptions import Unauthorized, NotFound
	jr.jasperserverlib = True
except:
	jr.jasperserverlib = False

from frappe.utils import pprint_dict

from frappe import _
import frappe

import json
from io import BytesIO
import inspect

from jasper_erpnext_report.utils.file import JasperXmlReport
import jasper_erpnext_report.utils.utils as utils

import JasperBase as Jb

_logger = frappe.get_logger("jasper_erpnext_report")

def _jasperserver(fn):
	"""
	decorator for jasperserver functions
	"""
	def innerfn(*args, **kwargs):
		newargs = {}
		me = args[0]
		try:
			fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
			for a in fnargs:
				if a in kwargs:
					newargs[a] = kwargs.get(a)
			fn_result = fn(*args, **newargs)
			me.update()
			return fn_result

		except Unauthorized as e:
			me._timeout()
			fn_result = fn(*args, **newargs)
			me.update()
			utils.jaspersession_set_value("last_jasper_session_timeout", frappe.utils.now())
			return fn_result
		except Exception as e:
			print "Problems: {}\n".format(e)
			_logger.error(_("Problems {}".format(e)))

	return innerfn

class JasperServer(Jb.JasperBase):
	def __init__(self, doc=None):
		self.is_login = False
		self.session = frappe.local.jasper_session = frappe._dict({'session': frappe._dict({})})
		super(JasperServer, self).__init__(doc)
		self.check_session()

	def check_session(self):
		if not jr.jasperserverlib:
			return
		if self.data['data'] and self.data['data'].get('cookie', None):
			self.resume_connection()
		else:
			self.login()

	def login(self):
		self.connect()
		if not self.in_jasper_session() and self.user == "Administrator":
			sessionId = "login_error"
			msg = _("Jasper Server is down. Please check Jasper Server or change to local report only (you will need pyhton module pyjnius).")
			title = _("Jasper Server, login error")
			self.send_mail_and_logger(sessionId, msg, title)
			#last_login = utils.jaspersession_get_value("login_error")
			#if not last_login:
			#	last_login = utils.add_to_time_str(hours=-5)
			#time_diff = frappe.utils.time_diff_in_hours(frappe.utils.now(), last_login)
			#if time_diff >= 4:
			#	self.send_email(_("Jasper Server is down. Please check Jasper Server or change to local report only (you will need pyhton module pyjnius)."), _("Jasper Server, login error"))
			#	utils.jaspersession_set_value("login_error", frappe.utils.now())
			#	_logger.error(_("Jasper Server is down. Please check Jasper Server or change to local report only (you will need pyhton module pyjnius)."))

	def in_jasper_session(self):
		try:
			session = bool(self.session.session.getSessionId())
		except:
			session = False

		return session

	def resume_connection(self):
		self.session = frappe.local.jasper_session = jasper.session(self.doc.get("jasper_server_url"), resume=True)
		if self.session:
			self.session.resume(self.data['data']['cookie'])
			self.is_login = True

	def connect(self):
		if self.user=="Guest":
			return

		try:
			if not self.doc:
				self.get_jasperconfig_from_db()

			self.session = frappe.local.jasper_session = jasper.session(self.doc.get("jasper_server_url"), self.doc.get("jasper_username"), self.doc.get("jasper_server_password"))

			self.update_cookie()
			self.is_login = True

		except Exception as e:
			self.is_login = False
			sessionId = "connect_error"
			msg = _("Jasper Server, login error. Reason: {}".format(e))
			title = _("Jasper Server, login error")
			self.send_mail_and_logger(sessionId, msg, title)
			#cur_user = "no_reply@gmail.com" if self.user == "Administrator" else self.user
			#last_conn = utils.jaspersession_get_value("connect_error")
			#if not last_conn:
			#	last_conn = utils.add_to_time_str(hours=-5)
			#time_diff = frappe.utils.time_diff_in_hours(frappe.utils.now(), last_conn)
			#if time_diff >= 4:
			#	self.send_email(_("Jasper Server, login error. Reason: {}").format(e), _("Jasper Server, login error"), user=cur_user)
			#	utils.jaspersession_set_value("connect_error", frappe.utils.now())
			#	_logger.error(_("Jasper Server, login error. Reason: {}").format(e))

	def send_mail_and_logger(self, sessionId, msg, title, log=True):
		cur_user = "no_reply@gmail.com" if self.user == "Administrator" else self.user
		last_err = utils.jaspersession_get_value(sessionId)
		if not last_err:
			last_err = utils.add_to_time_str(hours=-5)
		time_diff = frappe.utils.time_diff_in_hours(frappe.utils.now(), last_err)
		print "last_err 4 {} time_diff {} site {}".format(last_err, time_diff, getattr(frappe.local, 'site', None))
		if time_diff >= 4:
			self.send_email(msg, title, user=cur_user)
			utils.jaspersession_set_value(sessionId, frappe.utils.now())
			if log:
				_logger.error(msg)

	def logout(self):
		if self.session:
			self.session.logout()

	def _timeout(self):
		try:
			self.logout()
			self.login()
		except:
			_logger.error(_("_login: JasperServerSession Error while timeout and login"))

		_logger.info("_timeout JasperServerSession login successfuly {0}".format(self.doc))

	def update_cookie(self):
		self.data['data']['cookie'] = frappe.local.jasper_session.session.getSessionId() if frappe.local.jasper_session.session else {}
		self.update(force_cache=False, force_db=True)

	def get_server_info(self):
		details = Details(self.session)
		serverInfo = details.serverInfo()
		return serverInfo

	def get_jrxml_from_server(self, uri):
		f = DownloadBinary(self.session, uri)
		f.downloadBinary(file=True)
		return f.getFileContent()

	def import_all_jasper_reports(self, data, force=True):
		if self.is_login:
			reports = self.get_reports_list_from_server(force=force)
			docs = utils.do_doctype_from_jasper(data, reports, force=True)
			utils.import_all_jasper_remote_reports(docs, force)

	@_jasperserver
	def get_reports_list_from_server(self, force=False):
		ret = {}
		s = Search(self.session)
		result = s.search(path=self.doc.get("jasper_report_root_path"), type="reportUnit")
		reports = result.getDescriptor().json_descriptor()
		for report in reports:
			ics = []
			d = Details(self.session, report.get("uri"))
			r_details = d.details(expanded=False)
			r_param = r_details.getDescriptor().json_descriptor()
			uri = r_param[0].get('jrxml').get('jrxmlFileReference').get('uri')
			file_content = self.get_jrxml_from_server(uri)
			xmldoc = JasperXmlReport(BytesIO(file_content))
			params = xmldoc.get_params()
			query = self.get_query_jrxmlFile_from_server(file_content)
			for param in params:
				pname = param.xpath('./@name')
				pclass = param.xpath('./@class')
				ptype = pclass[0].split(".")
				c = len(ptype) - 1
				ics.append({"label":pname[0], "type":ptype[c].lower()})
			updateDate = report.get("updateDate", None)
			if not updateDate:
				updateDate = report.get("creationDate", frappe.utils.now())

			ret[report.get("label")] = {"uri":report.get("uri"), "inputControls": ics, "updateDate": updateDate, "queryString": query}

		return ret

	def run_remote_report_async(self, path, doc, data=None, params=None, pformat="pdf", ncopies=1):
		"""
		resps = []
		data = self.run_report_async(doc, data=data, params=params)
		if data.get('jasper_report_type', None) == "Form" or doc.jasper_report_type == "Form":
			ids = data.get('ids')
			for id in ids:
				data['ids'] = [id]
				resps.append(self._run_report_async(path, doc, data=data, params=params, pformat=pformat, ncopies=ncopies))
		else:
			resps.append(self._run_report_async(path, doc, data=data, params=params, pformat=pformat, ncopies=ncopies))
		cresp = self.prepareCollectResponse(resps)
		cresp["origin"] = "server"
		return [cresp]
		"""
		cresp = self.prepare_report_async(path, doc, data=data, params=params, pformat=pformat, ncopies=ncopies, for_all_sites=0)
		cresp["origin"] = "server"
		return [cresp]

	#run reports with http POST and run async and sync
	def _run_report_async(self, path, doc, data=None, params=None, pformat="pdf", ncopies=1, for_all_sites=0):
		data = data or {}
		pram, pram_server, copies = self.do_params(data, params, pformat)
		pram_copy_index = copies.get("pram_copy_index", -1)
		pram_copy_page_index = copies.get("pram_copy_page_index", -1)
		resp = []

		pram.extend(self.get_param_hook(doc, data, pram_server))

		copies = [_("Original"), _("Duplicated"), _("Triplicate")]

		for m in range(ncopies):
			npram = copy.deepcopy(pram)
			if pram_copy_index != -1:
				values = pram[pram_copy_index].get("value","")
				print "values in range 3 {}".format(values)
				if not values or not values[0]:
					npram[pram_copy_index]['value'] = [copies[m]]
				else:
					npram[pram_copy_index]['value'] = [frappe.utils.strip(values[m], ' \t\n\r')]

			if pram_copy_page_index != -1:
				npram[pram_copy_page_index]['value'] = [str(m) + _(" of ") + str(ncopies)]
			result = self.run_async(path, npram, data.get("report_name"), pformat=pformat)
			if result:
				requestId = result.get('requestId')
				reqDbObj = {"data":{"result": result, "report_name": data.get("report_name"), "last_updated": frappe.utils.now(),'session_expiry': utils.get_expiry_period()}}
				self.insert_jasper_reqid_record(requestId, reqDbObj)
				resp.append({"requestId":requestId, "report_name": data.get("report_name"), "status": result.get('status')})
			else:
				frappe.msgprint(_("There was an error in report request."),raise_exception=True)
		#TODO get reqId for check report state
		return resp

	@_jasperserver
	def run_async(self, path, pram, report_name, pformat="pdf"):
		rs = ReportingService(self.session)
		rr = ReportExecutionRequest()
		rr.setReportUnitUri(path)
		rr.setOutputFormat(pformat)
		rr.setParameters(pram)
		rr.setAsync(True)
		if pformat == "html":
			rr.setAttachmentsPrefix("./images/")
			#rr.setIgnorePagination(True)
		try:
			req = rs.newReportExecutionRequest(rr)
			res = req.run().getResult("content")
			res = json.loads(res)
			resp = self.prepareResponse(res, res.get("requestId"))
			resp["status"] = res.get("status")
			return resp
		except Exception as e:
			if isinstance(e, Unauthorized):

				raise Unauthorized
			else:
				frappe.throw(_("Error in report %s, server error is: %s." % (self.doc.jasper_report_name, e)))

	@_jasperserver
	def polling(self, reqId):
		res = []
		try:
			rs = ReportingService(self.session)
			rexecreq = rs.reportExecutionRequest(reqId)
			status = rexecreq.status().content
			status = json.loads(status)
			if status.get("value") == "ready":
				detail = self.reportExecDetail(rexecreq)
				res = self.prepareResponse(detail, reqId)
				if res.get('status', "") == "not ready":
					res = self.prepareResponse({}, reqId)
				else:
					res["status"] = "ready"
			elif status.get("value") == "failed":
				frappe.throw(_("Error in report %s, server error is: %s." % (self.doc.jasper_report_name, status['errorDescriptor'].get('message'))))
			else:
				res = self.prepareResponse({}, reqId)
		except NotFound:
			frappe.throw(_("Not Found."))
		return res

	#rexecreq : ReportExecutionRequestBuilder instance
	#check if the report is ready and get the expids for ready reports
	def reportExecDetail(self, rexecreq):
		return rexecreq.executionDetails()

	#rs: ReportingService instance
	#get the report with reqId for the given expId
	@_jasperserver
	def getReport(self, reqId, expId):
		try:
			rs = ReportingService(self.session)
			rexecreq = rs.reportExecutionRequest(reqId, expId)
			report = rexecreq.export().outputResource().getResult("content")
			return report
		except NotFound:
			frappe.throw(_("Not Found."))

	def getAttachment(self, reqId, expId, attachmentId):
		try:
			rs = ReportingService(self.session)
			rexecreq = rs.reportExecutionRequest(reqId, expId)
			content = rexecreq.export().attachment(attachmentId)
			return content
		except NotFound:
			frappe.throw(_("Not Found."))
<|endoftext|>


<commit_before>#
# Jasy - JavaScript Tooling Refined
# Copyright 2010 Sebastian Werner
#

import logging, os, json, xml.etree.ElementTree
from jasy.core.File import *
from jasy.core.Profiler import *
import jasy.core.Info

def store(locale):
    Parser(locale).export()

class Parser():
    def __init__(self, locale):
        pstart()
        logging.info("Preparing locale %s" % locale)
        splits = locale.split("_")
        language = splits[0]
        territory = splits[1] if len(splits) > 1 else None

        self.__locale = locale
        self.__data = {}

        # Add info section

        self.__data["info"] = {
            "name" : locale,
            "language" : language,
            "territory" : territory
        }

        # Add keys
        path = "%s.xml" % os.path.join(jasy.core.Info.cldrData("keys"), language)
        tree = xml.etree.ElementTree.parse(path)
        self.__data["keys"] = {
            "short" : { key.get("type"): key.text for key in tree.findall("/keys/short/key") },
            "full" : { key.get("type"): key.text for key in tree.findall("/keys/full/key") }
        }

        # Add main CLDR data: Fallback chain for locales
        main = jasy.core.Info.cldrData("main")
        while True:
            path = "%s.xml" % os.path.join(main, locale)
            tree = xml.etree.ElementTree.parse(path)

            self.__addDisplayNames(tree)
            self.__addDelimiters(tree)
            self.__addCalendars(tree)
            self.__addNumbers(tree)

            if "_" in locale:
                locale = locale[:locale.rindex("_")]
            else:
                break

        # Add supplemental CLDR data
        supplemental = jasy.core.Info.cldrData("supplemental")

        pstop()

    def export(self):
        project = jasy.core.Info.localeProject(self.__locale)
        logging.info("Writing result...")
        pstart()

        writefile(os.path.join(project, "manifest.cfg"), "[main]\nname = cldr\nkind = basic\n")
        self.__exportRecurser(self.__data, "locale", project)
        pstop()

    def __exportRecurser(self, data, prefix, project):
        for key in data:
            value = data[key]
            name = "%s.%s" % (prefix, key)

            firstIsDict = False
            for childKey in value:
                if type(value[childKey]) == dict:
                    firstIsDict = True
                    break

            if firstIsDict:
                self.__exportRecurser(value, name, project)
            else:
                result = "// Automatically generated by Jasy\ndeclare(\"%s\", %s);" % (name, json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False))
                filename = "%s.js" % name.replace(".", os.path.sep)

                writefile(os.path.join(project, "src", filename), result)

    def __getStore(self, parent, name):
        """ Manages data fields """

        if not name in parent:
            store = {}
            parent[name] = store
        else:
            store = parent[name]

        return store

    def __addDisplayNames(self, tree):
        """ Adds CLDR display names section """

        display = self.__getStore(self.__data, "display")

        for key in ["languages", "scripts", "territories", "variants", "keys", "types", "measurementSystemNames", "codePatterns"]:
            # make it a little bit shorter, there is not really any conflict potential
            if key == "measurementSystemNames":
                store = self.__getStore(display, "measurement")
            else:
                store = self.__getStore(display, key)

            for element in tree.findall("/localeDisplayNames/%s/*" % key):
                if not element.get("draft"):
                    field = element.get("type")
                    if not field in store:
                        store[field] = element.text

    def __addDelimiters(self, tree):
        """ Adds CLDR delimiters """

        delimiters = self.__getStore(self.__data, "delimiters")

        for element in tree.findall("/delimiters/*"):
            if not element.get("draft"):
                field = element.tag
                if not field in delimiters:
                    delimiters[field] = element.text

    def __addCalendars(self, tree, key="dates/calendars"):
        """ Loops through all CLDR calendars and adds them """

        calendars = self.__getStore(self.__data, "calendars")

        for element in tree.findall("/%s/*" % key):
            if not element.get("draft"):
                self.__addCalendar(calendars, element)

    def __addCalendar(self, store, element):
        """ Adds data from a CLDR calendar section """

        calendar = self.__getStore(store, element.get("type"))

        # Months Widths
        if element.find("months/monthContext/monthWidth"):
            months = self.__getStore(calendar, "months")
            for child in element.findall("months/monthContext/monthWidth"):
                if not child.get("draft"):
                    format = child.get("type")
                    if not format in months:
                        months[format] = {}

                    for month in child.findall("month"):
                        if not month.get("draft"):
                            name = month.get("type")
                            if not name in months[format]:
                                months[format][name] = month.text

        # Day Widths
        if element.find("days/dayContext/dayWidth"):
            days = self.__getStore(calendar, "days")
            for child in element.findall("days/dayContext/dayWidth"):
                if not child.get("draft"):
                    format = child.get("type")
                    if not format in days:
                        days[format] = {}

                    for day in child.findall("day"):
                        if not day.get("draft"):
                            name = day.get("type")
                            if not name in days[format]:
                                days[format][name] = day.text

        # Quarter Widths
        if element.find("quarters/quarterContext/quarterWidth"):
            quarters = self.__getStore(calendar, "quarters")
            for child in element.findall("quarters/quarterContext/quarterWidth"):
                if not child.get("draft"):
                    format = child.get("type")
                    if not format in quarters:
                        quarters[format] = {}

                    for quarter in child.findall("quarter"):
                        if not quarter.get("draft"):
                            name = quarter.get("type")
                            if not name in quarters[format]:
                                quarters[format][name] = quarter.text

        # Date Formats
        if element.find("dateFormats/dateFormatLength"):
            dateFormats = self.__getStore(calendar, "dates")
            for child in element.findall("dateFormats/dateFormatLength"):
                if not child.get("draft"):
                    format = child.get("type")
                    text = child.find("dateFormat/pattern").text
                    if not format in dateFormats:
                        dateFormats[format] = text

        # Time Formats
        if element.find("timeFormats/timeFormatLength"):
            timeFormats = self.__getStore(calendar, "times")
            for child in element.findall("timeFormats/timeFormatLength"):
                if not child.get("draft"):
                    format = child.get("type")
                    text = child.find("timeFormat/pattern").text
                    if not format in timeFormats:
                        timeFormats[format] = text

        # Fields
        if element.find("fields/field"):
            fields = self.__getStore(calendar, "fields")
            for child in element.findall("fields/field"):
                if not child.get("draft"):
                    format = child.get("type")
                    for nameChild in child.findall("displayName"):
                        if not nameChild.get("draft"):
                            text = nameChild.text
                            if not format in fields:
                                fields[format] = text
                            break

        # Relative
        if element.find("fields/field"):
            relatives = self.__getStore(calendar, "relatives")
            for child in element.findall("fields/field"):
                if not child.get("draft"):
                    format = child.get("type")
                    if child.find("relative"):
                        relativeField = self.__getStore(relatives, format)
                        for relChild in child.findall("relative"):
                            if not relChild.get("draft"):
                                pos = relChild.get("type")
                                text = relChild.text
                                if not pos in relativeField:
                                    relativeField[pos] = text

    def __addNumbers(self, tree):
        store = self.__getStore(self.__data, "numbers")

        # Symbols
        symbols = self.__getStore(store, "symbols")
        for element in tree.findall("numbers/symbols/*"):
            if not element.get("draft"):
                field = element.tag
                if not field in store:
                    symbols[field] = element.text

        # Formats
        if not "format" in store:
            store["format"] = {}

        for format in ["decimal", "scientific", "percent", "currency"]:
            if not format in store["format"]:
                for element in tree.findall("numbers//%sFormat/pattern" % format):
                    store["format"][format] = element.text

        # Currencies
        currencies = self.__getStore(store, "currencies")
        for child in tree.findall("numbers/currencies/currency"):
            if not child.get("draft"):
                short = child.get("type")
                for nameChild in child.findall("displayName"):
                    if not nameChild.get("draft"):
                        text = nameChild.text
                        if not format in currencies:
                            currencies[short] = text
                        break
                <commit_msg>Changed some keys so singular. Added missing section datetimeFormat<commit_after>#
# Jasy - JavaScript Tooling Refined
# Copyright 2010 Sebastian Werner
#

import logging, os, json, xml.etree.ElementTree
from jasy.core.File import *
from jasy.core.Profiler import *
import jasy.core.Info

def store(locale):
    Parser(locale).export()

class Parser():
    def __init__(self, locale):
        pstart()
        logging.info("Preparing locale %s" % locale)
        splits = locale.split("_")
        language = splits[0]
        territory = splits[1] if len(splits) > 1 else None

        self.__locale = locale
        self.__data = {}

        # Add info section

        self.__data["info"] = {
            "name" : locale,
            "language" : language,
            "territory" : territory
        }

        # Add keys
        path = "%s.xml" % os.path.join(jasy.core.Info.cldrData("keys"), language)
        tree = xml.etree.ElementTree.parse(path)
        self.__data["keys"] = {
            "short" : { key.get("type"): key.text for key in tree.findall("/keys/short/key") },
            "full" : { key.get("type"): key.text for key in tree.findall("/keys/full/key") }
        }

        # Add main CLDR data: Fallback chain for locales
        main = jasy.core.Info.cldrData("main")
        while True:
            path = "%s.xml" % os.path.join(main, locale)
            tree = xml.etree.ElementTree.parse(path)

            self.__addDisplayNames(tree)
            self.__addDelimiters(tree)
            self.__addCalendars(tree)
            self.__addNumbers(tree)

            if "_" in locale:
                locale = locale[:locale.rindex("_")]
            else:
                break

        # Add supplemental CLDR data
        supplemental = jasy.core.Info.cldrData("supplemental")

        pstop()

    def export(self):
        project = jasy.core.Info.localeProject(self.__locale)
        logging.info("Writing result...")
        pstart()

        writefile(os.path.join(project, "manifest.cfg"), "[main]\nname = cldr\nkind = basic\n")
        self.__exportRecurser(self.__data, "locale", project)
        pstop()

    def __exportRecurser(self, data, prefix, project):
        for key in data:
            value = data[key]
            name = "%s.%s" % (prefix, key)

            firstIsDict = False
            for childKey in value:
                if type(value[childKey]) == dict:
                    firstIsDict = True
                    break

            if firstIsDict:
                self.__exportRecurser(value, name, project)
            else:
                result = "// Automatically generated by Jasy\ndeclare(\"%s\", %s);" % (name, json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False))
                filename = "%s.js" % name.replace(".", os.path.sep)

                writefile(os.path.join(project, "src", filename), result)

    def __getStore(self, parent, name):
        """ Manages data fields """

        if not name in parent:
            store = {}
            parent[name] = store
        else:
            store = parent[name]

        return store

    def __addDisplayNames(self, tree):
        """ Adds CLDR display names section """

        display = self.__getStore(self.__data, "display")

        for key in ["languages", "scripts", "territories", "variants", "keys", "types", "measurementSystemNames"]:
            # make it a little bit shorter, there is not really any conflict potential
            if key == "measurementSystemNames":
                store = self.__getStore(display, "measurement")
            else:
                # remove last character "s" to force singular
                store = self.__getStore(display, key[:-1])

            for element in tree.findall("/localeDisplayNames/%s/*" % key):
                if not element.get("draft"):
                    field = element.get("type")
                    if not field in store:
                        store[field] = element.text

    def __addDelimiters(self, tree):
        """ Adds CLDR delimiters """

        delimiters = self.__getStore(self.__data, "delimiter")

        for element in tree.findall("/delimiters/*"):
            if not element.get("draft"):
                field = element.tag
                if not field in delimiters:
                    delimiters[field] = element.text

    def __addCalendars(self, tree, key="dates/calendars"):
        """ Loops through all CLDR calendars and adds them """

        calendars = self.__getStore(self.__data, "calendar")

        for element in tree.findall("/%s/*" % key):
            if not element.get("draft"):
                self.__addCalendar(calendars, element)

    def __addCalendar(self, store, element):
        """ Adds data from a CLDR calendar section """

        calendar = self.__getStore(store, element.get("type"))

        # Months Widths
        if element.find("months/monthContext/monthWidth"):
            months = self.__getStore(calendar, "month")
            for child in element.findall("months/monthContext/monthWidth"):
                if not child.get("draft"):
                    format = child.get("type")
                    if not format in months:
                        months[format] = {}

                    for month in child.findall("month"):
                        if not month.get("draft"):
                            name = month.get("type")
                            if not name in months[format]:
                                months[format][name] = month.text

        # Day Widths
        if element.find("days/dayContext/dayWidth"):
            days = self.__getStore(calendar, "day")
            for child in element.findall("days/dayContext/dayWidth"):
                if not child.get("draft"):
                    format = child.get("type")
                    if not format in days:
                        days[format] = {}

                    for day in child.findall("day"):
                        if not day.get("draft"):
                            name = day.get("type")
                            if not name in days[format]:
                                days[format][name] = day.text

        # Quarter Widths
        if element.find("quarters/quarterContext/quarterWidth"):
            quarters = self.__getStore(calendar, "quarter")
            for child in element.findall("quarters/quarterContext/quarterWidth"):
                if not child.get("draft"):
                    format = child.get("type")
                    if not format in quarters:
                        quarters[format] = {}

                    for quarter in child.findall("quarter"):
                        if not quarter.get("draft"):
                            name = quarter.get("type")
                            if not name in quarters[format]:
                                quarters[format][name] = quarter.text

        # Date Formats
        if element.find("dateFormats/dateFormatLength"):
            dateFormats = self.__getStore(calendar, "date")
            for child in element.findall("dateFormats/dateFormatLength"):
                if not child.get("draft"):
                    format = child.get("type")
                    text = child.find("dateFormat/pattern").text
                    if not format in dateFormats:
                        dateFormats[format] = text

        # Time Formats
        if element.find("timeFormats/timeFormatLength"):
            timeFormats = self.__getStore(calendar, "time")
            for child in element.findall("timeFormats/timeFormatLength"):
                if not child.get("draft"):
                    format = child.get("type")
                    text = child.find("timeFormat/pattern").text
                    if not format in timeFormats:
                        timeFormats[format] = text

        # DateTime Formats
        if element.find("dateTimeFormats/availableFormats"):
            datetime = self.__getStore(calendar, "datetime")
            for child in element.findall("dateTimeFormats/availableFormats/dateFormatItem"):
                if not child.get("draft"):
                    format = child.get("id")
                    text = child.text
                    if not format in datetime:
                        datetime[format] = text

        # Fields
        if element.find("fields/field"):
            fields = self.__getStore(calendar, "field")
            for child in element.findall("fields/field"):
                if not child.get("draft"):
                    format = child.get("type")
                    for nameChild in child.findall("displayName"):
                        if not nameChild.get("draft"):
                            text = nameChild.text
                            if not format in fields:
                                fields[format] = text
                            break

        # Relative
        if element.find("fields/field"):
            relatives = self.__getStore(calendar, "relative")
            for child in element.findall("fields/field"):
                if not child.get("draft"):
                    format = child.get("type")
                    if child.find("relative"):
                        relativeField = self.__getStore(relatives, format)
                        for relChild in child.findall("relative"):
                            if not relChild.get("draft"):
                                pos = relChild.get("type")
                                text = relChild.text
                                if not pos in relativeField:
                                    relativeField[pos] = text

    def __addNumbers(self, tree):
        store = self.__getStore(self.__data, "numbers")

        # Symbols
        symbols = self.__getStore(store, "symbol")
        for element in tree.findall("numbers/symbol/*"):
            if not element.get("draft"):
                field = element.tag
                if not field in store:
                    symbols[field] = element.text

        # Formats
        if not "format" in store:
            store["format"] = {}

        for format in ["decimal", "scientific", "percent", "currency"]:
            if not format in store["format"]:
                for element in tree.findall("numbers//%sFormat/pattern" % format):
                    store["format"][format] = element.text

        # Currencies
        currencies = self.__getStore(store, "currency")
        for child in tree.findall("numbers/currencies/currency"):
            if not child.get("draft"):
                short = child.get("type")
                for nameChild in child.findall("displayName"):
                    if not nameChild.get("draft"):
                        text = nameChild.text
                        if not format in currencies:
                            currencies[short] = text
                        break
                <|endoftext|>


<commit_before># Chord classification
#
# The task is to classify chords (or more precisely pitch class sets) based on chromagram features.
#
# We use a the whole Beatles dataset (ie. many songs).
#
# The task is in fact multilabel classification, since each pitch class is generally independent.

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import arrow
import os
import scipy.signal
import scipy.misc

from sklearn.metrics import hamming_loss, accuracy_score

from keras.models import Sequential
from keras.layers.core import Dense, Activation, Flatten, Dropout
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers.normalization import BatchNormalization
from keras.callbacks import ModelCheckpoint

## Data loading

dataset_file = 'data/beatles/ml_dataset/chromagram_block=4096_hop=2048_bins=-48,67_div=1/dataset.npz'
dataset = np.load(dataset_file)

X_train, Y_train, X_valid, Y_valid, X_test, Y_test = \
    dataset['X_train'], dataset['Y_train'], \
    dataset['X_valid'], dataset['Y_valid'], \
    dataset['X_test'], dataset['Y_test']

## Data preprocessing

### Features

# scaler = MinMaxScaler()
# X = scaler.fit_transform(features).astype('float32')

# let's rescale the features manually so that the're the same in all songs
# the range (in dB) is -120 to X.shape[1] (= 115)
def normalize(X):
    return (X.astype('float32') - 120) / (X.shape[1] - 120)

X_train = normalize(X_train)
X_valid = normalize(X_valid)
X_test = normalize(X_test)

for d in [X_train, X_valid, X_test, Y_train, Y_valid, Y_test]:
    print(d.shape)

# reshape for 1D convolution
def conv_reshape(X):
    return X.reshape(X.shape[0], X.shape[1], 1)

X_conv_train = conv_reshape(X_train)
X_conv_valid = conv_reshape(X_valid)
X_conv_test = conv_reshape(X_test)

## Model training and evaluation

model_dir = 'data/beatles/models'
os.makedirs(model_dir, exist_ok=True)

def new_model_id():
    return 'model_%s' % arrow.get().format('YYYY-MM-DD-HH-mm-ss')

def save_model_arch(model_id, model):
    arch_file = '%s/%s_arch.yaml' % (model_dir, model_id)
    print('architecture:', arch_file)
    open(arch_file, 'w').write(model.to_yaml())

def weights_file(model_id):
    return '%s/%s_weights.h5' % (model_dir, model_id)

def report_model_parameters(model):
    print('number of parameters:', model.count_params())
    print('weights:', [w.shape for w in model.get_weights()])

# #### Notes
#
# - the last layer has to be sigmoid, not softmax
#   - since each output label should be independent a multiple can be active at the same time
# - very sparse inputs can easily saturate sigmoid activation if it's near the first layer
# - class_mode='binary' for multi-label classification
# - predict_classes() then returns a binary vector
# - loss: MAE or binary_crossentropy?
#   - why binary_crossentropy gives worse accuracy than MAE?
#   - binary_crossentropy works ok
# - problems with loss going to NAN after the first training iteration
#   - optimizer clipnorm doesn't help
#   - BatchNormalization doesn't help
#     - BatchNormalization between convolution and activation works
# - BatchNormalization might be useful
# - be aware to use scaled inputs, not raw ones

model_id = new_model_id()
print('model id:', model_id)

model = Sequential()

model.add(Convolution1D(10, 3, input_shape=(X_train.shape[1], 1)))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Convolution1D(10, 3))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(MaxPooling1D(2, 2))

model.add(Flatten())

model.add(Dense(20))
model.add(Activation('relu'))

model.add(Dense(20))
model.add(Activation('relu'))

model.add(Dense(12))
model.add(Activation('sigmoid'))

report_model_parameters(model)

print('compiling the model')
model.compile(class_mode='binary', loss='binary_crossentropy', optimizer='adam')

save_model_arch(model_id, model)

print('training the model')
checkpointer = ModelCheckpoint(filepath=weights_file(model_id), verbose=1, save_best_only=True)

training_hist = model.fit(X_conv_train, Y_train, nb_epoch=10, callbacks=[checkpointer])

def report_training_curve(training_hist):
    losses = training_hist.history['loss']
    print('last loss:', losses[-1])
    plt.figure()
    plt.plot(losses)
    plt.xlabel('epochs')
    plt.ylabel('training loss')
    plt.title('%s - training curve' % model_id)
    plt.suptitle('last loss: %s' % losses[-1])
    plt.savefig(model_dir+'/'+model_id+'_training_losses.png')
    pd.DataFrame({'training_loss': losses}).to_csv(model_dir+'/'+model_id+'_training_losses.tsv', index=None)

report_training_curve(training_hist)

def model_report_multilabel(model_predict, X_train, Y_train, X_valid, Y_valid):
    def report_dataset(X, y_true, title):
        y_pred = model_predict(X)
        print(title + ' accuracy (exatch match):', accuracy_score(y_true, y_pred))
        print(title + ' hamming score (non-exatch match):', 1 - hamming_loss(y_true, y_pred))

    report_dataset(X_train, Y_train, 'training')
    report_dataset(X_valid, Y_valid, 'validation')

model_report_multilabel(model.predict_classes, X_conv_train, Y_train, X_conv_valid, Y_valid)

# visualization

def plot_labels(l, title, fifths=False, resample=True, exact=False):
    if fifths:
        l = l[:,np.arange(12)*7 % 12]
    l = l.T

    file = model_dir+'/'+model_id+'_'+title+'.png'

    if exact:
        scipy.misc.imsave(file, l)
    else:
        if resample:
            l = scipy.signal.resample(l, 200, axis=1)
        plt.figure(figsize=(20, 2))
        plt.imshow(l, cmap='gray', interpolation='none')
        plt.tight_layout()
        plt.savefig(file)

# # true labels
# plot_labels(labels_pcs, 'true')
# plot_labels(labels_pcs, 'exact_true', exact=True)
#
# # predicted labels
# labels_pred_full = model.predict_classes(conv_reshape(X))
# plot_labels(labels_pred_full, 'pred')
# plot_labels(labels_pred_full, 'exact_pred', exact=True)
#
# # difference
# plot_labels(labels_pcs - labels_pred_full, 'diff')
# plot_labels(labels_pcs - labels_pred_full, 'exact_diff', exact=True)

# plot_labels(labels_pred_full[:100], resample=False)
# plot_labels(labels_pcs[:100] - labels_pred_full[:100], resample=False)

# in case of input features with original time order we can apply median filter:
# medfilt(labels_pred_full, (15, 1))

def plot_labels_true_pred_diff():
    def plot2d(x):
        plt.imshow(scipy.signal.resample(x.T, 200, axis=1), cmap='gray', interpolation='none')
    plt.figure(figsize=(20, 6))
    ax = plt.subplot(3,1,1)
    plot2d(labels_pcs)
    ax.set_title('true')
    ax = plt.subplot(3,1,2)
    plot2d(labels_pred_full)
    ax.set_title('predicted')
    ax = plt.subplot(3,1,3)
    plot2d(labels_pred_full - labels_pcs)
    ax.set_title('difference')
    plt.tight_layout()
<commit_msg>Choose a MPL backend that doesn't need X.<commit_after># Chord classification
#
# The task is to classify chords (or more precisely pitch class sets) based on chromagram features.
#
# We use a the whole Beatles dataset (ie. many songs).
#
# The task is in fact multilabel classification, since each pitch class is generally independent.

import numpy as np
import pandas as pd
import matplotlib as mpl
# do not use Qt/X that require $DISPLAY
mpl.use('Agg')
import matplotlib.pyplot as plt
import arrow
import os
import scipy.signal
import scipy.misc

from sklearn.metrics import hamming_loss, accuracy_score

from keras.models import Sequential
from keras.layers.core import Dense, Activation, Flatten, Dropout
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers.normalization import BatchNormalization
from keras.callbacks import ModelCheckpoint

## Data loading

dataset_file = 'data/beatles/ml_dataset/chromagram_block=4096_hop=2048_bins=-48,67_div=1/dataset.npz'
dataset = np.load(dataset_file)

X_train, Y_train, X_valid, Y_valid, X_test, Y_test = \
    dataset['X_train'], dataset['Y_train'], \
    dataset['X_valid'], dataset['Y_valid'], \
    dataset['X_test'], dataset['Y_test']

## Data preprocessing

### Features

# scaler = MinMaxScaler()
# X = scaler.fit_transform(features).astype('float32')

# let's rescale the features manually so that the're the same in all songs
# the range (in dB) is -120 to X.shape[1] (= 115)
def normalize(X):
    return (X.astype('float32') - 120) / (X.shape[1] - 120)

X_train = normalize(X_train)
X_valid = normalize(X_valid)
X_test = normalize(X_test)

for d in [X_train, X_valid, X_test, Y_train, Y_valid, Y_test]:
    print(d.shape)

# reshape for 1D convolution
def conv_reshape(X):
    return X.reshape(X.shape[0], X.shape[1], 1)

X_conv_train = conv_reshape(X_train)
X_conv_valid = conv_reshape(X_valid)
X_conv_test = conv_reshape(X_test)

## Model training and evaluation

model_dir = 'data/beatles/models'
os.makedirs(model_dir, exist_ok=True)

def new_model_id():
    return 'model_%s' % arrow.get().format('YYYY-MM-DD-HH-mm-ss')

def save_model_arch(model_id, model):
    arch_file = '%s/%s_arch.yaml' % (model_dir, model_id)
    print('architecture:', arch_file)
    open(arch_file, 'w').write(model.to_yaml())

def weights_file(model_id):
    return '%s/%s_weights.h5' % (model_dir, model_id)

def report_model_parameters(model):
    print('number of parameters:', model.count_params())
    print('weights:', [w.shape for w in model.get_weights()])

# #### Notes
#
# - the last layer has to be sigmoid, not softmax
#   - since each output label should be independent a multiple can be active at the same time
# - very sparse inputs can easily saturate sigmoid activation if it's near the first layer
# - class_mode='binary' for multi-label classification
# - predict_classes() then returns a binary vector
# - loss: MAE or binary_crossentropy?
#   - why binary_crossentropy gives worse accuracy than MAE?
#   - binary_crossentropy works ok
# - problems with loss going to NAN after the first training iteration
#   - optimizer clipnorm doesn't help
#   - BatchNormalization doesn't help
#     - BatchNormalization between convolution and activation works
# - BatchNormalization might be useful
# - be aware to use scaled inputs, not raw ones

model_id = new_model_id()
print('model id:', model_id)

model = Sequential()

model.add(Convolution1D(10, 3, input_shape=(X_train.shape[1], 1)))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Convolution1D(10, 3))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(MaxPooling1D(2, 2))

model.add(Flatten())

model.add(Dense(20))
model.add(Activation('relu'))

model.add(Dense(20))
model.add(Activation('relu'))

model.add(Dense(12))
model.add(Activation('sigmoid'))

report_model_parameters(model)

print('compiling the model')
model.compile(class_mode='binary', loss='binary_crossentropy', optimizer='adam')

save_model_arch(model_id, model)

print('training the model')
checkpointer = ModelCheckpoint(filepath=weights_file(model_id), verbose=1, save_best_only=True)

training_hist = model.fit(X_conv_train, Y_train, nb_epoch=10, callbacks=[checkpointer])

def report_training_curve(training_hist):
    losses = training_hist.history['loss']
    print('last loss:', losses[-1])
    plt.figure()
    plt.plot(losses)
    plt.xlabel('epochs')
    plt.ylabel('training loss')
    plt.title('%s - training curve' % model_id)
    plt.suptitle('last loss: %s' % losses[-1])
    plt.savefig(model_dir+'/'+model_id+'_training_losses.png')
    pd.DataFrame({'training_loss': losses}).to_csv(model_dir+'/'+model_id+'_training_losses.tsv', index=None)

report_training_curve(training_hist)

def model_report_multilabel(model_predict, X_train, Y_train, X_valid, Y_valid):
    def report_dataset(X, y_true, title):
        y_pred = model_predict(X)
        print(title + ' accuracy (exatch match):', accuracy_score(y_true, y_pred))
        print(title + ' hamming score (non-exatch match):', 1 - hamming_loss(y_true, y_pred))

    report_dataset(X_train, Y_train, 'training')
    report_dataset(X_valid, Y_valid, 'validation')

model_report_multilabel(model.predict_classes, X_conv_train, Y_train, X_conv_valid, Y_valid)

# visualization

def plot_labels(l, title, fifths=False, resample=True, exact=False):
    if fifths:
        l = l[:,np.arange(12)*7 % 12]
    l = l.T

    file = model_dir+'/'+model_id+'_'+title+'.png'

    if exact:
        scipy.misc.imsave(file, l)
    else:
        if resample:
            l = scipy.signal.resample(l, 200, axis=1)
        plt.figure(figsize=(20, 2))
        plt.imshow(l, cmap='gray', interpolation='none')
        plt.tight_layout()
        plt.savefig(file)

# # true labels
# plot_labels(labels_pcs, 'true')
# plot_labels(labels_pcs, 'exact_true', exact=True)
#
# # predicted labels
# labels_pred_full = model.predict_classes(conv_reshape(X))
# plot_labels(labels_pred_full, 'pred')
# plot_labels(labels_pred_full, 'exact_pred', exact=True)
#
# # difference
# plot_labels(labels_pcs - labels_pred_full, 'diff')
# plot_labels(labels_pcs - labels_pred_full, 'exact_diff', exact=True)

# plot_labels(labels_pred_full[:100], resample=False)
# plot_labels(labels_pcs[:100] - labels_pred_full[:100], resample=False)

# in case of input features with original time order we can apply median filter:
# medfilt(labels_pred_full, (15, 1))

def plot_labels_true_pred_diff():
    def plot2d(x):
        plt.imshow(scipy.signal.resample(x.T, 200, axis=1), cmap='gray', interpolation='none')
    plt.figure(figsize=(20, 6))
    ax = plt.subplot(3,1,1)
    plot2d(labels_pcs)
    ax.set_title('true')
    ax = plt.subplot(3,1,2)
    plot2d(labels_pred_full)
    ax.set_title('predicted')
    ax = plt.subplot(3,1,3)
    plot2d(labels_pred_full - labels_pcs)
    ax.set_title('difference')
    plt.tight_layout()
<|endoftext|>


<commit_before>"""
Container for fieldlines in three-dimensional magnetic skeleton
"""
import numpy as np
from scipy.interpolate import splev, splprep, interp1d
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.utils.console import ProgressBar
import asdf
import zarr

from synthesizAR import Loop
from synthesizAR.visualize import plot_fieldlines
from synthesizAR.atomic import Element

class Skeleton(object):
    """
    Construct magnetic field skeleton fieldlines

    Parameters
    ----------
    loops : `list`
        List of `Loop` objects

    Examples
    --------
    >>> import synthesizAR
    >>> import astropy.units as u
    >>> loop = synthesizAR.Loop('loop', SkyCoord(x=[1,4]*u.Mm, y=[2,5]*u.Mm, z=[3,6]*u.Mm,frame='heliographic_stonyhurst', representation_type='cartesian'), [1e2,1e3] * u.G)
    >>> field = synthesizAR.Skeleton([loop,])
    """

    def __init__(self, loops):
        self.loops = loops

    @classmethod
    def from_coordinates(cls, coordinates, field_strengths):
        """
        Construct `Skeleton` from list of coordinates and field strengths

        Parameters
        ----------
        coordinates : `list`
            List of `~astropy.coordinates.SkyCoord` loop coordinates
        field_strengths : `list`
            List of `~astropy.units.Quantity` scalar magnetic field strength along the loop
        """
        loops = []
        for i, (coord, mag) in enumerate(zip(coordinates, field_strengths)):
            loops.append(Loop(f'loop{i:06d}', coord, mag))
        return cls(loops)

    def __repr__(self):
        return f'''synthesizAR Skeleton Object
------------------------
Number of loops: {len(self.loops)}'''

    def to_asdf(self, filename):
        """
        Serialize this instance of `Skeleton` to an ASDF file
        """
        tree = {}
        for l in self.loops:
            tree[l.name] = {
                'field_strength': l.field_strength,
                'coordinate': l.coordinate,
                'model_results_filename': l.model_results_filename,
            }
        with asdf.AsdfFile(tree) as asdf_file:
            asdf_file.write_to(filename)

    @classmethod
    def from_asdf(cls, filename):
        """
        Restore a `Skeleton` instance from an ASDF file

        Examples
        --------
        >>> import synthesizAR
        >>> restored_field = synthesizAR.Skeleton.from_asdf('/path/to/skeleton.asdf') # doctest: +SKIP
        """
        exclude_keys = ['asdf_library', 'history']
        loops = []
        with asdf.open(filename, mode='r', copy_arrays=True) as af:
            for k in af.keys():
                if k in exclude_keys:
                    continue
                model_results_filename = af.tree[k].get('model_results_filename', None)
                loops.append(Loop(
                    k,
                    SkyCoord(af.tree[k]['coordinate']),
                    af.tree[k]['field_strength'],
                    model_results_filename=model_results_filename,
                ))
        return cls(loops)

    @u.quantity_input
    def refine_loops(self, delta_s: u.cm):
        """
        Interpolate loop coordinates and field strengths to a specified spatial resolution
        and return a new `Skeleton` object.

        This can be important in order to ensure that an adequate number of points are used
        to represent each fieldline when binning intensities onto the instrument grid.
        """
        new_loops = []
        for l in self.loops:
            tck, _ = splprep(l.coordinate.cartesian.xyz.value, u=l.field_aligned_coordinate_norm)
            new_s = np.arange(0, l.length.to(u.Mm).value, delta_s.to(u.Mm).value) * u.Mm
            x, y, z = splev((new_s/l.length).decompose(), tck)
            unit = l.coordinate.cartesian.xyz.unit
            new_coord = SkyCoord(x=x*unit, y=y*unit, z=z*unit, frame=l.coordinate.frame,
                                 representation_type=l.coordinate.representation_type)
            f_B = interp1d(l.field_aligned_coordinate.to(u.Mm), l.field_strength)
            new_field_strength = f_B(new_s.to(u.Mm)) * l.field_strength.unit
            new_loops.append(Loop(l.name, new_coord, new_field_strength,
                                  model_results_filename=l.model_results_filename))

        return Skeleton(new_loops)

    @property
    def all_coordinates(self):
        """
        Coordinates for all loops in the skeleton.

        .. note:: This should be treated as a collection of points and NOT a
                  continuous structure.
        """
        return SkyCoord([l.coordinate for l in self.loops],
                        frame=self.loops[0].coordinate.frame,
                        representation_type=self.loops[0].coordinate.representation_type)

    @property
    def all_coordinates_centers(self):
        """
        Coordinates for all grid cell centers of all loops in the skeleton

        .. note:: This should be treated as a collection of points and NOT a
                  continuous structure.
        """
        return SkyCoord([l.coordinate_center for l in self.loops],
                        frame=self.loops[0].coordinate_center.frame,
                        representation_type=self.loops[0].coordinate_center.representation_type)

    def interpolate_loop_coordinates(self):
        """
        Interpolate all loop coordinates to ensure each is represented by a suitable number
        of points.
        """
        raise NotImplementedError

    def peek(self, **kwargs):
        """
        Plot loop coordinates on the solar disk.

        See Also
        --------
        synthesizAR.visualize.plot_fieldlines
        """
        plot_fieldlines(*[_.coordinate for _ in self.loops], **kwargs)

    def configure_loop_simulations(self, interface, **kwargs):
        """
        Configure hydrodynamic simulations for each loop object
        """
        with ProgressBar(len(self.loops), ipython_widget=kwargs.get('notebook', True)) as progress:
            for loop in self.loops:
                interface.configure_input(loop)
                progress.update()

    def load_loop_simulations(self, interface, filename, **kwargs):
        """
        Load in loop parameters from hydrodynamic results.
        """
        root = zarr.open(store=filename, mode='w', **kwargs)
        with ProgressBar(len(self.loops)) as progress:
            for loop in self.loops:
                loop.model_results_filename = filename
                # Load in parameters from interface
                (time, electron_temperature, ion_temperature,
                    density, velocity) = interface.load_results(loop)
                # Convert velocity to loop coordinate system
                # NOTE: the direction is evaluated at the left edges + the last right edge.
                # But the velocity is evaluated at the center of each cell so we need
                # to interpolate the direction to the cell centers for each component
                s = loop.field_aligned_coordinate.to(u.Mm).value
                s_center = loop.field_aligned_coordinate_center.to(u.Mm).value
                s_hat = loop.coordinate_direction
                velocity_x = velocity * interp1d(s, s_hat[0, :])(s_center)
                velocity_y = velocity * interp1d(s, s_hat[1, :])(s_center)
                velocity_z = velocity * interp1d(s, s_hat[2, :])(s_center)
                # Write to file
                grp = root.create_group(loop.name)
                grp.attrs['simulation_type'] = interface.name
                # time
                dset_time = grp.create_dataset('time', data=time.value)
                dset_time.attrs['unit'] = time.unit.to_string()
                # NOTE: Set the chunk size such that accessing all entries for a given timestep
                # is the most efficient pattern.
                chunks = (None,) + s_center.shape
                # electron temperature
                dset_electron_temperature = grp.create_dataset(
                    'electron_temperature', data=electron_temperature.value, chunks=chunks)
                dset_electron_temperature.attrs['unit'] = electron_temperature.unit.to_string()
                # ion temperature
                dset_ion_temperature = grp.create_dataset(
                    'ion_temperature', data=ion_temperature.value, chunks=chunks)
                dset_ion_temperature.attrs['unit'] = ion_temperature.unit.to_string()
                # number density
                dset_density = grp.create_dataset('density', data=density.value, chunks=chunks)
                dset_density.attrs['unit'] = density.unit.to_string()
                # field-aligned velocity
                dset_velocity = grp.create_dataset('velocity', data=velocity.value, chunks=chunks)
                dset_velocity.attrs['unit'] = velocity.unit.to_string()
                dset_velocity.attrs['note'] = 'Velocity in the field-aligned direction'
                # Cartesian xyz velocity
                dset_velocity_x = grp.create_dataset(
                    'velocity_x', data=velocity_x.value, chunks=chunks)
                dset_velocity_x.attrs['unit'] = velocity_x.unit.to_string()
                dset_velocity_x.attrs['note'] = 'x-component of velocity in HEEQ coordinates'
                dset_velocity_y = grp.create_dataset(
                    'velocity_y', data=velocity_y.value, chunks=chunks)
                dset_velocity_y.attrs['unit'] = velocity_y.unit.to_string()
                dset_velocity_y.attrs['note'] = 'y-component of velocity in HEEQ coordinates'
                dset_velocity_z = grp.create_dataset(
                    'velocity_z', data=velocity_z.value, chunks=chunks)
                dset_velocity_z.attrs['unit'] = velocity_z.unit.to_string()
                dset_velocity_z.attrs['note'] = 'z-component of velocity in HEEQ coordinates'
                progress.update()

    def load_ionization_fractions(self, emission_model, interface=None, **kwargs):
        """
        Load the ionization fractions for each ion in the emission model.

        Parameters
        ----------
        emission_model : `synthesizAR.atomic.EmissionModel`
        interface : optional
            A model interface. Only necessary if loading the ionization fractions
            from the model

        If the model interface provides a method for loading the population fraction
        from the model, use that to get the population fractions. Otherwise, compute
        the ion population fractions in equilibrium. This should be done after
        calling `load_loop_simulations`.
        """
        root = zarr.open(store=self.loops[0].model_results_filename, mode='a', **kwargs)
        # Check if we can load from the model
        FROM_MODEL = False
        if interface is not None and hasattr(interface, 'load_ionization_fraction'):
            frac = interface.load_ionization_fraction(self.loops[0], emission_model[0])
            # Some models may optionally output the ionization fractions such that
            # they will have this method, but it may not return anything
            if frac is not None:
                FROM_MODEL = True
        # Get the unique elements from all of our ions
        elements = list(set([ion.element_name for ion in emission_model]))
        elements = [Element(e, emission_model.temperature) for e in elements]
        for el in elements:
            ions = [i for i in emission_model if i.element_name == el.element_name]
            if not FROM_MODEL:
                ioneq = el.equilibrium_ionization()
            for loop in self.loops:
                chunks = (None,) + loop.field_aligned_coordinate_center.shape
                if not FROM_MODEL:
                    frac_el = interp1d(el.temperature,
                                       ioneq,
                                       axis=0,
                                       kind='linear',
                                       fill_value='extrapolate')(loop.electron_temperature)
                if 'ionization_fraction' in root[loop.name]:
                    grp = root[f'{loop.name}/ionization_fraction']
                else:
                    grp = root[loop.name].create_group('ionization_fraction')
                for ion in ions:
                    if FROM_MODEL:
                        frac = interface.load_ionization_fraction(loop, ion)
                        desc = f'Ionization fraction of {ion.ion_name} as computed by {interface.name}'
                    else:
                        frac = frac_el[:, :, ion.charge_state]
                        desc = f'Ionization fraction of {ion.ion_name} in equilibrium.'
                    dset = grp.create_dataset(f'{ion.ion_name}', data=frac, chunks=chunks)
                    dset.attrs['unit'] = ''
                    dset.attrs['description'] = desc
<commit_msg>remove duplicate function stub<commit_after>"""
Container for fieldlines in three-dimensional magnetic skeleton
"""
import numpy as np
from scipy.interpolate import splev, splprep, interp1d
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.utils.console import ProgressBar
import asdf
import zarr

from synthesizAR import Loop
from synthesizAR.visualize import plot_fieldlines
from synthesizAR.atomic import Element

class Skeleton(object):
    """
    Construct magnetic field skeleton fieldlines

    Parameters
    ----------
    loops : `list`
        List of `Loop` objects

    Examples
    --------
    >>> import synthesizAR
    >>> import astropy.units as u
    >>> loop = synthesizAR.Loop('loop', SkyCoord(x=[1,4]*u.Mm, y=[2,5]*u.Mm, z=[3,6]*u.Mm,frame='heliographic_stonyhurst', representation_type='cartesian'), [1e2,1e3] * u.G)
    >>> field = synthesizAR.Skeleton([loop,])
    """

    def __init__(self, loops):
        self.loops = loops

    @classmethod
    def from_coordinates(cls, coordinates, field_strengths):
        """
        Construct `Skeleton` from list of coordinates and field strengths

        Parameters
        ----------
        coordinates : `list`
            List of `~astropy.coordinates.SkyCoord` loop coordinates
        field_strengths : `list`
            List of `~astropy.units.Quantity` scalar magnetic field strength along the loop
        """
        loops = []
        for i, (coord, mag) in enumerate(zip(coordinates, field_strengths)):
            loops.append(Loop(f'loop{i:06d}', coord, mag))
        return cls(loops)

    def __repr__(self):
        return f'''synthesizAR Skeleton Object
------------------------
Number of loops: {len(self.loops)}'''

    def to_asdf(self, filename):
        """
        Serialize this instance of `Skeleton` to an ASDF file
        """
        tree = {}
        for l in self.loops:
            tree[l.name] = {
                'field_strength': l.field_strength,
                'coordinate': l.coordinate,
                'model_results_filename': l.model_results_filename,
            }
        with asdf.AsdfFile(tree) as asdf_file:
            asdf_file.write_to(filename)

    @classmethod
    def from_asdf(cls, filename):
        """
        Restore a `Skeleton` instance from an ASDF file

        Examples
        --------
        >>> import synthesizAR
        >>> restored_field = synthesizAR.Skeleton.from_asdf('/path/to/skeleton.asdf') # doctest: +SKIP
        """
        exclude_keys = ['asdf_library', 'history']
        loops = []
        with asdf.open(filename, mode='r', copy_arrays=True) as af:
            for k in af.keys():
                if k in exclude_keys:
                    continue
                model_results_filename = af.tree[k].get('model_results_filename', None)
                loops.append(Loop(
                    k,
                    SkyCoord(af.tree[k]['coordinate']),
                    af.tree[k]['field_strength'],
                    model_results_filename=model_results_filename,
                ))
        return cls(loops)

    @u.quantity_input
    def refine_loops(self, delta_s: u.cm):
        """
        Interpolate loop coordinates and field strengths to a specified spatial resolution
        and return a new `Skeleton` object.

        This can be important in order to ensure that an adequate number of points are used
        to represent each fieldline when binning intensities onto the instrument grid.
        """
        new_loops = []
        for l in self.loops:
            tck, _ = splprep(l.coordinate.cartesian.xyz.value, u=l.field_aligned_coordinate_norm)
            new_s = np.arange(0, l.length.to(u.Mm).value, delta_s.to(u.Mm).value) * u.Mm
            x, y, z = splev((new_s/l.length).decompose(), tck)
            unit = l.coordinate.cartesian.xyz.unit
            new_coord = SkyCoord(x=x*unit, y=y*unit, z=z*unit, frame=l.coordinate.frame,
                                 representation_type=l.coordinate.representation_type)
            f_B = interp1d(l.field_aligned_coordinate.to(u.Mm), l.field_strength)
            new_field_strength = f_B(new_s.to(u.Mm)) * l.field_strength.unit
            new_loops.append(Loop(l.name, new_coord, new_field_strength,
                                  model_results_filename=l.model_results_filename))

        return Skeleton(new_loops)

    @property
    def all_coordinates(self):
        """
        Coordinates for all loops in the skeleton.

        .. note:: This should be treated as a collection of points and NOT a
                  continuous structure.
        """
        return SkyCoord([l.coordinate for l in self.loops],
                        frame=self.loops[0].coordinate.frame,
                        representation_type=self.loops[0].coordinate.representation_type)

    @property
    def all_coordinates_centers(self):
        """
        Coordinates for all grid cell centers of all loops in the skeleton

        .. note:: This should be treated as a collection of points and NOT a
                  continuous structure.
        """
        return SkyCoord([l.coordinate_center for l in self.loops],
                        frame=self.loops[0].coordinate_center.frame,
                        representation_type=self.loops[0].coordinate_center.representation_type)

    def peek(self, **kwargs):
        """
        Plot loop coordinates on the solar disk.

        See Also
        --------
        synthesizAR.visualize.plot_fieldlines
        """
        plot_fieldlines(*[_.coordinate for _ in self.loops], **kwargs)

    def configure_loop_simulations(self, interface, **kwargs):
        """
        Configure hydrodynamic simulations for each loop object
        """
        with ProgressBar(len(self.loops), ipython_widget=kwargs.get('notebook', True)) as progress:
            for loop in self.loops:
                interface.configure_input(loop)
                progress.update()

    def load_loop_simulations(self, interface, filename, **kwargs):
        """
        Load in loop parameters from hydrodynamic results.
        """
        root = zarr.open(store=filename, mode='w', **kwargs)
        with ProgressBar(len(self.loops)) as progress:
            for loop in self.loops:
                loop.model_results_filename = filename
                # Load in parameters from interface
                (time, electron_temperature, ion_temperature,
                    density, velocity) = interface.load_results(loop)
                # Convert velocity to loop coordinate system
                # NOTE: the direction is evaluated at the left edges + the last right edge.
                # But the velocity is evaluated at the center of each cell so we need
                # to interpolate the direction to the cell centers for each component
                s = loop.field_aligned_coordinate.to(u.Mm).value
                s_center = loop.field_aligned_coordinate_center.to(u.Mm).value
                s_hat = loop.coordinate_direction
                velocity_x = velocity * interp1d(s, s_hat[0, :])(s_center)
                velocity_y = velocity * interp1d(s, s_hat[1, :])(s_center)
                velocity_z = velocity * interp1d(s, s_hat[2, :])(s_center)
                # Write to file
                grp = root.create_group(loop.name)
                grp.attrs['simulation_type'] = interface.name
                # time
                dset_time = grp.create_dataset('time', data=time.value)
                dset_time.attrs['unit'] = time.unit.to_string()
                # NOTE: Set the chunk size such that accessing all entries for a given timestep
                # is the most efficient pattern.
                chunks = (None,) + s_center.shape
                # electron temperature
                dset_electron_temperature = grp.create_dataset(
                    'electron_temperature', data=electron_temperature.value, chunks=chunks)
                dset_electron_temperature.attrs['unit'] = electron_temperature.unit.to_string()
                # ion temperature
                dset_ion_temperature = grp.create_dataset(
                    'ion_temperature', data=ion_temperature.value, chunks=chunks)
                dset_ion_temperature.attrs['unit'] = ion_temperature.unit.to_string()
                # number density
                dset_density = grp.create_dataset('density', data=density.value, chunks=chunks)
                dset_density.attrs['unit'] = density.unit.to_string()
                # field-aligned velocity
                dset_velocity = grp.create_dataset('velocity', data=velocity.value, chunks=chunks)
                dset_velocity.attrs['unit'] = velocity.unit.to_string()
                dset_velocity.attrs['note'] = 'Velocity in the field-aligned direction'
                # Cartesian xyz velocity
                dset_velocity_x = grp.create_dataset(
                    'velocity_x', data=velocity_x.value, chunks=chunks)
                dset_velocity_x.attrs['unit'] = velocity_x.unit.to_string()
                dset_velocity_x.attrs['note'] = 'x-component of velocity in HEEQ coordinates'
                dset_velocity_y = grp.create_dataset(
                    'velocity_y', data=velocity_y.value, chunks=chunks)
                dset_velocity_y.attrs['unit'] = velocity_y.unit.to_string()
                dset_velocity_y.attrs['note'] = 'y-component of velocity in HEEQ coordinates'
                dset_velocity_z = grp.create_dataset(
                    'velocity_z', data=velocity_z.value, chunks=chunks)
                dset_velocity_z.attrs['unit'] = velocity_z.unit.to_string()
                dset_velocity_z.attrs['note'] = 'z-component of velocity in HEEQ coordinates'
                progress.update()

    def load_ionization_fractions(self, emission_model, interface=None, **kwargs):
        """
        Load the ionization fractions for each ion in the emission model.

        Parameters
        ----------
        emission_model : `synthesizAR.atomic.EmissionModel`
        interface : optional
            A model interface. Only necessary if loading the ionization fractions
            from the model

        If the model interface provides a method for loading the population fraction
        from the model, use that to get the population fractions. Otherwise, compute
        the ion population fractions in equilibrium. This should be done after
        calling `load_loop_simulations`.
        """
        root = zarr.open(store=self.loops[0].model_results_filename, mode='a', **kwargs)
        # Check if we can load from the model
        FROM_MODEL = False
        if interface is not None and hasattr(interface, 'load_ionization_fraction'):
            frac = interface.load_ionization_fraction(self.loops[0], emission_model[0])
            # Some models may optionally output the ionization fractions such that
            # they will have this method, but it may not return anything
            if frac is not None:
                FROM_MODEL = True
        # Get the unique elements from all of our ions
        elements = list(set([ion.element_name for ion in emission_model]))
        elements = [Element(e, emission_model.temperature) for e in elements]
        for el in elements:
            ions = [i for i in emission_model if i.element_name == el.element_name]
            if not FROM_MODEL:
                ioneq = el.equilibrium_ionization()
            for loop in self.loops:
                chunks = (None,) + loop.field_aligned_coordinate_center.shape
                if not FROM_MODEL:
                    frac_el = interp1d(el.temperature,
                                       ioneq,
                                       axis=0,
                                       kind='linear',
                                       fill_value='extrapolate')(loop.electron_temperature)
                if 'ionization_fraction' in root[loop.name]:
                    grp = root[f'{loop.name}/ionization_fraction']
                else:
                    grp = root[loop.name].create_group('ionization_fraction')
                for ion in ions:
                    if FROM_MODEL:
                        frac = interface.load_ionization_fraction(loop, ion)
                        desc = f'Ionization fraction of {ion.ion_name} as computed by {interface.name}'
                    else:
                        frac = frac_el[:, :, ion.charge_state]
                        desc = f'Ionization fraction of {ion.ion_name} in equilibrium.'
                    dset = grp.create_dataset(f'{ion.ion_name}', data=frac, chunks=chunks)
                    dset.attrs['unit'] = ''
                    dset.attrs['description'] = desc
<|endoftext|>