<commit_before># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe
from frappe import _, _dict
from frappe.utils import (flt, getdate, get_first_day, get_last_day,
	add_months, add_days, formatdate)

def get_period_list(fiscal_year, periodicity, from_beginning=False):
	"""Get a list of dict {"to_date": to_date, "key": key, "label": label}
		Periodicity can be (Yearly, Quarterly, Monthly)"""

	start_date, end_date = frappe.db.get_value("Fiscal Year", fiscal_year, ["year_start_date", "year_end_date"])
	start_date = getdate(start_date)
	end_date = getdate(end_date)

	if periodicity == "Yearly":
		period_list = [_dict({"to_date": end_date, "key": fiscal_year, "label": fiscal_year})]
	else:
		months_to_add = {
			"Half-yearly": 6,
			"Quarterly": 3,
			"Monthly": 1
		}[periodicity]

		period_list = []

		# start with first day, so as to avoid year to_dates like 2-April if ever they occur
		to_date = get_first_day(start_date)

		for i in xrange(12 / months_to_add):
			to_date = add_months(to_date, months_to_add)

			if to_date == get_first_day(to_date):
				# if to_date is the first day, get the last day of previous month
				to_date = add_days(to_date, -1)
			else:
				# to_date should be the last day of the new to_date's month
				to_date = get_last_day(to_date)

			if to_date <= end_date:
				# the normal case
				period_list.append(_dict({ "to_date": to_date }))

				# if it ends before a full year
				if to_date == end_date:
					break

			else:
				# if a fiscal year ends before a 12 month period
				period_list.append(_dict({ "to_date": end_date }))
				break

	# common processing
	for opts in period_list:
		key = opts["to_date"].strftime("%b_%Y").lower()
		label = formatdate(opts["to_date"], "MMM YYYY")
		opts.update({
			"key": key.replace(" ", "_").replace("-", "_"),
			"label": label,
			"year_start_date": start_date,
			"year_end_date": end_date
		})

		if from_beginning:
			# set start date as None for all fiscal periods, used in case of Balance Sheet
			opts["from_date"] = None
		else:
			opts["from_date"] = start_date

	return period_list

def get_data(company, root_type, balance_must_be, period_list, ignore_closing_entries=False):
	accounts = get_accounts(company, root_type)
	if not accounts:
		return None

	accounts, accounts_by_name = filter_accounts(accounts)
	gl_entries_by_account = get_gl_entries(company, period_list[0]["from_date"], period_list[-1]["to_date"],
		accounts[0].lft, accounts[0].rgt, ignore_closing_entries=ignore_closing_entries)

	calculate_values(accounts, gl_entries_by_account, period_list)
	accumulate_values_into_parents(accounts, accounts_by_name, period_list)
	out = prepare_data(accounts, balance_must_be, period_list)

	if out:
		add_total_row(out, balance_must_be, period_list)

	return out

def calculate_values(accounts, gl_entries_by_account, period_list):
	for d in accounts:
		for name in ([d.name] + (d.collapsed_children or [])):
			for entry in gl_entries_by_account.get(name, []):
				for period in period_list:
					entry.posting_date = getdate(entry.posting_date)

					# check if posting date is within the period
					if entry.posting_date <= period.to_date:
						d[period.key] = d.get(period.key, 0.0) + flt(entry.debit) - flt(entry.credit)

def accumulate_values_into_parents(accounts, accounts_by_name, period_list):
	"""accumulate children's values in parent accounts"""
	for d in reversed(accounts):
		if d.parent_account:
			for period in period_list:
				accounts_by_name[d.parent_account][period.key] = accounts_by_name[d.parent_account].get(period.key, 0.0) + \
					d.get(period.key, 0.0)

def prepare_data(accounts, balance_must_be, period_list):
	out = []
	year_start_date = period_list[0]["year_start_date"].strftime("%Y-%m-%d")
	year_end_date = period_list[-1]["year_end_date"].strftime("%Y-%m-%d")

	for d in accounts:
		# add to output
		has_value = False
		row = {
			"account_name": d.account_name,
			"account": d.name,
			"parent_account": d.parent_account,
			"indent": flt(d.indent),
			"from_date": year_start_date,
			"to_date": year_end_date
		}
		for period in period_list:
			if d.get(period.key):
				# change sign based on Debit or Credit, since calculation is done using (debit - credit)
				d[period.key] *= (1 if balance_must_be=="Debit" else -1)

			row[period.key] = flt(d.get(period.key, 0.0), 3)

			if abs(row[period.key]) >= 0.005:
				# ignore zero values
				has_value = True

		if has_value:
			out.append(row)

	return out

def add_total_row(out, balance_must_be, period_list):
	row = {
		"account_name": _("Total ({0})").format(balance_must_be),
		"account": None
	}
	for period in period_list:
		row[period.key] = out[0].get(period.key, 0.0)
		out[0][period.key] = ""

	out.append(row)

	# blank row after Total
	out.append({})

def get_accounts(company, root_type):
	# root lft, rgt
	root_account = frappe.db.sql("""select lft, rgt from `tabAccount`
		where company=%s and root_type=%s order by lft limit 1""",
		(company, root_type), as_dict=True)

	if not root_account:
		return None

	lft, rgt = root_account[0].lft, root_account[0].rgt

	accounts = frappe.db.sql("""select * from `tabAccount`
		where company=%(company)s and lft >= %(lft)s and rgt <= %(rgt)s order by lft""",
		{ "company": company, "lft": lft, "rgt": rgt }, as_dict=True)

	return accounts

def filter_accounts(accounts, depth=10):
	parent_children_map = {}
	accounts_by_name = {}
	for d in accounts:
		accounts_by_name[d.name] = d
		parent_children_map.setdefault(d.parent_account or None, []).append(d)

	filtered_accounts = []
	def add_to_list(parent, level):
		if level < depth:
			for child in (parent_children_map.get(parent) or []):
				child.indent = level
				filtered_accounts.append(child)
				add_to_list(child.name, level + 1)

		else:
			# include all children at level lower than the depth
			parent_account = accounts_by_name[parent]
			parent_account["collapsed_children"] = []
			for d in accounts:
				if d.lft > parent_account.lft and d.rgt < parent_account.rgt:
					parent_account["collapsed_children"].append(d.name)

	add_to_list(None, 0)

	return filtered_accounts, accounts_by_name

def get_gl_entries(company, from_date, to_date, root_lft, root_rgt, ignore_closing_entries=False):
	"""Returns a dict like { "account": [gl entries], ... }"""
	additional_conditions = []

	if ignore_closing_entries:
		additional_conditions.append("and ifnull(voucher_type, '')!='Period Closing Voucher'")

	if from_date:
		additional_conditions.append("and posting_date >= %(from_date)s")

	gl_entries = frappe.db.sql("""select * from `tabGL Entry`
		where company=%(company)s
		{additional_conditions}
		and posting_date <= %(to_date)s
		and account in (select name from `tabAccount`
			where lft >= %(lft)s and rgt <= %(rgt)s)
		order by account, posting_date""".format(additional_conditions="\n".join(additional_conditions)),
		{
			"company": company,
			"from_date": from_date,
			"to_date": to_date,
			"lft": root_lft,
			"rgt": root_rgt
		},
		as_dict=True)

	gl_entries_by_account = {}
	for entry in gl_entries:
		gl_entries_by_account.setdefault(entry.account, []).append(entry)

	return gl_entries_by_account

def get_columns(period_list):
	columns = [{
		"fieldname": "account",
		"label": _("Account"),
		"fieldtype": "Link",
		"options": "Account",
		"width": 300
	}]
	for period in period_list:
		columns.append({
			"fieldname": period.key,
			"label": period.label,
			"fieldtype": "Currency",
			"width": 150
		})

	return columns
<commit_msg>fiscal year validation in financial statement<commit_after># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe
from frappe import _, _dict
from frappe.utils import (flt, getdate, get_first_day, get_last_day,
	add_months, add_days, formatdate)

def get_period_list(fiscal_year, periodicity, from_beginning=False):
	"""Get a list of dict {"to_date": to_date, "key": key, "label": label}
		Periodicity can be (Yearly, Quarterly, Monthly)"""

	fy_start_end_date = frappe.db.get_value("Fiscal Year", fiscal_year, ["year_start_date", "year_end_date"])
	if not fy_start_end_date:
		frappe.throw(_("Fiscal Year {0} not found.").format(fiscal_year))

	start_date = getdate(fy_start_end_date[0])
	end_date = getdate(fy_start_end_date[1])

	if periodicity == "Yearly":
		period_list = [_dict({"to_date": end_date, "key": fiscal_year, "label": fiscal_year})]
	else:
		months_to_add = {
			"Half-yearly": 6,
			"Quarterly": 3,
			"Monthly": 1
		}[periodicity]

		period_list = []

		# start with first day, so as to avoid year to_dates like 2-April if ever they occur
		to_date = get_first_day(start_date)

		for i in xrange(12 / months_to_add):
			to_date = add_months(to_date, months_to_add)

			if to_date == get_first_day(to_date):
				# if to_date is the first day, get the last day of previous month
				to_date = add_days(to_date, -1)
			else:
				# to_date should be the last day of the new to_date's month
				to_date = get_last_day(to_date)

			if to_date <= end_date:
				# the normal case
				period_list.append(_dict({ "to_date": to_date }))

				# if it ends before a full year
				if to_date == end_date:
					break

			else:
				# if a fiscal year ends before a 12 month period
				period_list.append(_dict({ "to_date": end_date }))
				break

	# common processing
	for opts in period_list:
		key = opts["to_date"].strftime("%b_%Y").lower()
		label = formatdate(opts["to_date"], "MMM YYYY")
		opts.update({
			"key": key.replace(" ", "_").replace("-", "_"),
			"label": label,
			"year_start_date": start_date,
			"year_end_date": end_date
		})

		if from_beginning:
			# set start date as None for all fiscal periods, used in case of Balance Sheet
			opts["from_date"] = None
		else:
			opts["from_date"] = start_date

	return period_list

def get_data(company, root_type, balance_must_be, period_list, ignore_closing_entries=False):
	accounts = get_accounts(company, root_type)
	if not accounts:
		return None

	accounts, accounts_by_name = filter_accounts(accounts)
	gl_entries_by_account = get_gl_entries(company, period_list[0]["from_date"], period_list[-1]["to_date"],
		accounts[0].lft, accounts[0].rgt, ignore_closing_entries=ignore_closing_entries)

	calculate_values(accounts, gl_entries_by_account, period_list)
	accumulate_values_into_parents(accounts, accounts_by_name, period_list)
	out = prepare_data(accounts, balance_must_be, period_list)

	if out:
		add_total_row(out, balance_must_be, period_list)

	return out

def calculate_values(accounts, gl_entries_by_account, period_list):
	for d in accounts:
		for name in ([d.name] + (d.collapsed_children or [])):
			for entry in gl_entries_by_account.get(name, []):
				for period in period_list:
					entry.posting_date = getdate(entry.posting_date)

					# check if posting date is within the period
					if entry.posting_date <= period.to_date:
						d[period.key] = d.get(period.key, 0.0) + flt(entry.debit) - flt(entry.credit)

def accumulate_values_into_parents(accounts, accounts_by_name, period_list):
	"""accumulate children's values in parent accounts"""
	for d in reversed(accounts):
		if d.parent_account:
			for period in period_list:
				accounts_by_name[d.parent_account][period.key] = accounts_by_name[d.parent_account].get(period.key, 0.0) + \
					d.get(period.key, 0.0)

def prepare_data(accounts, balance_must_be, period_list):
	out = []
	year_start_date = period_list[0]["year_start_date"].strftime("%Y-%m-%d")
	year_end_date = period_list[-1]["year_end_date"].strftime("%Y-%m-%d")

	for d in accounts:
		# add to output
		has_value = False
		row = {
			"account_name": d.account_name,
			"account": d.name,
			"parent_account": d.parent_account,
			"indent": flt(d.indent),
			"from_date": year_start_date,
			"to_date": year_end_date
		}
		for period in period_list:
			if d.get(period.key):
				# change sign based on Debit or Credit, since calculation is done using (debit - credit)
				d[period.key] *= (1 if balance_must_be=="Debit" else -1)

			row[period.key] = flt(d.get(period.key, 0.0), 3)

			if abs(row[period.key]) >= 0.005:
				# ignore zero values
				has_value = True

		if has_value:
			out.append(row)

	return out

def add_total_row(out, balance_must_be, period_list):
	row = {
		"account_name": _("Total ({0})").format(balance_must_be),
		"account": None
	}
	for period in period_list:
		row[period.key] = out[0].get(period.key, 0.0)
		out[0][period.key] = ""

	out.append(row)

	# blank row after Total
	out.append({})

def get_accounts(company, root_type):
	# root lft, rgt
	root_account = frappe.db.sql("""select lft, rgt from `tabAccount`
		where company=%s and root_type=%s order by lft limit 1""",
		(company, root_type), as_dict=True)

	if not root_account:
		return None

	lft, rgt = root_account[0].lft, root_account[0].rgt

	accounts = frappe.db.sql("""select * from `tabAccount`
		where company=%(company)s and lft >= %(lft)s and rgt <= %(rgt)s order by lft""",
		{ "company": company, "lft": lft, "rgt": rgt }, as_dict=True)

	return accounts

def filter_accounts(accounts, depth=10):
	parent_children_map = {}
	accounts_by_name = {}
	for d in accounts:
		accounts_by_name[d.name] = d
		parent_children_map.setdefault(d.parent_account or None, []).append(d)

	filtered_accounts = []
	def add_to_list(parent, level):
		if level < depth:
			for child in (parent_children_map.get(parent) or []):
				child.indent = level
				filtered_accounts.append(child)
				add_to_list(child.name, level + 1)

		else:
			# include all children at level lower than the depth
			parent_account = accounts_by_name[parent]
			parent_account["collapsed_children"] = []
			for d in accounts:
				if d.lft > parent_account.lft and d.rgt < parent_account.rgt:
					parent_account["collapsed_children"].append(d.name)

	add_to_list(None, 0)

	return filtered_accounts, accounts_by_name

def get_gl_entries(company, from_date, to_date, root_lft, root_rgt, ignore_closing_entries=False):
	"""Returns a dict like { "account": [gl entries], ... }"""
	additional_conditions = []

	if ignore_closing_entries:
		additional_conditions.append("and ifnull(voucher_type, '')!='Period Closing Voucher'")

	if from_date:
		additional_conditions.append("and posting_date >= %(from_date)s")

	gl_entries = frappe.db.sql("""select * from `tabGL Entry`
		where company=%(company)s
		{additional_conditions}
		and posting_date <= %(to_date)s
		and account in (select name from `tabAccount`
			where lft >= %(lft)s and rgt <= %(rgt)s)
		order by account, posting_date""".format(additional_conditions="\n".join(additional_conditions)),
		{
			"company": company,
			"from_date": from_date,
			"to_date": to_date,
			"lft": root_lft,
			"rgt": root_rgt
		},
		as_dict=True)

	gl_entries_by_account = {}
	for entry in gl_entries:
		gl_entries_by_account.setdefault(entry.account, []).append(entry)

	return gl_entries_by_account

def get_columns(period_list):
	columns = [{
		"fieldname": "account",
		"label": _("Account"),
		"fieldtype": "Link",
		"options": "Account",
		"width": 300
	}]
	for period in period_list:
		columns.append({
			"fieldname": period.key,
			"label": period.label,
			"fieldtype": "Currency",
			"width": 150
		})

	return columns
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os
import argparse
import re

import markdown2

class FileQueue(object):
    def __init__(self):
        self.queue = [] # [path, pointer]
        self.last_used = 0

    def add(self, file_path, file_pointer=None):
        for item in self.queue:
            if item[0] == file_path:
                return # Already on queue

        if not file_pointer:
            try:
                file_pointer = open(file_path, 'r')
            except: #FIXME
                print("Reference not found: " + path)

        self.queue.append([file_path, file_pointer])

    def get(self):
        if self.has_items:
            next_item = self.queue[self.last_used]
            self.last_used += 1
            return next_item
        else:
            return None

    def has_items(self):
        return self.last_used < len(self.queue)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='A simple tool to merge many MarkDown files into one.')

    parser.add_argument('-n', '--no-separators', dest='use_separators', action='store_false',
                        help='Merges documents without a separator between them.')

    parser.add_argument('-f', '--follow-links', dest='follow_links', action='store_true',
                        help='Attach linked MarkDown documents to the queue.')

    parser.add_argument('-q', '--html-only', dest='create_md', action='store_false',
                        help='Outputs only an HTML with the merged documents.')

    parser.add_argument('-m', '--md-only', dest='create_html', action='store_false',
                        help='Outputs only a MarkDown file with the merged documents.')

    parser.add_argument('-p', '--plain', dest='use_template', action='store_false',
                        help='Plain HTML output, without template.')

    parser.add_argument('-o', '--output', metavar='filename', dest='output', type=str, default="output")

    parser.add_argument('-c', metavar='style.css', dest='css', type=str,
                        default="github-markdown-css/github-markdown.css",
                        help='Custom stylesheet path')

    parser.add_argument('-t', '--title', metavar='"Document title"', dest='title', type=str, default="Document")

    parser.add_argument('files', metavar='file', type=file, nargs='+',
                        help='MarkDown file(s) to merge.')

    args = parser.parse_args()

    if args.follow_links:
        inline_links = re.compile(r'(?<=\]\().+\.md(?=\))', re.MULTILINE)  #

    file_queue = FileQueue()
    for file_pointer in args.files:
        full_path = os.path.abspath(file_pointer.name)
        file_queue.add(full_path, file_pointer)

    md_mesh = ""
    while file_queue.has_items():
        (path, md_file) = file_queue.get()
        if args.use_separators and md_mesh:
            md_mesh += "\n\n---\n\n"

        new_file = md_file.read()
        md_file.close()

        if args.follow_links:
            links = inline_links.findall(new_file)
            if links:
                base_path = os.path.dirname(os.path.abspath(md_file.name))
                for link in links:
                    file_queue.add(base_path + '/' + link)

        md_mesh += new_file

    args.output = re.sub(r'\.(md|html)$', '', args.output, flags=re.IGNORECASE)
    md_path = args.output + '.md'
    html_path = args.output + '.html'

    if args.create_md:
        md_output = open(md_path, 'w')
        md_output.write(md_mesh)
        md_output.close()

    if args.create_html:
        raw_html = markdown2.markdown(md_mesh, extras=['tables', 'fenced-code-blocks'])
        html_output = open(html_path, 'w')

        if args.use_template:
            t = open('template.html', 'r')
            template = t.read()
            t.close()

            template = template.replace('{{ title }}', args.title)
            if args.css:
                try:
                    css_file = open(args.css, 'r')
                    css = css_file.read()
                    css_file.close()
                except:
                    css = ""
                template = template.replace('{{ css }}', css)
            template = template.replace('{{ content }}', raw_html)

            html_output.write(template)

        else:
            html_output.write(raw_html)

        html_output.close()<commit_msg>Added comments to the code. Better variable names. Fixed tiny bugs.<commit_after>#!/usr/bin/env python

import sys
import os
import argparse
import re

import markdown2

class FileQueue(object):
    def __init__(self):
        self.queue = [] # [path, pointer]
        self.last_used = 0

    def add(self, file_path, file_pointer=None):
        for item in self.queue:
            if item[0] == file_path:
                return # Already on queue

        if not file_pointer:
            try:
                file_pointer = open(file_path, 'r')
            except: #FIXME
                print("Reference not found: " + path)

        self.queue.append([file_path, file_pointer])

    def get(self):
        if self.has_items:
            next_item = self.queue[self.last_used]
            self.last_used += 1
            return next_item
        else:
            return None

    def has_items(self):
        return self.last_used < len(self.queue)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='A simple tool to merge many MarkDown files into one.')

    parser.add_argument('-n', '--no-separators', dest='use_separators', action='store_false',
                        help='Merges documents without a separator between them.')

    parser.add_argument('-f', '--follow-links', dest='follow_links', action='store_true',
                        help='Attach linked MarkDown documents to the queue.')

    parser.add_argument('-q', '--html-only', dest='create_md', action='store_false',
                        help='Outputs only an HTML with the merged documents.')

    parser.add_argument('-m', '--md-only', dest='create_html', action='store_false',
                        help='Outputs only a MarkDown file with the merged documents.')

    parser.add_argument('-p', '--plain', dest='use_template', action='store_false',
                        help='Plain HTML output, without template.')

    parser.add_argument('-o', '--output', metavar='filename', dest='output', type=str, default="output")

    parser.add_argument('-c', metavar='style.css', dest='css', type=str,
                        default="github-markdown-css/github-markdown.css",
                        help='Custom stylesheet path')

    parser.add_argument('-t', '--title', metavar='"Document title"', dest='title', type=str, default="Document")

    parser.add_argument('files', metavar='file', type=file, nargs='+',
                        help='MarkDown file(s) to merge.')

    args = parser.parse_args()

    # Creates the RegEx to match other md documents
    if args.follow_links:
        inline_links = re.compile(r'(?<=\]\().+\.md(?=\))', re.MULTILINE)

    # Creates file queue and populates it
    file_queue = FileQueue()
    for file_pointer in args.files:
        full_path = os.path.abspath(file_pointer.name)
        file_queue.add(full_path, file_pointer)

    md_mesh = "" # where the compiled md is stored

    while file_queue.has_items():
        (path, md_file_pointer) = file_queue.get()
        if args.use_separators and md_mesh:
            md_mesh += "\n\n---\n\n"

        item_content = md_file_pointer.read()
        md_file_pointer.close()

        if args.follow_links:
            links = inline_links.findall(item_content)
            if links:
                # Adds new documents to the queue, considering their full path
                base_path = os.path.dirname(os.path.abspath(md_file_pointer.name))
                for link in links:
                    file_queue.add(base_path + '/' + link) # FIXME: relative paths may cause duplicates

        md_mesh += item_content

    # Cleans filename and prepares the names for both possible outputs
    args.output = re.sub(r'\.(md|html)$', '', args.output, flags=re.IGNORECASE)
    md_path = args.output + '.md'
    html_path = args.output + '.html'

    if args.create_md:
        md_output = open(md_path, 'w')
        md_output.write(md_mesh)
        md_output.close()

    if args.create_html:
        #Parsing
        raw_html = markdown2.markdown(md_mesh, extras=['tables', 'fenced-code-blocks'])

        html_output = open(html_path, 'w')

        if args.use_template:
            t = open('template.html', 'r')
            template = t.read()
            t.close()

            template = template.replace('{{ title }}', args.title)

            # Puts the CSS inline
            if args.css:
                try:
                    css_file = open(args.css, 'r')
                    css = css_file.read()
                    css_file.close()
                except:
                    css = ""
                template = template.replace('{{ css }}', css, 1)

            template = template.replace('{{ content }}', raw_html)

            html_output.write(template)

        else:
            html_output.write(raw_html)

        html_output.close()
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2017, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import logging
import numpy
from openquake.baselib.python3compat import decode
from openquake.commonlib import writers
from openquake.risklib import riskinput, scientific

def get_loss_builder(dstore):
    """
    :param dstore: datastore for an event based risk calculation
    :returns: a LossesByPeriodBuilder instance
    """
    oq = dstore['oqparam']
    weights = dstore['realizations']['weight']
    eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
    num_events = dstore['gmdata']['events']
    periods = oq.return_periods or scientific.return_periods(
        eff_time, num_events.max())
    return scientific.LossesByPeriodBuilder(
        numpy.array(periods), oq.loss_dt(), weights, num_events,
        eff_time, oq.risk_investigation_time or oq.hazard_investigation_time)

class LossCurveExporter(object):
    """
    Exporter for the loss curves. The most important method is
    `.export(export_type, what)` where `export_type` is a string like 'csv',
    and `what` is a string called export specifier. Here are some examples
    for the export specifier:

    rlzs/sid-42   # export loss curves of site #42 for all realizations
    rlz-003/sid-42   # export all loss curves of site #42, realization #3
    stats/sid-42   # export statistical loss curves of site #42
    mean/sid-42   # export mean loss curves of site #42
    quantile-0.1/sid-42   # export quantile loss curves of site #42

    rlzs/ref-a1          # export loss curves of asset a1 for all realizations
    rlz-003/ref-a1      # export loss curves of asset a1, realization 3
    stats/ref-a1        # export statistical loss curves of asset a1
    mean/ref-a1         # export mean loss curves of asset a1
    quantile-0.1/ref-a1 # export quantile loss curves of asset a1
    """
    def __init__(self, dstore):
        self.dstore = dstore
        try:
            self.builder = get_loss_builder(dstore)
        except KeyError:  # no 'agg_loss_table' for non event_based_risk
            pass
        self.assetcol = dstore['assetcol']
        arefs = [decode(aref) for aref in self.dstore['asset_refs']]
        self.str2asset = {arefs[asset.idx]: asset for asset in self.assetcol}
        self.asset_refs = self.dstore['asset_refs'].value
        self.loss_types = dstore.get_attr('composite_risk_model', 'loss_types')
        self.R = len(dstore['realizations'])

    def parse(self, what):
        """
        :param what:
            can be 'rlz-1/ref-asset1', 'rlz-2/sid-1', ...
        """
        if '/' not in what:
            key, spec = what, ''
        else:
            key, spec = what.split('/')
        if spec and not spec.startswith(('ref-', 'sid-')):
            raise ValueError('Wrong specification in %s' % what)
        elif spec == '':  # export losses for all assets
            aids = []
            arefs = []
            for aid, rec in enumerate(self.assetcol.array):
                aids.append(aid)
                arefs.append(self.asset_refs[rec['idx']])
        elif spec.startswith('sid-'):  # passed the site ID
            sid = int(spec[4:])
            aids = []
            arefs = []
            for aid, rec in enumerate(self.assetcol.array):
                if rec['site_id'] == sid:
                    aids.append(aid)
                    arefs.append(self.asset_refs[rec['idx']])
        elif spec.startswith('ref-'):  # passed the asset name
            arefs = [spec[4:]]
            aids = [self.str2asset[arefs[0]].ordinal]
        else:
            raise ValueError('Wrong specification in %s' % what)
        return aids, arefs, spec or 'all', key

    def export_csv(self, spec, asset_refs, curves_dict):
        """
        :param asset_ref: name of the asset
        :param curves_dict: a dictionary tag -> loss curves
        """
        writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
        ebr = hasattr(self, 'builder')
        for key in sorted(curves_dict):
            recs = curves_dict[key]
            data = [['asset', 'loss_type', 'loss', 'period' if ebr else 'poe']]
            for loss_type in self.loss_types:
                if ebr:  # event_based_risk
                    array = recs[loss_type]  # shape (A, P) loss_dt
                    periods = self.builder.return_periods
                    for aref, losses in zip(asset_refs, array):
                        for period, loss in zip(periods, losses):
                            data.append((aref, loss_type, loss, period))
                else:  # classical_risk
                    array = recs[loss_type]  # shape (A,) loss_curve_dt
                    for aref, losses, poes in zip(
                            asset_refs, array['losses'], array['poes']):
                        for loss, poe in zip(losses, poes):
                            data.append((aref, loss_type, loss, poe))
            dest = self.dstore.build_fname(
                'loss_curves', '%s-%s' % (spec, key), 'csv')
            writer.save(data, dest)
        return writer.getsaved()

    def export(self, export_type, what):
        """
        :param export_type: 'csv', 'json', ...
        :param what: string describing what to export
        :returns: list of exported file names
        """
        aids, arefs, spec, key = self.parse(what)
        if key.startswith('rlz'):
            curves = self.export_curves_rlzs(aids, key)
        else:  # statistical exports
            curves = self.export_curves_stats(aids, key)
        return getattr(self, 'export_' + export_type)(spec, arefs, curves)

    def export_curves_rlzs(self, aids, key):
        """
        :returns: a dictionary key -> record of dtype loss_curve_dt
        """
        if 'loss_curves-rlzs' in self.dstore:  # classical_risk
            data = self.dstore['loss_curves-rlzs'][aids]  # shape (A, R)
            if key.startswith('rlz-'):
                rlzi = int(key[4:])
                return {key: data[:, rlzi]}
            # else key == 'rlzs', returns all data
            return {'rlz-%03d' % rlzi: data[:, rlzi] for rlzi in range(self.R)}

        # otherwise event_based
        avalues = self.assetcol.values(aids)
        lrgetter = riskinput.LossRatiosGetter(self.dstore, aids)
        build = self.builder.build_rlz
        if key.startswith('rlz-'):
            rlzi = int(key[4:])
            ratios = lrgetter.get(rlzi)
            return {'rlz-%03d' % rlzi: build(avalues, ratios, rlzi)}
        else:  # key is 'rlzs', return a dictionary will all realizations
            # this may be disabled in the future unless an asset is specified
            dic = {}
            for rlzi in range(self.R):
                dic['rlz-%03d' % rlzi] = build(
                    avalues, lrgetter.get(rlzi), rlzi)
            return dic

    def export_curves_stats(self, aids, key):
        """
        :returns: a dictionary rlzi -> record of dtype loss_curve_dt
        """
        oq = self.dstore['oqparam']
        num_rlzs = len(self.dstore['realizations'])
        if num_rlzs == 1:
            logging.error('There is a single realization, there are no stats')
            return {}
        stats = oq.risk_stats()  # pair (name, func)
        stat2idx = {stat[0]: s for s, stat in enumerate(stats)}
        if 'loss_curves-stats' in self.dstore:  # classical_risk
            dset = self.dstore['loss_curves-stats']
            data = dset[aids]  # shape (A, S)
            if key == 'stats':
                return {stat[0]: data[:, s] for s, stat in enumerate(stats)}
            else:  # a specific statistics
                return {key: data[:, stat2idx[key]]}
        elif 'curves-stats' in self.dstore:  # event_based_risk
            dset = self.dstore['curves-stats']
            data = dset[aids]
            if key == 'stats':
                return {stat[0]: data[:, s] for s, stat in enumerate(stats)}
            else:  # a specific statistics
                return {key: data[:, stat2idx[key]]}
        else:
            raise KeyError('no loss curves in %s' % self.dstore)
<commit_msg>Small fix [skip hazardlib] [demos]<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2017, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import logging
import numpy
from openquake.baselib.python3compat import decode
from openquake.commonlib import writers
from openquake.risklib import riskinput, scientific

def get_loss_builder(dstore):
    """
    :param dstore: datastore for an event based risk calculation
    :returns: a LossesByPeriodBuilder instance
    """
    oq = dstore['oqparam']
    weights = dstore['realizations']['weight']
    eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
    num_events = dstore['gmdata']['events']
    periods = oq.return_periods or scientific.return_periods(
        eff_time, num_events.max())
    return scientific.LossesByPeriodBuilder(
        numpy.array(periods), oq.loss_dt(), weights, num_events,
        eff_time, oq.investigation_time)

class LossCurveExporter(object):
    """
    Exporter for the loss curves. The most important method is
    `.export(export_type, what)` where `export_type` is a string like 'csv',
    and `what` is a string called export specifier. Here are some examples
    for the export specifier:

    rlzs/sid-42   # export loss curves of site #42 for all realizations
    rlz-003/sid-42   # export all loss curves of site #42, realization #3
    stats/sid-42   # export statistical loss curves of site #42
    mean/sid-42   # export mean loss curves of site #42
    quantile-0.1/sid-42   # export quantile loss curves of site #42

    rlzs/ref-a1          # export loss curves of asset a1 for all realizations
    rlz-003/ref-a1      # export loss curves of asset a1, realization 3
    stats/ref-a1        # export statistical loss curves of asset a1
    mean/ref-a1         # export mean loss curves of asset a1
    quantile-0.1/ref-a1 # export quantile loss curves of asset a1
    """
    def __init__(self, dstore):
        self.dstore = dstore
        try:
            self.builder = get_loss_builder(dstore)
        except KeyError:  # no 'agg_loss_table' for non event_based_risk
            pass
        self.assetcol = dstore['assetcol']
        arefs = [decode(aref) for aref in self.dstore['asset_refs']]
        self.str2asset = {arefs[asset.idx]: asset for asset in self.assetcol}
        self.asset_refs = self.dstore['asset_refs'].value
        self.loss_types = dstore.get_attr('composite_risk_model', 'loss_types')
        self.R = len(dstore['realizations'])

    def parse(self, what):
        """
        :param what:
            can be 'rlz-1/ref-asset1', 'rlz-2/sid-1', ...
        """
        if '/' not in what:
            key, spec = what, ''
        else:
            key, spec = what.split('/')
        if spec and not spec.startswith(('ref-', 'sid-')):
            raise ValueError('Wrong specification in %s' % what)
        elif spec == '':  # export losses for all assets
            aids = []
            arefs = []
            for aid, rec in enumerate(self.assetcol.array):
                aids.append(aid)
                arefs.append(self.asset_refs[rec['idx']])
        elif spec.startswith('sid-'):  # passed the site ID
            sid = int(spec[4:])
            aids = []
            arefs = []
            for aid, rec in enumerate(self.assetcol.array):
                if rec['site_id'] == sid:
                    aids.append(aid)
                    arefs.append(self.asset_refs[rec['idx']])
        elif spec.startswith('ref-'):  # passed the asset name
            arefs = [spec[4:]]
            aids = [self.str2asset[arefs[0]].ordinal]
        else:
            raise ValueError('Wrong specification in %s' % what)
        return aids, arefs, spec or 'all', key

    def export_csv(self, spec, asset_refs, curves_dict):
        """
        :param asset_ref: name of the asset
        :param curves_dict: a dictionary tag -> loss curves
        """
        writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
        ebr = hasattr(self, 'builder')
        for key in sorted(curves_dict):
            recs = curves_dict[key]
            data = [['asset', 'loss_type', 'loss', 'period' if ebr else 'poe']]
            for loss_type in self.loss_types:
                if ebr:  # event_based_risk
                    array = recs[loss_type]  # shape (A, P) loss_dt
                    periods = self.builder.return_periods
                    for aref, losses in zip(asset_refs, array):
                        for period, loss in zip(periods, losses):
                            data.append((aref, loss_type, loss, period))
                else:  # classical_risk
                    array = recs[loss_type]  # shape (A,) loss_curve_dt
                    for aref, losses, poes in zip(
                            asset_refs, array['losses'], array['poes']):
                        for loss, poe in zip(losses, poes):
                            data.append((aref, loss_type, loss, poe))
            dest = self.dstore.build_fname(
                'loss_curves', '%s-%s' % (spec, key), 'csv')
            writer.save(data, dest)
        return writer.getsaved()

    def export(self, export_type, what):
        """
        :param export_type: 'csv', 'json', ...
        :param what: string describing what to export
        :returns: list of exported file names
        """
        aids, arefs, spec, key = self.parse(what)
        if key.startswith('rlz'):
            curves = self.export_curves_rlzs(aids, key)
        else:  # statistical exports
            curves = self.export_curves_stats(aids, key)
        return getattr(self, 'export_' + export_type)(spec, arefs, curves)

    def export_curves_rlzs(self, aids, key):
        """
        :returns: a dictionary key -> record of dtype loss_curve_dt
        """
        if 'loss_curves-rlzs' in self.dstore:  # classical_risk
            data = self.dstore['loss_curves-rlzs'][aids]  # shape (A, R)
            if key.startswith('rlz-'):
                rlzi = int(key[4:])
                return {key: data[:, rlzi]}
            # else key == 'rlzs', returns all data
            return {'rlz-%03d' % rlzi: data[:, rlzi] for rlzi in range(self.R)}

        # otherwise event_based
        avalues = self.assetcol.values(aids)
        lrgetter = riskinput.LossRatiosGetter(self.dstore, aids)
        build = self.builder.build_rlz
        if key.startswith('rlz-'):
            rlzi = int(key[4:])
            ratios = lrgetter.get(rlzi)
            return {'rlz-%03d' % rlzi: build(avalues, ratios, rlzi)}
        else:  # key is 'rlzs', return a dictionary will all realizations
            # this may be disabled in the future unless an asset is specified
            dic = {}
            for rlzi in range(self.R):
                dic['rlz-%03d' % rlzi] = build(
                    avalues, lrgetter.get(rlzi), rlzi)
            return dic

    def export_curves_stats(self, aids, key):
        """
        :returns: a dictionary rlzi -> record of dtype loss_curve_dt
        """
        oq = self.dstore['oqparam']
        num_rlzs = len(self.dstore['realizations'])
        if num_rlzs == 1:
            logging.error('There is a single realization, there are no stats')
            return {}
        stats = oq.risk_stats()  # pair (name, func)
        stat2idx = {stat[0]: s for s, stat in enumerate(stats)}
        if 'loss_curves-stats' in self.dstore:  # classical_risk
            dset = self.dstore['loss_curves-stats']
            data = dset[aids]  # shape (A, S)
            if key == 'stats':
                return {stat[0]: data[:, s] for s, stat in enumerate(stats)}
            else:  # a specific statistics
                return {key: data[:, stat2idx[key]]}
        elif 'curves-stats' in self.dstore:  # event_based_risk
            dset = self.dstore['curves-stats']
            data = dset[aids]
            if key == 'stats':
                return {stat[0]: data[:, s] for s, stat in enumerate(stats)}
            else:  # a specific statistics
                return {key: data[:, stat2idx[key]]}
        else:
            raise KeyError('no loss curves in %s' % self.dstore)
<|endoftext|>


<commit_before># Copyright © 2017-2019 Zuercher Hochschule fuer Angewandte Wissenschaften.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import connexion

from esm.controllers import _version_ok
from esm.models import BindingRequest
from esm.models import BindingResponse
from esm.models import Empty
from esm.models import LastOperation
from esm.models import ServiceInstance
from esm.models import ServiceRequest
from esm.models import ServiceResponse
from esm.models import ServiceType
from esm.models import UpdateOperationResponse
from esm.models import UpdateRequest

from adapters.datasource import STORE as store
from adapters.resources import EPM as epm

# from datetime import date, datetime
# from typing import List, Dict
# from six import iteritems
# from ..util import deserialize_date, deserialize_datetime

def create_service_instance(instance_id, service, accept_incomplete=None):
    """
    Provisions a service instance
    When the broker receives a provision request from a client, it should synchronously take whatever action is
    necessary to create a new service resource for the developer. The result of provisioning varies by service
    type, although there are a few common actions that work for many services. Supports asynchronous operations.'
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for
    future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param service: Service information.
    :type service: dict | bytes
    :param accept_incomplete: Indicates that the client is supporting asynchronous operations
    :type accept_incomplete: bool

    :rtype: ServiceResponse
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        if connexion.request.is_json:
            service = ServiceRequest.from_dict(connexion.request.get_json())
        else:
            return "Supplied body content is not or is mal-formed JSON", 400

        svc_type = store.get_service_instance(service.service_id)
        if len(svc_type) == 1:
            return 'Service instance with id {id} already exists'.format(id=service.service_id), 409
        # look up manifest based on plan id
        # based on the manifest type, select the driver
        # send the manifest for creation to the target system
        # store the ID along with refs to service, plan and manifest

        # get the manifest for the service/plan
        # TODO some validation required here to ensure it's the right svc/plan
        svc_type = store.get_service(service.service_id)[0]
        if svc_type is None:
            return 'Unrecognised service requested to be instantiated', 404

        plans = svc_type.plans
        plan = [p for p in plans if p.id == service.plan_id]
        if len(plan) <= 0:
            return 'no plan found.', 404

        mani = store.get_manifest(plan_id=plan[0].id)[0]

        if accept_incomplete:  # given docker-compose runs in detached mode this is not needed - only timing can verify
            # XXX put this in a thread to allow for asynch processing?
            epm.create(instance_id=instance_id, content=mani.manifest_content, c_type=mani.manifest_type)
        else:
            epm.create(instance_id=instance_id, content=mani.manifest_content, c_type=mani.manifest_type)

        last_op = LastOperation(  # stored within the service instance doc
            state='creating',
            description='service instance is being created'
        )

        # store the instance Id with manifest id
        srv_inst = ServiceInstance(
            service_type=svc_type,
            state=last_op,
            context={
                'id': instance_id,
                'manifest_id': mani.id,
            }
        )

        store.add_service_instance(srv_inst)

        if accept_incomplete:
            store.add_last_operation(instance_id=instance_id, last_operation=last_op)

        return 'created', 200

def deprovision_service_instance(instance_id, service_id, plan_id, accept_incomplete=None):
    """
    Deprovisions a service instance.
    'When a broker receives a deprovision request from a client, it should delete any resources it
    created during the provision. Usually this means that all resources are immediately reclaimed for
    future provisions.'
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used
    for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param service_id: service ID to be deprovisioned
    :type service_id: str
    :param plan_id: plan ID of the service to be deprovisioned
    :type plan_id: str
    :param accept_incomplete: Indicates that the client is supporting asynchronous operations
    :type accept_incomplete: bool

    :rtype: UpdateOperationResponse
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        # XXX if there's bindings remove first?
        # XXX what about undo?
        # check that the instance exists first
        instance = store.get_service_instance(instance_id=instance_id)
        if len(instance) == 1:
            epm.delete(instance_id=instance_id)
            store.delete_service_instance(instance_id)
            # we don't delete the last_operation explicitly as its embedded in the service_instance document
            # store.delete_last_operation(instance_id)

            return Empty(), 200
        else:
            return Empty(), 404

def instance_info(instance_id):
    """
    Returns information about the service instance.
    Returns information about the service instance. This is a simple read operation against the broker database and
    is provided as a developer/consumer convienence.
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used
    for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str

    :rtype: ServiceType
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        # service instance should already be recorded
        srv_inst = store.get_service_instance(instance_id)
        if len(srv_inst) < 1:
            return 'no service instance found.', 404
        srv_inst = srv_inst[0]

        # get the latest info
        inst_info = epm.info(instance_id=instance_id)

        if inst_info['srv_inst.state.state'] == 'failed':
            # try epm.delete(instance_id=instance_id)?
            return 'There has been a failure in creating the service instance.', 500

        # TODO need a converged state model for the SM
        srv_inst.state.state = inst_info['srv_inst.state.state']
        srv_inst.state.description = inst_info['srv_inst.state.description']

        # don't need you any more, buh-bye!
        del inst_info['srv_inst.state.state']
        del inst_info['srv_inst.state.description']

        # merge the two context dicts
        srv_inst.context = {**srv_inst.context, **inst_info}

        # update the service instance record - there should be an asynch method doing the update - event based
        store.add_service_instance(srv_inst)

        return srv_inst, 200

def last_operation_status(instance_id, service_id=None, plan_id=None, operation=None):
    """
    Gets the current state of the last operation upon the specified resource.
    When a broker returns status code 202 ACCEPTED for provision, update, or deprovision, the client will
    begin to poll the /v2/service_instances/:guid/last_operation endpoint to obtain the state of the last requested
    operation. The broker response must contain the field state and an optional field description.
    :param instance_id: The instance_id of a service instance is provided by the client. This ID will be used for
    future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.
    :type instance_id: str
    :param service_id: ID of the service from the catalog.
    :type service_id: str
    :param plan_id: ID of the plan from the catalog.
    :type plan_id: str
    :param operation: A broker-provided identifier for the operation. When a value for operation is included
    with asynchronous responses for Provision, Update, and Deprovision requests, the broker client should provide
    the same value using this query parameter as a URL-encoded string.;
    :type operation: str

    :rtype: LastOperation
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        # just re-use the method and return it's content and http status code.
        # version check not required here as it's done in the proxied call
        srv_inst, code = instance_info(instance_id=instance_id)
        if code == 404:
            return srv_inst + 'No service status therefore.', code
        else:
            return srv_inst.state, code

def service_bind(instance_id, binding_id, binding):
    """
    Binds to a service
    When the broker receives a bind request from the client, it should return information which helps an application to utilize the provisioned resource. This information is generically referred to as credentials. Applications should be issued unique credentials whenever possible, so one application access can be revoked without affecting other bound applications.
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param binding_id: The binding_id of a service binding is provided by the Cloud Controller.
    :type binding_id: str
    :param binding:
    :type binding: dict | bytes

    :rtype: BindingResponse
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        if connexion.request.is_json:
            binding = BindingRequest.from_dict(connexion.request.get_json())
        else:
            return "Supplied body content is not or is mal-formed JSON", 400
        return 'Not implemented :-(', 501

def service_unbind(instance_id, binding_id, service_id, plan_id):
    """
    Unbinds a service
    When a broker receives an unbind request from the client, it should delete any resources it created in bind. Usually this means that an application immediately cannot access the resource.
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param binding_id: The binding_id of a service binding is provided by the Cloud Controller.
    :type binding_id: str
    :param service_id: ID of the service from the catalog.
    :type service_id: str
    :param plan_id: ID of the plan from the catalog.
    :type plan_id: str

    :rtype: Empty
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        return 'Not implemented :-(', 501

def update_service_instance(instance_id, plan, accept_incomplete=None):
    """
    Updating a Service Instance
    Brokers that implement this endpoint can enable users to modify attributes of an existing service instance. The first attribute supports users modifying is the service plan. This effectively enables users to upgrade or downgrade their service instance to other plans. To see how users make these requests.'
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param plan: New Plan information.
    :type plan: dict | bytes
    :param accept_incomplete: Indicates that the client is supporting asynchronous operations
    :type accept_incomplete: bool

    :rtype: Empty
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        if connexion.request.is_json:
            plan = UpdateRequest.from_dict(connexion.request.get_json())
        else:
            return "Supplied body content is not or is mal-formed JSON", 400
        return 'Not implemented :-(', 501
<commit_msg>refactored to support the removal of Store dependency in the resource manager<commit_after># Copyright © 2017-2019 Zuercher Hochschule fuer Angewandte Wissenschaften.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import connexion

from esm.controllers import _version_ok
from esm.models import BindingRequest
from esm.models import BindingResponse
from esm.models import Empty
from esm.models import LastOperation
from esm.models import ServiceInstance
from esm.models import ServiceRequest
from esm.models import ServiceResponse
from esm.models import ServiceType
from esm.models import UpdateOperationResponse
from esm.models import UpdateRequest

from adapters.datasource import STORE as store
from adapters.resources import RM as rm

# from datetime import date, datetime
# from typing import List, Dict
# from six import iteritems
# from ..util import deserialize_date, deserialize_datetime

def create_service_instance(instance_id, service, accept_incomplete=None):
    """
    Provisions a service instance
    When the broker receives a provision request from a client, it should synchronously take whatever action is
    necessary to create a new service resource for the developer. The result of provisioning varies by service
    type, although there are a few common actions that work for many services. Supports asynchronous operations.'
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for
    future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param service: Service information.
    :type service: dict | bytes
    :param accept_incomplete: Indicates that the client is supporting asynchronous operations
    :type accept_incomplete: bool

    :rtype: ServiceResponse
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        if connexion.request.is_json:
            service = ServiceRequest.from_dict(connexion.request.get_json())
        else:
            return "Supplied body content is not or is mal-formed JSON", 400

        svc_type = store.get_service_instance(service.service_id)
        if len(svc_type) == 1:
            return 'Service instance with id {id} already exists'.format(id=service.service_id), 409
        # look up manifest based on plan id
        # based on the manifest type, select the driver
        # send the manifest for creation to the target system
        # store the ID along with refs to service, plan and manifest

        # get the manifest for the service/plan
        # TODO some validation required here to ensure it's the right svc/plan
        svc_type = store.get_service(service.service_id)[0]
        if svc_type is None:
            return 'Unrecognised service requested to be instantiated', 404

        plans = svc_type.plans
        plan = [p for p in plans if p.id == service.plan_id]
        if len(plan) <= 0:
            return 'no plan found.', 404

        mani = store.get_manifest(plan_id=plan[0].id)[0]

        if accept_incomplete:  # given docker-compose runs in detached mode this is not needed - only timing can verify
            # XXX put this in a thread to allow for asynch processing?
            rm.create(instance_id=instance_id, content=mani.manifest_content, c_type=mani.manifest_type)
        else:
            rm.create(instance_id=instance_id, content=mani.manifest_content, c_type=mani.manifest_type)

        last_op = LastOperation(  # stored within the service instance doc
            state='creating',
            description='service instance is being created'
        )

        # store the instance Id with manifest id
        srv_inst = ServiceInstance(
            service_type=svc_type,
            state=last_op,
            context={
                'id': instance_id,
                'manifest_id': mani.id,
            }
        )

        store.add_service_instance(srv_inst)

        if accept_incomplete:
            store.add_last_operation(instance_id=instance_id, last_operation=last_op)

        return 'created', 200

def deprovision_service_instance(instance_id, service_id, plan_id, accept_incomplete=None):
    """
    Deprovisions a service instance.
    'When a broker receives a deprovision request from a client, it should delete any resources it
    created during the provision. Usually this means that all resources are immediately reclaimed for
    future provisions.'
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used
    for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param service_id: service ID to be deprovisioned
    :type service_id: str
    :param plan_id: plan ID of the service to be deprovisioned
    :type plan_id: str
    :param accept_incomplete: Indicates that the client is supporting asynchronous operations
    :type accept_incomplete: bool

    :rtype: UpdateOperationResponse
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        # XXX if there's bindings remove first?
        # XXX what about undo?
        # check that the instance exists first
        instance = store.get_service_instance(instance_id=instance_id)
        if len(instance) == 1:
            mani_id = instance[0].context['manifest_id']
            mani = store.get_manifest(manifest_id=mani_id)
            if len(mani) < 1:
                return 'no service manifest found.', 404

            rm.delete(instance_id=instance_id, manifest_type=mani[0].manifest_type)
            store.delete_service_instance(instance_id)
            # we don't delete the last_operation explicitly as its embedded in the service_instance document
            # store.delete_last_operation(instance_id)

            return Empty(), 200
        else:
            return Empty(), 404

def instance_info(instance_id):
    """
    Returns information about the service instance.
    Returns information about the service instance. This is a simple read operation against the broker database and
    is provided as a developer/consumer convienence.
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used
    for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str

    :rtype: ServiceType
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        # service instance should already be recorded
        srv_inst = store.get_service_instance(instance_id)
        if len(srv_inst) < 1:
            return 'no service instance found.', 404
        srv_inst = srv_inst[0]

        # get the latest info
        mani_id = srv_inst.context['manifest_id']
        mani = store.get_manifest(manifest_id=mani_id)
        if len(mani) < 1:
            return 'no manifest found.', 404
        inst_info = rm.info(instance_id=instance_id, manifest_type=mani[0].manifest_type)

        if inst_info['srv_inst.state.state'] == 'failed':
            # try epm.delete(instance_id=instance_id)?
            return 'There has been a failure in creating the service instance.', 500

        # TODO need a converged state model for the SM
        srv_inst.state.state = inst_info['srv_inst.state.state']
        srv_inst.state.description = inst_info['srv_inst.state.description']

        # don't need you any more, buh-bye!
        del inst_info['srv_inst.state.state']
        del inst_info['srv_inst.state.description']

        # merge the two context dicts
        srv_inst.context = {**srv_inst.context, **inst_info}

        # update the service instance record - there should be an asynch method doing the update - event based
        store.add_service_instance(srv_inst)

        return srv_inst, 200

def last_operation_status(instance_id, service_id=None, plan_id=None, operation=None):
    """
    Gets the current state of the last operation upon the specified resource.
    When a broker returns status code 202 ACCEPTED for provision, update, or deprovision, the client will
    begin to poll the /v2/service_instances/:guid/last_operation endpoint to obtain the state of the last requested
    operation. The broker response must contain the field state and an optional field description.
    :param instance_id: The instance_id of a service instance is provided by the client. This ID will be used for
    future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.
    :type instance_id: str
    :param service_id: ID of the service from the catalog.
    :type service_id: str
    :param plan_id: ID of the plan from the catalog.
    :type plan_id: str
    :param operation: A broker-provided identifier for the operation. When a value for operation is included
    with asynchronous responses for Provision, Update, and Deprovision requests, the broker client should provide
    the same value using this query parameter as a URL-encoded string.;
    :type operation: str

    :rtype: LastOperation
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        # just re-use the method and return it's content and http status code.
        # version check not required here as it's done in the proxied call
        srv_inst, code = instance_info(instance_id=instance_id)
        if code == 404:
            return srv_inst + 'No service status therefore.', code
        else:
            return srv_inst.state, code

def service_bind(instance_id, binding_id, binding):
    """
    Binds to a service
    When the broker receives a bind request from the client, it should return information which helps an application to utilize the provisioned resource. This information is generically referred to as credentials. Applications should be issued unique credentials whenever possible, so one application access can be revoked without affecting other bound applications.
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param binding_id: The binding_id of a service binding is provided by the Cloud Controller.
    :type binding_id: str
    :param binding:
    :type binding: dict | bytes

    :rtype: BindingResponse
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        if connexion.request.is_json:
            binding = BindingRequest.from_dict(connexion.request.get_json())
        else:
            return "Supplied body content is not or is mal-formed JSON", 400
        return 'Not implemented :-(', 501

def service_unbind(instance_id, binding_id, service_id, plan_id):
    """
    Unbinds a service
    When a broker receives an unbind request from the client, it should delete any resources it created in bind. Usually this means that an application immediately cannot access the resource.
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param binding_id: The binding_id of a service binding is provided by the Cloud Controller.
    :type binding_id: str
    :param service_id: ID of the service from the catalog.
    :type service_id: str
    :param plan_id: ID of the plan from the catalog.
    :type plan_id: str

    :rtype: Empty
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        return 'Not implemented :-(', 501

def update_service_instance(instance_id, plan, accept_incomplete=None):
    """
    Updating a Service Instance
    Brokers that implement this endpoint can enable users to modify attributes of an existing service instance. The first attribute supports users modifying is the service plan. This effectively enables users to upgrade or downgrade their service instance to other plans. To see how users make these requests.'
    :param instance_id: 'The instance_id of a service instance is provided by the client. This ID will be used for future requests (bind and deprovision), so the broker must use it to correlate the resource it creates.'
    :type instance_id: str
    :param plan: New Plan information.
    :type plan: dict | bytes
    :param accept_incomplete: Indicates that the client is supporting asynchronous operations
    :type accept_incomplete: bool

    :rtype: Empty
    """
    ok, message, code = _version_ok()
    if not ok:
        return message, code
    else:
        if connexion.request.is_json:
            plan = UpdateRequest.from_dict(connexion.request.get_json())
        else:
            return "Supplied body content is not or is mal-formed JSON", 400
        return 'Not implemented :-(', 501
<|endoftext|>


<commit_before>from wsgiref.simple_server import make_server
import pprint
import urlparse
import os
from named_entity import *

def simple_app(environ, start_response):
    """Simplest possible application object"""
    status = '200 OK'
    response_headers = [('Content-type', 'text/plain')]
    start_response(status, response_headers)
    ret = ['Hello world!']
    if 'ner' in environ['PATH_INFO']:
        query = urlparse.parse_qs(environ['QUERY_STRING'])
        if 'text' in query:
            ret.append(do_ner(query['text'][0]))
    else:
        ret.append("query "+environ['QUERY_STRING'])
        ret.append("path "+environ['PATH_INFO'])
#    ret.append(pprint.pformat(environ))
    return ["\n\n".join(ret)]

port = int(os.getenv('PORT'))

httpd = make_server('', port, simple_app)
print "Serving HTTP on port %d..."%port

# Respond to requests until process is killed
httpd.serve_forever()
<commit_msg>added default port<commit_after>from wsgiref.simple_server import make_server
import pprint
import urlparse
import os
from named_entity import *

def simple_app(environ, start_response):
    """Simplest possible application object"""
    status = '200 OK'
    response_headers = [('Content-type', 'text/plain')]
    start_response(status, response_headers)
    ret = ['Hello world!']
    if 'ner' in environ['PATH_INFO']:
        query = urlparse.parse_qs(environ['QUERY_STRING'])
        if 'text' in query:
            ret.append(do_ner(query['text'][0]))
    else:
        ret.append("query "+environ['QUERY_STRING'])
        ret.append("path "+environ['PATH_INFO'])
#    ret.append(pprint.pformat(environ))
    return ["\n\n".join(ret)]

port = int(os.getenv('PORT', '8000'))

httpd = make_server('', port, simple_app)
print "Serving HTTP on port %d..."%port

# Respond to requests until process is killed
httpd.serve_forever()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import sys

import mapnik2

import constants

class InvarUtility(object):
    description = ''
    epilog = ''
    override_flags = ''

    def __init__(self):
        """
        Perform argument processing and other setup for a InvarUtility.
        """
        self._init_common_parser()
        self.add_arguments()
        self.args = self.argparser.parse_args()
        self._install_exception_handler()

        mapnik2.register_fonts('/Library/Fonts/')
        mapnik2.register_fonts('/usr/share/fonts')

    def add_arguments(self):
        """
        Called upon initialization once the parser for common arguments has been constructed.

        Should be overriden by individual utilities.
        """
        raise NotImplementedError('add_arguments must be provided by each subclass of InvarUtility.')

    def main(self):
        """
        Main loop of the utility.

        Should be overriden by individual utilities and explicitly called by the executing script.
        """
        raise NotImplementedError(' must be provided by each subclass of InvarUtility.')

    def _init_common_parser(self):
        """
        Prepare a base argparse argument parser so that flags are consistent across different shell command tools.
        """
        self.argparser = argparse.ArgumentParser(description=self.description, epilog=self.epilog)

        self.argparser.add_argument('config', help="Mapnik2 XML configuration file.")
        self.argparser.add_argument('output_dir', help="Destination directory for output.")
        self.argparser.add_argument('-p', '--process_count', help="Number of rendering processes to create.", type=int, default=constants.DEFAULT_PROCESS_COUNT)
        self.argparser.add_argument('-w', '--width', help="Width of images to render.", type=int, default=constants.DEFAULT_WIDTH)
        self.argparser.add_argument('-t', '--height', help="Height of images to render.", type=int, default=constants.DEFAULT_HEIGHT)

    def _install_exception_handler(self):
        """
        Installs a replacement for sys.excepthook, which handles pretty-printing uncaught exceptions.
        """
        def handler(t, value, traceback):
            if self.args.verbose:
                sys.__excepthook__(t, value, traceback)
            else:
                print value

        sys.excepthook = handler

<commit_msg>Added flag for custom font paths. Closes #2.<commit_after>#!/usr/bin/env python

import argparse
import sys

import mapnik2

import constants

class InvarUtility(object):
    description = ''
    epilog = ''
    override_flags = ''

    def __init__(self):
        """
        Perform argument processing and other setup for a InvarUtility.
        """
        self._init_common_parser()
        self.add_arguments()
        self.args = self.argparser.parse_args()
        self._install_exception_handler()

        for font_path in self.args.font_paths:
            mapnik2.register_fonts(font_path)

    def add_arguments(self):
        """
        Called upon initialization once the parser for common arguments has been constructed.

        Should be overriden by individual utilities.
        """
        raise NotImplementedError('add_arguments must be provided by each subclass of InvarUtility.')

    def main(self):
        """
        Main loop of the utility.

        Should be overriden by individual utilities and explicitly called by the executing script.
        """
        raise NotImplementedError(' must be provided by each subclass of InvarUtility.')

    def _init_common_parser(self):
        """
        Prepare a base argparse argument parser so that flags are consistent across different shell command tools.
        """
        self.argparser = argparse.ArgumentParser(description=self.description, epilog=self.epilog)

        self.argparser.add_argument('config', help="Mapnik2 XML configuration file.")
        self.argparser.add_argument('output_dir', help="Destination directory for output.")
        self.argparser.add_argument('-p', '--process_count', help="Number of rendering processes to create.", type=int, default=constants.DEFAULT_PROCESS_COUNT)
        self.argparser.add_argument('-w', '--width', help="Width of images to render.", type=int, default=constants.DEFAULT_WIDTH)
        self.argparser.add_argument('-t', '--height', help="Height of images to render.", type=int, default=constants.DEFAULT_HEIGHT)
        self.argparser.add_argument('--font-path', help='Add a directory to the paths which Mapnik will search by fonts.', dest='font_paths', action='append', default=['/Library/Fonts/', '/usr/share/fonts'])

    def _install_exception_handler(self):
        """
        Installs a replacement for sys.excepthook, which handles pretty-printing uncaught exceptions.
        """
        def handler(t, value, traceback):
            if self.args.verbose:
                sys.__excepthook__(t, value, traceback)
            else:
                print value

        sys.excepthook = handler

<|endoftext|>


<commit_before>import unittest
from src.dojo import Dojo

class TestCreateRoom (unittest.TestCase):
    def test_create_room_successfully(self):
        my_class_instance = Dojo()
        initial_room_count = len(my_class_instance.all_rooms)
        blue_office = my_class_instance.create_room("office", "Blue")
        self.assertTrue(blue_office)
        new_room_count = len(my_class_instance.all_rooms)
        self.assertEqual(new_room_count - initial_room_count, 1)

    def test_create_rooms_successfully(self):
        my_class_instance = Dojo()
        initial_room_count = len(my_class_instance.all_rooms)
        offices = my_class_instance.create_room("office", "Blue", "Black", "Brown")
        self.assertTrue(offices)
        new_room_count = len(my_class_instance.all_rooms)
        self.assertEqual(new_room_count - initial_room_count, 3)

    def test_person_added_to_system(self):
        initial_person_count = len(self.dojo.all_people)
        person = self.dojo.add_person("Neil", "Armstrong", "Staff")
        self.assertTrue(person)
        new_person_count = len(self.dojo.all_people)
        self.assertEqual(new_person_count - initial_person_count, 1)

    def test_person_has_been_assigned_office(self):
        person = self.dojo.add_person("Neil", "Armstrong", "Staff")
        self.assertTrue(person)
        self.assertTrue(self.dojo.all_people[-1].has_office)<commit_msg>Add test to check that person has been given living_space<commit_after>import unittest
from src.dojo import Dojo

class TestCreateRoom (unittest.TestCase):
    def test_create_room_successfully(self):
        my_class_instance = Dojo()
        initial_room_count = len(my_class_instance.all_rooms)
        blue_office = my_class_instance.create_room("office", "Blue")
        self.assertTrue(blue_office)
        new_room_count = len(my_class_instance.all_rooms)
        self.assertEqual(new_room_count - initial_room_count, 1)

    def test_create_rooms_successfully(self):
        my_class_instance = Dojo()
        initial_room_count = len(my_class_instance.all_rooms)
        offices = my_class_instance.create_room("office", "Blue", "Black", "Brown")
        self.assertTrue(offices)
        new_room_count = len(my_class_instance.all_rooms)
        self.assertEqual(new_room_count - initial_room_count, 3)

    def test_person_added_to_system(self):
        initial_person_count = len(self.dojo.all_people)
        person = self.dojo.add_person("Neil", "Armstrong", "Staff")
        self.assertTrue(person)
        new_person_count = len(self.dojo.all_people)
        self.assertEqual(new_person_count - initial_person_count, 1)

    def test_person_has_been_assigned_office(self):
        person = self.dojo.add_person("Neil", "Armstrong", "Staff")
        self.assertTrue(person)
        self.assertTrue(self.dojo.all_people[-1].has_office)

    def test_person_has_been_assigned_living_space(self):
        person = self.dojo.add_person("Eden", "Hazard", "Fellow", "Y")
        self.assertTrue(person)
        self.assertTrue(self.dojo.all_people[-1].has_living_space)<|endoftext|>


<commit_before>#!/usr/bin/env python3

from argparse import ArgumentParser
import subprocess
import shlex
import yaml

def evaluate(env, item):
    if 'cmd' in item:
        subprocess.call(env + item['cmd'], shell=True)
    if 'men' in item:
        print(item['men'])
        men_args = [subprocess.check_output(
            '%s echo %s' % (env, a), shell=True).decode('utf-8').strip('\n')
            for a in shlex.split(item['men'])]
        args.start = men_args[0]
        args.arg = men_args[1:]
        print(men_args)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('infile')
    parser.add_argument('-m', '--menu', default='dmenu -i -l 20')
    parser.add_argument('start', nargs='?', default='main')
    parser.add_argument('arg', nargs='*', default=[])
    args = parser.parse_args()

    f = open(args.infile)
    data = yaml.load(f)
    f.close()

    while args.start:
        menudata = data[args.start]
        args.start = None

        env = ''
        if 'args' in menudata:
            n = min(len(menudata['args']), len(args.arg))
            for x in range(n):
                env += '%s="%s";' % (menudata['args'][x], args.arg[x])
        items = ''
        item_dict = {}
        if 'items' in menudata:
            for item in menudata['items']:
                if 'txt' in item:
                    key = subprocess.check_output(
                            'echo %s' % item['txt'], shell=True).decode('utf-8')
                    items += key
                    item_dict[key] = item
                elif 'gen' in item:
                    gen_items = subprocess.check_output(
                            '%s echo "`%s`"' % (env, item['gen']),
                            shell=True).decode('utf-8')
                    items += gen_items
                    for line in gen_items.split('\n'):
                        item_dict[line + '\n'] = item
        items = items.replace('"', '\\"').strip()
        try:
            selection = subprocess.check_output(
                    'echo "%s" | %s' % (items, args.menu),
                    shell=True).decode('utf-8')
        except subprocess.CalledProcessError:
            selection = ''

        env += 'item="%s";' % selection.strip()
        if selection in item_dict:
            evaluate(env, item_dict[selection])
        else:
            evaluate(env, menudata)
<commit_msg>Removed print statements<commit_after>#!/usr/bin/env python3

from argparse import ArgumentParser
import subprocess
import shlex
import yaml

def evaluate(env, item):
    if 'cmd' in item:
        subprocess.call(env + item['cmd'], shell=True)
    if 'men' in item:
        men_args = [subprocess.check_output(
            '%s echo %s' % (env, a), shell=True).decode('utf-8').strip('\n')
            for a in shlex.split(item['men'])]
        args.start = men_args[0]
        args.arg = men_args[1:]

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('infile')
    parser.add_argument('-m', '--menu', default='dmenu -i -l 20')
    parser.add_argument('start', nargs='?', default='main')
    parser.add_argument('arg', nargs='*', default=[])
    args = parser.parse_args()

    f = open(args.infile)
    data = yaml.load(f)
    f.close()

    while args.start:
        menudata = data[args.start]
        args.start = None

        env = ''
        if 'args' in menudata:
            n = min(len(menudata['args']), len(args.arg))
            for x in range(n):
                env += '%s="%s";' % (menudata['args'][x], args.arg[x])
        items = ''
        item_dict = {}
        if 'items' in menudata:
            for item in menudata['items']:
                if 'txt' in item:
                    key = subprocess.check_output(
                            'echo %s' % item['txt'], shell=True).decode('utf-8')
                    items += key
                    item_dict[key] = item
                elif 'gen' in item:
                    gen_items = subprocess.check_output(
                            '%s echo "`%s`"' % (env, item['gen']),
                            shell=True).decode('utf-8')
                    items += gen_items
                    for line in gen_items.split('\n'):
                        item_dict[line + '\n'] = item
        items = items.replace('"', '\\"').strip()
        try:
            selection = subprocess.check_output(
                    'echo "%s" | %s' % (items, args.menu),
                    shell=True).decode('utf-8')
        except subprocess.CalledProcessError:
            selection = ''

        env += 'item="%s";' % selection.strip()
        if selection in item_dict:
            evaluate(env, item_dict[selection])
        else:
            evaluate(env, menudata)
<|endoftext|>