<commit_before># -*- coding: utf-8 -*-
import base64
import datetime
import json
import sys
from classes.transactions_explorer_data_type import TransactionsExplorerDataType
from classes.service import Service

def setup_data_types():
    # Each service contains 2 different ways of returning data: quarterly,
    # and seasonally adjusted. Each of the 2 ways of returning data is
    # available for 5 time periods.

    # Seasonally adjusted data has a duration of 1 year and contains 4 metrics.
    # Every year the spreadsheet takes the latest quarterly data and adds it to
    # that of the previous 3 quarters to come up with new seasonally adjusted
    # data point.

    seasonally_adjusted = TransactionsExplorerDataType(
        'seasonally-adjusted', '{period} {metric}',
        {
            u'Vol.': 'number_of_transactions',
            u'CPT': 'cost_per_transaction',
            u'Digital vol.': 'number_of_digital_transactions',
            u'Digital CPT': 'digital_cost_per_transaction',
            # The following two fields are not stored in the spreadsheet,
            # they are calculated below
            u'Digital take-up': 'digital_takeup',
            u'Total cost': 'total_cost',
        },
        {
            '2012-Q4': {
                '_start_at': datetime.datetime(2011, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2012, 04, 01, 0, 0),
            },
            '2013-Q1': {
                '_start_at': datetime.datetime(2012, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 01, 01, 0, 0),
            },
            '2013-Q2': {
                '_start_at': datetime.datetime(2012, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 04, 01, 0, 0),
            },
            '2013-Q3': {
                '_start_at': datetime.datetime(2012, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 07, 01, 0, 0),
            },
            '2013-Q4': {
                '_start_at': datetime.datetime(2012, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 10, 01, 0, 0),
            },
            '2014-Q1': {
                '_start_at': datetime.datetime(2013, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 01, 01, 0, 0),
            },
            '2014-Q2': {
                '_start_at': datetime.datetime(2013, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 04, 01, 0, 0),
            },
            '2014-Q3': {
                '_start_at': datetime.datetime(2013, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 07, 01, 0, 0),
            },
            '2014-Q4': {
                '_start_at': datetime.datetime(2013, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 10, 01, 0, 0),
            },
        },
        'year'
    )

    # Quarterly data is more granular than seasonally-adjusted data but only
    # contains 2 metrics rather than 4. As the name implies, it has a duration
    # of 3 months.

    quarterly = TransactionsExplorerDataType(
        'quarterly', '{metric}\n{period}',
        {
            u'Total transactions': 'number_of_transactions',
            u'Digital transactions': 'number_of_digital_transactions',
            # The following field is not stored in the spreadsheet,
            # it is calculated below
            u'Digital take-up': 'digital_takeup',
        },
        {
            'Jul - Sep 2012': {
                '_start_at': datetime.datetime(2012, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2012, 10, 01, 0, 0),
            },
            'Oct - Dec 2012': {
                '_start_at': datetime.datetime(2012, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 01, 01, 0, 0),
            },
            'Jan - Mar 2013': {
                '_start_at': datetime.datetime(2013, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 04, 01, 0, 0),
            },
            'Apr - Jun 2013': {
                '_start_at': datetime.datetime(2013, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 07, 01, 0, 0),
            },
            'Jul - Sep 2013': {
                '_start_at': datetime.datetime(2013, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 10, 01, 0, 0),
            },
            'Oct - Dec 2013': {
                '_start_at': datetime.datetime(2013, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 01, 01, 0, 0),
            },
            'Jan - Mar 2014': {
                '_start_at': datetime.datetime(2014, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 04, 01, 0, 0),
            },
            'Apr - Jun 2014': {
                '_start_at': datetime.datetime(2014, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 07, 01, 0, 0),
            },
            'Jul - Sep 2014': {
                '_start_at': datetime.datetime(2014, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 10, 01, 0, 0),
            },
        },
        'quarter'
    )

    return [seasonally_adjusted, quarterly]

def process(data):
    """Sets up the class instances we need to process and outputs JSON"""

    data_types = setup_data_types()

    formatted_data = []

    for index, service in enumerate(data):
        service = Service(index, service)

        for data_type in data_types:
            for period in data_type.periods:
                start_at = data_type.get_period_start_date(period)
                end_at = data_type.get_period_end_date(period)
                period_duration = data_type.period_duration
                service_id = service.identifier()

                datum_id = str(start_at) + str(end_at) + service_id.encode('utf-8')

                datum = {
                    '_id': base64.b64encode(datum_id),
                    '_timestamp': start_at,
                    'end_at': end_at,
                    'period': period_duration,
                    'service_id': service_id,
                    'type': data_type.title,
                }

                for metric in data_type.metrics:

                    metric_key = data_type.get_key(metric, period)

                    if service.attribute_exists(metric_key):
                        metric_value = service.get_datum(metric_key)
                    else:
                        if metric == 'Total cost':
                            number_of_transactions = service.get_datum(data_type.get_key('Vol.', period))
                            cpt = service.get_datum(data_type.get_key(u'CPT', period))

                            if number_of_transactions == None or cpt == None:
                                metric_value = None
                            else:
                                metric_value = (number_of_transactions * cpt)

                        elif metric == 'Digital take-up':
                            number_of_transactions = service.get_datum(data_type.get_key(data_type.get_spreadsheet_title_from_metric('number_of_transactions'), period))
                            number_of_digital_transactions = service.get_datum(data_type.get_key(data_type.get_spreadsheet_title_from_metric('number_of_digital_transactions'), period))

                            if not number_of_transactions:
                                metric_value = None
                            elif number_of_digital_transactions == 0:
                                metric_value = 0
                            elif number_of_digital_transactions == None:
                                metric_value = None
                            else:
                                metric_value = (number_of_digital_transactions / (number_of_transactions + 0.0))

                    tidy_metric_name = data_type.metrics[metric]
                    datum[tidy_metric_name] = metric_value

                formatted_data.append(datum)

    return formatted_data
<commit_msg>Explain odd start + end at for seasonally-adjusted<commit_after># -*- coding: utf-8 -*-
import base64
import datetime
import json
import sys
from classes.transactions_explorer_data_type import TransactionsExplorerDataType
from classes.service import Service

def setup_data_types():
    # Each service contains 2 different ways of returning data: quarterly,
    # and seasonally adjusted. Each of the 2 ways of returning data is
    # available for 5 time periods.

    # Seasonally adjusted data has a duration of 1 year and contains 4 metrics.
    # Every year the spreadsheet takes the latest quarterly data and adds it to
    # that of the previous 3 quarters to come up with new seasonally adjusted
    # data point.

    # For seasonally adjusted data, the keys used to extract the data
    # (e.g. 2014-Q4) represent when the data was *requested* not the
    # period which the data is for. The period the data is for is normally
    # the one before the one implied by the key. As a result the start at
    # for 2014-Q4 is October 2013 rather than January 2014.

    # 2012-Q4 is a special case of trial data collection and so does not fall
    # into the normal pattern. It was requested in 2012-Q4 but represents the
    # time period below (April 2011 to March 2012)

    seasonally_adjusted = TransactionsExplorerDataType(
        'seasonally-adjusted', '{period} {metric}',
        {
            u'Vol.': 'number_of_transactions',
            u'CPT': 'cost_per_transaction',
            u'Digital vol.': 'number_of_digital_transactions',
            u'Digital CPT': 'digital_cost_per_transaction',
            # The following two fields are not stored in the spreadsheet,
            # they are calculated below
            u'Digital take-up': 'digital_takeup',
            u'Total cost': 'total_cost',
        },
        {
            '2012-Q4': {
                '_start_at': datetime.datetime(2011, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2012, 04, 01, 0, 0),
            },
            '2013-Q1': {
                '_start_at': datetime.datetime(2012, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 01, 01, 0, 0),
            },
            '2013-Q2': {
                '_start_at': datetime.datetime(2012, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 04, 01, 0, 0),
            },
            '2013-Q3': {
                '_start_at': datetime.datetime(2012, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 07, 01, 0, 0),
            },
            '2013-Q4': {
                '_start_at': datetime.datetime(2012, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 10, 01, 0, 0),
            },
            '2014-Q1': {
                '_start_at': datetime.datetime(2013, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 01, 01, 0, 0),
            },
            '2014-Q2': {
                '_start_at': datetime.datetime(2013, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 04, 01, 0, 0),
            },
            '2014-Q3': {
                '_start_at': datetime.datetime(2013, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 07, 01, 0, 0),
            },
            '2014-Q4': {
                '_start_at': datetime.datetime(2013, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 10, 01, 0, 0),
            },
        },
        'year'
    )

    # Quarterly data is more granular than seasonally-adjusted data but only
    # contains 2 metrics rather than 4. As the name implies, it has a duration
    # of 3 months.

    quarterly = TransactionsExplorerDataType(
        'quarterly', '{metric}\n{period}',
        {
            u'Total transactions': 'number_of_transactions',
            u'Digital transactions': 'number_of_digital_transactions',
            # The following field is not stored in the spreadsheet,
            # it is calculated below
            u'Digital take-up': 'digital_takeup',
        },
        {
            'Jul - Sep 2012': {
                '_start_at': datetime.datetime(2012, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2012, 10, 01, 0, 0),
            },
            'Oct - Dec 2012': {
                '_start_at': datetime.datetime(2012, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 01, 01, 0, 0),
            },
            'Jan - Mar 2013': {
                '_start_at': datetime.datetime(2013, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 04, 01, 0, 0),
            },
            'Apr - Jun 2013': {
                '_start_at': datetime.datetime(2013, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 07, 01, 0, 0),
            },
            'Jul - Sep 2013': {
                '_start_at': datetime.datetime(2013, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2013, 10, 01, 0, 0),
            },
            'Oct - Dec 2013': {
                '_start_at': datetime.datetime(2013, 10, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 01, 01, 0, 0),
            },
            'Jan - Mar 2014': {
                '_start_at': datetime.datetime(2014, 01, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 04, 01, 0, 0),
            },
            'Apr - Jun 2014': {
                '_start_at': datetime.datetime(2014, 04, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 07, 01, 0, 0),
            },
            'Jul - Sep 2014': {
                '_start_at': datetime.datetime(2014, 07, 01, 0, 0),
                '_end_at': datetime.datetime(2014, 10, 01, 0, 0),
            },
        },
        'quarter'
    )

    return [seasonally_adjusted, quarterly]

def process(data):
    """Sets up the class instances we need to process and outputs JSON"""

    data_types = setup_data_types()

    formatted_data = []

    for index, service in enumerate(data):
        service = Service(index, service)

        for data_type in data_types:
            for period in data_type.periods:
                start_at = data_type.get_period_start_date(period)
                end_at = data_type.get_period_end_date(period)
                period_duration = data_type.period_duration
                service_id = service.identifier()

                datum_id = str(start_at) + str(end_at) + service_id.encode('utf-8')

                datum = {
                    '_id': base64.b64encode(datum_id),
                    '_timestamp': start_at,
                    'end_at': end_at,
                    'period': period_duration,
                    'service_id': service_id,
                    'type': data_type.title,
                }

                for metric in data_type.metrics:

                    metric_key = data_type.get_key(metric, period)

                    if service.attribute_exists(metric_key):
                        metric_value = service.get_datum(metric_key)
                    else:
                        if metric == 'Total cost':
                            number_of_transactions = service.get_datum(data_type.get_key('Vol.', period))
                            cpt = service.get_datum(data_type.get_key(u'CPT', period))

                            if number_of_transactions == None or cpt == None:
                                metric_value = None
                            else:
                                metric_value = (number_of_transactions * cpt)

                        elif metric == 'Digital take-up':
                            number_of_transactions = service.get_datum(data_type.get_key(data_type.get_spreadsheet_title_from_metric('number_of_transactions'), period))
                            number_of_digital_transactions = service.get_datum(data_type.get_key(data_type.get_spreadsheet_title_from_metric('number_of_digital_transactions'), period))

                            if not number_of_transactions:
                                metric_value = None
                            elif number_of_digital_transactions == 0:
                                metric_value = 0
                            elif number_of_digital_transactions == None:
                                metric_value = None
                            else:
                                metric_value = (number_of_digital_transactions / (number_of_transactions + 0.0))

                    tidy_metric_name = data_type.metrics[metric]
                    datum[tidy_metric_name] = metric_value

                formatted_data.append(datum)

    return formatted_data
<|endoftext|>


<commit_before>#!/usr/bin/env python2
"""
quattorpoddoc generates markdown documentation from:
 - configuration-modules-core perl documentation
 - configuration-modules-grid perl documentation
 - CAF perl documentation
 - CCM perl documentation
 - schema pan annotations
 and creates a index for the website on http://quattor.org.

@author: Wouter Depypere (Ghent University)

"""
import sys
import os
import re
from vsc.utils.generaloption import simple_option
from vsc.utils import fancylogger
from vsc.utils.run import run_asyncloop
import tempfile
from lxml import etree
import shutil

MAILREGEX = re.compile(("([a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`"
                        "{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|"
                        "\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)"))
PATHREGEX = re.compile(r'(\s+)((?:/[\w{}]+)+\.?\w*)(\s*)')
EXAMPLEMAILS = ["example", "username", "system.admin"]
logger = fancylogger.getLogger()
DOCDIR = "docs"

REPOMAP = {
    "configuration-modules-core": {
        "sitesubdir": "components",
        "target": "/NCM/Component/"
        },
    "configuration-modules-grid":{
        "sitesubdir": "components-grid",
        "target": "/NCM/Component/"
        },
    "CAF": {
        "sitesubdir": "CAF",
        "target": "/CAF/"
        },
    "CCM": {
        "sitesubdir": "CCM",
        "target": "EDG/WP4/CCM/"
        },
    }

def maven_clean_compile(repo_location, repository):
    """
    Executes mvn clean and mvn compile in the given modules_location.
    """
    repoloc = os.path.join(repo_location, repository)
    logger.info("Doing maven clean compile in %s." % repoloc)
    output = run_asyncloop("mvn clean compile", startpath=repoloc)
    logger.debug(output)

def generate_mds(repo, sources, location):
    """
    Takes a list of components with podfiles and generates a md file for it.
    """
    logger.info("Generating md files.")
    mdfiles = set()

    comppath = os.path.join(location, DOCDIR, REPOMAP[repo]['sitesubdir'])
    if not os.path.exists(comppath):
        os.makedirs(comppath)

    for source in sources:
        tpl = False
        if source.endswith(".tpl"):
            logger.warning("%s file extension is '.tpl', should become '.pan'." % source)
            tpl = True

        if source.endswith(".pan") or tpl:
            mdfile = create_md_from_pan(source, comppath)
            if mdfile is not None:
                mdfiles.add(mdfile)
        else:
            sourcename = source.split(REPOMAP[repo]['target'])[-1]
            mdfile = os.path.splitext(sourcename)[0].replace("/", "::").lower() + ".md"

            convert_pod_to_markdown(source, os.path.join(comppath, mdfile))
            mdfiles.add(mdfile)

    logger.info("Written %s md files." % len(mdfiles))
    return mdfiles

def find_description(sub, namespace):
    """
    Search for the desc tag, even if it is in documentation tag.
    """
    doc = sub.find("./%sdocumentation/%sdesc" % (namespace, namespace))
    if doc is None:
        doc = sub.find("./%sdesc" % namespace)
    return doc

def create_md_from_pan(source, comppath):
    """
    Takes a pan schema, creates the pan annotations and parses them to markdown.
    """
    modname = os.path.basename(os.path.split(source)[0])
    templatename = os.path.splitext(os.path.basename(source))[0]
    mdfile = "%s::%s.md" % (modname, templatename)
    tmpdir = tempfile.mkdtemp()
    logger.debug("Temporary directory: %s" % tmpdir)
    panccommand = ["panc-annotations", "--output-dir", tmpdir, "--base-dir"]
    panccommand.extend(os.path.split(source))
    output = run_asyncloop(panccommand)
    logger.debug(output)
    namespace = "{http://quattor.org/pan/annotations}"

    tpl = "%s.pan.annotation.xml" % templatename
    xml = etree.parse(os.path.join(tmpdir, tpl))
    root = xml.getroot()

    if len(root) == 0:
        logger.debug("%s would be empty, skipping it." % mdfile)
        return None

    stypes = root.findall('%stype' % namespace)
    deffunctions = root.findall('%sfunction' % namespace)

    if len(stypes) == 0 and len(deffunctions) == 0:
        logger.debug("%s has no usable content, skipping it." % mdfile)
        return None

    mdtext = []

    mdtext.append("# Types\n")
    for stype in stypes:
        name = stype.get('name')
        mdtext.append("- /software/%s/%s" % (modname, name))

        doc = find_description(stype, namespace)
        if doc is not None:
            mdtext.append("%s- decription: %s" % (" "*4, doc.text))

        for field in stype.findall(".//%sfield" % namespace):
            mdtext.append("%s- /software/%s/%s/%s" % (" "*4, modname, name, field.get('name')))

            doct = find_description(field, namespace)
            if doct is not None:
                mdtext.append("%s- description: %s" % (" "*8, doct.text))

            required = field.get('required')
            if required == "true":
                mdtext.append("%s- required" % (" "*8))
            else:
                mdtext.append("%s- optional" % (" "*8))

            for basetype in field.findall(".//%sbasetype" % namespace):
                fieldtype = basetype.get('name')
                mdtext.append("%s- type: %s" % (" "*8, fieldtype))
                if fieldtype == "long" and basetype.get('range'):
                    fieldrange = basetype.get('range')
                    mdtext.append("%s- range: %s" % (" "*8, fieldrange))
            mdtext.append("\n")

    if deffunctions:
        mdtext.append("\n# Functions\n")
        root.findall('%sfunction' % namespace)
        for fnname in deffunctions:
            name = fnname.get('name')
            mdtext.append("- %s" % name)

            doc = find_description(fnname, namespace)
            if doc is not None:
                mdtext.append("%s- description: %s " % (" "*4, doc.text))

            for arg in fnname.findall(".//%sarg" % namespace):
                mdtext.append("%s- arg: %s" % (" "*4, arg.text))

    with open(os.path.join(comppath, mdfile), "w") as fih:
        fih.write("\n".join(mdtext))

    logger.debug("Removing temporary directory: %s" % tmpdir)
    shutil.rmtree(tmpdir)
    return mdfile

def convert_pod_to_markdown(podfile, outputfile):
    """
    Takes a podfile and converts it to a markdown with the help of pod2markdown.
    """
    logger.debug("Running pod2markdown on %s." % podfile)
    output = run_asyncloop("pod2markdown %s" % podfile)
    logger.debug("writing output to %s." % outputfile)
    logger.debug(output)
    with open(outputfile, "w") as fih:
        fih.write(output[1])

def generate_toc(mdfiles, outputloc, indexname):
    """
    Generates a TOC for the parsed components.
    """
    logger.info("Generating TOC as %s." % os.path.join(outputloc, indexname))

    with open(os.path.join(outputloc, indexname), "w") as fih:
        fih.write("site_name: Quattor Documentation\n\n")
        fih.write("theme: 'readthedocs'\n\n")
        fih.write("pages:\n")
        fih.write("- introduction: 'index.md'\n")

        for subdivision in sorted(mdfiles.keys()):
            fih.write("- %s:\n" % subdivision)
            for page in sorted(mdfiles[subdivision]):
                linkname = page.split(".")[0]
                linktopage = "%s/%s" % (subdivision, page)
                write_if_exists(outputloc, linktopage, linkname, fih)
        fih.write("\n")

def write_if_exists(outputloc, linktopage, linkname, fih):
    """
    Checks if the MD exists before adding it to the TOC.
    """
    if os.path.exists(os.path.join(outputloc, DOCDIR, linktopage)):
        logger.debug("Adding %s to toc." % linkname)
        fih.write("    - %s: '%s'\n" % (linkname, linktopage))
    else:
        logger.warn("Expected %s but it does not exist. Not adding to toc."
                    % os.path.join(outputloc, DOCDIR, linktopage))

def remove_mail(mdfiles, outputloc):
    """
    Removes the email addresses from the markdown files.
    """
    logger.info("Removing emailaddresses from md files.")
    counter = 0
    for subdivision in mdfiles.keys():
        for mdfile in mdfiles[subdivision]:
            mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile)
            with open(mdfileloc, 'r') as fih:
                mdcontent = fih.read()
            replace = False
            for email in re.findall(MAILREGEX, mdcontent):
                logger.debug("Found %s." % email[0])
                replace = True
                if email[0].startswith('//'):
                    replace = False
                for ignoremail in EXAMPLEMAILS:
                    if ignoremail in email[0]:
                        replace = False

            if replace:
                logger.debug("Removed it from line.")
                mdcontent = mdcontent.replace(email[0], '')
                with open(mdfileloc, 'w') as fih:
                    fih.write(mdcontent)
                counter += 1
    logger.info("Removed %s email addresses." % counter)

def remove_whitespace(mdfiles, outputloc):
    """
    Removes extra whitespace (\n\n\n).
    """
    logger.info("Removing extra whitespace from md files.")
    counter = 0
    for subdivision in mdfiles.keys():
        for mdfile in mdfiles[subdivision]:
            mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile)
            with open(mdfileloc, 'r') as fih:
                mdcontent = fih.read()
            if '\n\n\n' in mdcontent:
                logger.debug("Removing whitespace in %s." % mdfile)
                mdcontent = mdcontent.replace('\n\n\n', '\n')
                with open(mdfileloc, 'w') as fih:
                    fih.write(mdcontent)
                counter += 1
    logger.info("Removed extra whitespace from %s files." % counter)

def decrease_title_size(mdfiles, outputloc):
    """
    Makes titles smaller, e.g. replace "# " with "### ".
    """
    logger.info("Downsizing titles in md files.")
    counter = 0
    for subdivision in mdfiles.keys():
        for mdfile in mdfiles[subdivision]:
            mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile)
            with open(mdfileloc, 'r') as fih:
                mdcontent = fih.read()
            if '# ' in mdcontent:
                logger.debug("Making titles smaller in %s." % mdfile)
                mdcontent = mdcontent.replace('# ', '### ')
                with open(mdfileloc, 'w') as fih:
                    fih.write(mdcontent)
                counter += 1
    logger.info("Downsized titles in %s files." % counter)

def remove_headers(mdfiles, outputloc):
    """
    Removes MAINTAINER and AUTHOR headers from md files.
    """
    logger.info("Removing AUTHOR and MAINTAINER headers from md files.")
    counter = 0
    for subdivision in mdfiles.keys():
        for mdfile in mdfiles[subdivision]:
            mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile)
            with open(mdfileloc, 'r') as fih:
                mdcontent = fih.read()
            if '# MAINTAINER' in mdcontent:
                logger.debug("Removing # MAINTAINER in %s." % mdfile)
                mdcontent = mdcontent.replace('# MAINTAINER', '')
                with open(mdfileloc, 'w') as fih:
                    fih.write(mdcontent)
                counter += 1
            if '# AUTHOR' in mdcontent:
                logger.debug("Removing # AUTHOR in %s." % mdfile)
                mdcontent = mdcontent.replace('# AUTHOR', '')
                with open(mdfileloc, 'w') as fih:
                    fih.write(mdcontent)
                counter += 1

    logger.info("Removed %s unused headers." % counter)

def codify_paths(mdfiles, outputloc):
    """
    Puts paths inside code tags
    """
    logger.info("Putting paths inside code tags.")
    counter = 0
    for subdivision in mdfiles.keys():
        for mdfile in mdfiles[subdivision]:
            mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile)
            with open(mdfileloc, 'r') as fih:
                mdcontent = fih.read()

            logger.debug("Tagging paths in %s." % mdfile)
            mdcontent, counter = PATHREGEX.subn(r'\1`\2`\3', mdcontent)
            with open(mdfileloc, 'w') as fih:
                fih.write(mdcontent)

    logger.info("Code tagged %s paths." % counter)

def check_input_and_commands(sourceloc, outputloc, runmaven):
    """
    Check if the directories are in place.
    Check if the required binaries are in place.
    """
    logger.info("Checking if the given paths exist.")
    if not sourceloc:
        logger.error("Repo location not specified.")
        sys.exit(1)
    if not outputloc:
        logger.error("output location not specified")
        sys.exit(1)
    if not os.path.exists(sourceloc):
        logger.error("Repo location %s does not exist" % sourceloc)
        sys.exit(1)
    for repo in REPOMAP.keys():
        if not os.path.exists(os.path.join(sourceloc, repo)):
            logger.error("Repo location %s does not exist" % os.path.join(sourceloc, repo))
            sys.exit(1)
    if not os.path.exists(outputloc):
        logger.error("Output location %s does not exist" % outputloc)
        sys.exit(1)

    logger.info("Checking if required binaries are installed.")
    if runmaven:
        if not which("mvn"):
            logger.error("The command mvn is not available on this system, please install maven.")
            sys.exit(1)
    if not which("pod2markdown"):
        logger.error("The command pod2markdown is not available on this system, please install pod2markdown.")
        sys.exit(1)

def list_perl_modules(module_location):
    """
    return a list of perl modules in module_location.
    Try to filter out duplicates, pod takes precedence over pm.
    """
    finallist = []
    for path, _, files in os.walk(module_location):
        if not is_wanted_dir(path, files):
            continue

        for mfile in files:
            if is_wanted_file(path, mfile):
                fname = os.path.join(path, mfile)
                duplicate = ""
                if "doc/pod" in fname:
                    duplicate = fname.replace('doc/pod', 'lib/perl')
                    if mfile.endswith('.pod'):
                        duplicate = duplicate.replace(".pod", ".pm")
                    if duplicate in finallist:
                        finallist[finallist.index(duplicate)] = fname
                        continue

                duplicate = ""
                if "lib/perl" in fname:
                    duplicate = fname.replace('lib/perl', 'doc/pod')
                if mfile.endswith('.pm'):
                    duplicate = duplicate.replace(".pm", ".pod")
                    if duplicate not in finallist:
                        finallist.append(fname)
                    else:
                        continue
                finallist.append(os.path.join(path, mfile))

    return finallist

def is_wanted_dir(path, files):
    """
    Check if the directory matches required criteria:
     - It must be in 'target directory'
     - It should contain files
     - it shoud contain one of the following in the path:
        - doc/pod
        - lib/perl
        - pan
    """
    if 'target' not in path: return False
    if len(files) == 0: return False
    if True not in [substr in path for substr in ["doc/pod", "lib/perl", "pan"]]: return False
    return True

def is_wanted_file(path, filename):
    """
    Check if the file matches one of the criteria:
     - a perl file based on extension
     - schema.pan
     - a perl file based on shebang
    """
    if True in [filename.endswith(ext) for ext in [".pod", ".pm", ".pl"]]: return True
    if filename.endswith(".pan"): return True
    if len(filename.split(".")) < 2:
        with open(os.path.join(path, filename), 'r') as pfile:
            if 'perl' in pfile.readline(): return True
    return False

def which(command):
    """
    Check if given command is available for the current user on this system.
    """
    found = False
    for direct in os.getenv("PATH").split(':'):
        if os.path.exists(os.path.join(direct, command)):
            found = True

    return found

def main(repoloc, outputloc):
    """
    Main run of the script.
    """
    mdfiles = {}
    for repo in REPOMAP.keys():
        logger.info("Processing %s." % repo)
        if GO.options.maven_compile:
            logger.info("Doing maven clean and compile.")
            maven_clean_compile(repoloc, repo)
        else:
            logger.info("Skipping maven clean and compile.")

        pmodules = list_perl_modules(os.path.join(repoloc, repo))
        mdfiles[REPOMAP[repo]['sitesubdir']] = generate_mds(repo, pmodules, outputloc)

    generate_toc(mdfiles, outputloc, GO.options.index_name)

    if GO.options.remove_emails:
        remove_mail(mdfiles, outputloc)

    if GO.options.remove_headers:
        remove_headers(mdfiles, outputloc)

    if GO.options.small_titles:
        decrease_title_size(mdfiles, outputloc)

    if GO.options.remove_whitespace:
        remove_whitespace(mdfiles, outputloc)

    if GO.options.codify_paths:
        codify_paths(mdfiles, outputloc)

if __name__ == '__main__':
    OPTIONS = {
        'modules_location': ('The location of the repo checkout.', None, 'store', None, 'm'),
        'output_location': ('The location where the output markdown files should be written to.', None, 'store', None, 'o'),
        'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None, 'store_true', False, 'c'),
        'index_name': ('Filename for the index/toc for the components.', None, 'store', 'mkdocs.yml', 'i'),
        'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'),
        'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'),
        'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'),
        'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's'),
        'codify_paths': ('Put paths inside code tags.', None, 'store_true', True, 'p'),
    }
    GO = simple_option(OPTIONS)
    logger.info("Starting main.")

    check_input_and_commands(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile)

    main(GO.options.modules_location, GO.options.output_location)

    logger.info("Done.")
<commit_msg>remove old quattorpoddoc script.<commit_after><|endoftext|>


<commit_before>import delphin_6_automation.database_interactions.sampling_interactions

__author__ = "Christian Kongsgaard"
__license__ = 'MIT'

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules
import sys
import os

# RiBuild Modules
from delphin_6_automation.logging.ribuild_logger import ribuild_logger
from delphin_6_automation.sampling import sampling
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions import sampling_interactions

# Logger
logger = ribuild_logger(__name__)

# -------------------------------------------------------------------------------------------------------------------- #
# RIBuild

def main():
    print_header()
    menu()

def print_header():
    print('---------------------------------------------------')
    print('|                                                  |')
    print('|           RiBuild EU Research Project            |')
    print('|           for Hygrothermal Simulations           |')
    print('|                                                  |')
    print('|                 WORK IN PROGRESS                 |')
    print('|               Sampling Environment               |')
    print('|                                                  |')
    print('---------------------------------------------------')

def menu():
    print('')
    print('------------------- SAMPLING MENU ---------------------')
    print('')
    print("Available Actions:")
    print("[a] Start Sampling")
    print("[b] Create Sampling Scheme")
    print("[c] View Current Samples")
    print("[x] Exit")

    choice = input("> ").strip().lower()

    if choice == 'a':
        logger.info('starting sampling')
        sampling_scheme_id = input("Define sampling scheme ID >")
        logger.info(sampling_scheme_id)

        print('Starting sampling\n')
        sampling_worker(sampling_scheme_id)

    elif choice == 'b':
        scheme = sampling.create_sampling_scheme(os.path.dirname(__file__))
        scheme_id = sampling_interactions.upload_sampling_scheme(scheme)
        print(f'Created sampling and uploaded it with ID: {scheme_id}')
        logger.info(f'Created sampling and uploaded it with ID: {scheme_id}')

    elif choice == 'c':
        print('Not implemented')
        pass

    elif choice == 'x':
        print("Goodbye")

def sampling_worker(scheme_id):
    scheme = sampling_interactions.download_sampling_scheme(scheme_id)
    sample_iteration = 0
    current_error = 0
    convergence = False

    while not convergence:
        print(f'Running sampling iteration #{sample_iteration}')
        logger.info(f'Running sampling iteration #{sample_iteration}')

        samples = sampling.load_existing_samples(scheme_id)
        new_samples = sampling.create_samples(scheme, samples)
        sampling_document = sampling_interactions.upload_samples(new_samples, sample_iteration)
        delphin_ids = sampling.create_delphin_projects(scheme, new_samples)
        sampling_interactions.add_delphin_to_sampling(sampling_document, delphin_ids)
        simulation_interactions.wait_until_simulated(delphin_ids)
        current_error = sampling.calculate_error(delphin_ids)
        sampling_interactions.upload_standard_error(sampling_document, current_error)

        print(f'Standard Error at iteration {sample_iteration} is: {current_error}')
        logger.info(f'Standard Error at iteration {sample_iteration} is: {current_error}')

        convergence = sampling.check_convergence(scheme, current_error)
        sample_iteration += 1

    print(f'Convergence reached at iteration #{sample_iteration}')
    logger.info(f'Convergence reached at iteration #{sample_iteration}')
    print('Exits. Bye')
    sys.exit()
<commit_msg>bug fix<commit_after>import delphin_6_automation.database_interactions.sampling_interactions

__author__ = "Christian Kongsgaard"
__license__ = 'MIT'

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules
import sys
import os

# RiBuild Modules
from delphin_6_automation.logging.ribuild_logger import ribuild_logger
from delphin_6_automation.sampling import sampling
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions import sampling_interactions

# Logger
logger = ribuild_logger(__name__)

# -------------------------------------------------------------------------------------------------------------------- #
# RIBuild

def main():
    print_header()
    menu()

def print_header():
    print('---------------------------------------------------')
    print('|                                                  |')
    print('|           RiBuild EU Research Project            |')
    print('|           for Hygrothermal Simulations           |')
    print('|                                                  |')
    print('|                 WORK IN PROGRESS                 |')
    print('|               Sampling Environment               |')
    print('|                                                  |')
    print('---------------------------------------------------')

def menu():
    print('')
    print('------------------- SAMPLING MENU ---------------------')
    print('')
    print("Available Actions:")
    print("[a] Start Sampling")
    print("[b] Create Sampling Scheme")
    print("[c] View Current Samples")
    print("[x] Exit")

    choice = input("> ").strip().lower()

    if choice == 'a':
        logger.info('starting sampling')
        sampling_scheme_id = input("Define sampling scheme ID >")
        logger.info(sampling_scheme_id)

        print('Starting sampling\n')
        sampling_worker(sampling_scheme_id)

    elif choice == 'b':
        scheme = sampling.create_sampling_scheme(os.path.dirname(__file__))
        scheme_id = sampling_interactions.upload_sampling_scheme(scheme)
        print(f'Created sampling and uploaded it with ID: {scheme_id}')
        logger.info(f'Created sampling and uploaded it with ID: {scheme_id}')

    elif choice == 'c':
        print('Not implemented')
        pass

    elif choice == 'x':
        print("Goodbye")

def sampling_worker(scheme_id):
    scheme = sampling_interactions.download_sampling_scheme(scheme_id)
    sample_iteration = 0
    convergence = False

    while not convergence:
        print(f'Running sampling iteration #{sample_iteration}')
        logger.info(f'Running sampling iteration #{sample_iteration}')

        samples = sampling.load_existing_samples(scheme_id)
        new_samples = sampling.create_samples(scheme, samples)
        sampling_document = sampling_interactions.upload_samples(new_samples, sample_iteration)
        delphin_ids = sampling.create_delphin_projects(scheme, new_samples)
        sampling_interactions.add_delphin_to_sampling(sampling_document, delphin_ids)
        simulation_interactions.wait_until_simulated(delphin_ids)
        current_error = sampling.calculate_error(delphin_ids)
        sampling_interactions.upload_standard_error(sampling_document, current_error)

        print(f'Standard Error at iteration {sample_iteration} is: {current_error}')
        logger.info(f'Standard Error at iteration {sample_iteration} is: {current_error}')

        convergence = sampling.check_convergence(scheme, current_error)
        sample_iteration += 1

    print(f'Convergence reached at iteration #{sample_iteration}')
    logger.info(f'Convergence reached at iteration #{sample_iteration}')
    print('Exits. Bye')
    sys.exit()
<|endoftext|>


<commit_before>from sklearn.base import BaseEstimator, ClassifierMixin
import fasttext

class FastTextClassifier(BaseEstimator,ClassifierMixin):
"""Base classiifer of Fasttext estimator"""

	def __init__(self,lpr='__label__',lr=0.1,lru=100,dim=100,ws=5,epoch=5,minc=1,neg=5,ngram=1,loss='softmax',nbucket=0,minn=0,maxn=0):
		"""
		label_prefix   			label prefix ['__label__']
		lr             			learning rate [0.1]
		lr_update_rate 			change the rate of updates for the learning rate [100]
		dim            			size of word vectors [100]
		ws             			size of the context window [5]
		epoch          			number of epochs [5]
		min_count      			minimal number of word occurences [1]
		neg            			number of negatives sampled [5]
		word_ngrams    			max length of word ngram [1]
		loss           			loss function {ns, hs, softmax} [softmax]
		bucket         			number of buckets [0]
		minn           			min length of char ngram [0]
		maxn           			min length of char ngram [0]
		"""
		self.label_prefix=lpr
		self.lr=lr
		self.lr_update_rate=lru
		self.dim=dim
		self.ws=ws
		self.epoch=epoch
		self.min_count=minc
		self.neg=neg
		self.word_ngrams=ngram
		self.loss=loss
		self.bucket=bucket
		self.minn=minn
		self.maxn=maxn

	def fit(self,X,y):
		'train a classifier and return the model'
		pass
	def predict(self,X):
		'get class labels for the classifier'
		pass
	def report(self,model=None):
		'prints classification report'
		pass
	def predict_proba(self,X,y):
		'return predicted probabilities'
		pass
	def getlabels(self):
		'retuns classlabels'
		pass
	def getproperties(self):
                '''
                Input: Nothing, other than object self pointer
                Return: None , prints the descriptions of the model hyperparameters
                '''
                print("The model has following hyperparameters as part of its specification")
                print("Label prefix used : "+str(self.label_prefix)
                print("Learning rate :"+ str(lr))
                print("Learning rate update after "+str(self.lr_update_rate)+"iterations")
                print("Embedding size: "+str(self.dim))
                print("Epochs :"+ str(self.epochs)
                print("minimal number of word occurences: "+self.min_count)
                print("number of negatives sampled :"+str(self.neg))
                print("max length of word ngram "+str(self.word_ngrams))
                print("loss function: "+str(self.loss))
                print("number of buckets "+str(self.bucket))
                print("min length of char ngram:"+str(self.minn))
                print("min length of char ngram"+ str(self.maxn))
                return(None)

	def loadpretrained(self,X):
		'returns the model with pretrained weights'
		pass

class SkipgramFastText(BaseEstimator,ClassifierMixin):

	def __init__(self,lpr='__label__',lr=0.1,lru=100,dim=100,ws=5,epoch=5,minc=1,neg=5,ngram=1,\
loss='softmax',nbucket=0,minn=0,maxn=0,th=12,t=0.0001,verbosec=0,encoding='utf-8'):
			"""
			lr             learning rate [0.05]
			lr_update_rate change the rate of updates for the learning rate [100]
			dim            size of word vectors [100]
			ws             size of the context window [5]
			epoch          number of epochs [5]
			min_count      minimal number of word occurences [5]
			neg            number of negatives sampled [5]
			word_ngrams    max length of word ngram [1]
			loss           loss function {ns, hs, softmax} [ns]
			bucket         number of buckets [2000000]
			minn           min length of char ngram [3]
			maxn           max length of char ngram [6]
			thread         number of threads [12]
			t              sampling threshold [0.0001]
			silent         disable the log output from the C++ extension [1]
			encoding       specify input_file encoding [utf-8]
			"""
			self.lr=lr
			self.lr_update_rate=lru
			self.dim=dim
			self.ws=ws
			self.epoch=epoch
			self.min_count=minc
			self.neg=neg
			self.word_ngrams=ngram
			self.loss=loss
			self.bucket=bucket
			self.minn=minn
			self.maxn=maxn
			self.n_thread=th
			self.samplet=t
			self.silent=verbosec
			self.enc=encoding
	def fit(self,X,modelname='model'):
		pass
	def getproperties(self):
                '''
                Input: Nothing, other than object self pointer
                Return: None , prints the descriptions of the model hyperparameters
                '''
                print("The model has following hyperparameters as part of its specification")
                print("Learning rate :"+ str(lr))
                print("Learning rate update after "+str(self.lr_update_rate)+"iterations")
                print("Embedding size: "+str(self.dim))
                print("Epochs :"+ str(self.epochs)
                print("minimal number of word occurences: "+self.min_count)
                print("number of negatives sampled :"+str(self.neg))
                print("max length of word ngram "+str(self.word_ngrams))
                print("loss function: "+str(self.loss))
                print("number of buckets "+str(self.bucket))
                print("min length of char ngram:"+str(self.minn))
                print("min length of char ngram"+ str(self.maxn))
                print("number of threads: "+str(self.n_thread))
                print("sampling threshold"+str(self.samplet))
                print("Verbose log output from the C++ extension enable=1/disble=0:"+ str(self.silent))
                print("input_file encoding :"+str(self.enc))
                return None

	def getwords(self):
		'return word list'
		pass
	def getvector(self):
		'return embedding'
		pass

class cbowFastText((BaseEstimator,ClassifierMixin):
	def __init__(self,lpr='__label__',lr=0.1,lru=100,dim=100,ws=5,epoch=5,minc=1,neg=5,ngram=1,\
loss='softmax',nbucket=0,minn=0,maxn=0,th=12,t=0.0001,verbosec=0,encoding='utf-8'):			"""
			lr             learning rate [0.05]
			lr_update_rate change the rate of updates for the learning rate [100]
			dim            size of word vectors [100]
			ws             size of the context window [5]
			epoch          number of epochs [5]
			min_count      minimal number of word occurences [5]
			neg            number of negatives sampled [5]
			word_ngrams    max length of word ngram [1]
			loss           loss function {ns, hs, softmax} [ns]
			bucket         number of buckets [2000000]
			minn           min length of char ngram [3]
			maxn           max length of char ngram [6]
			thread         number of threads [12]
			t              sampling threshold [0.0001]
			silent         disable the log output from the C++ extension [1]
			encoding       specify input_file encoding [utf-8]

			"""
			self.lr=lr
			self.lr_update_rate=lru
			self.dim=dim
			self.ws=ws
			self.epoch=epoch
			self.min_count=minc
			self.neg=neg
			self.word_ngrams=ngram
			self.loss=loss
			self.bucket=bucket
			self.minn=minn
			self.maxn=maxn
			self.n_thread=th
			self.samplet=t
			self.silent=verbosec
			self.enc=encoding
	def fit(self,X,modelname='model'):
		pass
	def getproperties(self):
		'''
                Input: Nothing, other than object self pointer
                Return: None , prints the descriptions of the model hyperparameters
                '''
                print("The model has following hyperparameters as part of its specification")
                print("Learning rate :"+ str(lr))
                print("Learning rate update after "+str(self.lr_update_rate)+"iterations")
                print("Embedding size: "+str(self.dim))
                print("Epochs :"+ str(self.epochs)
                print("minimal number of word occurences: "+self.min_count)
                print("number of negatives sampled :"+str(self.neg))
                print("max length of word ngram "+str(self.word_ngrams))
                print("loss function: "+str(self.loss))
                print("number of buckets "+str(self.bucket))
                print("min length of char ngram:"+str(self.minn))
                print("min length of char ngram"+ str(self.maxn))
                print("number of threads: "+str(self.n_thread))
                print("sampling threshold"+str(self.samplet))
                print("Verbose log output from the C++ extension enable=1/disble=0:"+ str(self.silent))
                print("input_file encoding :"+str(self.enc))

	def getwords(self):
		'return word list'
		pass
	def getvector(self):
		'return embedding'
		pass

<commit_msg>Added text classifier api<commit_after>from sklearn.base import BaseEstimator, ClassifierMixin
import fasttext as ft
from sklearn.metrics import classification_report

class FastTextClassifier(BaseEstimator,ClassifierMixin):
"""Base classiifer of Fasttext estimator"""

	def __init__(self,lpr='__label__',lr=0.1,lru=100,dim=100,ws=5,epoch=5,minc=1,neg=5,ngram=1,loss='softmax',nbucket=0,minn=0,maxn=0,thread=2,silent=1,output="model"):
		"""
		label_prefix   			label prefix ['__label__']
		lr             			learning rate [0.1]
		lr_update_rate 			change the rate of updates for the learning rate [100]
		dim            			size of word vectors [100]
		ws             			size of the context window [5]
		epoch          			number of epochs [5]
		min_count      			minimal number of word occurences [1]
		neg            			number of negatives sampled [5]
		word_ngrams    			max length of word ngram [1]
		loss           			loss function {ns, hs, softmax} [softmax]
		bucket         			number of buckets [0]
		minn           			min length of char ngram [0]
		maxn           			min length of char ngram [0]
		todo : Recheck need of some of the variables, present in default classifier
		"""

		self.label_prefix=lpr
		self.lr=lr
		self.lr_update_rate=lru
		self.dim=dim
		self.ws=ws
		self.epoch=epoch
		self.min_count=minc
		self.neg=neg
		self.word_ngrams=ngram
		self.loss=loss
		self.bucket=bucket
		self.minn=minn
		self.maxn=maxn
		self.thread=thread
		self.silent=silent
		self.classifier=None
		self.result=None

		self.output=output

	def fit(self,input_file):
                '''
                Input: takes input file in format
                returns classifier object
                to do: add option to feed list of X and Y or file
                '''

                self.classifier = ft.supervised(input_file, self.output, dim=self.dim, lr=self.lr, epoch=self.epoch, min_count=self.min_count, word_ngrams=self.word_ngrams, bucket=self.bucket, thread=self.thread, silent=self.silent, label_prefix=self.lpr)
                self.labels=
                return(self.classisifer)

	def predict(self,test_file,csvflag=True,reports=False):
                '''
                Input: Takes input test finle in format
                return results object
                to do: add unit tests using sentiment analysis dataset
                to do: Add K best labels options for csvflag = False
                to do: add report option
                '''
                try:
                    if type(test_file) == 'list' and csvflag=False:
                        self.result=self.classifier.predict(test_file)
                    else:
                        print "Error in input"
                    if csvflag:
                            self.result=self.classifier.test(test_file)
                except:
                    print("Exception in predict call error in format of test_file/input sentence list")
                return(self.result)

	def report(self,ytrue,ypred):
                '''
                Input: predicted and true labels
                return reort of classification
                to do: add label option and unit testing

                '''
                print(classification_report(ytrue,ypred))
                return None

	def predict_proba(self,X):
                '''
                Input: List of sentences
                return reort of classification
                to do: check output of classifier predct_proba add label option and unit testing
                '''
                labels=self.classifier.predict_proba(X)
                return(labels)

	def getlabels(self):
                '''
                Input: None
                returns: Class labels in dataset
                to do : check need of the this funcion
                '''
		return(self.classifier.labels)

	def getproperties(self):

                '''
                Input: Nothing, other than object self pointer
                Return: None , prints the descriptions of the model hyperparameters
                '''

                print("The model has following hyperparameters as part of its specification")
                print("Label prefix used : "+str(self.label_prefix)
                print("Learning rate :"+ str(lr))
                print("Learning rate update after "+str(self.lr_update_rate)+"iterations")
                print("Embedding size: "+str(self.dim))
                print("Epochs :"+ str(self.epochs)
                print("minimal number of word occurences: "+self.min_count)
                print("number of negatives sampled :"+str(self.neg))
                print("max length of word ngram "+str(self.word_ngrams))
                print("loss function: "+str(self.loss))
                print("number of buckets "+str(self.bucket))
                print("min length of char ngram:"+str(self.minn))
                print("min length of char ngram"+ str(self.maxn))
                return(None)

	def loadpretrained(self,X):
		'returns the model with pretrained weights'
		pass

class SkipgramFastText(BaseEstimator,ClassifierMixin):

	def __init__(self,lpr='__label__',lr=0.1,lru=100,dim=100,ws=5,epoch=5,minc=1,neg=5,ngram=1,\
loss='softmax',nbucket=0,minn=0,maxn=0,th=12,t=0.0001,verbosec=0,encoding='utf-8'):
			"""
			lr             learning rate [0.05]
			lr_update_rate change the rate of updates for the learning rate [100]
			dim            size of word vectors [100]
			ws             size of the context window [5]
			epoch          number of epochs [5]
			min_count      minimal number of word occurences [5]
			neg            number of negatives sampled [5]
			word_ngrams    max length of word ngram [1]
			loss           loss function {ns, hs, softmax} [ns]
			bucket         number of buckets [2000000]
			minn           min length of char ngram [3]
			maxn           max length of char ngram [6]
			thread         number of threads [12]
			t              sampling threshold [0.0001]
			silent         disable the log output from the C++ extension [1]
			encoding       specify input_file encoding [utf-8]
			"""
			self.lr=lr
			self.lr_update_rate=lru
			self.dim=dim
			self.ws=ws
			self.epoch=epoch
			self.min_count=minc
			self.neg=neg
			self.word_ngrams=ngram
			self.loss=loss
			self.bucket=bucket
			self.minn=minn
			self.maxn=maxn
			self.n_thread=th
			self.samplet=t
			self.silent=verbosec
			self.enc=encoding
	def fit(self,X,modelname='model'):
		pass
	def getproperties(self):
                '''
                Input: Nothing, other than object self pointer
                Return: None , prints the descriptions of the model hyperparameters
                '''
                print("The model has following hyperparameters as part of its specification")
                print("Learning rate :"+ str(lr))
                print("Learning rate update after "+str(self.lr_update_rate)+"iterations")
                print("Embedding size: "+str(self.dim))
                print("Epochs :"+ str(self.epochs)
                print("minimal number of word occurences: "+self.min_count)
                print("number of negatives sampled :"+str(self.neg))
                print("max length of word ngram "+str(self.word_ngrams))
                print("loss function: "+str(self.loss))
                print("number of buckets "+str(self.bucket))
                print("min length of char ngram:"+str(self.minn))
                print("min length of char ngram"+ str(self.maxn))
                print("number of threads: "+str(self.n_thread))
                print("sampling threshold"+str(self.samplet))
                print("Verbose log output from the C++ extension enable=1/disble=0:"+ str(self.silent))
                print("input_file encoding :"+str(self.enc))
                return None

	def getwords(self):
		'return word list'
		pass
	def getvector(self):
		'return embedding'
		pass

class cbowFastText((BaseEstimator,ClassifierMixin):
	def __init__(self,lpr='__label__',lr=0.1,lru=100,dim=100,ws=5,epoch=5,minc=1,neg=5,ngram=1,\
loss='softmax',nbucket=0,minn=0,maxn=0,th=12,t=0.0001,verbosec=0,encoding='utf-8'):			"""
			lr             learning rate [0.05]
			lr_update_rate change the rate of updates for the learning rate [100]
			dim            size of word vectors [100]
			ws             size of the context window [5]
			epoch          number of epochs [5]
			min_count      minimal number of word occurences [5]
			neg            number of negatives sampled [5]
			word_ngrams    max length of word ngram [1]
			loss           loss function {ns, hs, softmax} [ns]
			bucket         number of buckets [2000000]
			minn           min length of char ngram [3]
			maxn           max length of char ngram [6]
			thread         number of threads [12]
			t              sampling threshold [0.0001]
			silent         disable the log output from the C++ extension [1]
			encoding       specify input_file encoding [utf-8]

			"""
			self.lr=lr
			self.lr_update_rate=lru
			self.dim=dim
			self.ws=ws
			self.epoch=epoch
			self.min_count=minc
			self.neg=neg
			self.word_ngrams=ngram
			self.loss=loss
			self.bucket=bucket
			self.minn=minn
			self.maxn=maxn
			self.n_thread=th
			self.samplet=t
			self.silent=verbosec
			self.enc=encoding
	def fit(self,X,modelname='model'):
		pass
	def getproperties(self):
		'''
                Input: Nothing, other than object self pointer
                Return: None , prints the descriptions of the model hyperparameters
                '''
                print("The model has following hyperparameters as part of its specification")
                print("Learning rate :"+ str(lr))
                print("Learning rate update after "+str(self.lr_update_rate)+"iterations")
                print("Embedding size: "+str(self.dim))
                print("Epochs :"+ str(self.epochs)
                print("minimal number of word occurences: "+self.min_count)
                print("number of negatives sampled :"+str(self.neg))
                print("max length of word ngram "+str(self.word_ngrams))
                print("loss function: "+str(self.loss))
                print("number of buckets "+str(self.bucket))
                print("min length of char ngram:"+str(self.minn))
                print("min length of char ngram"+ str(self.maxn))
                print("number of threads: "+str(self.n_thread))
                print("sampling threshold"+str(self.samplet))
                print("Verbose log output from the C++ extension enable=1/disble=0:"+ str(self.silent))
                print("input_file encoding :"+str(self.enc))

	def getwords(self):
		'return word list'
		pass
	def getvector(self):
		'return embedding'
		pass

<|endoftext|>


<commit_before># coding=utf-8

# A simple demonstration of how to load a QGIS project and then
# show it in a widget.
# This code is public domain, use if for any purpose you see fit.
# Tim Sutton 2015

import os
import sys
from qgis.core import (
    QgsProject, QgsComposition, QgsApplication)
from qgis.gui import QgsMapCanvas, QgsLayerTreeMapCanvasBridge
from PyQt4.QtCore import QFileInfo
from PyQt4.QtXml import QDomDocument

gui_flag = True
app = QgsApplication(sys.argv, gui_flag)

# Make sure QGIS_PREFIX_PATH is set in your env if needed!
app.initQgis()
project_path = os.path.dirname(__file__) + os.path.sep + 'test.qgs'
template_path = os.path.abspath(os.path.join(
    os.path.dirname(__file__),
    'test.qpt'
))

def make_pdf():
    canvas = QgsMapCanvas()
    # Load our project
    QgsProject.instance().read(QFileInfo(project_path))
    bridge = QgsLayerTreeMapCanvasBridge(
        QgsProject.instance().layerTreeRoot(), canvas)
    bridge.setCanvasLayers()
    if canvas.layerCount() < 1:
        print 'No layers loaded from this project, exiting.'
        return
    print canvas.mapSettings().extent().toString()
    template_file = file(template_path)
    template_content = template_file.read()
    template_file.close()
    document = QDomDocument()
    document.setContent(template_content)
    composition = QgsComposition(canvas.mapSettings())
    # You can use this to replace any string like this [key]
    # in the template with a new value. e.g. to replace
    # [date] pass a map like this {'date': '1 Jan 2012'}
    substitution_map = {
        'DATE_TIME_START': 'foo',
        'DATE_TIME_END': 'bar'}
    composition.loadFromTemplate(document, substitution_map)
    # You must set the id in the template
    map_item = composition.getComposerItemById('map')
    map_item.setMapCanvas(canvas)
    map_item.zoomToExtent(canvas.extent())
    # You must set the id in the template
    legend_item = composition.getComposerItemById('legend')
    legend_item.updateLegend()
    composition.refreshItems()
    composition.exportAsPDF('/tmp/test2.pdf')
    QgsProject.instance().clear()

make_pdf()

<commit_msg>Tweaks to pdf generator<commit_after># coding=utf-8

# A simple demonstration of how to load a QGIS project and then
# show it in a widget.
# This code is public domain, use if for any purpose you see fit.
# Tim Sutton 2015

import os
import sys
from qgis.core import (
    QgsProject, QgsComposition, QgsApplication, QgsProviderRegistry)
from qgis.gui import QgsMapCanvas, QgsLayerTreeMapCanvasBridge
from PyQt4.QtCore import QFileInfo
from PyQt4.QtXml import QDomDocument

gui_flag = True
app = QgsApplication(sys.argv, gui_flag)

# Make sure QGIS_PREFIX_PATH is set in your env if needed!
app.initQgis()

print 'Available providers:'
print QgsProviderRegistry.instance().pluginList()

project_path = os.path.join(
    os.path.dirname(__file__),
    os.path.pardir,
    'jk-floods.qgs')
template_path = os.path.abspath(os.path.join(
    os.path.dirname(__file__),
    'test.qpt'
))

def make_pdf():
    canvas = QgsMapCanvas()
    # Load our project
    QgsProject.instance().read(QFileInfo(project_path))
    bridge = QgsLayerTreeMapCanvasBridge(
        QgsProject.instance().layerTreeRoot(), canvas)
    bridge.setCanvasLayers()
    if canvas.layerCount() < 1:
        print 'No layers loaded from this project, exiting.'
        return
    print canvas.mapSettings().extent().toString()
    template_file = file(template_path)
    template_content = template_file.read()
    template_file.close()
    document = QDomDocument()
    document.setContent(template_content)
    composition = QgsComposition(canvas.mapSettings())
    # You can use this to replace any string like this [key]
    # in the template with a new value. e.g. to replace
    # [date] pass a map like this {'date': '1 Jan 2012'}
    substitution_map = {
        'DATE_TIME_START': 'foo',
        'DATE_TIME_END': 'bar'}
    composition.loadFromTemplate(document, substitution_map)
    # You must set the id in the template
    map_item = composition.getComposerItemById('map')
    map_item.setMapCanvas(canvas)
    map_item.zoomToExtent(canvas.extent())
    # You must set the id in the template
    legend_item = composition.getComposerItemById('legend')
    legend_item.updateLegend()
    composition.refreshItems()
    composition.exportAsPDF('/tmp/test2.pdf')
    QgsProject.instance().clear()

make_pdf()

<|endoftext|>


<commit_before>from django.db import models
from django.db.models.signals import pre_save
from decimal import Decimal
import math
import urllib
import urlparse
import io
import json
from roygbiv import Roygbiv
from colourlens.utils import ArtColour

TWOPLACES = Decimal(10) ** -2

class Colour(models.Model):
    """
    A named-colour from a select palette with defined RGB colours
    """
    name = models.CharField(blank=False, max_length=20, db_index=True)
    hex_value = models.CharField(blank=True, max_length=10, unique=True)
    red = models.IntegerField(blank=False, null=True)
    green = models.IntegerField(blank=False, null=True)
    blue = models.IntegerField(blank=False, null=True)
    hue = models.IntegerField(blank=True, null=True, db_index=True)
    elite = models.BooleanField(default=False)

    def __unicode__(self):
        return u"%s" % (self.name)

class Artwork(models.Model):
    """
    An image of an artwork number with an accession (reference) number
    """

    accession_number = models.CharField(unique=True, max_length=100)
    title = models.CharField(blank=True, max_length=100)
    artist = models.CharField(blank=True, max_length=100)
    year = models.IntegerField(blank=True, null=True)
    url = models.URLField(blank=True)
    image_url = models.URLField(blank=True)
    institution = models.CharField(blank=True, max_length=10, db_index=True)
    proportions = models.TextField(blank=True)
    colours = models.ManyToManyField(Colour, through="ColourDistance")

    def colour_parts(self):

        if self.proportions:
            return json.loads(self.proportions)
        else:
            return []

    @classmethod
    def from_url(cls, accession_number, institution, image_url):
        try:
            response = urllib.urlopen(image_url)
            im_bytes = response.read()
        except IOError, e:
            return None
        aw = Artwork.from_file(
            accession_number,
            institution,
            io.BytesIO(im_bytes)
        )
        if not aw:
            return None
        aw.image_url = image_url
        aw.save()
        return aw

    @classmethod
    def from_file(cls, accession_number, institution, filename):
        aw, cr = Artwork.objects.get_or_create(
            institution=institution,
            accession_number=accession_number
        )
        aw.colourdistance_set.all().delete()
        aw.title = institution + accession_number
        aw.image_url = "file://%s" % filename
        try:
            roy_im = Roygbiv(filename)
        except IOError, e:
            return None
        p = roy_im.get_palette()
        rgbs = []
        preselected = []
        for palette_colour in p.colors:
            prominence = round(palette_colour.prominence, 3)
            c = ArtColour(*palette_colour.value, prominence=prominence)
            c.hex_me_up()
            css, cr = Colour.objects.get_or_create(hex_value=c.css['hex'])
            css.name = c.css['name']
            css.hue = c.css['hue']
            dist = Decimal(c.css['distance']).quantize(TWOPLACES)
            prom = Decimal(c.css['prominence']).quantize(TWOPLACES)
            cd, cr = ColourDistance.objects.get_or_create(colour=css,
                                                          artwork=aw)
            if cr:
                cd.distance = dist
                cd.prominence = prom
            else:
                if cd.prominence:
                    total_area = cd.prominence + prom
                    d1 = cd.distance * (cd.prominence / total_area)
                    d2 = dist * (prom / total_area)
                    cd.distance = d1 + d2
                    cd.prominence = total_area
            cd.save()
            css.save()
        aw.save()
        return aw

    def __unicode__(self):
        return u"%s" % (self.accession_number)

class ColourDistance(models.Model):
    """
    Join Model with distance field added to show proximity to colour
    """
    artwork = models.ForeignKey(Artwork)
    colour = models.ForeignKey(Colour)
    red = models.IntegerField(blank=False, null=True)
    green = models.IntegerField(blank=False, null=True)
    blue = models.IntegerField(blank=False, null=True)
    distance = models.DecimalField(max_digits=5, decimal_places=2,
                                   null=True, db_index=True)
    prominence = models.DecimalField(max_digits=3, decimal_places=2,
                                     null=True, db_index=True)
    presence = models.DecimalField(max_digits=5, decimal_places=2, null=True,
                                   db_index=True)

    class Meta:
        unique_together = ('artwork', 'colour')

    def __unicode__(self):
        return u"s%s %s d:%s p:%s" % (self.artwork, self.colour,
                                      self.distance, self.prominence)

def artwork_proportions(sender, instance, **kwargs):
    if instance.colourdistance_set.count() > 0:
        total_area = reduce(lambda x,y: x+y, [cd.prominence for cd in instance.colourdistance_set.all()])
        colour_list = [
            (cd.colour.hex_value, float(cd.prominence * (100/total_area)))
            for cd in instance.colourdistance_set.all()
        ]
        instance.proportions = json.dumps(colour_list)
    else:
        instance.proportions = json.dumps([])

def calculate_colour_presence(sender, instance, **kwargs):
    """
    Calculates colour "presence" in image. This is distance as
    a factor of colour area
    """
    if instance.prominence < 0.01:
        instance.prominence = 0.01
    if instance.distance and instance.prominence:

        dist = Decimal(instance.distance)
        prom = Decimal(instance.prominence)
        distsqrt = math.sqrt(dist)
        presence = prom * 100 / 4 / Decimal(distsqrt)
        instance.presence = Decimal(presence).quantize(TWOPLACES)

pre_save.connect(calculate_colour_presence, sender=ColourDistance)
pre_save.connect(artwork_proportions, sender=Artwork)
<commit_msg>Exception printing<commit_after>from django.db import models
from django.db.models.signals import pre_save
from decimal import Decimal
import math
import urllib
import urlparse
import io
import json
from roygbiv import Roygbiv
from colourlens.utils import ArtColour

TWOPLACES = Decimal(10) ** -2

class Colour(models.Model):
    """
    A named-colour from a select palette with defined RGB colours
    """
    name = models.CharField(blank=False, max_length=20, db_index=True)
    hex_value = models.CharField(blank=True, max_length=10, unique=True)
    red = models.IntegerField(blank=False, null=True)
    green = models.IntegerField(blank=False, null=True)
    blue = models.IntegerField(blank=False, null=True)
    hue = models.IntegerField(blank=True, null=True, db_index=True)
    elite = models.BooleanField(default=False)

    def __unicode__(self):
        return u"%s" % (self.name)

class Artwork(models.Model):
    """
    An image of an artwork number with an accession (reference) number
    """

    accession_number = models.CharField(unique=True, max_length=100)
    title = models.CharField(blank=True, max_length=100)
    artist = models.CharField(blank=True, max_length=100)
    year = models.IntegerField(blank=True, null=True)
    url = models.URLField(blank=True)
    image_url = models.URLField(blank=True)
    institution = models.CharField(blank=True, max_length=10, db_index=True)
    proportions = models.TextField(blank=True)
    colours = models.ManyToManyField(Colour, through="ColourDistance")

    def colour_parts(self):

        if self.proportions:
            return json.loads(self.proportions)
        else:
            return []

    @classmethod
    def from_url(cls, accession_number, institution, image_url):
        try:
            response = urllib.urlopen(image_url)
            im_bytes = response.read()
        except IOError, e:
            print e
            return None
        aw = Artwork.from_file(
            accession_number,
            institution,
            io.BytesIO(im_bytes)
        )
        if not aw:
            return None
        aw.image_url = image_url
        aw.save()
        return aw

    @classmethod
    def from_file(cls, accession_number, institution, filename):
        aw, cr = Artwork.objects.get_or_create(
            institution=institution,
            accession_number=accession_number
        )
        aw.colourdistance_set.all().delete()
        aw.title = institution + accession_number
        aw.image_url = "file://%s" % filename
        try:
            roy_im = Roygbiv(filename)
        except IOError, e:
            print e
            return None
        p = roy_im.get_palette()
        rgbs = []
        preselected = []
        for palette_colour in p.colors:
            prominence = round(palette_colour.prominence, 3)
            c = ArtColour(*palette_colour.value, prominence=prominence)
            c.hex_me_up()
            css, cr = Colour.objects.get_or_create(hex_value=c.css['hex'])
            css.name = c.css['name']
            css.hue = c.css['hue']
            dist = Decimal(c.css['distance']).quantize(TWOPLACES)
            prom = Decimal(c.css['prominence']).quantize(TWOPLACES)
            cd, cr = ColourDistance.objects.get_or_create(colour=css,
                                                          artwork=aw)
            if cr:
                cd.distance = dist
                cd.prominence = prom
            else:
                if cd.prominence:
                    total_area = cd.prominence + prom
                    d1 = cd.distance * (cd.prominence / total_area)
                    d2 = dist * (prom / total_area)
                    cd.distance = d1 + d2
                    cd.prominence = total_area
            cd.save()
            css.save()
        aw.save()
        return aw

    def __unicode__(self):
        return u"%s" % (self.accession_number)

class ColourDistance(models.Model):
    """
    Join Model with distance field added to show proximity to colour
    """
    artwork = models.ForeignKey(Artwork)
    colour = models.ForeignKey(Colour)
    red = models.IntegerField(blank=False, null=True)
    green = models.IntegerField(blank=False, null=True)
    blue = models.IntegerField(blank=False, null=True)
    distance = models.DecimalField(max_digits=5, decimal_places=2,
                                   null=True, db_index=True)
    prominence = models.DecimalField(max_digits=3, decimal_places=2,
                                     null=True, db_index=True)
    presence = models.DecimalField(max_digits=5, decimal_places=2, null=True,
                                   db_index=True)

    class Meta:
        unique_together = ('artwork', 'colour')

    def __unicode__(self):
        return u"s%s %s d:%s p:%s" % (self.artwork, self.colour,
                                      self.distance, self.prominence)

def artwork_proportions(sender, instance, **kwargs):
    if instance.colourdistance_set.count() > 0:
        total_area = reduce(lambda x,y: x+y, [cd.prominence for cd in instance.colourdistance_set.all()])
        colour_list = [
            (cd.colour.hex_value, float(cd.prominence * (100/total_area)))
            for cd in instance.colourdistance_set.all()
        ]
        instance.proportions = json.dumps(colour_list)
    else:
        instance.proportions = json.dumps([])

def calculate_colour_presence(sender, instance, **kwargs):
    """
    Calculates colour "presence" in image. This is distance as
    a factor of colour area
    """
    if instance.prominence < 0.01:
        instance.prominence = 0.01
    if instance.distance and instance.prominence:

        dist = Decimal(instance.distance)
        prom = Decimal(instance.prominence)
        distsqrt = math.sqrt(dist)
        presence = prom * 100 / 4 / Decimal(distsqrt)
        instance.presence = Decimal(presence).quantize(TWOPLACES)

pre_save.connect(calculate_colour_presence, sender=ColourDistance)
pre_save.connect(artwork_proportions, sender=Artwork)
<|endoftext|>


<commit_before>import numpy as np
import inspect
import os

from astropy.cosmology import FlatwCDM
import pandas as pd
import pickle
from dessn.models.d_simple_stan.load_correction_data import load_correction_supernova
from dessn.models.d_simple_stan.truth import get_truths_labels_significance
from scipy.stats import norm

def load_fit_snana_correction(n_sne):
    print("Getting SNANA dummy data")
    supernovae = load_correction_supernova(correction_source="snana")
    redshifts = supernovae["redshifts"][:n_sne]
    apparents = supernovae["apparents"][:n_sne]
    stretches = supernovae["stretches"][:n_sne]
    colours = supernovae["colours"][:n_sne]
    masses = supernovae["masses"][:n_sne]

    obs_mBx1c_cov = []
    obs_mBx1c = []
    for mb, x1, c in zip(apparents, stretches, colours):
        vector = np.array([mb, x1, c])
        # Add intrinsic scatter to the mix
        diag = np.array([0.05, 0.2, 0.05]) ** 2
        cov = np.diag(diag)
        vector += np.random.multivariate_normal([0, 0, 0], cov)
        obs_mBx1c_cov.append(cov)
        obs_mBx1c.append(vector)

    covs = np.array(obs_mBx1c_cov)

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": covs,
        "redshifts": redshifts,
        "mass": masses,
        "deta_dcalib": np.zeros((n_sne, 3, 4))
    }

def load_fit_snana_diff(n_sne):
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    data_file = this_dir + "/data/snana_diff_cosmology.npy"
    data = np.load(data_file)
    np.random.shuffle(data)

    supernovae = {
        "masses": np.ones(n_sne),
        "redshifts": data[:n_sne, 0],
        "apparents": data[:n_sne, 1],
        "stretches": data[:n_sne, 2],
        "colours": data[:n_sne, 3],
        "smear": data[:n_sne, 4]
    }

    # Correct for the way snana does intrinsic dispersion
    supernovae["apparents"] += supernovae["smear"]
    supernovae["existing_prob"] = norm.logpdf(supernovae["colours"], 0, 0.1) \
                                  + norm.logpdf(supernovae["stretches"], 0, 1) \
                                  + norm.logpdf(supernovae["smear"], 0, 0.1)

    redshifts = supernovae["redshifts"]
    apparents = supernovae["apparents"]
    stretches = supernovae["stretches"]
    colours = supernovae["colours"]
    masses = supernovae["masses"]

    obs_mBx1c_cov = []
    obs_mBx1c = []
    for mb, x1, c in zip(apparents, stretches, colours):
        vector = np.array([mb, x1, c])
        # Add intrinsic scatter to the mix
        diag = np.array([0.05, 0.2, 0.05]) ** 2
        cov = np.diag(diag)
        vector += np.random.multivariate_normal([0, 0, 0], cov)
        obs_mBx1c_cov.append(cov)
        obs_mBx1c.append(vector)

    covs = np.array(obs_mBx1c_cov)

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": covs,
        "redshifts": redshifts,
        "mass": masses,
        "deta_dcalib": np.zeros((n_sne, 3, 4))
    }

def get_fitres_data():
    print("Getting data from Fitres file")
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    fitres_file = os.path.abspath(this_dir + "/data/FITOPT000.FITRES")
    dataframe = pd.read_csv(fitres_file, sep='\s+', skiprows=5, comment="#")
    data = dataframe.to_records()

    n_sne = data.size
    obs_mBx1c = data[['mB', 'x1', 'c']].view((float, 3))

    zs = data["zCMB"]
    masses = data["HOST_LOGMASS"]
    gtz = masses > 0
    masses = gtz * masses
    mbs = data["mB"]
    x0s = data["x0"]
    x1s = data["x1"]
    cs = data["c"]

    mbse = data["mBERR"]
    x1se = data["x1ERR"]
    cse = data["cERR"]

    cov_x1_c = data["COV_x1_c"]
    cov_x0_c = data["COV_c_x0"]
    cov_x1_x0 = data["COV_x1_x0"]

    covs = []
    for mb, x0, x1, c, mbe, x1e, ce, cx1c, cx0c, cx1x0 in zip(mbs, x0s, x1s, cs, mbse, x1se, cse, cov_x1_c, cov_x0_c,
                                                              cov_x1_x0):
        cmbx1 = -5 * cx1x0 / (2 * x0 * np.log(10))
        cmbc = -5 * cx0c / (2 * x0 * np.log(10))
        cov = np.array([[mbe * mbe, cmbx1, cmbc], [cmbx1, x1e * x1e, cx1c], [cmbc, cx1c, ce * ce]])
        covs.append(cov)

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": covs,
        "redshifts": zs,
        "mass": masses,
        "deta_dcalib": np.zeros((n_sne, 3, 4))
    }

def get_sncosmo_pickle_data(n_sne):
    print("Getting data from sncosmo supernovae pickle")
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    pickle_file = os.path.abspath(this_dir + "/data/supernovae.pickle")
    with open(pickle_file, 'rb') as pkl:
        supernovae = pickle.load(pkl)
    passed = [s for s in supernovae if s["pc"]]
    np.random.shuffle(passed)
    passed = passed[:n_sne]
    return {
        "n_sne": n_sne,
        "obs_mBx1c": [s["parameters"] for s in passed],
        "obs_mBx1c_cov": [s["covariance"] for s in passed],
        "redshifts": np.array([s["z"] for s in passed]),
        "mass": np.array([s["m"] for s in passed]),
        "deta_dcalib": [s["dp"] for s in passed]
    }

def get_physical_data(n_sne):
    print("Getting simple data")
    vals = get_truths_labels_significance()
    mapping = {k[0]: k[1] for k in vals}

    obs_mBx1c = []
    obs_mBx1c_cov = []
    obs_mBx1c_cor = []
    deta_dcalib = []

    redshifts = np.linspace(0.05, 1.1, n_sne)
    cosmology = FlatwCDM(70.0, mapping["Om"]) #, w0=mapping["w"])
    dist_mod = cosmology.distmod(redshifts).value

    redshift_pre_comp = 0.9 + np.power(10, 0.95 * redshifts)
    alpha = mapping["alpha"]
    beta = mapping["beta"]
    dscale = mapping["dscale"]
    dratio = mapping["dratio"]
    p_high_masses = np.random.uniform(low=0.0, high=1.0, size=dist_mod.size)
    means = np.array([mapping["mean_MB"], mapping["mean_x1"], mapping["mean_c"]])
    sigmas = np.array([mapping["sigma_MB"], mapping["sigma_x1"], mapping["sigma_c"]])
    sigmas_mat = np.dot(sigmas[:, None], sigmas[None, :])
    correlations = np.dot(mapping["intrinsic_correlation"], mapping["intrinsic_correlation"].T)
    pop_cov = correlations * sigmas_mat
    for zz, mu, p in zip(redshift_pre_comp, dist_mod, p_high_masses):

        # Generate the actual mB, x1 and c values
        MB, x1, c = np.random.multivariate_normal(means, pop_cov)
        mass_correction = dscale * (1.9 * (1 - dratio) / zz + dratio)
        mb = MB + mu - alpha * x1 + beta * c - mass_correction * p
        vector = np.array([mb, x1, c])
        # Add intrinsic scatter to the mix
        diag = np.array([0.05, 0.3, 0.05]) ** 2
        cov = np.diag(diag)
        vector += np.random.multivariate_normal([0, 0, 0], cov)
        cor = cov / np.sqrt(np.diag(cov))[None, :] / np.sqrt(np.diag(cov))[:, None]
        obs_mBx1c_cor.append(cor)
        obs_mBx1c_cov.append(cov)
        obs_mBx1c.append(vector)
        deta_dcalib.append(np.ones((3, 4)))

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": obs_mBx1c_cov,
        "deta_dcalib": deta_dcalib,
        "redshifts": redshifts,
        "mass": p_high_masses
    }

def get_snana_data():
    print("Getting SNANA data from TEST_DATA_17677")
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    filename = this_dir + "/data/des_sim.pickle"
    print("Getting SNANA data")
    with open(filename, 'rb') as f:
        data = pickle.load(f)
    return data
<commit_msg>greppable debug message<commit_after>import numpy as np
import inspect
import os

from astropy.cosmology import FlatwCDM
import pandas as pd
import pickle
from dessn.models.d_simple_stan.load_correction_data import load_correction_supernova
from dessn.models.d_simple_stan.truth import get_truths_labels_significance
from scipy.stats import norm

def load_fit_snana_correction(n_sne):
    print("Getting SNANA dummy data")
    supernovae = load_correction_supernova(correction_source="snana")
    redshifts = supernovae["redshifts"][:n_sne]
    apparents = supernovae["apparents"][:n_sne]
    stretches = supernovae["stretches"][:n_sne]
    colours = supernovae["colours"][:n_sne]
    masses = supernovae["masses"][:n_sne]
    print("SHUFFLE: ", apparents.mean(), colours.mean(), stretches.mean())
    obs_mBx1c_cov = []
    obs_mBx1c = []
    for mb, x1, c in zip(apparents, stretches, colours):
        vector = np.array([mb, x1, c])
        # Add intrinsic scatter to the mix
        diag = np.array([0.05, 0.2, 0.05]) ** 2
        cov = np.diag(diag)
        vector += np.random.multivariate_normal([0, 0, 0], cov)
        obs_mBx1c_cov.append(cov)
        obs_mBx1c.append(vector)

    covs = np.array(obs_mBx1c_cov)

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": covs,
        "redshifts": redshifts,
        "mass": masses,
        "deta_dcalib": np.zeros((n_sne, 3, 4))
    }

def load_fit_snana_diff(n_sne):
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    data_file = this_dir + "/data/snana_diff_cosmology.npy"
    data = np.load(data_file)
    np.random.shuffle(data)

    supernovae = {
        "masses": np.ones(n_sne),
        "redshifts": data[:n_sne, 0],
        "apparents": data[:n_sne, 1],
        "stretches": data[:n_sne, 2],
        "colours": data[:n_sne, 3],
        "smear": data[:n_sne, 4]
    }

    # Correct for the way snana does intrinsic dispersion
    supernovae["apparents"] += supernovae["smear"]
    supernovae["existing_prob"] = norm.logpdf(supernovae["colours"], 0, 0.1) \
                                  + norm.logpdf(supernovae["stretches"], 0, 1) \
                                  + norm.logpdf(supernovae["smear"], 0, 0.1)

    redshifts = supernovae["redshifts"]
    apparents = supernovae["apparents"]
    stretches = supernovae["stretches"]
    colours = supernovae["colours"]
    masses = supernovae["masses"]

    obs_mBx1c_cov = []
    obs_mBx1c = []
    for mb, x1, c in zip(apparents, stretches, colours):
        vector = np.array([mb, x1, c])
        # Add intrinsic scatter to the mix
        diag = np.array([0.05, 0.2, 0.05]) ** 2
        cov = np.diag(diag)
        vector += np.random.multivariate_normal([0, 0, 0], cov)
        obs_mBx1c_cov.append(cov)
        obs_mBx1c.append(vector)

    covs = np.array(obs_mBx1c_cov)

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": covs,
        "redshifts": redshifts,
        "mass": masses,
        "deta_dcalib": np.zeros((n_sne, 3, 4))
    }

def get_fitres_data():
    print("Getting data from Fitres file")
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    fitres_file = os.path.abspath(this_dir + "/data/FITOPT000.FITRES")
    dataframe = pd.read_csv(fitres_file, sep='\s+', skiprows=5, comment="#")
    data = dataframe.to_records()

    n_sne = data.size
    obs_mBx1c = data[['mB', 'x1', 'c']].view((float, 3))

    zs = data["zCMB"]
    masses = data["HOST_LOGMASS"]
    gtz = masses > 0
    masses = gtz * masses
    mbs = data["mB"]
    x0s = data["x0"]
    x1s = data["x1"]
    cs = data["c"]

    mbse = data["mBERR"]
    x1se = data["x1ERR"]
    cse = data["cERR"]

    cov_x1_c = data["COV_x1_c"]
    cov_x0_c = data["COV_c_x0"]
    cov_x1_x0 = data["COV_x1_x0"]

    covs = []
    for mb, x0, x1, c, mbe, x1e, ce, cx1c, cx0c, cx1x0 in zip(mbs, x0s, x1s, cs, mbse, x1se, cse, cov_x1_c, cov_x0_c,
                                                              cov_x1_x0):
        cmbx1 = -5 * cx1x0 / (2 * x0 * np.log(10))
        cmbc = -5 * cx0c / (2 * x0 * np.log(10))
        cov = np.array([[mbe * mbe, cmbx1, cmbc], [cmbx1, x1e * x1e, cx1c], [cmbc, cx1c, ce * ce]])
        covs.append(cov)

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": covs,
        "redshifts": zs,
        "mass": masses,
        "deta_dcalib": np.zeros((n_sne, 3, 4))
    }

def get_sncosmo_pickle_data(n_sne):
    print("Getting data from sncosmo supernovae pickle")
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    pickle_file = os.path.abspath(this_dir + "/data/supernovae.pickle")
    with open(pickle_file, 'rb') as pkl:
        supernovae = pickle.load(pkl)
    passed = [s for s in supernovae if s["pc"]]
    np.random.shuffle(passed)
    passed = passed[:n_sne]
    return {
        "n_sne": n_sne,
        "obs_mBx1c": [s["parameters"] for s in passed],
        "obs_mBx1c_cov": [s["covariance"] for s in passed],
        "redshifts": np.array([s["z"] for s in passed]),
        "mass": np.array([s["m"] for s in passed]),
        "deta_dcalib": [s["dp"] for s in passed]
    }

def get_physical_data(n_sne):
    print("Getting simple data")
    vals = get_truths_labels_significance()
    mapping = {k[0]: k[1] for k in vals}

    obs_mBx1c = []
    obs_mBx1c_cov = []
    obs_mBx1c_cor = []
    deta_dcalib = []

    redshifts = np.linspace(0.05, 1.1, n_sne)
    cosmology = FlatwCDM(70.0, mapping["Om"]) #, w0=mapping["w"])
    dist_mod = cosmology.distmod(redshifts).value

    redshift_pre_comp = 0.9 + np.power(10, 0.95 * redshifts)
    alpha = mapping["alpha"]
    beta = mapping["beta"]
    dscale = mapping["dscale"]
    dratio = mapping["dratio"]
    p_high_masses = np.random.uniform(low=0.0, high=1.0, size=dist_mod.size)
    means = np.array([mapping["mean_MB"], mapping["mean_x1"], mapping["mean_c"]])
    sigmas = np.array([mapping["sigma_MB"], mapping["sigma_x1"], mapping["sigma_c"]])
    sigmas_mat = np.dot(sigmas[:, None], sigmas[None, :])
    correlations = np.dot(mapping["intrinsic_correlation"], mapping["intrinsic_correlation"].T)
    pop_cov = correlations * sigmas_mat
    for zz, mu, p in zip(redshift_pre_comp, dist_mod, p_high_masses):

        # Generate the actual mB, x1 and c values
        MB, x1, c = np.random.multivariate_normal(means, pop_cov)
        mass_correction = dscale * (1.9 * (1 - dratio) / zz + dratio)
        mb = MB + mu - alpha * x1 + beta * c - mass_correction * p
        vector = np.array([mb, x1, c])
        # Add intrinsic scatter to the mix
        diag = np.array([0.05, 0.3, 0.05]) ** 2
        cov = np.diag(diag)
        vector += np.random.multivariate_normal([0, 0, 0], cov)
        cor = cov / np.sqrt(np.diag(cov))[None, :] / np.sqrt(np.diag(cov))[:, None]
        obs_mBx1c_cor.append(cor)
        obs_mBx1c_cov.append(cov)
        obs_mBx1c.append(vector)
        deta_dcalib.append(np.ones((3, 4)))

    return {
        "n_sne": n_sne,
        "obs_mBx1c": obs_mBx1c,
        "obs_mBx1c_cov": obs_mBx1c_cov,
        "deta_dcalib": deta_dcalib,
        "redshifts": redshifts,
        "mass": p_high_masses
    }

def get_snana_data():
    print("Getting SNANA data from TEST_DATA_17677")
    this_dir = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
    filename = this_dir + "/data/des_sim.pickle"
    print("Getting SNANA data")
    with open(filename, 'rb') as f:
        data = pickle.load(f)
    return data
<|endoftext|>