<commit_before># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

from accelerator.models import (
    JUDGING_FEEDBACK_STATUS_COMPLETE,
    JudgeApplicationFeedback,
    JudgeRoundCommitment,
    Scenario,
)
from collections import defaultdict
from django.db.models import Sum, F
from impact.v1.helpers.criterion_option_spec_helper import (
    CriterionOptionSpecHelper,
)
from impact.v1.helpers.criterion_helper import CriterionHelper
from impact.v1.helpers.judge_gender_criterion_helper import (
    JudgeGenderCriterionHelper,
)
from impact.v1.helpers.judge_role_criterion_helper import (
    JudgeRoleCriterionHelper,
)
from impact.v1.helpers.matching_industry_criterion_helper import (
    MatchingIndustryCriterionHelper,
)
from impact.v1.helpers.matching_program_criterion_helper import (
    MatchingProgramCriterionHelper,
)
from accelerator_abstract.models.base_core_profile import (
    UI_GENDER_CHOICES
)

CriterionHelper.register_helper(
    JudgeGenderCriterionHelper, "judge", "gender")
CriterionHelper.register_helper(
    JudgeRoleCriterionHelper, "judge", "role")
CriterionHelper.register_helper(
    MatchingIndustryCriterionHelper, "matching", "industry")
CriterionHelper.register_helper(
    MatchingProgramCriterionHelper, "matching", "program")

class OptionAnalysis(object):
    _judge_to_count = None
    expert_profile = "judge__expertprofile__"
    industry_judge_field = expert_profile + "primary_industry__name"
    program_judge_field = expert_profile + "home_program_family__name"
    gender_judge_field = expert_profile + "gender"
    role_judge_field = expert_profile + "expert_category__name"

    def __init__(self,
                 apps,
                 app_ids,
                 judging_round,
                 application_counts,
                 criterion_helpers):
        self.criterion_helpers = criterion_helpers
        self.option_spec = None
        self.judging_round = judging_round
        self.cycle = self.judging_round.program.cycle
        self.helper = None
        self.apps = apps
        self.app_ids = app_ids
        self.application_counts = application_counts
        self.criterion_total_capacities = {}
        self.judge_to_capacity_cache = None
        self.criterion_total_functions = {
            ("judge", "gender"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.gender_judge_field,
            },
            ("reads", "reads"): {
                'function': None,
            },
            ("judge", "role"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.role_judge_field,
            },
            ("matching", "industry"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.industry_judge_field,
            },
            ("matching", "program"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.program_judge_field,
            }
        }

        active_scenarios = Scenario.objects.filter(
            judging_round=judging_round, is_active=True)
        self.jpa = JudgeApplicationFeedback.objects.filter(
            application__in=apps,
            panel__judgepanelassignment__scenario__in=active_scenarios)
        self.completed_feedbacks = self.jpa.filter(
            feedback_status=JUDGING_FEEDBACK_STATUS_COMPLETE)
        self.application_criteria_read_state_cache = {}
        self.gender_match_dict = dict(
            (value.lower(), key) for key, value in UI_GENDER_CHOICES)

    def analyses(self, option_spec):
        self.option_spec = option_spec
        self.helper = CriterionOptionSpecHelper(
            option_spec, self.criterion_helpers)
        return [self.analysis(option) for option in self.find_options()]

    def analysis(self, option_name):
        result = {
            "criterion_option_spec_id": self.option_spec.id,
            "criterion_name": self.option_spec.criterion.name,
            "criterion_type": self.option_spec.criterion.type,
            "criterion_id": self.option_spec.criterion.id,
            "option": option_name,
            "weight": self.option_spec.weight,
            "count": self.option_spec.count,
        }
        result.update(self.calc_needs(option_name))
        result.update(self.calc_capacity(option_name))
        return result

    def find_options(self):
        return self.helper.options(self.apps)

    def calc_needs(self, option_name):
        needs_dist = self.calc_needs_distribution(option_name)
        read_count = self.option_spec.count
        return {
            "needs_distribution": needs_dist,
            "total_required_reads": read_count * sum(needs_dist.values()),
            "completed_required_reads": sum(
                [min(read_count, read_count - k) * v
                 for (k, v) in needs_dist.items()]),
            "satisfied_apps": sum(
                [v for (k, v) in needs_dist.items() if k <= 0]),
            "needy_apps": sum(
                [v for (k, v) in needs_dist.items() if k > 0]),
            "remaining_needed_reads": sum(
                [v*k for (k, v) in needs_dist.items() if k > 0])
        }

    def calc_needs_distribution(self, option_name):
        app_counts = self.application_criteria_read_state(
            self.completed_feedbacks,
            option_name=option_name)
        counts = defaultdict(int)
        criterion_name = self.option_spec.criterion.name
        for count in app_counts.values():
            total = (
                count[criterion_name].get(option_name, 0)
                if criterion_name != "reads" else count[criterion_name]
            )
            counts[total] += 1

        read_count = 0
        for count_number in counts:
            if count_number > 0:
                read_count = read_count + counts[count_number]

        unread_count = (
            self.helper.app_count(self.apps, option_name) - read_count)
        if unread_count != 0:
            counts[0] = unread_count
        expected_count = self.option_spec.count
        return {expected_count - k: v for (k, v) in counts.items()}

    def calc_capacity(self, option_name):
        commitments = JudgeRoundCommitment.objects.filter(
            judging_round=self.judging_round)
        criteria_function = self.criterion_total_functions[
            (self.option_spec.criterion.type, self.option_spec.criterion.name)
        ]['function']
        total_capacity = (
            criteria_function(option_name)
            if criteria_function is not None
            else self.helper.total_capacity(
                commitments=commitments,
                option_name=option_name)
        )
        remaining_capacity = self.remaining_capacity(
            self.application_counts,
            self.option_spec,
            option_name)
        return {
            "total_capacity": total_capacity,
            "remaining_capacity": remaining_capacity,
        }

    def general_criterion_total_capacity(self, option):
        option_name = self.option_spec.criterion.name
        field = self.criterion_total_functions[
            (self.option_spec.criterion.type, option_name)
        ]['filter_field']

        if option_name == "gender":
            option = self.gender_match_dict[option]

        if self.criterion_total_capacities.get(option_name) is None:
            capacities = JudgeRoundCommitment.objects.filter(
                judging_round=self.judging_round) \
                .values(field) \
                .annotate(total=Sum("capacity"))
            result = {
                cap[field]: cap["total"]
                for cap in capacities}
            self.criterion_total_capacities[option_name] = result

        key_exists = self.criterion_total_capacities[option_name].get(option)
        return 0 if not key_exists else self.criterion_total_capacities[
            option_name][option]

    def remaining_capacity(self, assignment_counts, option_spec, option):
        if self.judge_to_capacity_cache is None:
            self.judge_to_capacity_cache = JudgeRoundCommitment.objects.filter(
                judging_round=self.judging_round).values(
                    "judge_id",
                    self.industry_judge_field,
                    self.program_judge_field,
                    self.gender_judge_field,
                    self.role_judge_field,
                    "capacity"
                ).annotate(
                    industry=F(self.industry_judge_field),
                    program=F(self.program_judge_field),
                    gender=F(self.gender_judge_field),
                    role=F(self.role_judge_field),
                )

        result = 0
        option_name = option_spec.criterion.name

        for judge in self.judge_to_capacity_cache:
            gender_option = 'prefer not to state'
            if option_name == "gender":
                gender_option = self.gender_match_dict[option]

            if (
                option_name == "reads" or
                judge[option_name] == option or
                judge[option_name] == gender_option
            ):
                result += max(0, judge['capacity'] - assignment_counts.get(
                    judge['judge_id'], 0))
        return result

    def application_criteria_read_state(self, feedbacks, option_name):
        criterion_name = self.option_spec.criterion.name

        if criterion_name == "gender":
            option_name = self.gender_match_dict[option_name]

        if not self.application_criteria_read_state_cache:
            ids_cache_value = {}
            db_values = feedbacks.values(
                "application_id",
                "judge_id",
                self.industry_judge_field,
                self.program_judge_field,
                self.gender_judge_field,
                self.role_judge_field,
            ).annotate(
                industry=F(self.industry_judge_field),
                program=F(self.program_judge_field),
                gender=F(self.gender_judge_field),
                role=F(self.role_judge_field),
            )

            for db_value in db_values:
                app_id = db_value["application_id"]
                if ids_cache_value.get(app_id) is None:
                    ids_cache_value[app_id] = {
                        "industry": {},
                        "program": {},
                        "gender": {},
                        "role": {},
                        "reads": 0
                    }

                industry_value = ids_cache_value[app_id]["industry"].get(
                    db_value["industry"])
                ids_cache_value[app_id]["industry"][db_value["industry"]] = (
                    1 if industry_value is None else industry_value + 1)

                program_value = ids_cache_value[app_id]["program"].get(
                    db_value["program"])
                ids_cache_value[app_id]["program"][db_value["program"]] = (
                    1 if program_value is None else program_value + 1)

                gender_value = ids_cache_value[app_id]["gender"].get(
                    db_value["gender"])
                ids_cache_value[app_id]["gender"][db_value["gender"]] = (
                    1 if gender_value is None else gender_value + 1)

                role_value = ids_cache_value[app_id]["role"].get(
                    db_value["role"])
                ids_cache_value[app_id]["role"][db_value["role"]] = (
                    1 if role_value is None else role_value + 1)

                ids_cache_value[app_id]["reads"] += 1
                self.application_criteria_read_state_cache = ids_cache_value

        return self.application_criteria_read_state_cache

def feedbacks_for_judging_round(judging_round, apps):
    scenarios = Scenario.objects.filter(
        judging_round=judging_round, is_active=True)
    return JudgeApplicationFeedback.objects.filter(
        application__in=apps,
        panel__judgepanelassignment__scenario__in=scenarios)
<commit_msg>[AC-5981] handle edge case<commit_after># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

from accelerator.models import (
    JUDGING_FEEDBACK_STATUS_COMPLETE,
    JudgeApplicationFeedback,
    JudgeRoundCommitment,
    Scenario,
)
from collections import defaultdict
from django.db.models import Sum, F
from impact.v1.helpers.criterion_option_spec_helper import (
    CriterionOptionSpecHelper,
)
from impact.v1.helpers.criterion_helper import CriterionHelper
from impact.v1.helpers.judge_gender_criterion_helper import (
    JudgeGenderCriterionHelper,
)
from impact.v1.helpers.judge_role_criterion_helper import (
    JudgeRoleCriterionHelper,
)
from impact.v1.helpers.matching_industry_criterion_helper import (
    MatchingIndustryCriterionHelper,
)
from impact.v1.helpers.matching_program_criterion_helper import (
    MatchingProgramCriterionHelper,
)
from accelerator_abstract.models.base_core_profile import (
    UI_GENDER_CHOICES
)

CriterionHelper.register_helper(
    JudgeGenderCriterionHelper, "judge", "gender")
CriterionHelper.register_helper(
    JudgeRoleCriterionHelper, "judge", "role")
CriterionHelper.register_helper(
    MatchingIndustryCriterionHelper, "matching", "industry")
CriterionHelper.register_helper(
    MatchingProgramCriterionHelper, "matching", "program")

class OptionAnalysis(object):
    _judge_to_count = None
    expert_profile = "judge__expertprofile__"
    industry_judge_field = expert_profile + "primary_industry__name"
    program_judge_field = expert_profile + "home_program_family__name"
    gender_judge_field = expert_profile + "gender"
    role_judge_field = expert_profile + "expert_category__name"

    def __init__(self,
                 apps,
                 app_ids,
                 judging_round,
                 application_counts,
                 criterion_helpers):
        self.criterion_helpers = criterion_helpers
        self.option_spec = None
        self.judging_round = judging_round
        self.cycle = self.judging_round.program.cycle
        self.helper = None
        self.apps = apps
        self.app_ids = app_ids
        self.application_counts = application_counts
        self.criterion_total_capacities = {}
        self.judge_to_capacity_cache = None
        self.criterion_total_functions = {
            ("judge", "gender"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.gender_judge_field,
            },
            ("reads", "reads"): {
                'function': None,
            },
            ("judge", "role"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.role_judge_field,
            },
            ("matching", "industry"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.industry_judge_field,
            },
            ("matching", "program"): {
                'function': self.general_criterion_total_capacity,
                'filter_field': self.program_judge_field,
            }
        }

        active_scenarios = Scenario.objects.filter(
            judging_round=judging_round, is_active=True)
        self.jpa = JudgeApplicationFeedback.objects.filter(
            application__in=apps,
            panel__judgepanelassignment__scenario__in=active_scenarios)
        self.completed_feedbacks = self.jpa.filter(
            feedback_status=JUDGING_FEEDBACK_STATUS_COMPLETE)
        self.application_criteria_read_state_cache = {}
        self.gender_match_dict = dict(
            (value.lower(), key) for key, value in UI_GENDER_CHOICES)

    def analyses(self, option_spec):
        self.option_spec = option_spec
        self.helper = CriterionOptionSpecHelper(
            option_spec, self.criterion_helpers)
        return [self.analysis(option) for option in self.find_options()]

    def analysis(self, option_name):
        result = {
            "criterion_option_spec_id": self.option_spec.id,
            "criterion_name": self.option_spec.criterion.name,
            "criterion_type": self.option_spec.criterion.type,
            "criterion_id": self.option_spec.criterion.id,
            "option": option_name,
            "weight": self.option_spec.weight,
            "count": self.option_spec.count,
        }
        result.update(self.calc_needs(option_name))
        result.update(self.calc_capacity(option_name))
        return result

    def find_options(self):
        return self.helper.options(self.apps)

    def calc_needs(self, option_name):
        needs_dist = self.calc_needs_distribution(option_name)
        read_count = self.option_spec.count
        return {
            "needs_distribution": needs_dist,
            "total_required_reads": read_count * sum(needs_dist.values()),
            "completed_required_reads": sum(
                [min(read_count, read_count - k) * v
                 for (k, v) in needs_dist.items()]),
            "satisfied_apps": sum(
                [v for (k, v) in needs_dist.items() if k <= 0]),
            "needy_apps": sum(
                [v for (k, v) in needs_dist.items() if k > 0]),
            "remaining_needed_reads": sum(
                [v*k for (k, v) in needs_dist.items() if k > 0])
        }

    def calc_needs_distribution(self, option_name):
        app_counts = self.application_criteria_read_state(
            self.completed_feedbacks,
            option_name=option_name)
        counts = defaultdict(int)
        criterion_name = self.option_spec.criterion.name
        for count in app_counts.values():
            total = (
                count[criterion_name].get(option_name, 0)
                if criterion_name != "reads" else count[criterion_name]
            )
            counts[total] += 1

        read_count = 0
        for count_number in counts:
            if count_number > 0:
                read_count = read_count + counts[count_number]

        unread_count = (
            self.helper.app_count(self.apps, option_name) - read_count)
        if unread_count != 0:
            counts[0] = unread_count
        expected_count = self.option_spec.count
        return {expected_count - k: v for (k, v) in counts.items()}

    def calc_capacity(self, option_name):
        commitments = JudgeRoundCommitment.objects.filter(
            judging_round=self.judging_round)
        criteria_function = self.criterion_total_functions[
            (self.option_spec.criterion.type, self.option_spec.criterion.name)
        ]['function']
        total_capacity = (
            criteria_function(option_name)
            if criteria_function is not None
            else self.helper.total_capacity(
                commitments=commitments,
                option_name=option_name)
        )
        remaining_capacity = self.remaining_capacity(
            self.application_counts,
            self.option_spec,
            option_name)
        return {
            "total_capacity": total_capacity,
            "remaining_capacity": remaining_capacity,
        }

    def general_criterion_total_capacity(self, option):
        option_name = self.option_spec.criterion.name
        field = self.criterion_total_functions[
            (self.option_spec.criterion.type, option_name)
        ]['filter_field']

        if option_name == "gender":
            option = self.gender_match_dict.get(option)

        if self.criterion_total_capacities.get(option_name) is None:
            capacities = JudgeRoundCommitment.objects.filter(
                judging_round=self.judging_round) \
                .values(field) \
                .annotate(total=Sum("capacity"))
            result = {
                cap[field]: cap["total"]
                for cap in capacities}
            self.criterion_total_capacities[option_name] = result

        key_exists = self.criterion_total_capacities[option_name].get(option)
        return 0 if not key_exists else self.criterion_total_capacities[
            option_name][option]

    def remaining_capacity(self, assignment_counts, option_spec, option):
        if self.judge_to_capacity_cache is None:
            self.judge_to_capacity_cache = JudgeRoundCommitment.objects.filter(
                judging_round=self.judging_round).values(
                    "judge_id",
                    self.industry_judge_field,
                    self.program_judge_field,
                    self.gender_judge_field,
                    self.role_judge_field,
                    "capacity"
                ).annotate(
                    industry=F(self.industry_judge_field),
                    program=F(self.program_judge_field),
                    gender=F(self.gender_judge_field),
                    role=F(self.role_judge_field),
                )

        result = 0
        option_name = option_spec.criterion.name

        for judge in self.judge_to_capacity_cache:
            gender_option = 'prefer not to state'
            if option_name == "gender":
                gender_option = self.gender_match_dict.get(option)

            if (
                option_name == "reads" or
                judge[option_name] == option or
                judge[option_name] == gender_option
            ):
                result += max(0, judge['capacity'] - assignment_counts.get(
                    judge['judge_id'], 0))
        return result

    def application_criteria_read_state(self, feedbacks, option_name):
        criterion_name = self.option_spec.criterion.name

        if not self.application_criteria_read_state_cache:
            ids_cache_value = {}
            db_values = feedbacks.values(
                "application_id",
                "judge_id",
                self.industry_judge_field,
                self.program_judge_field,
                self.gender_judge_field,
                self.role_judge_field,
            ).annotate(
                industry=F(self.industry_judge_field),
                program=F(self.program_judge_field),
                gender=F(self.gender_judge_field),
                role=F(self.role_judge_field),
            )

            for db_value in db_values:
                app_id = db_value["application_id"]
                if ids_cache_value.get(app_id) is None:
                    ids_cache_value[app_id] = {
                        "industry": {},
                        "program": {},
                        "gender": {},
                        "role": {},
                        "reads": 0
                    }

                industry_value = ids_cache_value[app_id]["industry"].get(
                    db_value["industry"])
                ids_cache_value[app_id]["industry"][db_value["industry"]] = (
                    1 if industry_value is None else industry_value + 1)

                program_value = ids_cache_value[app_id]["program"].get(
                    db_value["program"])
                ids_cache_value[app_id]["program"][db_value["program"]] = (
                    1 if program_value is None else program_value + 1)

                gender_value = ids_cache_value[app_id]["gender"].get(
                    db_value["gender"])
                ids_cache_value[app_id]["gender"][db_value["gender"]] = (
                    1 if gender_value is None else gender_value + 1)

                role_value = ids_cache_value[app_id]["role"].get(
                    db_value["role"])
                ids_cache_value[app_id]["role"][db_value["role"]] = (
                    1 if role_value is None else role_value + 1)

                ids_cache_value[app_id]["reads"] += 1
                self.application_criteria_read_state_cache = ids_cache_value

        return self.application_criteria_read_state_cache

def feedbacks_for_judging_round(judging_round, apps):
    scenarios = Scenario.objects.filter(
        judging_round=judging_round, is_active=True)
    return JudgeApplicationFeedback.objects.filter(
        application__in=apps,
        panel__judgepanelassignment__scenario__in=scenarios)
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import sys
import json
import argparse

TEMP_NETCFG_FILE = '/tmp/bmv2-demo-cfg.json'
BASE_LONGITUDE = -115
SWITCH_BASE_LATITUDE = 25
HOST_BASE_LATITUDE = 28
BASE_SHIFT = 8
VLAN_NONE = -1
DEFAULT_SW_BW = 50
DEFAULT_HOST_BW = 25

if 'ONOS_ROOT' not in os.environ:
    print "Environment var $ONOS_ROOT not set"
    exit()
else:
    ONOS_ROOT = os.environ["ONOS_ROOT"]
    sys.path.append(ONOS_ROOT + "/tools/dev/mininet")
if 'RUN_PACK_PATH' not in os.environ:
    print "Environment var $RUN_PACK_PATH not set"
    exit()
else:
    RUN_PACK_PATH = os.environ["RUN_PACK_PATH"]

from onos import ONOSCluster, ONOSCLI
from bmv2 import ONOSBmv2Switch

from itertools import combinations
from time import sleep
from subprocess import call

from mininet.cli import CLI
from mininet.link import TCLink
from mininet.log import setLogLevel
from mininet.net import Mininet
from mininet.node import RemoteController, Host
from mininet.topo import Topo

class ClosTopo(Topo):
    "2 stage Clos topology"

    def __init__(self, pipeconfId="", **opts):
        # Initialize topology and default options
        Topo.__init__(self, **opts)

        bmv2SwitchIds = ["s11", "s12", "s13", "s21", "s22", "s23"]
        bmv2Switches = {}

        for switchId in bmv2SwitchIds:
            deviceId=int(switchId[1:])
            # Use first number in device id to calculate latitude (row number)
            latitude = SWITCH_BASE_LATITUDE + (deviceId // 10) * BASE_SHIFT

            # Use second number in device id to calculate longitude (column number)
            longitude = BASE_LONGITUDE + (deviceId % 10) * BASE_SHIFT
            bmv2Switches[switchId] = self.addSwitch(switchId,
                                                    cls=ONOSBmv2Switch,
                                                    loglevel="warn",
                                                    deviceId=deviceId,
                                                    netcfg=False,
                                                    longitude=longitude,
                                                    latitude=latitude,
                                                    pipeconfId=pipeconfId)

        for i in (1, 2, 3):
            for j in (1, 2, 3):
                if i == j:
                    # 2 links
                    self.addLink(bmv2Switches["s1%d" % i], bmv2Switches["s2%d" % j],
                                 cls=TCLink, bw=DEFAULT_SW_BW)
                    self.addLink(bmv2Switches["s1%d" % i], bmv2Switches["s2%d" % j],
                                 cls=TCLink, bw=DEFAULT_SW_BW)
                else:
                    self.addLink(bmv2Switches["s1%d" % i], bmv2Switches["s2%d" % j],
                                 cls=TCLink, bw=DEFAULT_SW_BW)

        for hostId in (1, 2, 3):
            host = self.addHost("h%d" % hostId,
                                cls=DemoHost,
                                ip="10.0.0.%d/24" % hostId,
                                mac='00:00:00:00:00:%02x' % hostId)
            self.addLink(host, bmv2Switches["s1%d" % hostId], cls=TCLink, bw=DEFAULT_HOST_BW)

class DemoHost(Host):
    "Demo host"

    def __init__(self, name, inNamespace=True, **params):
        Host.__init__(self, name, inNamespace=inNamespace, **params)
        self.exectoken = "/tmp/mn-exec-token-host-%s" % name
        self.cmd("touch %s" % self.exectoken)

    def config(self, **params):
        r = super(Host, self).config(**params)

        for off in ["rx", "tx", "sg"]:
            cmd = "/sbin/ethtool --offload %s %s off" % (self.defaultIntf(), off)
            self.cmd(cmd)

        # disable IPv6
        self.cmd("sysctl -w net.ipv6.conf.all.disable_ipv6=1")
        self.cmd("sysctl -w net.ipv6.conf.default.disable_ipv6=1")
        self.cmd("sysctl -w net.ipv6.conf.lo.disable_ipv6=1")

        return r

    def startPingBg(self, h):
        self.cmd(self.getInfiniteCmdBg("ping -i0.5 %s" % h.IP()))
        self.cmd(self.getInfiniteCmdBg("arping -w5000000 %s" % h.IP()))

    def startIperfServer(self):
        self.cmd(self.getInfiniteCmdBg("iperf3 -s"))

    def startIperfClient(self, h, flowBw="512k", numFlows=5, duration=5):
        iperfCmd = "iperf3 -c{} -b{} -P{} -t{}".format(h.IP(), flowBw, numFlows, duration)
        self.cmd(self.getInfiniteCmdBg(iperfCmd, sleep=0))

    def stop(self):
        self.cmd("killall iperf3")
        self.cmd("killall ping")
        self.cmd("killall arping")

    def describe(self):
        print "**********"
        print self.name
        print "default interface: %s\t%s\t%s" % (
            self.defaultIntf().name,
            self.defaultIntf().IP(),
            self.defaultIntf().MAC()
        )
        print "**********"

    def getInfiniteCmdBg(self, cmd, logfile="/dev/null", sleep=1):
        return "(while [ -e {} ]; " \
               "do {}; " \
               "sleep {}; " \
               "done;) > {} 2>&1 &".format(self.exectoken, cmd, sleep, logfile)

    def getCmdBg(self, cmd, logfile="/dev/null"):
        return "{} > {} 2>&1 &".format(cmd, logfile)

def generateNetcfg(onosIp, net):
    netcfg = { 'devices': {}, 'links': {}, 'hosts': {}}
    # Device configs
    for sw in net.switches:
        srcIp = sw.getSourceIp(onosIp)
        netcfg['devices'][sw.onosDeviceId] = sw.getDeviceConfig(srcIp)

    hostLocations = {}
    # Link configs
    for link in net.links:
        switchPort = link.intf1.name.split('-')
        sw1Name = switchPort[0] # s11
        port1Name = switchPort[1] # eth0
        port1 = port1Name[3:]
        switchPort = link.intf2.name.split('-')
        sw2Name = switchPort[0]
        port2Name = switchPort[1]
        port2 = port2Name[3:]
        sw1 = net[sw1Name]
        sw2 = net[sw2Name]
        if isinstance(sw1, Host):
            # record host location and ignore it
            # e.g. {'h1': 'device:bmv2:11'}
            hostLocations[sw1.name] = '%s/%s' % (sw2.onosDeviceId, port2)
            continue

        if isinstance(sw2, Host):
            # record host location and ignore it
            # e.g. {'h1': 'device:bmv2:11'}
            hostLocations[sw2.name] = '%s/%s' % (sw1.onosDeviceId, port1)
            continue

        linkId = '%s/%s-%s/%s' % (sw1.onosDeviceId, port1, sw2.onosDeviceId, port2)
        netcfg['links'][linkId] = {
            'basic': {
                'type': 'DIRECT',
                'bandwidth': 50
            }
        }

    # Host configs
    longitude = BASE_LONGITUDE
    for host in net.hosts:
        longitude = longitude + BASE_SHIFT
        hostDefaultIntf = host.defaultIntf()
        hostMac = host.MAC(hostDefaultIntf)
        hostIp = host.IP(hostDefaultIntf)
        hostId = '%s/%d' % (hostMac, VLAN_NONE)
        location = hostLocations[host.name]

        # use host Id to generate host location
        hostConfig = {
            'basic': {
                'locations': [location],
                'ips': [hostIp],
                'name': host.name,
                'latitude': HOST_BASE_LATITUDE,
                'longitude': longitude
            }
        }
        netcfg['hosts'][hostId] = hostConfig

    print "Writing network config to %s" % TEMP_NETCFG_FILE
    with open(TEMP_NETCFG_FILE, 'w') as tempFile:
        json.dump(netcfg, tempFile)

def main(args):
    setLogLevel('debug')
    if not args.onos_ip:
        controller = ONOSCluster('c0', 3)
        onosIp = controller.nodes()[0].IP()
    else:
        controller = RemoteController('c0', ip=args.onos_ip)
        onosIp = args.onos_ip

    topo = ClosTopo(pipeconfId=args.pipeconf_id)

    net = Mininet(topo=topo, build=False, controller=[controller])

    net.build()
    net.start()

    print "Network started"

    # Generate background traffic.
    sleep(3)
    for (h1, h2) in combinations(net.hosts, 2):
        h1.startPingBg(h2)
        h2.startPingBg(h1)

    print "Background ping started"

    for h in net.hosts:
        h.startIperfServer()

    print "Iperf servers started"

    # sleep(4)
    # print "Starting traffic from h1 to h3..."
    # net.hosts[0].startIperfClient(net.hosts[-1], flowBw="200k", numFlows=100, duration=10)

    generateNetcfg(onosIp, net)

    print "Uploading netcfg..."
    call(("%s/onos-netcfg" % RUN_PACK_PATH, onosIp, TEMP_NETCFG_FILE))

    if not args.onos_ip:
        ONOSCLI(net)
    else:
        CLI(net)

    net.stop()
    call(("rm", "-f", TEMP_NETCFG_FILE))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='BMv2 mininet demo script (2-stage Clos topology)')
    parser.add_argument('--onos-ip', help='ONOS-BMv2 controller IP address',
                        type=str, action="store", required=False)
    parser.add_argument('--pipeconf-id', help='Pipeconf ID for switches',
                        type=str, action="store", required=False, default='')
    args = parser.parse_args()
    setLogLevel('info')
    main(args)
<commit_msg>Prettified netcfg JSON produced by bmv2-demo.py<commit_after>#!/usr/bin/python

import os
import sys
import json
import argparse

TEMP_NETCFG_FILE = '/tmp/bmv2-demo-cfg.json'
BASE_LONGITUDE = -115
SWITCH_BASE_LATITUDE = 25
HOST_BASE_LATITUDE = 28
BASE_SHIFT = 8
VLAN_NONE = -1
DEFAULT_SW_BW = 50
DEFAULT_HOST_BW = 25

if 'ONOS_ROOT' not in os.environ:
    print "Environment var $ONOS_ROOT not set"
    exit()
else:
    ONOS_ROOT = os.environ["ONOS_ROOT"]
    sys.path.append(ONOS_ROOT + "/tools/dev/mininet")
if 'RUN_PACK_PATH' not in os.environ:
    print "Environment var $RUN_PACK_PATH not set"
    exit()
else:
    RUN_PACK_PATH = os.environ["RUN_PACK_PATH"]

from onos import ONOSCluster, ONOSCLI
from bmv2 import ONOSBmv2Switch

from itertools import combinations
from time import sleep
from subprocess import call

from mininet.cli import CLI
from mininet.link import TCLink
from mininet.log import setLogLevel
from mininet.net import Mininet
from mininet.node import RemoteController, Host
from mininet.topo import Topo

class ClosTopo(Topo):
    "2 stage Clos topology"

    def __init__(self, pipeconfId="", **opts):
        # Initialize topology and default options
        Topo.__init__(self, **opts)

        bmv2SwitchIds = ["s11", "s12", "s13", "s21", "s22", "s23"]
        bmv2Switches = {}

        for switchId in bmv2SwitchIds:
            deviceId = int(switchId[1:])
            # Use first number in device id to calculate latitude (row number)
            latitude = SWITCH_BASE_LATITUDE + (deviceId // 10) * BASE_SHIFT

            # Use second number in device id to calculate longitude (column number)
            longitude = BASE_LONGITUDE + (deviceId % 10) * BASE_SHIFT
            bmv2Switches[switchId] = self.addSwitch(switchId,
                                                    cls=ONOSBmv2Switch,
                                                    loglevel="warn",
                                                    deviceId=deviceId,
                                                    netcfg=False,
                                                    longitude=longitude,
                                                    latitude=latitude,
                                                    pipeconfId=pipeconfId)

        for i in (1, 2, 3):
            for j in (1, 2, 3):
                if i == j:
                    # 2 links
                    self.addLink(bmv2Switches["s1%d" % i], bmv2Switches["s2%d" % j],
                                 cls=TCLink, bw=DEFAULT_SW_BW)
                    self.addLink(bmv2Switches["s1%d" % i], bmv2Switches["s2%d" % j],
                                 cls=TCLink, bw=DEFAULT_SW_BW)
                else:
                    self.addLink(bmv2Switches["s1%d" % i], bmv2Switches["s2%d" % j],
                                 cls=TCLink, bw=DEFAULT_SW_BW)

        for hostId in (1, 2, 3):
            host = self.addHost("h%d" % hostId,
                                cls=DemoHost,
                                ip="10.0.0.%d/24" % hostId,
                                mac='00:00:00:00:00:%02x' % hostId)
            self.addLink(host, bmv2Switches["s1%d" % hostId], cls=TCLink, bw=DEFAULT_HOST_BW)

class DemoHost(Host):
    "Demo host"

    def __init__(self, name, inNamespace=True, **params):
        Host.__init__(self, name, inNamespace=inNamespace, **params)
        self.exectoken = "/tmp/mn-exec-token-host-%s" % name
        self.cmd("touch %s" % self.exectoken)

    def config(self, **params):
        r = super(Host, self).config(**params)

        for off in ["rx", "tx", "sg"]:
            cmd = "/sbin/ethtool --offload %s %s off" % (self.defaultIntf(), off)
            self.cmd(cmd)

        # disable IPv6
        self.cmd("sysctl -w net.ipv6.conf.all.disable_ipv6=1")
        self.cmd("sysctl -w net.ipv6.conf.default.disable_ipv6=1")
        self.cmd("sysctl -w net.ipv6.conf.lo.disable_ipv6=1")

        return r

    def startPingBg(self, h):
        self.cmd(self.getInfiniteCmdBg("ping -i0.5 %s" % h.IP()))
        self.cmd(self.getInfiniteCmdBg("arping -w5000000 %s" % h.IP()))

    def startIperfServer(self):
        self.cmd(self.getInfiniteCmdBg("iperf3 -s"))

    def startIperfClient(self, h, flowBw="512k", numFlows=5, duration=5):
        iperfCmd = "iperf3 -c{} -b{} -P{} -t{}".format(h.IP(), flowBw, numFlows, duration)
        self.cmd(self.getInfiniteCmdBg(iperfCmd, sleep=0))

    def stop(self):
        self.cmd("killall iperf3")
        self.cmd("killall ping")
        self.cmd("killall arping")

    def describe(self):
        print "**********"
        print self.name
        print "default interface: %s\t%s\t%s" % (
            self.defaultIntf().name,
            self.defaultIntf().IP(),
            self.defaultIntf().MAC()
        )
        print "**********"

    def getInfiniteCmdBg(self, cmd, logfile="/dev/null", sleep=1):
        return "(while [ -e {} ]; " \
               "do {}; " \
               "sleep {}; " \
               "done;) > {} 2>&1 &".format(self.exectoken, cmd, sleep, logfile)

    def getCmdBg(self, cmd, logfile="/dev/null"):
        return "{} > {} 2>&1 &".format(cmd, logfile)

def generateNetcfg(onosIp, net, args):
    netcfg = {'devices': {}, 'links': {}, 'hosts': {}}
    # Device configs
    for sw in net.switches:
        srcIp = sw.getSourceIp(onosIp)
        netcfg['devices'][sw.onosDeviceId] = sw.getDeviceConfig(srcIp)

    hostLocations = {}
    # Link configs
    for link in net.links:
        switchPort = link.intf1.name.split('-')
        sw1Name = switchPort[0]  # s11
        port1Name = switchPort[1]  # eth0
        port1 = port1Name[3:]
        switchPort = link.intf2.name.split('-')
        sw2Name = switchPort[0]
        port2Name = switchPort[1]
        port2 = port2Name[3:]
        sw1 = net[sw1Name]
        sw2 = net[sw2Name]
        if isinstance(sw1, Host):
            # record host location and ignore it
            # e.g. {'h1': 'device:bmv2:11'}
            hostLocations[sw1.name] = '%s/%s' % (sw2.onosDeviceId, port2)
            continue

        if isinstance(sw2, Host):
            # record host location and ignore it
            # e.g. {'h1': 'device:bmv2:11'}
            hostLocations[sw2.name] = '%s/%s' % (sw1.onosDeviceId, port1)
            continue

        linkId = '%s/%s-%s/%s' % (sw1.onosDeviceId, port1, sw2.onosDeviceId, port2)
        netcfg['links'][linkId] = {
            'basic': {
                'type': 'DIRECT',
                'bandwidth': 50
            }
        }

    # Host configs
    longitude = BASE_LONGITUDE
    for host in net.hosts:
        longitude = longitude + BASE_SHIFT
        hostDefaultIntf = host.defaultIntf()
        hostMac = host.MAC(hostDefaultIntf)
        hostIp = host.IP(hostDefaultIntf)
        hostId = '%s/%d' % (hostMac, VLAN_NONE)
        location = hostLocations[host.name]

        # use host Id to generate host location
        hostConfig = {
            'basic': {
                'locations': [location],
                'ips': [hostIp],
                'name': host.name,
                'latitude': HOST_BASE_LATITUDE,
                'longitude': longitude
            }
        }
        netcfg['hosts'][hostId] = hostConfig

    print "Writing network config to %s" % TEMP_NETCFG_FILE
    with open(TEMP_NETCFG_FILE, 'w') as tempFile:
        json.dump(netcfg, tempFile, indent=4)

def main(args):
    if not args.onos_ip:
        controller = ONOSCluster('c0', 3)
        onosIp = controller.nodes()[0].IP()
    else:
        controller = RemoteController('c0', ip=args.onos_ip)
        onosIp = args.onos_ip

    topo = ClosTopo(pipeconfId=args.pipeconf_id)

    net = Mininet(topo=topo, build=False, controller=[controller])

    net.build()
    net.start()

    print "Network started"

    # Generate background traffic.
    sleep(3)
    for (h1, h2) in combinations(net.hosts, 2):
        h1.startPingBg(h2)
        h2.startPingBg(h1)

    print "Background ping started"

    for h in net.hosts:
        h.startIperfServer()

    print "Iperf servers started"

    # sleep(4)
    # print "Starting traffic from h1 to h3..."
    # net.hosts[0].startIperfClient(net.hosts[-1], flowBw="200k", numFlows=100, duration=10)

    generateNetcfg(onosIp, net, args)

    print "Uploading netcfg..."
    call(("%s/onos-netcfg" % RUN_PACK_PATH, onosIp, TEMP_NETCFG_FILE))

    if not args.onos_ip:
        ONOSCLI(net)
    else:
        CLI(net)

    net.stop()
    call(("rm", "-f", TEMP_NETCFG_FILE))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='BMv2 mininet demo script (2-stage Clos topology)')
    parser.add_argument('--onos-ip', help='ONOS-BMv2 controller IP address',
                        type=str, action="store", required=False)
    parser.add_argument('--pipeconf-id', help='Pipeconf ID for switches',
                        type=str, action="store", required=False, default='')
    args = parser.parse_args()
    setLogLevel('info')
    main(args)
<|endoftext|>


<commit_before>from pyparsers.parser import MemeParser
from pyparsers.memetr import MemeTr
from pyparsers.astbuilder import *
import pyutils
from pyutils import bits
from pyutils import entries
import traceback
import os
import sys
import ejson
from . import comp_vmemory

class MMC(object):
    MAGIC_NUMBER = 0x420

    def __init__(self, cmodule):
        self.cmodule = cmodule

    def name_ptr_for(self, name, mmc):
        acc = 0
        for entry_name_t, bsize in mmc['names']:
            if entry_name_t[0:-1] == name:
                return self.HEADER_SIZE + acc
            acc += bsize
        raise Exception('entry {} not found in NAMES'.format(name))

    def create_mmc_struct(self, vmem):
        mmc = {'header':
               {'magic_number': None,
                'ot_size': None,
                'er_size': None,
                'st_size': None,
                'names_size': None},
               'names': [],
               'object_table': [],         # the loaded objects of this module
               'external_references': [],  # references to core objects
               'symbol_table': [],         # Symbol objects needed
               'reloc_table': []           # addresses within the OT that need relocation on load
            }

        self.HEADER_SIZE = len(mmc['header']) * bits.WSIZE
        self.cmodule.fill(vmem)

        mmc['header']['magic_number'] = self.MAGIC_NUMBER

        # mmc['names'] = [(name_t, bits.string_block_size(name_t)) for name_t in [name + '\0' for name in vmem.external_names()]]

        # labels = self.cmodule.entry_labels()
        names_list = [n + "\0" for n in vmem.external_names()]

        mmc['names'] = [(name_t, bits.string_block_size(name_t)) for name_t in names_list]
        mmc['header']['names_size'] = sum([x[1] for x in mmc['names']])

        base = self.HEADER_SIZE + mmc['header']['names_size']
        vmem.set_base(base)

        mmc['object_table'] = vmem.object_table()
        mmc['header']['ot_size'] = len(mmc['object_table'])

        for pair in vmem.external_references():
            mmc['external_references'].append(self.name_ptr_for(pair[0], mmc))
            mmc['external_references'].append(pair[1])

        mmc['header']['er_size'] = len(mmc['external_references']) * bits.WSIZE

        for pair in vmem.symbols_references():
            mmc['symbol_table'].append(self.name_ptr_for(pair[0], mmc))
            mmc['symbol_table'].append(pair[1])

        mmc['header']['st_size'] = len(mmc['symbol_table']) * bits.WSIZE

        mmc['reloc_table'] = vmem.reloc_table()

        return mmc

    def dump(self, filepath):
        vmem = comp_vmemory.CompVirtualMemory()
        mmc = self.create_mmc_struct(vmem)
        with open(filepath[:-2] + "mmc", "w") as fp:
            # header
            fp.write(bits.pack_word(mmc['header']['magic_number']))
            fp.write(bits.pack_word(mmc['header']['ot_size']))
            fp.write(bits.pack_word(mmc['header']['er_size']))
            fp.write(bits.pack_word(mmc['header']['st_size']))
            fp.write(bits.pack_word(mmc['header']['names_size']))

            # names
            for name, chunk_size in mmc['names']:
                text = name + ((chunk_size - len(name)) * '\0')
                fp.write(text)

            # object table
            for v in mmc['object_table']:
                fp.write(bits.pack_byte(v))

            # external references
            for v in mmc['external_references']:
                fp.write(bits.pack_word(v))

            # symbols
            for v in mmc['symbol_table']:
                fp.write(bits.pack_word(v))

            # reloc table
            for word in mmc['reloc_table']:
                fp.write(bits.pack_word(word))

class MMC_Fun(object):
    MAGIC_NUMBER = 0x420

    def __init__(self, cmod, cfun):
        self.cmod = cmod
        self.cfun = cfun

    def name_ptr_for(self, name, mmc):
        acc = 0
        for entry_name_t, bsize in mmc['names']:
            if entry_name_t[0:-1] == name:
                return self.HEADER_SIZE + acc
            acc += bsize
        raise Exception('entry {} not found in NAMES'.format(name))

    def create_mmc_struct(self, vmem):
        mmc = {'header':
               {'magic_number': None,
                'ot_size': None,
                'er_size': None,
                'st_size': None,
                'names_size': None,
                'cfun_addr': None},
               'names': [],
               'object_table': [],
               'external_references': [],
               'symbol_table': [],
               'reloc_table': []
            }

        self.HEADER_SIZE = len(mmc['header']) * bits.WSIZE
        self.cmod.fill(vmem)

        mmc['header']['magic_number'] = self.MAGIC_NUMBER

        names_list = set([n + "\0" for n in vmem.external_names()])

        mmc['names'] = [(name_t, bits.string_block_size(name_t)) for name_t in names_list]
        mmc['header']['names_size'] = sum([x[1] for x in mmc['names']])

        base = self.HEADER_SIZE + mmc['header']['names_size']
        vmem.set_base(base)

        mmc['object_table'] = vmem.object_table()
        mmc['header']['ot_size'] = len(mmc['object_table'])

        for pair in vmem.external_references():
            mmc['external_references'].append(self.name_ptr_for(pair[0], mmc))
            mmc['external_references'].append(pair[1])

        mmc['header']['er_size'] = len(mmc['external_references']) * bits.WSIZE

        for pair in vmem.symbols_references():
            mmc['symbol_table'].append(self.name_ptr_for(pair[0], mmc))
            mmc['symbol_table'].append(pair[1])

        mmc['header']['st_size'] = len(mmc['symbol_table']) * bits.WSIZE

        mmc['reloc_table'] = vmem.reloc_table()

        mmc['header']['cfun_addr'] = vmem.physical_address(self.cfun.oop)
        return mmc

    def dump(self):
        vmem = comp_vmemory.CompVirtualMemory()
        mmc = self.create_mmc_struct(vmem)

        fp = sys.stdout

        # header
        fp.write(bits.pack_word(mmc['header']['magic_number']))
        fp.write(bits.pack_word(mmc['header']['ot_size']))
        fp.write(bits.pack_word(mmc['header']['er_size']))
        fp.write(bits.pack_word(mmc['header']['st_size']))
        fp.write(bits.pack_word(mmc['header']['names_size']))
        fp.write(bits.pack_word(mmc['header']['cfun_addr']))

        # names
        for name, chunk_size in mmc['names']:
            text = name + ((chunk_size - len(name)) * '\0')
            fp.write(text)

        # object table
        for v in mmc['object_table']:
            fp.write(bits.pack_byte(v))

        # external references
        for v in mmc['external_references']:
            fp.write(bits.pack_word(v))

        # symbols
        for v in mmc['symbol_table']:
            fp.write(bits.pack_word(v))

        # reloc table
        for word in mmc['reloc_table']:
            fp.write(bits.pack_word(word))

class Compiler(ASTBuilder):
    def __init__(self):
        self.cmodule = None
        self.filepath = None

    def new_module(self):
        module_name =  os.path.splitext(os.path.basename(self.filepath))[0]
        self.cmodule = entries.CompiledModule(module_name)
        return self.cmodule

    def do_parse(self, parser, rule, *args):
        try:
            parser.has_fun_literal = pyutils.Flag() # pymeta uses eval() which disables assignment. This works around it
            return parser.apply(rule, *args)
        except Exception as err:
            if hasattr(err,'formatError'):
                print err.formatError(''.join(parser.input.data))
            else:
                print err, traceback.format_exc()
            sys.exit(1)

    def compile(self, filepath):
        self.filepath = filepath
        self.line_offset = 0
        self.parser = MemeParser(open(filepath, 'r').read())
        self.parser.i = self

        ast = self.do_parse(self.parser, 'start')[0]

        print ast

        self.parser = MemeTr([ast])
        self.parser.i = self

        self.do_parse(self.parser, "start")

        mmc = MMC(self.cmodule)
        mmc.dump(filepath)

    def compile_lambda(self, text, env_names):
        self.filepath = '<eval>'
        self.line_offset = 0

        self.parser = MemeParser(text)
        self.parser.i = self

        ast = self.do_parse(self.parser, 'cfunliteral_body')[0]
        # print ast

        cmod = entries.CompiledModule('dummy_module')
        cfun = cmod.new_function('<eval>', [])
        cfun.uses_env(True)
        cfun.declare_vars(env_names)

        self.parser = MemeTr([cfun, ast])
        self.parser.i = self

        self.do_parse(self.parser, 'cfunliteral')

        mmc = MMC_Fun(cmod, cfun)
        mmc.dump()

    def recompile_function(self, line, text):
        self.filepath = '<???>'
        self.line_offset = int(line)

        self.parser = MemeParser(text)
        self.parser.i = self

        ast = self.do_parse(self.parser, 'fun_rest', 'random_name')[0]
        # print ast

        # cmod = entries.CompiledModule('dummy_module')
        # cfun = cmod.new_function('<????????>', [])

        cmod = self.new_module()
        self.parser = MemeTr([cmod, ast])
        self.parser.i = self

        self.do_parse(self.parser, 'function_definition')

        cfun = cmod.functions['random_name']
        cfun.uses_env(False)
        mmc = MMC_Fun(cmod, cfun)
        mmc.dump()

def main(*paths):
    for path in paths:
        Compiler().compile(path)

def compile_lambda(js):
    # python -m pycompiler.compiler compile-lambda '{"text":"fun() {X.z}","env_names":""}'
    # python -m pycompiler.compiler compile-lambbd '{"text":"fun() {a}","env_names":["a"]}'
    js = ejson.loads(js)
    Compiler().compile_lambda(js['text'], js['env_names'])

def recompile_fun(js):
    # python -m pycompiler.compiler recompile-fun '{"text":"fun() {X.z}"}'
    js = ejson.loads(js)
    Compiler().recompile_function(js["start_line"], js['text'])

if __name__ == '__main__':
    from clime import Program
    prog = Program(debug=True, default='main')
    prog.main()
<commit_msg>pycompiler: fix error reporting when lists are parsed<commit_after>from pyparsers.parser import MemeParser
from pyparsers.memetr import MemeTr
from pyparsers.astbuilder import *
import pyutils
from pyutils import bits
from pyutils import entries
import traceback
import os
import sys
import ejson
from . import comp_vmemory

class MMC(object):
    MAGIC_NUMBER = 0x420

    def __init__(self, cmodule):
        self.cmodule = cmodule

    def name_ptr_for(self, name, mmc):
        acc = 0
        for entry_name_t, bsize in mmc['names']:
            if entry_name_t[0:-1] == name:
                return self.HEADER_SIZE + acc
            acc += bsize
        raise Exception('entry {} not found in NAMES'.format(name))

    def create_mmc_struct(self, vmem):
        mmc = {'header':
               {'magic_number': None,
                'ot_size': None,
                'er_size': None,
                'st_size': None,
                'names_size': None},
               'names': [],
               'object_table': [],         # the loaded objects of this module
               'external_references': [],  # references to core objects
               'symbol_table': [],         # Symbol objects needed
               'reloc_table': []           # addresses within the OT that need relocation on load
            }

        self.HEADER_SIZE = len(mmc['header']) * bits.WSIZE
        self.cmodule.fill(vmem)

        mmc['header']['magic_number'] = self.MAGIC_NUMBER

        # mmc['names'] = [(name_t, bits.string_block_size(name_t)) for name_t in [name + '\0' for name in vmem.external_names()]]

        # labels = self.cmodule.entry_labels()
        names_list = [n + "\0" for n in vmem.external_names()]

        mmc['names'] = [(name_t, bits.string_block_size(name_t)) for name_t in names_list]
        mmc['header']['names_size'] = sum([x[1] for x in mmc['names']])

        base = self.HEADER_SIZE + mmc['header']['names_size']
        vmem.set_base(base)

        mmc['object_table'] = vmem.object_table()
        mmc['header']['ot_size'] = len(mmc['object_table'])

        for pair in vmem.external_references():
            mmc['external_references'].append(self.name_ptr_for(pair[0], mmc))
            mmc['external_references'].append(pair[1])

        mmc['header']['er_size'] = len(mmc['external_references']) * bits.WSIZE

        for pair in vmem.symbols_references():
            mmc['symbol_table'].append(self.name_ptr_for(pair[0], mmc))
            mmc['symbol_table'].append(pair[1])

        mmc['header']['st_size'] = len(mmc['symbol_table']) * bits.WSIZE

        mmc['reloc_table'] = vmem.reloc_table()

        return mmc

    def dump(self, filepath):
        vmem = comp_vmemory.CompVirtualMemory()
        mmc = self.create_mmc_struct(vmem)
        with open(filepath[:-2] + "mmc", "w") as fp:
            # header
            fp.write(bits.pack_word(mmc['header']['magic_number']))
            fp.write(bits.pack_word(mmc['header']['ot_size']))
            fp.write(bits.pack_word(mmc['header']['er_size']))
            fp.write(bits.pack_word(mmc['header']['st_size']))
            fp.write(bits.pack_word(mmc['header']['names_size']))

            # names
            for name, chunk_size in mmc['names']:
                text = name + ((chunk_size - len(name)) * '\0')
                fp.write(text)

            # object table
            for v in mmc['object_table']:
                fp.write(bits.pack_byte(v))

            # external references
            for v in mmc['external_references']:
                fp.write(bits.pack_word(v))

            # symbols
            for v in mmc['symbol_table']:
                fp.write(bits.pack_word(v))

            # reloc table
            for word in mmc['reloc_table']:
                fp.write(bits.pack_word(word))

class MMC_Fun(object):
    MAGIC_NUMBER = 0x420

    def __init__(self, cmod, cfun):
        self.cmod = cmod
        self.cfun = cfun

    def name_ptr_for(self, name, mmc):
        acc = 0
        for entry_name_t, bsize in mmc['names']:
            if entry_name_t[0:-1] == name:
                return self.HEADER_SIZE + acc
            acc += bsize
        raise Exception('entry {} not found in NAMES'.format(name))

    def create_mmc_struct(self, vmem):
        mmc = {'header':
               {'magic_number': None,
                'ot_size': None,
                'er_size': None,
                'st_size': None,
                'names_size': None,
                'cfun_addr': None},
               'names': [],
               'object_table': [],
               'external_references': [],
               'symbol_table': [],
               'reloc_table': []
            }

        self.HEADER_SIZE = len(mmc['header']) * bits.WSIZE
        self.cmod.fill(vmem)

        mmc['header']['magic_number'] = self.MAGIC_NUMBER

        names_list = set([n + "\0" for n in vmem.external_names()])

        mmc['names'] = [(name_t, bits.string_block_size(name_t)) for name_t in names_list]
        mmc['header']['names_size'] = sum([x[1] for x in mmc['names']])

        base = self.HEADER_SIZE + mmc['header']['names_size']
        vmem.set_base(base)

        mmc['object_table'] = vmem.object_table()
        mmc['header']['ot_size'] = len(mmc['object_table'])

        for pair in vmem.external_references():
            mmc['external_references'].append(self.name_ptr_for(pair[0], mmc))
            mmc['external_references'].append(pair[1])

        mmc['header']['er_size'] = len(mmc['external_references']) * bits.WSIZE

        for pair in vmem.symbols_references():
            mmc['symbol_table'].append(self.name_ptr_for(pair[0], mmc))
            mmc['symbol_table'].append(pair[1])

        mmc['header']['st_size'] = len(mmc['symbol_table']) * bits.WSIZE

        mmc['reloc_table'] = vmem.reloc_table()

        mmc['header']['cfun_addr'] = vmem.physical_address(self.cfun.oop)
        return mmc

    def dump(self):
        vmem = comp_vmemory.CompVirtualMemory()
        mmc = self.create_mmc_struct(vmem)

        fp = sys.stdout

        # header
        fp.write(bits.pack_word(mmc['header']['magic_number']))
        fp.write(bits.pack_word(mmc['header']['ot_size']))
        fp.write(bits.pack_word(mmc['header']['er_size']))
        fp.write(bits.pack_word(mmc['header']['st_size']))
        fp.write(bits.pack_word(mmc['header']['names_size']))
        fp.write(bits.pack_word(mmc['header']['cfun_addr']))

        # names
        for name, chunk_size in mmc['names']:
            text = name + ((chunk_size - len(name)) * '\0')
            fp.write(text)

        # object table
        for v in mmc['object_table']:
            fp.write(bits.pack_byte(v))

        # external references
        for v in mmc['external_references']:
            fp.write(bits.pack_word(v))

        # symbols
        for v in mmc['symbol_table']:
            fp.write(bits.pack_word(v))

        # reloc table
        for word in mmc['reloc_table']:
            fp.write(bits.pack_word(word))

class Compiler(ASTBuilder):
    def __init__(self):
        self.cmodule = None
        self.filepath = None

    def new_module(self):
        module_name =  os.path.splitext(os.path.basename(self.filepath))[0]
        self.cmodule = entries.CompiledModule(module_name)
        return self.cmodule

    def do_parse(self, parser, rule, *args):
        try:
            parser.has_fun_literal = pyutils.Flag() # pymeta uses eval() which disables assignment. This works around it
            return parser.apply(rule, *args)
        except Exception as err:
            print '\n'
            if hasattr(err,'formatError') and isinstance(parser.input.data[0], (str, unicode)):
                print err.formatError(''.join(parser.input.data))
            elif hasattr(err,'formatError') :
                print err.formatError(parser.input.data)
            else:
                print err, traceback.format_exc()
            sys.exit(1)

    def compile(self, filepath):
        self.filepath = filepath
        self.line_offset = 0
        self.parser = MemeParser(open(filepath, 'r').read())
        self.parser.i = self

        ast = self.do_parse(self.parser, 'start')[0]

        print ast

        self.parser = MemeTr([ast])
        self.parser.i = self

        self.do_parse(self.parser, "start")

        mmc = MMC(self.cmodule)
        mmc.dump(filepath)

    def compile_lambda(self, text, env_names):
        self.filepath = '<eval>'
        self.line_offset = 0

        self.parser = MemeParser(text)
        self.parser.i = self

        ast = self.do_parse(self.parser, 'cfunliteral_body')[0]
        # print ast

        cmod = entries.CompiledModule('dummy_module')
        cfun = cmod.new_function('<eval>', [])
        cfun.uses_env(True)
        cfun.declare_vars(env_names)

        self.parser = MemeTr([cfun, ast])
        self.parser.i = self

        self.do_parse(self.parser, 'cfunliteral')

        mmc = MMC_Fun(cmod, cfun)
        mmc.dump()

    def recompile_function(self, line, text):
        self.filepath = '<???>'
        self.line_offset = int(line)

        self.parser = MemeParser(text)
        self.parser.i = self

        ast = self.do_parse(self.parser, 'fun_rest', 'random_name')[0]
        # print ast

        # cmod = entries.CompiledModule('dummy_module')
        # cfun = cmod.new_function('<????????>', [])

        cmod = self.new_module()
        self.parser = MemeTr([cmod, ast])
        self.parser.i = self

        self.do_parse(self.parser, 'function_definition')

        cfun = cmod.functions['random_name']
        cfun.uses_env(False)
        mmc = MMC_Fun(cmod, cfun)
        mmc.dump()

def main(*paths):
    for path in paths:
        Compiler().compile(path)

def compile_lambda(js):
    # python -m pycompiler.compiler compile-lambda '{"text":"fun() {X.z}","env_names":""}'
    # python -m pycompiler.compiler compile-lambbd '{"text":"fun() {a}","env_names":["a"]}'
    js = ejson.loads(js)
    Compiler().compile_lambda(js['text'], js['env_names'])

def recompile_fun(js):
    # python -m pycompiler.compiler recompile-fun '{"text":"fun() {X.z}"}'
    js = ejson.loads(js)
    Compiler().recompile_function(js["start_line"], js['text'])

if __name__ == '__main__':
    from clime import Program
    prog = Program(debug=True, default='main')
    prog.main()
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RPlotmo(RPackage):
    """plotmo: Plot a Model's Residuals, Response, and Partial Dependence
    Plots.

    Plot model surfaces for a wide variety of models using partial dependence
    plots and other techniques. Also plot model residuals and other information
    on the model."""

    homepage = "http://www.milbo.users.sonic.net/"
    url      = "https://cloud.r-project.org/src/contrib/plotmo_3.5.6.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/plotmo"

    version('3.5.6', sha256='78f08dc897136d21fa8ade2acb6290351b569d29eb0592c7074c0be3cf2aa594')

    depends_on('r@3.4.0:', type=('build', 'run'))
    depends_on('r-formula@1.2-3:', type=('build', 'run'))
    depends_on('r-plotrix', type=('build', 'run'))
    depends_on('r-teachingdemos', type=('build', 'run'))
<commit_msg>r-plotmo: add v3.6.0 (#20909)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RPlotmo(RPackage):
    """Plot a Model's Residuals, Response, and Partial Dependence Plots.

    Plot model surfaces for a wide variety of models using partial dependence
    plots and other techniques. Also plot model residuals and other information
    on the model."""

    homepage = "http://www.milbo.users.sonic.net/"
    url      = "https://cloud.r-project.org/src/contrib/plotmo_3.5.6.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/plotmo"

    version('3.6.0', sha256='c05afcc442f9542868beea5c3c40fb93b049f9b61c42725b2a1e2bc750c241e3')
    version('3.5.6', sha256='78f08dc897136d21fa8ade2acb6290351b569d29eb0592c7074c0be3cf2aa594')

    depends_on('r@3.4.0:', type=('build', 'run'))
    depends_on('r-formula@1.2-3:', type=('build', 'run'))
    depends_on('r-plotrix', type=('build', 'run'))
    depends_on('r-teachingdemos', type=('build', 'run'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Provides programs to process and analyze PROBA2/LYRA data."""
from __future__ import absolute_import, division, print_function

import datetime
import sys
from collections import OrderedDict

from matplotlib import pyplot as plt
from astropy.io import fits
import pandas

from sunpy.lightcurve import LightCurve
from sunpy.time import parse_time

from sunpy import config

from sunpy.extern.six.moves import urllib

TIME_FORMAT = config.get("general", "time_format")

__all__ = ['LYRALightCurve']

class LYRALightCurve(LightCurve):
    """
    Proba-2 LYRA LightCurve.

    LYRA (Large Yield RAdiometer) is an ultraviolet irradiance radiometer that
    observes the Sun in four passbands, chosen for their relevance to
    solar physics, aeronomy and space weather.
    LYRA is composed of three (redundant) units, each of them constituted of the
    same four channels:

        * 120-123 nm Lyman-alpha channel
        * 190-222 nm Herzberg continuum channel
        * Aluminium filter channel (17-80 nm + a contribution below 5 nm), including He II at 30.4 nm
        * Zirconium filter channel (6-20 nm + a contribution below 2 nm), rejecting He II

    LYRA can take data with cadences chosen in the 100Hz to 0.1Hz interval.

    PROBA2 was launched on 2 November 2009.

    This class can download and hold either Level 2 data (the default) which
    has sub-second resolution or Level 3 data which has one minute resolution.
    The level can be specified with the ``level`` keyword argument to
    `~sunpy.lightcurve.LyraLightCurve.create`.

    Examples
    --------
    >>> import sunpy
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create()
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create('~/Data/lyra/lyra_20110810-000000_lev2_std.fits')   # doctest: +SKIP
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create('2011/08/10')
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create('2011/08/10', level=3)
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create("http://proba2.oma.be/lyra/data/bsd/2011/08/10/lyra_20110810-000000_lev2_std.fits")
    >>> lyra.peek()   # doctest: +SKIP

    References
    ----------
    * `Proba2 SWAP Science Center <http://proba2.sidc.be/about/SWAP/>`_
    * `LYRA Data Homepage <http://proba2.sidc.be/data/LYRA>`_
    * `LYRA Instrument Homepage <http://proba2.sidc.be/about/LYRA>`_
    """

    def peek(self, names=3, **kwargs):
        """Plots the LYRA data. An example is shown below.

        .. plot::

            import sunpy.lightcurve
            from sunpy.data.sample import LYRA_LEVEL3_LIGHTCURVE
            lyra = sunpy.lightcurve.LYRALightCurve.create(LYRA_LEVEL3_LIGHTCURVE)
            lyra.peek()

        Parameters
        ----------
        names : int
            The number of columns to plot.

        **kwargs : dict
            Any additional plot arguments that should be used
            when plotting.

        Returns
        -------
        fig : `~matplotlib.Figure`
            A plot figure.
        """
        lyranames = (('Lyman alpha','Herzberg cont.','Al filter','Zr filter'),
                 ('120-123nm','190-222nm','17-80nm + <5nm','6-20nm + <2nm'))

        # Choose title if none was specified
        #if not kwargs.has_key("title"):
        #    if len(self.data.columns) > 1:
        #        kwargs['title'] = 'LYRA data'
        #    else:
        #        if self._filename is not None:
        #            base = self._filename
        #            kwargs['title'] = os.path.splitext(base)[0]
        #        else:
        #            kwargs['title'] = 'LYRA data'
        figure = plt.figure()
        plt.subplots_adjust(left=0.17,top=0.94,right=0.94,bottom=0.15)
        axes = plt.gca()

        axes = self.data.plot(ax=axes, subplots=True, sharex=True, **kwargs)

        for i, name in enumerate(self.data.columns):
            if names < 3:
                name = lyranames[names][i]
            else:
                name = lyranames[0][i] + ' \n (' + lyranames[1][i] + ')'
            axes[i].set_ylabel( "{name} \n (W/m**2)".format(name=name), fontsize=9.5)

        axes[0].set_title("LYRA ({0:{1}})".format(self.data.index[0],TIME_FORMAT))
        axes[-1].set_xlabel("Time")
        for axe in axes:
            axe.locator_params(axis='y',nbins=6)

        figure.show()

        return figure

    @staticmethod
    def _get_url_for_date(date, **kwargs):
        """Returns a URL to the LYRA data for the specified date"""
        dt = parse_time(date or datetime.datetime.utcnow())

        # Filename
        filename = "lyra_{0:%Y%m%d-}000000_lev{1:d}_std.fits".format(
            dt, kwargs.get('level', 2))
        # URL
        base_url = "http://proba2.oma.be/lyra/data/bsd/"
        url_path = urllib.parse.urljoin(dt.strftime('%Y/%m/%d/'), filename)
        return urllib.parse.urljoin(base_url, url_path)

    @classmethod
    def _get_default_uri(cls):
        """Returns URL for latest LYRA data"""
        return cls._get_url_for_date(datetime.datetime.utcnow())

    @staticmethod
    def _parse_fits(filepath):
        """Parses LYRA data from a FITS file"""
        # Open file with PyFITS
        hdulist = fits.open(filepath)
        fits_record = hdulist[1].data
        # secondary_header = hdulist[1].header

        # Start and end dates.  Different LYRA FITS files have
        # different tags for the date obs.
        if 'date-obs' in hdulist[0].header:
            start_str = hdulist[0].header['date-obs']
        elif 'date_obs' in hdulist[0].header:
            start_str = hdulist[0].header['date_obs']
        # end_str = hdulist[0].header['date-end']

        # start = datetime.datetime.strptime(start_str, '%Y-%m-%dT%H:%M:%S.%f')
        start = parse_time(start_str)
        # end = datetime.datetime.strptime(end_str, '%Y-%m-%dT%H:%M:%S.%f')

        # First column are times.  For level 2 data, the units are [s].
        # For level 3 data, the units are [min]
        if hdulist[1].header['TUNIT1'] == 's':
            times = [start + datetime.timedelta(seconds=n)
                     for n in fits_record.field(0)]
        elif hdulist[1].header['TUNIT1'] == 'MIN':
            times = [start + datetime.timedelta(minutes=int(n))
                     for n in fits_record.field(0)]
        else:
            raise ValueError("Time unit in LYRA fits file not recognised.  "
                             "Value = {0}".format(hdulist[1].header['TUNIT1']))

        # Rest of columns are the data
        table = {}

        for i, col in enumerate(fits_record.columns[1:-1]):
            # temporary patch for big-endian data bug on pandas 0.13
            if fits_record.field(i+1).dtype.byteorder == '>' and sys.byteorder =='little':
                table[col.name] = fits_record.field(i + 1).byteswap().newbyteorder()
            else:
                table[col.name] = fits_record.field(i + 1)

        # Return the header and the data
        data = pandas.DataFrame(table, index=times)
        data.sort_index(inplace=True)
        return OrderedDict(hdulist[0].header), data
<commit_msg>Make level 3 lyra docs clearer.<commit_after># -*- coding: utf-8 -*-
"""Provides programs to process and analyze PROBA2/LYRA data."""
from __future__ import absolute_import, division, print_function

import datetime
import sys
from collections import OrderedDict

from matplotlib import pyplot as plt
from astropy.io import fits
import pandas

from sunpy.lightcurve import LightCurve
from sunpy.time import parse_time

from sunpy import config

from sunpy.extern.six.moves import urllib

TIME_FORMAT = config.get("general", "time_format")

__all__ = ['LYRALightCurve']

class LYRALightCurve(LightCurve):
    """
    Proba-2 LYRA LightCurve.

    LYRA (Large Yield RAdiometer) is an ultraviolet irradiance radiometer that
    observes the Sun in four passbands, chosen for their relevance to
    solar physics, aeronomy and space weather.
    LYRA is composed of three (redundant) units, each of them constituted of the
    same four channels:

        * 120-123 nm Lyman-alpha channel
        * 190-222 nm Herzberg continuum channel
        * Aluminium filter channel (17-80 nm + a contribution below 5 nm), including He II at 30.4 nm
        * Zirconium filter channel (6-20 nm + a contribution below 2 nm), rejecting He II

    LYRA can take data with cadences chosen in the 100Hz to 0.1Hz interval.

    PROBA2 was launched on 2 November 2009.

    This class can download and hold either Level 2 data (the default) which
    has sub-second resolution or Level 3 data which is derived from Level 2
    data and has one minute resolution. The level can be specified with the
    ``level`` keyword argument to `~sunpy.lightcurve.LyraLightCurve.create`.

    Examples
    --------
    >>> import sunpy
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create()
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create('~/Data/lyra/lyra_20110810-000000_lev2_std.fits')   # doctest: +SKIP
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create('2011/08/10')
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create('2011/08/10', level=3)
    >>> lyra = sunpy.lightcurve.LYRALightCurve.create("http://proba2.oma.be/lyra/data/bsd/2011/08/10/lyra_20110810-000000_lev2_std.fits")
    >>> lyra.peek()   # doctest: +SKIP

    References
    ----------
    * `Proba2 SWAP Science Center <http://proba2.sidc.be/about/SWAP/>`_
    * `LYRA Data Homepage <http://proba2.sidc.be/data/LYRA>`_
    * `LYRA Instrument Homepage <http://proba2.sidc.be/about/LYRA>`_
    """

    def peek(self, names=3, **kwargs):
        """Plots the LYRA data. An example is shown below.

        .. plot::

            import sunpy.lightcurve
            from sunpy.data.sample import LYRA_LEVEL3_LIGHTCURVE
            lyra = sunpy.lightcurve.LYRALightCurve.create(LYRA_LEVEL3_LIGHTCURVE)
            lyra.peek()

        Parameters
        ----------
        names : int
            The number of columns to plot.

        **kwargs : dict
            Any additional plot arguments that should be used
            when plotting.

        Returns
        -------
        fig : `~matplotlib.Figure`
            A plot figure.
        """
        lyranames = (('Lyman alpha','Herzberg cont.','Al filter','Zr filter'),
                 ('120-123nm','190-222nm','17-80nm + <5nm','6-20nm + <2nm'))

        # Choose title if none was specified
        #if not kwargs.has_key("title"):
        #    if len(self.data.columns) > 1:
        #        kwargs['title'] = 'LYRA data'
        #    else:
        #        if self._filename is not None:
        #            base = self._filename
        #            kwargs['title'] = os.path.splitext(base)[0]
        #        else:
        #            kwargs['title'] = 'LYRA data'
        figure = plt.figure()
        plt.subplots_adjust(left=0.17,top=0.94,right=0.94,bottom=0.15)
        axes = plt.gca()

        axes = self.data.plot(ax=axes, subplots=True, sharex=True, **kwargs)

        for i, name in enumerate(self.data.columns):
            if names < 3:
                name = lyranames[names][i]
            else:
                name = lyranames[0][i] + ' \n (' + lyranames[1][i] + ')'
            axes[i].set_ylabel( "{name} \n (W/m**2)".format(name=name), fontsize=9.5)

        axes[0].set_title("LYRA ({0:{1}})".format(self.data.index[0],TIME_FORMAT))
        axes[-1].set_xlabel("Time")
        for axe in axes:
            axe.locator_params(axis='y',nbins=6)

        figure.show()

        return figure

    @staticmethod
    def _get_url_for_date(date, **kwargs):
        """Returns a URL to the LYRA data for the specified date"""
        dt = parse_time(date or datetime.datetime.utcnow())

        # Filename
        filename = "lyra_{0:%Y%m%d-}000000_lev{1:d}_std.fits".format(
            dt, kwargs.get('level', 2))
        # URL
        base_url = "http://proba2.oma.be/lyra/data/bsd/"
        url_path = urllib.parse.urljoin(dt.strftime('%Y/%m/%d/'), filename)
        return urllib.parse.urljoin(base_url, url_path)

    @classmethod
    def _get_default_uri(cls):
        """Returns URL for latest LYRA data"""
        return cls._get_url_for_date(datetime.datetime.utcnow())

    @staticmethod
    def _parse_fits(filepath):
        """Parses LYRA data from a FITS file"""
        # Open file with PyFITS
        hdulist = fits.open(filepath)
        fits_record = hdulist[1].data
        # secondary_header = hdulist[1].header

        # Start and end dates.  Different LYRA FITS files have
        # different tags for the date obs.
        if 'date-obs' in hdulist[0].header:
            start_str = hdulist[0].header['date-obs']
        elif 'date_obs' in hdulist[0].header:
            start_str = hdulist[0].header['date_obs']
        # end_str = hdulist[0].header['date-end']

        # start = datetime.datetime.strptime(start_str, '%Y-%m-%dT%H:%M:%S.%f')
        start = parse_time(start_str)
        # end = datetime.datetime.strptime(end_str, '%Y-%m-%dT%H:%M:%S.%f')

        # First column are times.  For level 2 data, the units are [s].
        # For level 3 data, the units are [min]
        if hdulist[1].header['TUNIT1'] == 's':
            times = [start + datetime.timedelta(seconds=n)
                     for n in fits_record.field(0)]
        elif hdulist[1].header['TUNIT1'] == 'MIN':
            times = [start + datetime.timedelta(minutes=int(n))
                     for n in fits_record.field(0)]
        else:
            raise ValueError("Time unit in LYRA fits file not recognised.  "
                             "Value = {0}".format(hdulist[1].header['TUNIT1']))

        # Rest of columns are the data
        table = {}

        for i, col in enumerate(fits_record.columns[1:-1]):
            # temporary patch for big-endian data bug on pandas 0.13
            if fits_record.field(i+1).dtype.byteorder == '>' and sys.byteorder =='little':
                table[col.name] = fits_record.field(i + 1).byteswap().newbyteorder()
            else:
                table[col.name] = fits_record.field(i + 1)

        # Return the header and the data
        data = pandas.DataFrame(table, index=times)
        data.sort_index(inplace=True)
        return OrderedDict(hdulist[0].header), data
<|endoftext|>


<commit_before>#!/bin/env python
from __future__ import unicode_literals,print_function
import six
import subprocess
import sys
import re
import h5py
import time
import numpy as np
from vmc_postproc import load

def vmc_exec(**kwargs):
    """
    arguments:
    - nprocs (default 4)`
    - hosts (default localhost)
    - slurmqueue (default False)
    - partition (default qwfall)
    - rundir (default .)
    plus all those given by runing vmc -h.
    returns a dictionnary with all measurements as
    asked using the kwargs.
    """
    opts=get_vmc_args()
    opts+=['nprocs','hosts']
    kwargs.setdefault('nprocs',4)
    kwargs.setdefault('hosts','localhost')
    kwargs.setdefault('dir','.')
    kwargs.setdefault('partition','qwfall')
    kwargs.setdefault('rundir','.')
    kwargs.setdefault('slurmqueue',False)
    add_keys=['nprocs','hosts','slurmqueue','partition','rundir']
    meas_trans={'meas_magnetization':'Magnetization',\
                'meas_projheis':'ProjHeis',\
                'meas_stagmagn':'StagMagn',\
                'meas_statspinstruct':'StatSpinStruct'}
    if kwargs.setdefault('stagflux_wav',False):
        meas_trans['meas_stagmagn']='StagMagnZ'
    vmcargs=[]
    for key,value in kwargs.items():
        if not key in add_keys:
            if not key in opts:
                raise RuntimeError('Unrecognized option \'{}\'.'.format(key))
            if type(value)==bool:
                if value:
                    vmcargs+=['--{}'.format(key)]
            else:
                vmcargs+=['--{key}={val}'.format(key=key,val=value)]
    if kwargs['slurmqueue']:
        batchscript="""#!/bin/bash
#SBATCH -n {nprocs}
#SBATCH -c 1
#SBATCH -J vmc_exec
#SBATCH -p {partition}
#SBATCH --error=vmc_exec-%j.stderr
#SBATCH --output=vmc_exec-%j.stdout

cd {rundir}

mpirun ./vmc --prefix=$SLURM_JOB_ID""".format(nprocs=kwargs['nprocs'],partition=kwargs['partition'],rundir=kwargs['rundir'])+vmcargs
        batch_file=open('vmc_exec_batch.sh','w')
        batch_file.write(batchscript)
        batchfile.close()
        slurmproc=subprocess.Popen(['sbatch','vmc_exec_batch.sh'],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        stdout,stderr=subprocess.communicate()
        if six.PY3:
            stdout=str(stdout,encoding='utf-8')
            stderr=str(stderr,encoding='utf-8')
        else:
            stdout=unicode(stdout,encoding='utf-8')
            stderr=unicode(stderr,encoding='utf-8')
        print(stderr)
        print(stdout)
        prefix=re.findall(r'Submitted batch job ([0-9]+)',stdout)
        done=False
        while not done:
            squeueproc=subprocess.Popen(['squeue','-h','-j',prefix],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
            stdout,stderr=squeueproc.communicate()
            done=(len(stdout)==0)
            time.sleep(10)
        print('Finished calculation '+prefix)
    else:
        vmcproc=subprocess.Popen(['mpiexec','-np',str(kwargs['nprocs']),'--host',kwargs['hosts'],'vmc']+vmcargs,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        stdout,stderr=vmcproc.communicate()
        if six.PY3:
            stdout=str(stdout,encoding='utf-8')
            stderr=str(stderr,encoding='utf-8')
        else:
            stdout=unicode(stdout,encoding='utf-8')
            stderr=unicode(stderr,encoding='utf-8')
        print(stderr)
        prefix=re.findall(r'output prefix=([0-9]+)',stdout)

    outq=dict()
    for meas in meas_trans.keys():
        if kwargs.setdefault(meas,False):
            outq[meas_trans[meas]]=load.get_quantity(kwargs['dir']+'/{prefix}-{measname}.h5'.format(prefix=prefix[0],measname=meas_trans[meas]),kwargs['samples'])
    return outq

def get_vmc_args():
    """
    get all arguments from the vmc command
    """
    vmcquery=subprocess.Popen(['vmc'],stdout=subprocess.PIPE)
    vmcargs,_=vmcquery.communicate()
    if six.PY3:
        vmcargs=str(vmcargs,encoding='utf-8')
    else:
        vmcargs=unicode(vmcargs,encoding='utf-8')
    opts=[]
    for l in vmcargs.split('\n'):
        opts+=re.findall(r'--([a-zA-Z0-9_]*)=?.*',l)
    return opts
<commit_msg>slurm bugs<commit_after>#!/bin/env python
from __future__ import unicode_literals,print_function
import six
import subprocess
import sys
import re
import h5py
import time
import numpy as np
from vmc_postproc import load

def vmc_exec(**kwargs):
    """
    arguments:
    - nprocs (default 4)`
    - hosts (default localhost)
    - slurmqueue (default False)
    - partition (default qwfall)
    - rundir (default .)
    plus all those given by runing vmc -h.
    returns a dictionnary with all measurements as
    asked using the kwargs.
    """
    opts=get_vmc_args()
    opts+=['nprocs','hosts']
    kwargs.setdefault('nprocs',4)
    kwargs.setdefault('hosts','localhost')
    kwargs.setdefault('dir','.')
    kwargs.setdefault('partition','qwfall')
    kwargs.setdefault('rundir','.')
    kwargs.setdefault('slurmqueue',False)
    add_keys=['nprocs','hosts','slurmqueue','partition','rundir']
    meas_trans={'meas_magnetization':'Magnetization',\
                'meas_projheis':'ProjHeis',\
                'meas_stagmagn':'StagMagn',\
                'meas_statspinstruct':'StatSpinStruct'}
    if kwargs.setdefault('stagflux_wav',False):
        meas_trans['meas_stagmagn']='StagMagnZ'
    vmcargs=[]
    for key,value in kwargs.items():
        if not key in add_keys:
            if not key in opts:
                raise RuntimeError('Unrecognized option \'{}\'.'.format(key))
            if type(value)==bool:
                if value:
                    vmcargs+=['--{}'.format(key)]
            else:
                vmcargs+=['--{key}={val}'.format(key=key,val=value)]
    if kwargs['slurmqueue']:
        vmcargsinline=''
        for a in vmcargs:
            vmcargsinline+=' '+a
        batchscript="""#!/bin/bash
#SBATCH -n {nprocs}
#SBATCH -c 1
#SBATCH -J vmc_exec
#SBATCH -p {partition}
#SBATCH --error=vmc_exec-%j.stderr
#SBATCH --output=vmc_exec-%j.stdout

cd {rundir}

mpirun ./vmc --prefix=$SLURM_JOB_ID""".format(nprocs=kwargs['nprocs'],partition=kwargs['partition'],rundir=kwargs['rundir'])+vmcargsinline
        batch_file=open('vmc_exec_batch.sh','w')
        batch_file.write(batchscript)
        batchfile.close()
        slurmproc=subprocess.Popen(['sbatch','vmc_exec_batch.sh'],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        stdout,stderr=subprocess.communicate()
        if six.PY3:
            stdout=str(stdout,encoding='utf-8')
            stderr=str(stderr,encoding='utf-8')
        else:
            stdout=unicode(stdout,encoding='utf-8')
            stderr=unicode(stderr,encoding='utf-8')
        print(stderr)
        print(stdout)
        prefix=re.findall(r'Submitted batch job ([0-9]+)',stdout)
        done=False
        while not done:
            squeueproc=subprocess.Popen(['squeue','-h','-j',prefix],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
            stdout,stderr=squeueproc.communicate()
            done=(len(stdout)==0)
            time.sleep(10)
        print('Finished calculation '+prefix)
    else:
        vmcproc=subprocess.Popen(['mpiexec','-np',str(kwargs['nprocs']),'--host',kwargs['hosts'],'vmc']+vmcargs,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        stdout,stderr=vmcproc.communicate()
        if six.PY3:
            stdout=str(stdout,encoding='utf-8')
            stderr=str(stderr,encoding='utf-8')
        else:
            stdout=unicode(stdout,encoding='utf-8')
            stderr=unicode(stderr,encoding='utf-8')
        print(stderr)
        prefix=re.findall(r'output prefix=([0-9]+)',stdout)

    outq=dict()
    for meas in meas_trans.keys():
        if kwargs.setdefault(meas,False):
            outq[meas_trans[meas]]=load.get_quantity(kwargs['dir']+'/{prefix}-{measname}.h5'.format(prefix=prefix[0],measname=meas_trans[meas]),kwargs['samples'])
    return outq

def get_vmc_args():
    """
    get all arguments from the vmc command
    """
    vmcquery=subprocess.Popen(['vmc'],stdout=subprocess.PIPE)
    vmcargs,_=vmcquery.communicate()
    if six.PY3:
        vmcargs=str(vmcargs,encoding='utf-8')
    else:
        vmcargs=unicode(vmcargs,encoding='utf-8')
    opts=[]
    for l in vmcargs.split('\n'):
        opts+=re.findall(r'--([a-zA-Z0-9_]*)=?.*',l)
    return opts
<|endoftext|>