<commit_before>#file : InMoov2.minimalTorso.py

# this will run with versions of MRL above 1695
# a very minimal script for InMoov
# although this script is very short you can still
# do voice control of a right Arm
# for any command which you say - you will be required to say a confirmation
# e.g. you say -> test stomach, InMoov will ask -> "Did you say test stomach?", you will need to
# respond with a confirmation ("yes","correct","yeah","ya")
from java.lang import String
from org.myrobotlab.service import Runtime
import urllib2
import os

# To set a directory
# Modify this line according to your directory and version of MRL
os.chdir("C:/myrobotlab/myrobotlab.1.0.107/audioFile/google/en_gb/audrey")

# the name of the local file
# remove the file if it already exist in the Audiofile directory
soundfilename="starting mouth.mp3";

try:
	mp3file = urllib2.urlopen('http://www.inmoov.fr/wp-content/uploads/2015/05/starting-mouth.mp3')
	output = open(soundfilename,'wb')
	output.write(mp3file.read())
	output.close()
except IOError:
	print "Check access right on the directory"
except Exception:
	print "Can't get the sound File ! Check internet Connexion"

leftPort = "COM20"  #modify port according to your board

i01 = Runtime.createAndStart("i01", "InMoov")
i01.startEar()

mouth = Runtime.createAndStart("mouth","Speech")
i01.startMouth()
##############
torso = i01.startTorso("COM20")  #modify port according to your board
# tweaking default torso settings
torso.topStom.setMinMax(0,180)
torso.topStom.map(0,180,67,110)
torso.midStom.setMinMax(0,180)
torso.midStom.map(0,180,60,120)
#torso.lowStom.setMinMax(0,180)
#torso.lowStom.map(0,180,60,110)
#torso.topStom.setRest(90)
#torso.midStom.setRest(90)
#torso.lowStom.setRest(90)

#################
# verbal commands
ear = i01.ear

ear.addCommand("attach everything", "i01", "attach")
ear.addCommand("disconnect everything", "i01", "detach")
ear.addCommand("attach torso", "i01.torso", "attach")
ear.addCommand("disconnect torso", "i01.torso", "detach")
ear.addCommand("rest", "python", "rest")
ear.addCommand("capture gesture", ear.getName(), "captureGesture")
ear.addCommand("manual", ear.getName(), "lockOutAllGrammarExcept", "voice control")
ear.addCommand("voice control", ear.getName(), "clearLock")
ear.addCommand("test your stomach", "python", "teststomach")

ear.addComfirmations("yes","correct","ya","yeah", "yes please", "yes of course")
ear.addNegations("no","wrong","nope","nah","no thank you", "no thanks")

ear.startListening()

def teststomach():
    i01.setTorsoSpeed(0.75,0.55,0.75)
    i01.moveTorso(90,90,90)
    sleep(2)
    i01.moveTorso(45,90,90)
    sleep(4)
    i01.moveTorso(90,90,90)
    sleep(2)
    i01.moveTorso(135,90,90)
    sleep(4)
    i01.moveTorso(90,90,90)
    sleep(2)
    i01.moveTorso(90,45,90)
    sleep(3)
    i01.moveTorso(90,135,90)
    sleep(3)
    i01.moveTorso(90,90,45)
    sleep(3)
    i01.moveTorso(90,90,135)
    sleep(3)<commit_msg>Update InMoov2.minimalTorso.py<commit_after>#file : InMoov2.minimalTorso.py

# this will run with versions of MRL 1.0.107
# a very minimal script for InMoov
# although this script is very short you can still
# do voice control of a right Arm
# for any command which you say - you will be required to say a confirmation
# e.g. you say -> test stomach, InMoov will ask -> "Did you say test stomach?", you will need to
# respond with a confirmation ("yes","correct","yeah","ya")
from java.lang import String
from org.myrobotlab.service import Runtime
import urllib2
import os

# To set a directory
# Modify this line according to your directory and version of MRL
os.chdir("C:/myrobotlab/myrobotlab.1.0.107/audioFile/google/en_gb/audrey")

# the name of the local file
# remove the file if it already exist in the Audiofile directory
soundfilename="starting mouth.mp3";

try:
	mp3file = urllib2.urlopen('http://www.inmoov.fr/wp-content/uploads/2015/05/starting-mouth.mp3')
	output = open(soundfilename,'wb')
	output.write(mp3file.read())
	output.close()
except IOError:
	print "Check access right on the directory"
except Exception:
	print "Can't get the sound File ! Check internet Connexion"

leftPort = "COM20"  #modify port according to your board

i01 = Runtime.createAndStart("i01", "InMoov")
i01.startEar()

mouth = Runtime.createAndStart("mouth","Speech")
i01.startMouth()
##############
torso = i01.startTorso("COM20")  #modify port according to your board
# tweaking default torso settings
torso.topStom.setMinMax(0,180)
torso.topStom.map(0,180,67,110)
torso.midStom.setMinMax(0,180)
torso.midStom.map(0,180,60,120)
#torso.lowStom.setMinMax(0,180)
#torso.lowStom.map(0,180,60,110)
#torso.topStom.setRest(90)
#torso.midStom.setRest(90)
#torso.lowStom.setRest(90)

#################
# verbal commands
ear = i01.ear

ear.addCommand("attach everything", "i01", "attach")
ear.addCommand("disconnect everything", "i01", "detach")
ear.addCommand("attach torso", "i01.torso", "attach")
ear.addCommand("disconnect torso", "i01.torso", "detach")
ear.addCommand("rest", "python", "rest")
ear.addCommand("capture gesture", ear.getName(), "captureGesture")
ear.addCommand("manual", ear.getName(), "lockOutAllGrammarExcept", "voice control")
ear.addCommand("voice control", ear.getName(), "clearLock")
ear.addCommand("test your stomach", "python", "teststomach")

ear.addComfirmations("yes","correct","ya","yeah", "yes please", "yes of course")
ear.addNegations("no","wrong","nope","nah","no thank you", "no thanks")

ear.startListening()

def teststomach():
    i01.setTorsoSpeed(0.75,0.55,0.75)
    i01.moveTorso(90,90,90)
    sleep(2)
    i01.moveTorso(45,90,90)
    sleep(4)
    i01.moveTorso(90,90,90)
    sleep(2)
    i01.moveTorso(135,90,90)
    sleep(4)
    i01.moveTorso(90,90,90)
    sleep(2)
    i01.moveTorso(90,45,90)
    sleep(3)
    i01.moveTorso(90,135,90)
    sleep(3)
    i01.moveTorso(90,90,45)
    sleep(3)
    i01.moveTorso(90,90,135)
    sleep(3)
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
from jinja2 import Environment, FileSystemLoader
import json
from libmozdata.bugzilla import Bugzilla
from libmozdata import utils as lmdutils
from auto_nag.bugzilla.utils import get_config_path
from auto_nag import mail, utils

# https://bugzilla.mozilla.org/buglist.cgi?keywords=topcrash%2C%20&keywords_type=allwords&bug_severity=major&bug_severity=normal&bug_severity=minor&bug_severity=trivial&bug_severity=enhancement&resolution=---&query_format=advanced
def get_bz_params():
    fields = ['id']
    params = {'include_fields': fields,
              'resolution': ['---'],
              "bug_severity": ["major", "normal", "minor", "trivial", "enhancement"],
              'keywords': 'topcrash',
              'keywords_type': 'allwords'
              }

    return params

def get_bugs():
    # the search query can be long to evaluate
    TIMEOUT = 240

    def bug_handler(bug, data):
        data.append(bug['id'])

    bugids = []
    Bugzilla(get_bz_params(),
             bughandler=bug_handler,
             bugdata=bugids,
             timeout=TIMEOUT).get_data().wait()

    return sorted(bugids)

def get_login_info():
    with open(get_config_path(), 'r') as In:
        return json.load(In)

def get_email(bztoken, date, dryrun):
    Bugzilla.TOKEN = bztoken
    bugids = get_bugs(date=date)
    if bugids:
        env = Environment(loader=FileSystemLoader('templates'))
        template = env.get_template('topcrash_bad_severity.html')
        body = template.render(date=date,
                               bugids=bugids)
        title = '[autonag] Bugs with topcrash keyword but incorrect severity {}'.format(date)
        return title, body
    return None, None

def send_email(date='today', dryrun=False):
    login_info = get_login_info()
    date = lmdutils.get_date(date)
    title, body = get_email(login_info['bz_api_key'], date, dryrun)
    if title:
        mail.send(login_info['ldap_username'],
                  utils.get_config('common', 'receivers', ['sylvestre@mozilla.com']),
                  title, body,
                  html=True, login=login_info, dryrun=dryrun)
    else:
        print('TOPCRASH_BAD_SEVERITY: No data for {}'.format(date))

if __name__ == '__main__':
    description = 'Get the top crashes bug without a proper severity'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-d', '--dryrun', dest='dryrun',
                        action='store_true', default=False,
                        help='Just do the query, and print emails to console without emailing anyone') # NOQA
    parser.add_argument('-D', '--date', dest='date',
                        action='store', default='today',
                        help='Date for the query')
    args = parser.parse_args()
    send_email(date=args.date, dryrun=args.dryrun)
<commit_msg>Fix a bug with date<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
from jinja2 import Environment, FileSystemLoader
import json
from libmozdata.bugzilla import Bugzilla
from libmozdata import utils as lmdutils
from auto_nag.bugzilla.utils import get_config_path
from auto_nag import mail, utils

# https://bugzilla.mozilla.org/buglist.cgi?keywords=topcrash%2C%20&keywords_type=allwords&bug_severity=major&bug_severity=normal&bug_severity=minor&bug_severity=trivial&bug_severity=enhancement&resolution=---&query_format=advanced
def get_bz_params():
    fields = ['id']
    params = {'include_fields': fields,
              'resolution': ['---'],
              "bug_severity": ["major", "normal", "minor", "trivial", "enhancement"],
              'keywords': 'topcrash',
              'keywords_type': 'allwords'
              }

    return params

def get_bugs():
    # the search query can be long to evaluate
    TIMEOUT = 240

    def bug_handler(bug, data):
        data.append(bug['id'])

    bugids = []
    Bugzilla(get_bz_params(),
             bughandler=bug_handler,
             bugdata=bugids,
             timeout=TIMEOUT).get_data().wait()

    return sorted(bugids)

def get_login_info():
    with open(get_config_path(), 'r') as In:
        return json.load(In)

def get_email(bztoken, date, dryrun):
    Bugzilla.TOKEN = bztoken
    bugids = get_bugs()
    if bugids:
        env = Environment(loader=FileSystemLoader('templates'))
        template = env.get_template('topcrash_bad_severity.html')
        body = template.render(date=date,
                               bugids=bugids)
        title = '[autonag] Bugs with topcrash keyword but incorrect severity {}'.format(date)
        return title, body
    return None, None

def send_email(date='today', dryrun=False):
    login_info = get_login_info()
    date = lmdutils.get_date(date)
    title, body = get_email(login_info['bz_api_key'], date, dryrun)
    if title:
        mail.send(login_info['ldap_username'],
                  utils.get_config('common', 'receivers', ['sylvestre@mozilla.com']),
                  title, body,
                  html=True, login=login_info, dryrun=dryrun)
    else:
        print('TOPCRASH_BAD_SEVERITY: No data for {}'.format(date))

if __name__ == '__main__':
    description = 'Get the top crashes bug without a proper severity'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-d', '--dryrun', dest='dryrun',
                        action='store_true', default=False,
                        help='Just do the query, and print emails to console without emailing anyone') # NOQA
    parser.add_argument('-D', '--date', dest='date',
                        action='store', default='today',
                        help='Date for the query')
    args = parser.parse_args()
    send_email(date=args.date, dryrun=args.dryrun)
<|endoftext|>


<commit_before>"""
Support for Prometheus metrics export.

For more details about this component, please refer to the documentation at
https://home-assistant.io/components/prometheus/
"""
import asyncio
import logging

import voluptuous as vol
from aiohttp import web

from homeassistant.components.http import HomeAssistantView
from homeassistant.components import recorder
from homeassistant.const import (CONF_DOMAINS, CONF_ENTITIES, CONF_EXCLUDE,
                                 CONF_INCLUDE, EVENT_STATE_CHANGED,
                                 TEMP_CELSIUS, TEMP_FAHRENHEIT)
from homeassistant import core as hacore
from homeassistant.helpers import state as state_helper

_LOGGER = logging.getLogger(__name__)

REQUIREMENTS = ['prometheus_client==0.0.19']

DOMAIN = 'prometheus'
DEPENDENCIES = ['http']

CONFIG_SCHEMA = vol.Schema({
    DOMAIN: recorder.FILTER_SCHEMA,
}, extra=vol.ALLOW_EXTRA)

API_ENDPOINT = '/api/prometheus'

def setup(hass, config):
    """Activate Prometheus component."""
    import prometheus_client

    hass.http.register_view(PrometheusView(prometheus_client))

    conf = config.get(DOMAIN, {})
    exclude = conf.get(CONF_EXCLUDE, {})
    include = conf.get(CONF_INCLUDE, {})
    metrics = Metrics(prometheus_client, exclude, include)

    hass.bus.listen(EVENT_STATE_CHANGED, metrics.handle_event)

    return True

class Metrics:
    """Model all of the metrics which should be exposed to Prometheus."""

    def __init__(self, prometheus_client, exclude, include):
        """Initialize Prometheus Metrics."""
        self.prometheus_client = prometheus_client
        self.exclude = exclude.get(CONF_ENTITIES, []) + \
            exclude.get(CONF_DOMAINS, [])
        self.include_domains = include.get(CONF_DOMAINS, [])
        self.include_entities = include.get(CONF_ENTITIES, [])
        self._metrics = {}

    def handle_event(self, event):
        """Listen for new messages on the bus, and add them to Prometheus."""
        state = event.data.get('new_state')
        if state is None:
            return

        entity_id = state.entity_id
        _LOGGER.debug("Handling state update for %s", entity_id)
        domain, _ = hacore.split_entity_id(entity_id)

        if entity_id in self.exclude:
            return
        if domain in self.exclude and entity_id not in self.include_entities:
            return
        if self.include_domains and domain not in self.include_domains:
            return
        if not self.exclude and (self.include_entities and
                                 entity_id not in self.include_entities):
            return

        handler = '_handle_' + domain

        if hasattr(self, handler):
            getattr(self, handler)(state)

    def _metric(self, metric, factory, documentation, labels=None):
        if labels is None:
            labels = ['entity', 'friendly_name']

        try:
            return self._metrics[metric]
        except KeyError:
            self._metrics[metric] = factory(metric, documentation, labels)
            return self._metrics[metric]

    @staticmethod
    def _labels(state):
        return {
            'entity': state.entity_id,
            'friendly_name': state.attributes.get('friendly_name'),
        }

    def _battery(self, state):
        if 'battery_level' in state.attributes:
            metric = self._metric(
                'battery_level_percent',
                self.prometheus_client.Gauge,
                'Battery level as a percentage of its capacity',
            )
            try:
                value = float(state.attributes['battery_level'])
                metric.labels(**self._labels(state)).set(value)
            except ValueError:
                pass

    def _handle_binary_sensor(self, state):
        metric = self._metric(
            'binary_sensor_state',
            self.prometheus_client.Gauge,
            'State of the binary sensor (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

    def _handle_device_tracker(self, state):
        metric = self._metric(
            'device_tracker_state',
            self.prometheus_client.Gauge,
            'State of the device tracker (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

    def _handle_light(self, state):
        metric = self._metric(
            'light_state',
            self.prometheus_client.Gauge,
            'Load level of a light (0..1)',
        )

        try:
            if 'brightness' in state.attributes:
                value = state.attributes['brightness'] / 255.0
            else:
                value = state_helper.state_as_number(state)
            value = value * 100
            metric.labels(**self._labels(state)).set(value)
        except ValueError:
            pass

    def _handle_lock(self, state):
        metric = self._metric(
            'lock_state',
            self.prometheus_client.Gauge,
            'State of the lock (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

    def _handle_sensor(self, state):
        _sensor_types = {
            TEMP_CELSIUS: (
                'temperature_c', self.prometheus_client.Gauge,
                'Temperature in degrees Celsius',
            ),
            TEMP_FAHRENHEIT: (
                'temperature_c', self.prometheus_client.Gauge,
                'Temperature in degrees Celsius',
            ),
            '%': (
                'relative_humidity', self.prometheus_client.Gauge,
                'Relative humidity (0..100)',
            ),
            'lux': (
                'light_lux', self.prometheus_client.Gauge,
                'Light level in lux',
            ),
            'kWh': (
                'electricity_used_kwh', self.prometheus_client.Gauge,
                'Electricity used by this device in KWh',
            ),
            'V': (
                'voltage', self.prometheus_client.Gauge,
                'Currently reported voltage in Volts',
            ),
            'W': (
                'electricity_usage_w', self.prometheus_client.Gauge,
                'Currently reported electricity draw in Watts',
            ),
        }

        unit = state.attributes.get('unit_of_measurement')
        metric = _sensor_types.get(unit)

        if metric is not None:
            metric = self._metric(*metric)
            try:
                value = state_helper.state_as_number(state)
                metric.labels(**self._labels(state)).set(value)
            except ValueError:
                pass

        self._battery(state)

    def _handle_switch(self, state):
        metric = self._metric(
            'switch_state',
            self.prometheus_client.Gauge,
            'State of the switch (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

class PrometheusView(HomeAssistantView):
    """Handle Prometheus requests."""

    url = API_ENDPOINT
    name = 'api:prometheus'

    def __init__(self, prometheus_client):
        """Initialize Prometheus view."""
        self.prometheus_client = prometheus_client

    @asyncio.coroutine
    def get(self, request):
        """Handle request for Prometheus metrics."""
        _LOGGER.debug('Received Prometheus metrics request')

        return web.Response(
            body=self.prometheus_client.generate_latest(),
            content_type="text/plain")
<commit_msg>prometheus: Convert fahrenheit to celsius (#8511)<commit_after>"""
Support for Prometheus metrics export.

For more details about this component, please refer to the documentation at
https://home-assistant.io/components/prometheus/
"""
import asyncio
import logging

import voluptuous as vol
from aiohttp import web

from homeassistant.components.http import HomeAssistantView
from homeassistant.components import recorder
from homeassistant.const import (CONF_DOMAINS, CONF_ENTITIES, CONF_EXCLUDE,
                                 CONF_INCLUDE, EVENT_STATE_CHANGED,
                                 TEMP_CELSIUS, TEMP_FAHRENHEIT)
from homeassistant import core as hacore
from homeassistant.helpers import state as state_helper
from homeassistant.util.temperature import fahrenheit_to_celsius

_LOGGER = logging.getLogger(__name__)

REQUIREMENTS = ['prometheus_client==0.0.19']

DOMAIN = 'prometheus'
DEPENDENCIES = ['http']

CONFIG_SCHEMA = vol.Schema({
    DOMAIN: recorder.FILTER_SCHEMA,
}, extra=vol.ALLOW_EXTRA)

API_ENDPOINT = '/api/prometheus'

def setup(hass, config):
    """Activate Prometheus component."""
    import prometheus_client

    hass.http.register_view(PrometheusView(prometheus_client))

    conf = config.get(DOMAIN, {})
    exclude = conf.get(CONF_EXCLUDE, {})
    include = conf.get(CONF_INCLUDE, {})
    metrics = Metrics(prometheus_client, exclude, include)

    hass.bus.listen(EVENT_STATE_CHANGED, metrics.handle_event)

    return True

class Metrics:
    """Model all of the metrics which should be exposed to Prometheus."""

    def __init__(self, prometheus_client, exclude, include):
        """Initialize Prometheus Metrics."""
        self.prometheus_client = prometheus_client
        self.exclude = exclude.get(CONF_ENTITIES, []) + \
            exclude.get(CONF_DOMAINS, [])
        self.include_domains = include.get(CONF_DOMAINS, [])
        self.include_entities = include.get(CONF_ENTITIES, [])
        self._metrics = {}

    def handle_event(self, event):
        """Listen for new messages on the bus, and add them to Prometheus."""
        state = event.data.get('new_state')
        if state is None:
            return

        entity_id = state.entity_id
        _LOGGER.debug("Handling state update for %s", entity_id)
        domain, _ = hacore.split_entity_id(entity_id)

        if entity_id in self.exclude:
            return
        if domain in self.exclude and entity_id not in self.include_entities:
            return
        if self.include_domains and domain not in self.include_domains:
            return
        if not self.exclude and (self.include_entities and
                                 entity_id not in self.include_entities):
            return

        handler = '_handle_' + domain

        if hasattr(self, handler):
            getattr(self, handler)(state)

    def _metric(self, metric, factory, documentation, labels=None):
        if labels is None:
            labels = ['entity', 'friendly_name']

        try:
            return self._metrics[metric]
        except KeyError:
            self._metrics[metric] = factory(metric, documentation, labels)
            return self._metrics[metric]

    @staticmethod
    def _labels(state):
        return {
            'entity': state.entity_id,
            'friendly_name': state.attributes.get('friendly_name'),
        }

    def _battery(self, state):
        if 'battery_level' in state.attributes:
            metric = self._metric(
                'battery_level_percent',
                self.prometheus_client.Gauge,
                'Battery level as a percentage of its capacity',
            )
            try:
                value = float(state.attributes['battery_level'])
                metric.labels(**self._labels(state)).set(value)
            except ValueError:
                pass

    def _handle_binary_sensor(self, state):
        metric = self._metric(
            'binary_sensor_state',
            self.prometheus_client.Gauge,
            'State of the binary sensor (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

    def _handle_device_tracker(self, state):
        metric = self._metric(
            'device_tracker_state',
            self.prometheus_client.Gauge,
            'State of the device tracker (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

    def _handle_light(self, state):
        metric = self._metric(
            'light_state',
            self.prometheus_client.Gauge,
            'Load level of a light (0..1)',
        )

        try:
            if 'brightness' in state.attributes:
                value = state.attributes['brightness'] / 255.0
            else:
                value = state_helper.state_as_number(state)
            value = value * 100
            metric.labels(**self._labels(state)).set(value)
        except ValueError:
            pass

    def _handle_lock(self, state):
        metric = self._metric(
            'lock_state',
            self.prometheus_client.Gauge,
            'State of the lock (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

    def _handle_sensor(self, state):
        _sensor_types = {
            TEMP_CELSIUS: (
                'temperature_c', self.prometheus_client.Gauge,
                'Temperature in degrees Celsius',
            ),
            TEMP_FAHRENHEIT: (
                'temperature_c', self.prometheus_client.Gauge,
                'Temperature in degrees Celsius',
            ),
            '%': (
                'relative_humidity', self.prometheus_client.Gauge,
                'Relative humidity (0..100)',
            ),
            'lux': (
                'light_lux', self.prometheus_client.Gauge,
                'Light level in lux',
            ),
            'kWh': (
                'electricity_used_kwh', self.prometheus_client.Gauge,
                'Electricity used by this device in KWh',
            ),
            'V': (
                'voltage', self.prometheus_client.Gauge,
                'Currently reported voltage in Volts',
            ),
            'W': (
                'electricity_usage_w', self.prometheus_client.Gauge,
                'Currently reported electricity draw in Watts',
            ),
        }

        unit = state.attributes.get('unit_of_measurement')
        metric = _sensor_types.get(unit)

        if metric is not None:
            metric = self._metric(*metric)
            try:
                value = state_helper.state_as_number(state)
                if unit == TEMP_FAHRENHEIT:
                    value = fahrenheit_to_celsius(value)
                metric.labels(**self._labels(state)).set(value)
            except ValueError:
                pass

        self._battery(state)

    def _handle_switch(self, state):
        metric = self._metric(
            'switch_state',
            self.prometheus_client.Gauge,
            'State of the switch (0/1)',
        )
        value = state_helper.state_as_number(state)
        metric.labels(**self._labels(state)).set(value)

class PrometheusView(HomeAssistantView):
    """Handle Prometheus requests."""

    url = API_ENDPOINT
    name = 'api:prometheus'

    def __init__(self, prometheus_client):
        """Initialize Prometheus view."""
        self.prometheus_client = prometheus_client

    @asyncio.coroutine
    def get(self, request):
        """Handle request for Prometheus metrics."""
        _LOGGER.debug('Received Prometheus metrics request')

        return web.Response(
            body=self.prometheus_client.generate_latest(),
            content_type="text/plain")
<|endoftext|>


<commit_before>import collections
import domino
import random

def randomized_hands():
    '''
    Returns 4 hands of 7 dominoes each by randomly shuffing the
    28 dominoes used in this variation of the dominoes game.
    '''
    dominoes = [domino.Domino(i, j) for i in range(7) for j in range(i, 7)]
    random.shuffle(dominoes)
    return [domino.Hand(dominoes[0:7]), domino.Hand(dominoes[7:14]),
            domino.Hand(dominoes[14:21]), domino.Hand(dominoes[21:28])]

'''
namedtuple to represent the result of a dominoes game.

The attributes are defined as follows:
    player (int): the last player to make a move.
    won (bool): True if the game ended due to an empty hand.
                False if the game ended due to being stuck
    points (int): the absolute value of this quantity indicates
                  the amount of points earned by the winning team.
                  This quantity is positive if the last player to
                  make a move is part of the winning team, and
                  negative otherwise. If it is 0, it means the
                  game ended in a tie.
'''
Result = collections.namedtuple('Result', ['player', 'won', 'points'])

class Game:
    '''
    Python class for objects that represent a dominoes game.

    This variation of the dominoes game is played
    using 28 dominoes, which use values from 0 to 6:
    [0|0][0|1][0|2][0|3][0|4][0|5][0|6]
    [1|1][1|2][1|3][1|4][1|5][1|6]
    [2|2][2|3][2|4][2|5][2|6]
    [3|3][3|4][3|5][3|6]
    [4|4][4|5][4|6]
    [5|5][5|6]
    [6|6]

    These dominoes are shuffled, and distributed evenly among
    4 players. These players then sit on the edges of a square.
    Players sitting opposite of each other are on the same team,
    and the center of the square is the game board. Throughout
    the game, each player will only be able to see their hand,
    the game board, and the amount of dominoes left in the hands
    of the other players. Note that no player can see the values
    on the dominoes in the hands of the other players.

    The 4 players will then take turns placing dominoes from their
    hands onto the game board. The game board consists of a chain
    of dominoes placed end to end such that the values on connected
    ends always match.

    Prior to distributing the dominoes, the 4 players will agree on
    which player will play first, either by designating a specific
    player or a specific domino that must be played first (often [6|6]).
    After the game starts, play proceeds clockwise.

    If a player is able to place a domino on the board, he/she must.
    Only if they have no possible moves, can the pass on their turn.

    The game ends either when a player runs out of dominoes or when no
    player can play a domino (in which case we say the game is stuck).

    If a player runs out of dominoes, his/her team will earn a number
    of points computed by adding all the values of all the dominoes
    remaining in the hands of the 3 other players.

    If the game is stuck, each team will add up all the values of
    all the dominoes remaining in their hands. The team with the
    lower score wins, and earns a number of points computed by
    adding both teams' scores. If both teams have the same score,
    the game is declared a tie, and neither team earns any points.

    :param Domino starting_domino: the domino that should be played
                                   to start the game. The player
                                   with this domino in their hand
                                   will play first.
    :param int starting_player: the player that should play first.
                                This value is ignored if a staring
                                domino is provided.
    '''
    def __init__(self, starting_domino=None, starting_player=0):
        self.board = domino.Board()

        self.hands = randomized_hands()

        if starting_domino is None:
            self.turn = starting_player
        else:
            self.turn = self._domino_hand(starting_domino)
            self.make_move(starting_domino, True)

        self.result = None

    def skinny_board(self):
        self.board = domino.SkinnyBoard.from_board(self.board)

    def _domino_hand(self, d):
        for i, hand in enumerate(self.hands):
            if d in hand:
                return i

    def _remaining_points(self):
        points = []
        for hand in self.hands:
            points.append(sum(d.first + d.second for d in hand))

        return points

    def _has_valid_move(self):
        if not self.board:
            return True

        for d in self.hands[self.turn]:
            if self.board.left_end() in d or \
               self.board.right_end() in d:
                return True

        return False

    def valid_moves(self):
        if not self.board:
            return [(d, True) for d in self.hands[self.turn]]

        moves = []
        for d in self.hands[self.turn]:
            if self.board.left_end() in d:
                moves.append((d, True))
            if self.board.right_end() in d and \
               self.board.left_end() != self.board.right_end():
                moves.append((d, False))

        return moves

    def make_move(self, d, left):
        self.hands[self.turn].play(d)

        try:
            if left:
                self.board.add_left(d)
            else:
                self.board.add_right(d)
        except domino.EndsMismatchException as error:
            self.hands[self.turn].draw(d)

            raise error

        if not self.hands[self.turn]:
            self.result = Result(self.turn, True, sum(self._remaining_points()))
            return self.result

        num_players = len(self.hands)
        stuck = True
        for _ in range(num_players):
            self.turn = (self.turn + 1) % num_players
            if self._has_valid_move():
                stuck = False
                break

        if stuck:
            player_points = self._remaining_points()
            team_points = [player_points[0] + player_points[2],
                           player_points[1] + player_points[3]]

            if team_points[0] < team_points[1]:
                self.result = Result(self.turn, False, pow(-1, self.turn) * sum(team_points))
            elif team_points[0] == team_points[1]:
                self.result = Result(self.turn, False, 0)
            else:
                self.result = Result(self.turn, False, pow(-1, self.turn + 1) * sum(team_points))

            return self.result

    def __str__(self):
        string_list = ['Board: {}'.format(self.board)]

        for i, hand in enumerate(self.hands):
            string_list.append("Player {}'s hand: {}".format(i, hand))

        if self.result is None:
            string_list.append("Player {}'s turn".format(self.turn))
        else:
            if self.result.won:
                string_list.append(
                    'Player {} won and scored {} points!'.format(self.result.player,
                                                                 self.result.points)
                )
            else:
                if self.result.points > 0:
                    string_list.append(
                        'Player {} stuck the game and won {} points!'.format(self.result.player,
                                                                             self.result.points)
                    )
                elif not self.result.points:
                    string_list.append(
                        'Player {} stuck the game and tied!'.format(self.result.player)
                    )
                else:
                    string_list.append(
                        'Player {} stuck the game and won'
                        ' the opposing team {} points!'.format(self.result.player,
                                                               -1 * self.result.points)
                    )

        return '\n'.join(string_list)

    def __repr__(self):
        return str(self)
<commit_msg>raising exception in _domino_hand() if domino is not found<commit_after>import collections
import domino
import random

def randomized_hands():
    '''
    Returns 4 hands of 7 dominoes each by randomly shuffing the
    28 dominoes used in this variation of the dominoes game.
    '''
    dominoes = [domino.Domino(i, j) for i in range(7) for j in range(i, 7)]
    random.shuffle(dominoes)
    return [domino.Hand(dominoes[0:7]), domino.Hand(dominoes[7:14]),
            domino.Hand(dominoes[14:21]), domino.Hand(dominoes[21:28])]

'''
namedtuple to represent the result of a dominoes game.

The attributes are defined as follows:
    player (int): the last player to make a move.
    won (bool): True if the game ended due to an empty hand.
                False if the game ended due to being stuck
    points (int): the absolute value of this quantity indicates
                  the amount of points earned by the winning team.
                  This quantity is positive if the last player to
                  make a move is part of the winning team, and
                  negative otherwise. If it is 0, it means the
                  game ended in a tie.
'''
Result = collections.namedtuple('Result', ['player', 'won', 'points'])

class Game:
    '''
    Python class for objects that represent a dominoes game.

    This variation of the dominoes game is played
    using 28 dominoes, which use values from 0 to 6:
    [0|0][0|1][0|2][0|3][0|4][0|5][0|6]
    [1|1][1|2][1|3][1|4][1|5][1|6]
    [2|2][2|3][2|4][2|5][2|6]
    [3|3][3|4][3|5][3|6]
    [4|4][4|5][4|6]
    [5|5][5|6]
    [6|6]

    These dominoes are shuffled, and distributed evenly among
    4 players. These players then sit on the edges of a square.
    Players sitting opposite of each other are on the same team,
    and the center of the square is the game board. Throughout
    the game, each player will only be able to see their hand,
    the game board, and the amount of dominoes left in the hands
    of the other players. Note that no player can see the values
    on the dominoes in the hands of the other players.

    The 4 players will then take turns placing dominoes from their
    hands onto the game board. The game board consists of a chain
    of dominoes placed end to end such that the values on connected
    ends always match.

    Prior to distributing the dominoes, the 4 players will agree on
    which player will play first, either by designating a specific
    player or a specific domino that must be played first (often [6|6]).
    After the game starts, play proceeds clockwise.

    If a player is able to place a domino on the board, he/she must.
    Only if they have no possible moves, can the pass on their turn.

    The game ends either when a player runs out of dominoes or when no
    player can play a domino (in which case we say the game is stuck).

    If a player runs out of dominoes, his/her team will earn a number
    of points computed by adding all the values of all the dominoes
    remaining in the hands of the 3 other players.

    If the game is stuck, each team will add up all the values of
    all the dominoes remaining in their hands. The team with the
    lower score wins, and earns a number of points computed by
    adding both teams' scores. If both teams have the same score,
    the game is declared a tie, and neither team earns any points.

    :param Domino starting_domino: the domino that should be played
                                   to start the game. The player
                                   with this domino in their hand
                                   will play first.
    :param int starting_player: the player that should play first.
                                This value is ignored if a staring
                                domino is provided.
    '''
    def __init__(self, starting_domino=None, starting_player=0):
        self.board = domino.Board()

        self.hands = randomized_hands()

        if starting_domino is None:
            self.turn = starting_player
        else:
            self.turn = self._domino_hand(starting_domino)
            self.make_move(starting_domino, True)

        self.result = None

    def skinny_board(self):
        self.board = domino.SkinnyBoard.from_board(self.board)

    def _domino_hand(self, d):
        for i, hand in enumerate(self.hands):
            if d in hand:
                return i

        raise domino.NoSuchDominoException('{} is not in any hand!'.format(d))

    def _remaining_points(self):
        points = []
        for hand in self.hands:
            points.append(sum(d.first + d.second for d in hand))

        return points

    def _has_valid_move(self):
        if not self.board:
            return True

        for d in self.hands[self.turn]:
            if self.board.left_end() in d or \
               self.board.right_end() in d:
                return True

        return False

    def valid_moves(self):
        if not self.board:
            return [(d, True) for d in self.hands[self.turn]]

        moves = []
        for d in self.hands[self.turn]:
            if self.board.left_end() in d:
                moves.append((d, True))
            if self.board.right_end() in d and \
               self.board.left_end() != self.board.right_end():
                moves.append((d, False))

        return moves

    def make_move(self, d, left):
        self.hands[self.turn].play(d)

        try:
            if left:
                self.board.add_left(d)
            else:
                self.board.add_right(d)
        except domino.EndsMismatchException as error:
            self.hands[self.turn].draw(d)

            raise error

        if not self.hands[self.turn]:
            self.result = Result(self.turn, True, sum(self._remaining_points()))
            return self.result

        num_players = len(self.hands)
        stuck = True
        for _ in range(num_players):
            self.turn = (self.turn + 1) % num_players
            if self._has_valid_move():
                stuck = False
                break

        if stuck:
            player_points = self._remaining_points()
            team_points = [player_points[0] + player_points[2],
                           player_points[1] + player_points[3]]

            if team_points[0] < team_points[1]:
                self.result = Result(self.turn, False, pow(-1, self.turn) * sum(team_points))
            elif team_points[0] == team_points[1]:
                self.result = Result(self.turn, False, 0)
            else:
                self.result = Result(self.turn, False, pow(-1, self.turn + 1) * sum(team_points))

            return self.result

    def __str__(self):
        string_list = ['Board: {}'.format(self.board)]

        for i, hand in enumerate(self.hands):
            string_list.append("Player {}'s hand: {}".format(i, hand))

        if self.result is None:
            string_list.append("Player {}'s turn".format(self.turn))
        else:
            if self.result.won:
                string_list.append(
                    'Player {} won and scored {} points!'.format(self.result.player,
                                                                 self.result.points)
                )
            else:
                if self.result.points > 0:
                    string_list.append(
                        'Player {} stuck the game and won {} points!'.format(self.result.player,
                                                                             self.result.points)
                    )
                elif not self.result.points:
                    string_list.append(
                        'Player {} stuck the game and tied!'.format(self.result.player)
                    )
                else:
                    string_list.append(
                        'Player {} stuck the game and won'
                        ' the opposing team {} points!'.format(self.result.player,
                                                               -1 * self.result.points)
                    )

        return '\n'.join(string_list)

    def __repr__(self):
        return str(self)
<|endoftext|>


<commit_before>import re
import os
from bibtexparser.bparser import BibTexParser

#Parse Bibtex file and return dictionary
class BibTex(object):

    def __init__(self,filename):
        self.filename = filename

    def __repr__(self):
        return 'BibTex File {} '.format(self.file)

    def fetch(self):
        if not os.path.exists(self.filename):
            raise Exception("Bibtex file {} does not exist".format(self.filename))
        if not os.path.isfile(self.filename):
            raise Exception("Bibtex file {} is not a file".format(self.filename))

    def parse(self):
        articles= list()
        with open(self.filename,'r') as bibfile:
            entries = BibTexParser(bibfile).get_entry_list()
            for entry in entries:
                article=dict()
                for kw,btpkw in zip(['title','author','abstract','url'],['title','author','abstract','link']):
                    article[kw]= entry[btpkw] if btpkw in entry else ''
                articles.append(article)
        return articles
<commit_msg>fixed typoe in bibtex<commit_after>import re
import os
from bibtexparser.bparser import BibTexParser

#Parse Bibtex file and return dictionary
class BibTex(object):

    def __init__(self,filename):
        self.filename = filename

    def __repr__(self):
        return 'BibTex File {} '.format(self.filename)

    def fetch(self):
        if not os.path.exists(self.filename):
            raise Exception("Bibtex file {} does not exist".format(self.filename))
        if not os.path.isfile(self.filename):
            raise Exception("Bibtex file {} is not a file".format(self.filename))

    def parse(self):
        articles= list()
        with open(self.filename,'r') as bibfile:
            entries = BibTexParser(bibfile).get_entry_list()
            for entry in entries:
                article=dict()
                for kw,btpkw in zip(['title','author','abstract','url'],['title','author','abstract','link']):
                    article[kw]= entry[btpkw] if btpkw in entry else ''
                articles.append(article)
        return articles
<|endoftext|>


<commit_before>import os
from app_config import AppConfig as ac
import subprocess

def create_video():
    count = 0
    num_frames_per_vid = 60
    images_folder = os.path.join(ac.CURRENT_PROJECT_PATH, "final_images")
    videos_folder = os.path.join(ac.CURRENT_PROJECT_PATH, "final_videos")

    # Make the videos folder if it doesn't exists
    if not os.path.exists(videos_folder):
        os.makedirs(videos_folder)
    if not os.path.exists(images_folder):
        os.makedirs(images_folder)
    db_path = os.path.join(ac.CURRENT_PROJECT_PATH, "run", "results.sqlite")
    delete_videos(videos_folder)

    while True:
        # Delete old images, and recreate them in the right place
        delete_images(images_folder)
        subprocess.call(["display-trajectories.py", "-i", ac.CURRENT_PROJECT_VIDEO_PATH,"-d", db_path, "-o", ac.CURRENT_PROJECT_PATH + "/homography/homography.txt", "-t", "object", "--save-images", "-f", str(count*num_frames_per_vid), "--last-frame", str((count + 1)*num_frames_per_vid - 1)])
        move_images_to_project_dir_folder(images_folder)

        # If we got to the end of the video, break
        if not images_exist(images_folder):
            print 'No more images'
            break

        # Get the frames, and create a short video out of them
        renumber_frames(images_folder, count*num_frames_per_vid)
        convert_frames_to_video(images_folder, videos_folder, "video-"+str(count)+".mp4")

        count += 1

    combine_videos(videos_folder, "final_videos")

def move_images_to_project_dir_folder(folder):
    images_folder = folder
    if not os.path.exists(images_folder):
        os.makedirs(images_folder)
    for file in os.listdir(os.getcwd()):
        if file[0:6] == 'image-' and file[-4:] == '.png':
            os.rename(file, os.path.join(images_folder, file))

def convert_frames_to_video(images_folder, videos_folder, filename):
    subprocess.call(["ffmpeg", "-framerate", "30", "-i", os.path.join(images_folder, "image-%d.png"), "-c:v", "libx264", "-pix_fmt", "yuv420p", os.path.join(videos_folder, filename)])

def delete_videos(folder):
    videos_folder = folder
    file_extensions = ['.mp4', '.mpg']

    for extension in file_extensions:
        if os.path.exists(videos_folder):
            for file in os.listdir(videos_folder):
                if file[0:6] == 'video-' and file[-4:] == extension:
                    os.remove(os.path.join(videos_folder, file))
                if file == 'output' + extension:
                    os.remove(os.path.join(videos_folder, file))
        for file in os.listdir(os.getcwd()):
            if file[0:6] == 'video-' and file[-4:] == extension:
                os.remove(os.path.join(os.getcwd(), file))
            if file == 'output' + extension:
                os.remove(os.path.join(videos_folder, file))

def delete_images(folder):
    images_folder = folder
    if os.path.exists(images_folder):
        for file in os.listdir(images_folder):
            if file[0:6] == 'image-' and file[-4:] == '.png':
                os.remove(os.path.join(images_folder, file))
    for file in os.listdir(os.getcwd()):
        if file[0:6] == 'image-' and file[-4:] == '.png':
            os.remove(os.path.join(os.getcwd(), file))

def move_videos_to_folder(folder):
    videos_folder = folder
    if not os.path.exists(videos_folder):
        os.makedirs(videos_folder)
    for file in os.listdir(os.getcwd()):
        if file[0:6] == 'video-' and file[-4:] == '.mp4':
            os.rename(file, os.path.join(videos_folder, file))

def images_exist(folder):
    images_folder = folder
    if os.path.exists(images_folder):
        for file in os.listdir(images_folder):
            if file[0:6] == 'image-' and file[-4:] == '.png':
                return True
    return False

def combine_videos(folder, filename):
    # The only way I could find to join videos was to convert the videos to .mpg format, and then join them.
    # This seems to be the only way to keep ffmpeg happy.
    videos_folder = folder
    convert_to_mpeg(videos_folder)

    # Using Popen seems to be necessary in order to pipe the output of one into the other
    p1 = subprocess.Popen(['cat']+get_videos(videos_folder), stdout=subprocess.PIPE)
    p2 = subprocess.Popen(['ffmpeg', '-f', 'mpeg', '-i', '-', '-qscale', '0', '-vcodec', 'mpeg4', os.path.join(videos_folder, 'output.mp4')], stdin=p1.stdout, stdout=subprocess.PIPE)
    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.

def convert_to_mpeg(folder):
    videos_folder = folder
    count = 0

    while os.path.exists(os.path.join(videos_folder, "video-"+str(count)+".mp4")):
        subprocess.call(['ffmpeg', '-i', os.path.join(videos_folder, 'video-'+str(count)+'.mp4'), '-qscale', '0', os.path.join(folder, "video-"+str(count)+".mpg")])
        count += 1

def get_videos(folder):
    videos_folder = folder
    count = 0
    videos = []

    while os.path.exists(os.path.join(videos_folder, "video-"+str(count)+".mpg")):
        videos.append(os.path.join(videos_folder, "video-"+str(count)+".mpg"))
        count += 1
    return videos

def renumber_frames(folder, frame):
    images_folder = folder
    print(images_folder)

    # Rename them all to 'new-image-x' in order to not interfere with the current 'image-x'
    for file in os.listdir(images_folder):
        if file[0:6] == 'image-' and file[-4:] == '.png':
            number = file[6:-4]
            new_number = int(number) - frame
            new_file = 'new-image-'+str(new_number)+'.png'
            os.rename(os.path.join(images_folder, file), os.path.join(images_folder, new_file))

    # Rename the 'new-image-x' to 'image-x'
    for file in os.listdir(images_folder):
        if file[0:10] == 'new-image-' and file[-4:] == '.png':
            new_file = file[4:]
            os.rename(os.path.join(images_folder, file), os.path.join(images_folder, new_file))

<commit_msg>Begin with mpg for combination<commit_after>import os
from app_config import AppConfig as ac
import subprocess

def create_video():
    count = 0
    num_frames_per_vid = 60
    images_folder = os.path.join(ac.CURRENT_PROJECT_PATH, "final_images")
    videos_folder = os.path.join(ac.CURRENT_PROJECT_PATH, "final_videos")

    # Make the videos folder if it doesn't exists
    if not os.path.exists(videos_folder):
        os.makedirs(videos_folder)
    if not os.path.exists(images_folder):
        os.makedirs(images_folder)
    db_path = os.path.join(ac.CURRENT_PROJECT_PATH, "run", "results.sqlite")
    delete_videos(videos_folder)

    while True:
        # Delete old images, and recreate them in the right place
        delete_images(images_folder)
        subprocess.call(["display-trajectories.py", "-i", ac.CURRENT_PROJECT_VIDEO_PATH,"-d", db_path, "-o", ac.CURRENT_PROJECT_PATH + "/homography/homography.txt", "-t", "object", "--save-images", "-f", str(count*num_frames_per_vid), "--last-frame", str((count + 1)*num_frames_per_vid - 1)])
        move_images_to_project_dir_folder(images_folder)

        # If we got to the end of the video, break
        if not images_exist(images_folder):
            print 'No more images'
            break

        # Get the frames, and create a short video out of them
        renumber_frames(images_folder, count*num_frames_per_vid)
        convert_frames_to_video(images_folder, videos_folder, "video-"+str(count)+".mpg")

        count += 1

    combine_videos(videos_folder, "final_videos")

def move_images_to_project_dir_folder(folder):
    images_folder = folder
    if not os.path.exists(images_folder):
        os.makedirs(images_folder)
    for file in os.listdir(os.getcwd()):
        if file[0:6] == 'image-' and file[-4:] == '.png':
            os.rename(file, os.path.join(images_folder, file))

def convert_frames_to_video(images_folder, videos_folder, filename):
    subprocess.call(["ffmpeg", "-framerate", "30", "-i", os.path.join(images_folder, "image-%d.png"), "-c:v", "libx264", "-pix_fmt", "yuv420p", os.path.join(videos_folder, filename)])

def delete_videos(folder):
    videos_folder = folder
    file_extensions = ['.mp4', '.mpg']

    for extension in file_extensions:
        if os.path.exists(videos_folder):
            for file in os.listdir(videos_folder):
                if file[0:6] == 'video-' and file[-4:] == extension:
                    os.remove(os.path.join(videos_folder, file))
                if file == 'output' + extension:
                    os.remove(os.path.join(videos_folder, file))
        for file in os.listdir(os.getcwd()):
            if file[0:6] == 'video-' and file[-4:] == extension:
                os.remove(os.path.join(os.getcwd(), file))
            if file == 'output' + extension:
                os.remove(os.path.join(videos_folder, file))

def delete_images(folder):
    images_folder = folder
    if os.path.exists(images_folder):
        for file in os.listdir(images_folder):
            if file[0:6] == 'image-' and file[-4:] == '.png':
                os.remove(os.path.join(images_folder, file))
    for file in os.listdir(os.getcwd()):
        if file[0:6] == 'image-' and file[-4:] == '.png':
            os.remove(os.path.join(os.getcwd(), file))

def move_videos_to_folder(folder):
    videos_folder = folder
    if not os.path.exists(videos_folder):
        os.makedirs(videos_folder)
    for file in os.listdir(os.getcwd()):
        if file[0:6] == 'video-' and file[-4:] == '.mp4':
            os.rename(file, os.path.join(videos_folder, file))

def images_exist(folder):
    images_folder = folder
    if os.path.exists(images_folder):
        for file in os.listdir(images_folder):
            if file[0:6] == 'image-' and file[-4:] == '.png':
                return True
    return False

def combine_videos(folder, filename):
    # The only way I could find to join videos was to convert the videos to .mpg format, and then join them.
    # This seems to be the only way to keep ffmpeg happy.
    videos_folder = folder

    # Using Popen seems to be necessary in order to pipe the output of one into the other
    p1 = subprocess.Popen(['cat']+get_videos(videos_folder), stdout=subprocess.PIPE)
    p2 = subprocess.Popen(['ffmpeg', '-f', 'mpeg', '-i', '-', '-qscale', '0', '-vcodec', 'mpeg4', os.path.join(videos_folder, 'output.mp4')], stdin=p1.stdout, stdout=subprocess.PIPE)
    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.

def convert_to_mpeg(folder):
    videos_folder = folder
    count = 0

    while os.path.exists(os.path.join(videos_folder, "video-"+str(count)+".mp4")):
        subprocess.call(['ffmpeg', '-i', os.path.join(videos_folder, 'video-'+str(count)+'.mp4'), '-qscale', '0', os.path.join(folder, "video-"+str(count)+".mpg")])
        count += 1

def get_videos(folder):
    videos_folder = folder
    count = 0
    videos = []

    while os.path.exists(os.path.join(videos_folder, "video-"+str(count)+".mpg")):
        videos.append(os.path.join(videos_folder, "video-"+str(count)+".mpg"))
        count += 1
    return videos

def renumber_frames(folder, frame):
    images_folder = folder
    print(images_folder)

    # Rename them all to 'new-image-x' in order to not interfere with the current 'image-x'
    for file in os.listdir(images_folder):
        if file[0:6] == 'image-' and file[-4:] == '.png':
            number = file[6:-4]
            new_number = int(number) - frame
            new_file = 'new-image-'+str(new_number)+'.png'
            os.rename(os.path.join(images_folder, file), os.path.join(images_folder, new_file))

    # Rename the 'new-image-x' to 'image-x'
    for file in os.listdir(images_folder):
        if file[0:10] == 'new-image-' and file[-4:] == '.png':
            new_file = file[4:]
            os.rename(os.path.join(images_folder, file), os.path.join(images_folder, new_file))

<|endoftext|>


<commit_before>from __future__ import division
import re
from itertools import product
from IPython import embed
import pandas as pd
import numpy as np
import gc

particle_vars = [u'pf', u'df', u'vt', u'vr', u'vc']
heat_vars = [u'ef']
momentum_vars = [u'vf']
store_format = 'table'

#'vti_GB', 'dfi_GB', 'vci_GB',
#       'pfi_GB', 'efi_GB',
#
#       'efe_GB', 'vce_GB', 'pfe_GB',
#       'vte_GB', 'dfe_GB'
             #'chie', 'ven', 'ver', 'vec']
def regime_filter(data, leq, less):
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    bool &= (data['efe_GB'] < less) & (data['efi_GB'] < less)
    bool &= (data['efe_GB'] >= leq) & (data['efi_GB'] >= leq)
    data = data.loc[bool]
    return data

def stability_filter(data):
    for col in data.columns:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] not in heat_vars + particle_vars + momentum_vars:
            print('skipping {!s}'.format(col))
            continue
        if splitted[2] == 'TEM':
            gam_filter = 'tem'
        elif splitted[2] == 'ITG':
            gam_filter = 'itg'
        elif splitted[2] == 'ETG':
            gam_filter = 'elec'
        elif splitted[0] in heat_vars and splitted[1] == 'e':
            gam_filter = 'multi'
        else:
            gam_filter = 'ion'

        pre = len(data[col])
        if gam_filter == 'ion':
            data[col] = data[col].loc[data['gam_leq_GB'] != 0]
        elif gam_filter == 'elec':
            data[col] = data[col].loc[data['gam_great_GB'] != 0]
        elif gam_filter == 'multi':
            data[col] = data[col].loc[(data['gam_leq_GB'] != 0) | (data['gam_great_GB'] != 0)]
        elif gam_filter == 'tem':
            data[col] = data[col].loc[data['TEM']]
        elif gam_filter == 'itg':
            data[col] = data[col].loc[data['ITG']]
        print('{:.2f}% of sane {!s:<9} points unstable at {!s:<5} scale'.format(np.sum(~data[col].isnull()) / pre * 100, col, gam_filter))
    return data

def filter_negative(data):
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    for col in data.columns:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] in heat_vars:
            bool &= (data[col] >= 0)
        elif splitted[0] in particle_vars:
            pass
    return bool

def filter_ck(data, bound):
    return (np.abs(data['cki']) < bound) & (np.abs(data['cke']) < bound)

def filter_totsep(data, septot_factor, startlen=None):
    if startlen is None:
        startlen = len(data)
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    for type, spec in product(particle_vars + heat_vars, ['i', 'e']):
        totname = type + spec + '_GB'
        if totname != 'vre_GB' and totname != 'vri_GB':
            if type in particle_vars or spec == 'i': # no ETG
                seps = ['ITG', 'TEM']
            else: # All modes
                seps = ['ETG', 'ITG', 'TEM']
            for sep in seps:
                sepname = type + spec + sep + '_GB'
                #sepflux += data[sepname]
                bool &= np.abs(data[sepname]) <= septot_factor * np.abs(data[totname])

            print('After filter {!s:<6} {!s:<6} {:.2f}% left'.format('septot', totname, 100*np.sum(bool)/startlen))
    return bool

def filter_ambipolar(data, bound):
    return (data['absambi'] < bound) & (data['absambi'] > 1/bound)

def filter_femtoflux(data, bound):
    fluxes = [col for col in data if len(re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)) > 1 if re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)[0] in particle_vars + heat_vars + momentum_vars]
    absflux = data[fluxes].abs()
    return ~((absflux < bound) & (absflux != 0)).any(axis=1)

def sanity_filter(data, ck_bound, septot_factor, ambi_bound, femto_bound, startlen=None):
    if startlen is None:
        startlen = len(data)
    # Throw away point if negative heat flux
    data = data.loc[filter_negative(data)]
    print('After filter {!s:<13} {:.2f}% left'.format('negative', 100*len(data)/startlen))
    gc.collect()

    # Throw away point if cke or cki too high
    data = data.loc[filter_ck(data, ck_bound)]
    print('After filter {!s:<13} {:.2f}% left'.format('ck', 100*len(data)/startlen))
    gc.collect()

    # Throw away point if sep flux is way higher than tot flux
    data = data.loc[filter_totsep(data, septot_factor, startlen=startlen)]
    print('After filter {!s:<13} {:.2f}% left'.format('septot', 100*len(data)/startlen))
    gc.collect()

    data = data.loc[filter_ambipolar(data, ambi_bound)]
    print('After filter {!s:<13} {:.2f}% left'.format('ambipolar', 100*len(data)/startlen))
    gc.collect()

    data = data.loc[filter_femtoflux(data, femto_bound)]
    print('After filter {!s:<13} {:.2f}% left'.format('femtoflux', 100*len(data)/startlen))
    gc.collect()

    # Alternatively:
    #data = data.loc[filter_negative(data) & filter_ck(data, ck_bound) & filter_totsep(data, septot_factor)]

    return data
    #for col in data.columns:
    #    splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
    #    if splitted[0] in particle_vars + heat_vars:
    #        if splitted[2] != '':
    #            data.loc[]

def separate_to_store(input, data, const, storename):
    store = pd.HDFStore(storename)
    store['input'] = input.loc[data.index]
    for col in data:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] in heat_vars + particle_vars + momentum_vars + ['gam_leq_GB', 'gam_less_GB']:
            store.put(col, data[col].dropna(), format=store_format)
    store.put('constants', const)
    store.close()

def create_divsum(store):
    for group in store:
        group = group[1:]
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)(_GB|SI|cm)').split(group)
        if splitted[0] in heat_vars and splitted[1] == 'i' and len(splitted) == 5:
            group2 = splitted[0] + 'e' + ''.join(splitted[2:])
            for name, set in [('_'.join([group, 'plus', group2]),
                              store[group] + store[group2]),
                              ('_'.join([group, 'div', group2]),
                               store[group] / store[group2])]:
                set.name = name
                store.put(set.name, set, format=store_format)
        if splitted[0] == 'pf' and splitted[1] == 'e' and len(splitted) == 5:
            group2 = 'efi' + ''.join(splitted[2:])
            group3 = 'efe' + ''.join(splitted[2:])
            for name, set in [
                ('_'.join([group, 'plus', group2, 'plus', group3]),
                 store[group] + store[group2] + store[group3]),
                ('_'.join([group, 'div', group2]),
                 store[group] / store[group2]),
                ('_'.join([group, 'div', group3]),
                 store[group] / store[group3])
            ]:
                print(name)
                set.name = name
                store.put(set.name, set, format=store_format)

def filter_9D_to_7D(input, Zeffx=1, Nustar=1e-3):
    if len(input.columns) != 9:
        print("Warning! This function assumes 9D input with ['Ati', 'Ate', 'An', 'qx', 'smag', 'x', 'Ti_Te', 'Zeffx', 'Nustar']")

    idx = input.index[(
        np.isclose(input['Zeffx'], Zeffx,     atol=1e-5, rtol=1e-3) &
        np.isclose(input['Nustar'], Nustar, atol=1e-5, rtol=1e-3)
    )]
    return idx

def filter_7D_to_4D(input, Ate=6.5, An=2, x=0.45):
    if len(input.columns) != 7:
        print("Warning! This function assumes 9D input with ['Ati', 'Ate', 'An', 'qx', 'smag', 'x', 'Ti_Te']")

    idx = input.index[(
        np.isclose(input['Ate'], Ate,     atol=1e-5, rtol=1e-3) &
        np.isclose(input['An'], An, atol=1e-5, rtol=1e-3) &
        np.isclose(input['x'], x, atol=1e-5, rtol=1e-3)
    )]
    return idx

def split_input(input, const):
    idx = {}
    consts = {9: const.copy(),
              7: const.copy(),
              4: const.copy()}
    idx[7] = filter_9D_to_7D(input)

    inputs = {9: input}
    idx[9] = input.index
    inputs[7] = input.loc[idx[7]]
    for name in ['Zeffx', 'Nustar']:
        consts[7][name] = inputs[7].head(1)[name]
    inputs[7].drop(['Zeffx', 'Nustar'], axis='columns', inplace=True)

    idx[4] = filter_7D_to_4D(inputs[7])
    inputs[4] = inputs[7].loc[idx[4]]
    for name in ['Ate', 'An', 'x']:
        consts[4][name] = inputs[4].head(1)[name]
    inputs[4].drop(['Ate', 'An', 'x'], axis='columns', inplace=True)

    return idx, inputs, consts

def split_sane(input, data, const):
    idx, inputs, consts = split_input(input, const)
    for dim in [7, 4]:
        print('splitting', dim)
        store = pd.HDFStore('sane_' + 'gen2_' + str(dim) + 'D_nions0_flat' + '_filter' + str(filter_num) + '.h5')
        store['/megarun1/flattened'] = data.loc[idx[dim]]
        store['/megarun1/input'] = inputs[dim]
        store['/megarun1/constants'] = consts[dim]
        store.close()

def split_subsets(input, data, const, frac=0.1):
    idx, inputs, consts = split_input(input, const)

    rand_index = pd.Int64Index(np.random.permutation(input.index))
    sep_index = int(frac * len(rand_index))
    idx['test'] = rand_index[:sep_index]
    idx['training'] = rand_index[sep_index:]

    for dim, set in product([9, 7, 4], ['test', 'training']):
        print(dim, set)
        store = pd.HDFStore(set + '_' + 'gen2_' + str(dim) + 'D_nions0_flat.h5')
        store['/megarun1/flattened'] = data.loc[idx[dim] & idx[set]]
        store['/megarun1/input'] = inputs[dim].loc[idx[set]]
        store['/megarun1/constants'] = consts[dim]
        store.close()

if __name__ == '__main__':
    dim = 9

    store_name = ''.join(['gen2_', str(dim), 'D_nions0_flat'])
    store = pd.HDFStore('../' + store_name + '.h5', 'r')

    input = store['/megarun1/input']
    data = store['/megarun1/flattened']

    startlen = len(data)
    data = sanity_filter(data, 50, 1.5, 1.5, 1e-4, startlen=startlen)
    data = regime_filter(data, 0, 100)
    gc.collect()
    input = input.loc[data.index]
    print('After filter {!s:<13} {:.2f}% left'.format('regime', 100*len(data)/startlen))
    filter_num = 7
    sane_store = pd.HDFStore('../sane_' + store_name + '_filter' + str(filter_num) + '.h5')
    sane_store['/megarun1/input'] = input
    sane_store['/megarun1/flattened'] = data
    const = sane_store['/megarun1/constants'] = store['/megarun1/constants']
    #input = sane_store['/megarun1/input']
    #data = sane_store['/megarun1/flattened']
    #const = sane_store['/megarun1/constants']
    split_sane(input, data, const)
    sane_store.close()
    split_subsets(input, data, const, frac=0.1)
    del data, input, const
    gc.collect()

    for dim, set in product([4, 7, 9], ['test', 'training']):
        print(dim, set)
        basename = set + '_' + 'gen2_' + str(dim) + 'D_nions0_flat.h5'
        store = pd.HDFStore(basename)
        data = store['/megarun1/flattened']
        input = store['/megarun1/input']
        const = store['/megarun1/constants']

        gam = data['gam_leq_GB']
        gam = gam[gam != 0]
        data = stability_filter(data)
        data.put('gam_leq_GB', gam, format=store_format)
        separate_to_store(input, data, const, 'unstable_' + basename)
    #separate_to_store(input, data, '../filtered_' + store_name + '_filter6')
<commit_msg>Added efe/efi to filtering<commit_after>from __future__ import division
import re
from itertools import product
from IPython import embed
import pandas as pd
import numpy as np
import gc

particle_vars = [u'pf', u'df', u'vt', u'vr', u'vc']
heat_vars = [u'ef']
momentum_vars = [u'vf']
store_format = 'table'

#'vti_GB', 'dfi_GB', 'vci_GB',
#       'pfi_GB', 'efi_GB',
#
#       'efe_GB', 'vce_GB', 'pfe_GB',
#       'vte_GB', 'dfe_GB'
             #'chie', 'ven', 'ver', 'vec']
def regime_filter(data, leq, less):
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    bool &= (data['efe_GB'] < less) & (data['efi_GB'] < less)
    bool &= (data['efe_GB'] >= leq) & (data['efi_GB'] >= leq)
    data = data.loc[bool]
    return data

def stability_filter(data):
    for col in data.columns:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] not in heat_vars + particle_vars + momentum_vars:
            print('skipping {!s}'.format(col))
            continue
        if splitted[2] == 'TEM':
            gam_filter = 'tem'
        elif splitted[2] == 'ITG':
            gam_filter = 'itg'
        elif splitted[2] == 'ETG':
            gam_filter = 'elec'
        elif splitted[0] in heat_vars and splitted[1] == 'e':
            gam_filter = 'multi'
        else:
            gam_filter = 'ion'

        pre = len(data[col])
        if gam_filter == 'ion':
            data[col] = data[col].loc[data['gam_leq_GB'] != 0]
        elif gam_filter == 'elec':
            data[col] = data[col].loc[data['gam_great_GB'] != 0]
        elif gam_filter == 'multi':
            data[col] = data[col].loc[(data['gam_leq_GB'] != 0) | (data['gam_great_GB'] != 0)]
        elif gam_filter == 'tem':
            data[col] = data[col].loc[data['TEM']]
        elif gam_filter == 'itg':
            data[col] = data[col].loc[data['ITG']]
        print('{:.2f}% of sane {!s:<9} points unstable at {!s:<5} scale'.format(np.sum(~data[col].isnull()) / pre * 100, col, gam_filter))
    return data

def filter_negative(data):
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    for col in data.columns:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] in heat_vars:
            bool &= (data[col] >= 0)
        elif splitted[0] in particle_vars:
            pass
    return bool

def filter_ck(data, bound):
    return (np.abs(data['cki']) < bound) & (np.abs(data['cke']) < bound)

def filter_totsep(data, septot_factor, startlen=None):
    if startlen is None:
        startlen = len(data)
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    for type, spec in product(particle_vars + heat_vars, ['i', 'e']):
        totname = type + spec + '_GB'
        if totname != 'vre_GB' and totname != 'vri_GB':
            if type in particle_vars or spec == 'i': # no ETG
                seps = ['ITG', 'TEM']
            else: # All modes
                seps = ['ETG', 'ITG', 'TEM']
            for sep in seps:
                sepname = type + spec + sep + '_GB'
                #sepflux += data[sepname]
                bool &= np.abs(data[sepname]) <= septot_factor * np.abs(data[totname])

            print('After filter {!s:<6} {!s:<6} {:.2f}% left'.format('septot', totname, 100*np.sum(bool)/startlen))
    return bool

def filter_ambipolar(data, bound):
    return (data['absambi'] < bound) & (data['absambi'] > 1/bound)

def filter_femtoflux(data, bound):
    fluxes = [col for col in data if len(re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)) > 1 if re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)[0] in particle_vars + heat_vars + momentum_vars]
    absflux = data[fluxes].abs()
    return ~((absflux < bound) & (absflux != 0)).any(axis=1)

def sanity_filter(data, ck_bound, septot_factor, ambi_bound, femto_bound, startlen=None):
    if startlen is None:
        startlen = len(data)
    # Throw away point if negative heat flux
    data = data.loc[filter_negative(data)]
    print('After filter {!s:<13} {:.2f}% left'.format('negative', 100*len(data)/startlen))
    gc.collect()

    # Throw away point if cke or cki too high
    data = data.loc[filter_ck(data, ck_bound)]
    print('After filter {!s:<13} {:.2f}% left'.format('ck', 100*len(data)/startlen))
    gc.collect()

    # Throw away point if sep flux is way higher than tot flux
    data = data.loc[filter_totsep(data, septot_factor, startlen=startlen)]
    print('After filter {!s:<13} {:.2f}% left'.format('septot', 100*len(data)/startlen))
    gc.collect()

    data = data.loc[filter_ambipolar(data, ambi_bound)]
    print('After filter {!s:<13} {:.2f}% left'.format('ambipolar', 100*len(data)/startlen))
    gc.collect()

    data = data.loc[filter_femtoflux(data, femto_bound)]
    print('After filter {!s:<13} {:.2f}% left'.format('femtoflux', 100*len(data)/startlen))
    gc.collect()

    # Alternatively:
    #data = data.loc[filter_negative(data) & filter_ck(data, ck_bound) & filter_totsep(data, septot_factor)]

    return data
    #for col in data.columns:
    #    splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
    #    if splitted[0] in particle_vars + heat_vars:
    #        if splitted[2] != '':
    #            data.loc[]

def separate_to_store(input, data, const, storename):
    store = pd.HDFStore(storename)
    store['input'] = input.loc[data.index]
    for col in data:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] in heat_vars + particle_vars + momentum_vars + ['gam_leq_GB', 'gam_less_GB']:
            store.put(col, data[col].dropna(), format=store_format)
    store.put('constants', const)
    store.close()

def create_divsum(store):
    for group in store:
        group = group[1:]
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)(_GB|SI|cm)').split(group)
        if splitted[0] in heat_vars and splitted[1] == 'i' and len(splitted) == 5:
            group2 = splitted[0] + 'e' + ''.join(splitted[2:])
            for name, set in [('_'.join([group, 'plus', group2]),
                              store[group] + store[group2]),
                              ('_'.join([group, 'div', group2]),
                               store[group] / store[group2]),
                              ('_'.join([group2, 'div', group]),
                               store[group2] / store[group])
                              ]:
                print(name)
                set.name = name
                store.put(set.name, set, format=store_format)
        if splitted[0] == 'pf' and splitted[1] == 'e' and len(splitted) == 5:
            group2 = 'efi' + ''.join(splitted[2:])
            group3 = 'efe' + ''.join(splitted[2:])
            for name, set in [
                ('_'.join([group, 'plus', group2, 'plus', group3]),
                 store[group] + store[group2] + store[group3]),
                ('_'.join([group, 'div', group2]),
                 store[group] / store[group2]),
                ('_'.join([group, 'div', group3]),
                 store[group] / store[group3])
            ]:
                print(name)
                set.name = name
                store.put(set.name, set, format=store_format)

def filter_9D_to_7D(input, Zeffx=1, Nustar=1e-3):
    if len(input.columns) != 9:
        print("Warning! This function assumes 9D input with ['Ati', 'Ate', 'An', 'qx', 'smag', 'x', 'Ti_Te', 'Zeffx', 'Nustar']")

    idx = input.index[(
        np.isclose(input['Zeffx'], Zeffx,     atol=1e-5, rtol=1e-3) &
        np.isclose(input['Nustar'], Nustar, atol=1e-5, rtol=1e-3)
    )]
    return idx

def filter_7D_to_4D(input, Ate=6.5, An=2, x=0.45):
    if len(input.columns) != 7:
        print("Warning! This function assumes 9D input with ['Ati', 'Ate', 'An', 'qx', 'smag', 'x', 'Ti_Te']")

    idx = input.index[(
        np.isclose(input['Ate'], Ate,     atol=1e-5, rtol=1e-3) &
        np.isclose(input['An'], An, atol=1e-5, rtol=1e-3) &
        np.isclose(input['x'], x, atol=1e-5, rtol=1e-3)
    )]
    return idx

def split_input(input, const):
    idx = {}
    consts = {9: const.copy(),
              7: const.copy(),
              4: const.copy()}
    idx[7] = filter_9D_to_7D(input)

    inputs = {9: input}
    idx[9] = input.index
    inputs[7] = input.loc[idx[7]]
    for name in ['Zeffx', 'Nustar']:
        consts[7][name] = inputs[7].head(1)[name]
    inputs[7].drop(['Zeffx', 'Nustar'], axis='columns', inplace=True)

    idx[4] = filter_7D_to_4D(inputs[7])
    inputs[4] = inputs[7].loc[idx[4]]
    for name in ['Ate', 'An', 'x']:
        consts[4][name] = inputs[4].head(1)[name]
    inputs[4].drop(['Ate', 'An', 'x'], axis='columns', inplace=True)

    return idx, inputs, consts

def split_sane(input, data, const):
    idx, inputs, consts = split_input(input, const)
    for dim in [7, 4]:
        print('splitting', dim)
        store = pd.HDFStore('sane_' + 'gen2_' + str(dim) + 'D_nions0_flat' + '_filter' + str(filter_num) + '.h5')
        store['/megarun1/flattened'] = data.loc[idx[dim]]
        store['/megarun1/input'] = inputs[dim]
        store['/megarun1/constants'] = consts[dim]
        store.close()

def split_subsets(input, data, const, frac=0.1):
    idx, inputs, consts = split_input(input, const)

    rand_index = pd.Int64Index(np.random.permutation(input.index))
    sep_index = int(frac * len(rand_index))
    idx['test'] = rand_index[:sep_index]
    idx['training'] = rand_index[sep_index:]

    for dim, set in product([9, 7, 4], ['test', 'training']):
        print(dim, set)
        store = pd.HDFStore(set + '_' + 'gen2_' + str(dim) + 'D_nions0_flat.h5')
        store['/megarun1/flattened'] = data.loc[idx[dim] & idx[set]]
        store['/megarun1/input'] = inputs[dim].loc[idx[set]]
        store['/megarun1/constants'] = consts[dim]
        store.close()

if __name__ == '__main__':
    dim = 9

    store_name = ''.join(['gen2_', str(dim), 'D_nions0_flat'])
    store = pd.HDFStore('../' + store_name + '.h5', 'r')

    input = store['/megarun1/input']
    data = store['/megarun1/flattened']

    startlen = len(data)
    data = sanity_filter(data, 50, 1.5, 1.5, 1e-4, startlen=startlen)
    data = regime_filter(data, 0, 100)
    gc.collect()
    input = input.loc[data.index]
    print('After filter {!s:<13} {:.2f}% left'.format('regime', 100*len(data)/startlen))
    filter_num = 7
    sane_store = pd.HDFStore('../sane_' + store_name + '_filter' + str(filter_num) + '.h5')
    sane_store['/megarun1/input'] = input
    sane_store['/megarun1/flattened'] = data
    const = sane_store['/megarun1/constants'] = store['/megarun1/constants']
    #input = sane_store['/megarun1/input']
    #data = sane_store['/megarun1/flattened']
    #const = sane_store['/megarun1/constants']
    split_sane(input, data, const)
    sane_store.close()
    split_subsets(input, data, const, frac=0.1)
    del data, input, const
    gc.collect()

    for dim, set in product([4, 7, 9], ['test', 'training']):
        print(dim, set)
        basename = set + '_' + 'gen2_' + str(dim) + 'D_nions0_flat.h5'
        store = pd.HDFStore(basename)
        data = store['/megarun1/flattened']
        input = store['/megarun1/input']
        const = store['/megarun1/constants']

        gam = data['gam_leq_GB']
        gam = gam[gam != 0]
        data = stability_filter(data)
        data.put('gam_leq_GB', gam, format=store_format)
        separate_to_store(input, data, const, 'unstable_' + basename)
    #separate_to_store(input, data, '../filtered_' + store_name + '_filter6')
<|endoftext|>


<commit_before>Package ('fontconfig', '2.7.3',
	configure_flags = '--disable-docs',
	sources = [
		'http://www.fontconfig.org/release/%{name}-%{version}.tar.gz'
	]
)
<commit_msg>[bundle] fix typo in fontconfig package<commit_after>Package ('fontconfig', '2.7.3',
	configure_flags = [ '--disable-docs' ],
	sources = [
		'http://www.fontconfig.org/release/%{name}-%{version}.tar.gz'
	]
)
<|endoftext|>