<commit_before>#!/usr/bin/env python

import logging
import time

import base
from kcaa import kcsapi
from kcaa import screens
import organizing

logger = logging.getLogger('kcaa.manipulators.shipyard')

class BuildShip(base.Manipulator):

    def run(self, fuel, ammo, steel, bauxite, grand, material):
        fuel = int(fuel)
        ammo = int(ammo)
        steel = int(steel)
        bauxite = int(bauxite)
        grand = grand is True or grand == 'true'
        material = int(material)
        logger.info('Buliding a {} ship with [{}, {}, {}, {}]'.format(
            'grand' if grand else 'normal', fuel, ammo, steel, bauxite))
        build_dock = self.objects.get('BuildDock')
        if not build_dock:
            logger.error('No build dock was found. Giving up.')
            return
        empty_slots = build_dock.empty_slots
        if not empty_slots:
            logger.error('No empty build slot was found.')
            return
        slot_index = empty_slots[0].id - 1
        if grand:
            if (fuel < 1500 or fuel > 7000 or fuel % 10 != 0 or
                    ammo < 1500 or ammo > 7000 or ammo % 10 != 0 or
                    steel < 2000 or steel > 7000 or steel % 10 != 0 or
                    bauxite < 1000 or bauxite > 7000 or bauxite % 10 != 0 or
                    material not in (1, 20, 100)):
                logger.error('Resource amount is invalid for grand building.')
                return
        else:
            if (fuel < 30 or fuel >= 1000 or
                    ammo < 30 or ammo >= 1000 or
                    steel < 30 or steel >= 1000 or
                    bauxite < 30 or bauxite >= 1000):
                logger.error('Resource amount is invalid for normal building.')
                return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.select_slot(slot_index)
        if grand:
            yield self.screen.try_grand_building()
            yield self.screen.set_material(material)
        yield self.screen.set_resource(grand, fuel, ammo, steel, bauxite)
        yield self.screen.confirm_building()

class BoostShipBuilding(base.Manipulator):

    def run(self, slot_id):
        slot_id = int(slot_id)
        self.require_objects(['BuildDock'])
        build_dock = self.objects['BuildDock']
        logger.info('Boosting the ship building at the slot {}'.format(
            slot_id))
        slot_index = slot_id - 1
        slot = build_dock.slots[slot_index]
        if slot.empty:
            raise Exception('Slot {} is empty.'.format(slot_id))
        now = long(1000 * time.time())
        if slot.completed(now):
            logger.info('Slot {} has completed the build.')
            return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.boost_build(slot_index)
        yield self.screen.confirm_boost()
        yield self.do_manipulator(ReceiveShip, slot_id=slot_id)

class ReceiveShip(base.Manipulator):

    def run(self, slot_id):
        slot_id = int(slot_id)
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.select_slot(slot_id - 1)
        yield self.screen.check_ship()
        yield self.do_manipulator(organizing.LockUniqueShips)

class AutoReceiveShips(base.AutoManipulator):

    # Ship buliding are completed on time? At least there is no harm on
    # checking the completion when the reported ETA has come.
    precursor_duration = 0

    @classmethod
    def required_objects(cls):
        return ['BuildDock']

    @classmethod
    def can_trigger(cls, owner):
        if not screens.in_category(owner.screen_id, screens.PORT):
            return
        build_dock = owner.objects.get('BuildDock')
        now = long(1000 * time.time()) + cls.precursor_duration
        slot_ids = []
        for slot in build_dock.slots:
            if slot.completed(now):
                slot_ids.append(slot.id)
        if slot_ids:
            return {'slot_ids': slot_ids}

    def run(self, slot_ids):
        yield 1.0
        for slot_id in slot_ids:
            yield self.do_manipulator(ReceiveShip, slot_id=slot_id)

class DevelopEquipment(base.Manipulator):

    def run(self, fuel, ammo, steel, bauxite):
        fuel = int(fuel)
        ammo = int(ammo)
        steel = int(steel)
        bauxite = int(bauxite)
        logger.info('Developing an equipment with [{}, {}, {}, {}]'.format(
            fuel, ammo, steel, bauxite))
        if (fuel < 10 or fuel > 300 or
                ammo < 10 or ammo > 300 or
                steel < 10 or steel > 300 or
                bauxite < 10 or bauxite > 300):
            logger.error('Resource amount is invalid for development.')
            return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.try_equipment_development()
        yield self.screen.set_development_resource(fuel, ammo, steel, bauxite)
        yield self.screen.confirm_development()
        yield self.screen.check_equipment()

class DissolveShip(base.Manipulator):

    def run(self, ship_id):
        ship_id = int(ship_id)
        ship_list = self.objects.get('ShipList')
        if not ship_list:
            logger.error('No ship list was found. Giving up.')
            return
        ship = ship_list.ships[str(ship_id)]
        logger.info('Dissolving a ship {} ({})'.format(
            ship.name.encode('utf8'), ship_id))
        # TODO: Check if the ship has a locked equipment.
        if ship.locked or ship.is_under_repair or ship.away_for_mission:
            logger.error('Ship {} ({}) is not ready for dissolution.'.format(
                ship.name.encode('utf8'), ship_id))
            return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.try_dissolution()
        page, index = ship_list.get_ship_position(ship_id)
        yield self.screen.select_page(page, ship_list.max_page)
        yield self.screen.select_ship(index)
        yield self.screen.confirm_dissolution()
        yield self.screen.unfocus_selection()

class AutoDissolveShips(base.AutoManipulator):

    @staticmethod
    def get_target_ship(ship_list, player_info):
        if player_info.max_ships - len(ship_list.ships) >= 5:
            return None
        target_ships = sorted(
            ship_list.dissolvable_ships(), kcsapi.ShipSorter.rebuilding_rank)
        return target_ships[0] if target_ships else None

    @classmethod
    def monitored_objects(cls):
        return ['ShipList', 'PlayerInfo']

    @classmethod
    def can_trigger(cls, owner):
        if (owner.screen_id != screens.PORT_MAIN and
                owner.screen_id != screens.PORT_SHIPYARD):
            return
        if (owner.manager.is_manipulator_scheduled('DissolveShip') or
                owner.manager.is_manipulator_scheduled('EnhanceBestShip') or
                owner.manager.is_manipulator_scheduled('AutoEnhanceBestShip')):
            return
        target_ship = AutoDissolveShips.get_target_ship(
            owner.objects['ShipList'], owner.objects['PlayerInfo'])
        if target_ship:
            return {}

    def run(self):
        target_ship = AutoDissolveShips.get_target_ship(
            self.objects['ShipList'], self.objects['PlayerInfo'])
        if target_ship:
            yield self.do_manipulator(DissolveShip, ship_id=target_ship.id)

class DissolveEquipment(base.Manipulator):

    def run(self, equipment_ids):
        if not isinstance(equipment_ids, list):
            equipment_ids = [int(equipment_id) for equipment_id in
                             equipment_ids.split(',')]
        equipment_list = self.objects.get('EquipmentList')
        if not equipment_list:
            logger.error('No equipment list was found. Giving up.')
            return
        ship_list = self.objects.get('ShipList')
        if not ship_list:
            logger.error('No ship list was found. Giving up.')
            return
        logger.info('Dissolving equipment {}'.format(', '.join(
            str(equipment_id) for equipment_id in equipment_ids)))
        for equipment_id in equipment_ids:
            if str(equipment_id) not in equipment_list.items:
                logger.error('Equipment ({}) is not found. Giving up.'.format(
                    equipment_id))
                return
            if equipment_list.items[str(equipment_id)].locked:
                logger.error('Equipment ({}) is locked. Giving up.'.format(
                    equipment_id))
                return
        unequipped_items = equipment_list.get_unequipped_items(ship_list)
        while equipment_ids:
            max_page = equipment_list.get_max_page(unequipped_items)
            first_page = max_page
            indices = []
            ids_to_dissolve = set()
            for equipment_id in equipment_ids:
                page, in_page_index = equipment_list.compute_page_position(
                    equipment_id, unequipped_items)
                if page is None:
                    logger.error('Eqiupment ({}) is equipped. Giving up.'
                                 .format(equipment_id))
                    return
                if page < first_page:
                    first_page = page
                    indices = [in_page_index]
                    ids_to_dissolve = set([equipment_id])
                elif page == first_page:
                    indices.append(in_page_index)
                    ids_to_dissolve.add(equipment_id)
            yield self.screen.change_screen(screens.PORT_SHIPYARD)
            yield self.screen.try_item_dissolution()
            yield self.screen.select_item_page(first_page, max_page)
            for in_page_index in indices:
                yield self.screen.select_item(in_page_index)
            yield self.screen.confirm_item_dissolution()
            yield self.screen.unfocus_selection()
            unequipped_items = [item for item in unequipped_items if
                                item.id not in ids_to_dissolve]
            equipment_ids = [equipment_id for equipment_id in equipment_ids if
                             equipment_id not in ids_to_dissolve]
<commit_msg>Recompute the receivable ships in AutoReceiveShips.<commit_after>#!/usr/bin/env python

import logging
import time

import base
from kcaa import kcsapi
from kcaa import screens
import organizing

logger = logging.getLogger('kcaa.manipulators.shipyard')

class BuildShip(base.Manipulator):

    def run(self, fuel, ammo, steel, bauxite, grand, material):
        fuel = int(fuel)
        ammo = int(ammo)
        steel = int(steel)
        bauxite = int(bauxite)
        grand = grand is True or grand == 'true'
        material = int(material)
        logger.info('Buliding a {} ship with [{}, {}, {}, {}]'.format(
            'grand' if grand else 'normal', fuel, ammo, steel, bauxite))
        build_dock = self.objects.get('BuildDock')
        if not build_dock:
            logger.error('No build dock was found. Giving up.')
            return
        empty_slots = build_dock.empty_slots
        if not empty_slots:
            logger.error('No empty build slot was found.')
            return
        slot_index = empty_slots[0].id - 1
        if grand:
            if (fuel < 1500 or fuel > 7000 or fuel % 10 != 0 or
                    ammo < 1500 or ammo > 7000 or ammo % 10 != 0 or
                    steel < 2000 or steel > 7000 or steel % 10 != 0 or
                    bauxite < 1000 or bauxite > 7000 or bauxite % 10 != 0 or
                    material not in (1, 20, 100)):
                logger.error('Resource amount is invalid for grand building.')
                return
        else:
            if (fuel < 30 or fuel >= 1000 or
                    ammo < 30 or ammo >= 1000 or
                    steel < 30 or steel >= 1000 or
                    bauxite < 30 or bauxite >= 1000):
                logger.error('Resource amount is invalid for normal building.')
                return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.select_slot(slot_index)
        if grand:
            yield self.screen.try_grand_building()
            yield self.screen.set_material(material)
        yield self.screen.set_resource(grand, fuel, ammo, steel, bauxite)
        yield self.screen.confirm_building()

class BoostShipBuilding(base.Manipulator):

    def run(self, slot_id):
        slot_id = int(slot_id)
        self.require_objects(['BuildDock'])
        build_dock = self.objects['BuildDock']
        logger.info('Boosting the ship building at the slot {}'.format(
            slot_id))
        slot_index = slot_id - 1
        slot = build_dock.slots[slot_index]
        if slot.empty:
            raise Exception('Slot {} is empty.'.format(slot_id))
        now = long(1000 * time.time())
        if slot.completed(now):
            logger.info('Slot {} has completed the build.')
            return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.boost_build(slot_index)
        yield self.screen.confirm_boost()
        yield self.do_manipulator(ReceiveShip, slot_id=slot_id)

class ReceiveShip(base.Manipulator):

    def run(self, slot_id):
        slot_id = int(slot_id)
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.select_slot(slot_id - 1)
        yield self.screen.check_ship()
        yield self.do_manipulator(organizing.LockUniqueShips)

class AutoReceiveShips(base.AutoManipulator):

    # Ship buliding are completed on time? At least there is no harm on
    # checking the completion when the reported ETA has come.
    precursor_duration = 0

    @classmethod
    def required_objects(cls):
        return ['BuildDock']

    @classmethod
    def get_receivable_slots(cls, objects):
        build_dock = objects.get('BuildDock')
        now = long(1000 * time.time()) + cls.precursor_duration
        return [slot for slot in build_dock.slots if slot.completed(now)]

    @classmethod
    def can_trigger(cls, owner):
        if not screens.in_category(owner.screen_id, screens.PORT):
            return
        if owner.manager.is_manipulator_scheduled('BoostShipBuilding'):
            return
        if AutoReceiveShips.get_receivable_slots(owner.objects):
            return {}

    def run(self):
        yield 1.0
        for slot in AutoReceiveShips.get_receivable_slots(self.objects):
            yield self.do_manipulator(ReceiveShip, slot_id=slot.id)

class DevelopEquipment(base.Manipulator):

    def run(self, fuel, ammo, steel, bauxite):
        fuel = int(fuel)
        ammo = int(ammo)
        steel = int(steel)
        bauxite = int(bauxite)
        logger.info('Developing an equipment with [{}, {}, {}, {}]'.format(
            fuel, ammo, steel, bauxite))
        if (fuel < 10 or fuel > 300 or
                ammo < 10 or ammo > 300 or
                steel < 10 or steel > 300 or
                bauxite < 10 or bauxite > 300):
            logger.error('Resource amount is invalid for development.')
            return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.try_equipment_development()
        yield self.screen.set_development_resource(fuel, ammo, steel, bauxite)
        yield self.screen.confirm_development()
        yield self.screen.check_equipment()

class DissolveShip(base.Manipulator):

    def run(self, ship_id):
        ship_id = int(ship_id)
        ship_list = self.objects.get('ShipList')
        if not ship_list:
            logger.error('No ship list was found. Giving up.')
            return
        ship = ship_list.ships[str(ship_id)]
        logger.info('Dissolving a ship {} ({})'.format(
            ship.name.encode('utf8'), ship_id))
        # TODO: Check if the ship has a locked equipment.
        if ship.locked or ship.is_under_repair or ship.away_for_mission:
            logger.error('Ship {} ({}) is not ready for dissolution.'.format(
                ship.name.encode('utf8'), ship_id))
            return
        yield self.screen.change_screen(screens.PORT_SHIPYARD)
        yield self.screen.try_dissolution()
        page, index = ship_list.get_ship_position(ship_id)
        yield self.screen.select_page(page, ship_list.max_page)
        yield self.screen.select_ship(index)
        yield self.screen.confirm_dissolution()
        yield self.screen.unfocus_selection()

class AutoDissolveShips(base.AutoManipulator):

    @staticmethod
    def get_target_ship(ship_list, player_info):
        if player_info.max_ships - len(ship_list.ships) >= 5:
            return None
        target_ships = sorted(
            ship_list.dissolvable_ships(), kcsapi.ShipSorter.rebuilding_rank)
        return target_ships[0] if target_ships else None

    @classmethod
    def monitored_objects(cls):
        return ['ShipList', 'PlayerInfo']

    @classmethod
    def can_trigger(cls, owner):
        if (owner.screen_id != screens.PORT_MAIN and
                owner.screen_id != screens.PORT_SHIPYARD):
            return
        if (owner.manager.is_manipulator_scheduled('DissolveShip') or
                owner.manager.is_manipulator_scheduled('EnhanceBestShip') or
                owner.manager.is_manipulator_scheduled('AutoEnhanceBestShip')):
            return
        target_ship = AutoDissolveShips.get_target_ship(
            owner.objects['ShipList'], owner.objects['PlayerInfo'])
        if target_ship:
            return {}

    def run(self):
        target_ship = AutoDissolveShips.get_target_ship(
            self.objects['ShipList'], self.objects['PlayerInfo'])
        if target_ship:
            yield self.do_manipulator(DissolveShip, ship_id=target_ship.id)

class DissolveEquipment(base.Manipulator):

    def run(self, equipment_ids):
        if not isinstance(equipment_ids, list):
            equipment_ids = [int(equipment_id) for equipment_id in
                             equipment_ids.split(',')]
        equipment_list = self.objects.get('EquipmentList')
        if not equipment_list:
            logger.error('No equipment list was found. Giving up.')
            return
        ship_list = self.objects.get('ShipList')
        if not ship_list:
            logger.error('No ship list was found. Giving up.')
            return
        logger.info('Dissolving equipment {}'.format(', '.join(
            str(equipment_id) for equipment_id in equipment_ids)))
        for equipment_id in equipment_ids:
            if str(equipment_id) not in equipment_list.items:
                logger.error('Equipment ({}) is not found. Giving up.'.format(
                    equipment_id))
                return
            if equipment_list.items[str(equipment_id)].locked:
                logger.error('Equipment ({}) is locked. Giving up.'.format(
                    equipment_id))
                return
        unequipped_items = equipment_list.get_unequipped_items(ship_list)
        while equipment_ids:
            max_page = equipment_list.get_max_page(unequipped_items)
            first_page = max_page
            indices = []
            ids_to_dissolve = set()
            for equipment_id in equipment_ids:
                page, in_page_index = equipment_list.compute_page_position(
                    equipment_id, unequipped_items)
                if page is None:
                    logger.error('Eqiupment ({}) is equipped. Giving up.'
                                 .format(equipment_id))
                    return
                if page < first_page:
                    first_page = page
                    indices = [in_page_index]
                    ids_to_dissolve = set([equipment_id])
                elif page == first_page:
                    indices.append(in_page_index)
                    ids_to_dissolve.add(equipment_id)
            yield self.screen.change_screen(screens.PORT_SHIPYARD)
            yield self.screen.try_item_dissolution()
            yield self.screen.select_item_page(first_page, max_page)
            for in_page_index in indices:
                yield self.screen.select_item(in_page_index)
            yield self.screen.confirm_item_dissolution()
            yield self.screen.unfocus_selection()
            unequipped_items = [item for item in unequipped_items if
                                item.id not in ids_to_dissolve]
            equipment_ids = [equipment_id for equipment_id in equipment_ids if
                             equipment_id not in ids_to_dissolve]
<|endoftext|>


<commit_before>"""
 Create a plot of today's estimated precipitation based on the Q3 data
"""

import datetime
import numpy as np
import os
import sys
sys.path.insert(0, '../mrms')
import util
import pytz
import gzip

from pyiem.plot import MapPlot

def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    # Start at 1 AM
    now = ts.replace(hour=1, minute=0)
    ets = now + datetime.timedelta(hours=24)
    interval = datetime.timedelta(hours=1)

    precip = np.zeros( (3500,7000) )
    lastts = None
    while now < ets:
        gmt = now.astimezone(pytz.timezone("UTC"))
        # Only need tile 1 and 2 to sufficiently do Iowa
        for tile in range(1,3):
            fn = util.get_fn('1hrad', gmt, tile)
            if os.path.isfile(fn):
                lastts = now
                tilemeta, val = util.reader(fn)
                ysz, xsz = np.shape(val)
                if tile == 1:
                    x0 = 0
                    y0 = 1750
                if tile == 2:
                    x0 = 3500
                    y0 = 1750
                precip[y0:(y0+ysz),x0:(x0+xsz)] += val
        now += interval

    lastts = lastts - datetime.timedelta(minutes=1)
    subtitle = "Total between 12:00 AM and %s" % (
                                            lastts.strftime("%H:%M %p %Z"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'
    pqstr = "plot %s %s00 iowa_q2_1d.png iowa_q2_1d.png png" % (routes,
            ts.strftime("%Y%m%d%H"), )
    m = MapPlot(title="%s NMQ Q3 Today's Precipitation" % (
                                                    ts.strftime("%-d %b %Y"),),
                subtitle=subtitle)

    clevs = np.arange(0,0.2,0.05)
    clevs = np.append(clevs, np.arange(0.2, 1.0, 0.1))
    clevs = np.append(clevs, np.arange(1.0, 5.0, 0.25))
    clevs = np.append(clevs, np.arange(5.0, 10.0, 1.0))
    clevs[0] = 0.01

    x,y = np.meshgrid(util.XAXIS, util.YAXIS)
    m.pcolormesh(x, y, precip / 24.5, clevs, units='inch')

    #map.drawstates(zorder=2)
    m.drawcounties()
    m.postprocess(pqstr=pqstr)

if __name__ == "__main__":
    ''' This is how we roll '''
    if len(sys.argv) == 4:
        date = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]),
                                 int(sys.argv[3]), 12, 0)
        realtime = False
    else:
        date = datetime.datetime.now()
        date = date - datetime.timedelta(minutes=60)
        date = date.replace(hour=12, minute=0, second=0, microsecond=0)
        realtime = True
    # Stupid pytz timezone dance
    date = date.replace(tzinfo=pytz.timezone("UTC"))
    date = date.astimezone(pytz.timezone("America/Chicago"))
    doday(date, realtime)
<commit_msg>fix timestamp label for 12hr clock<commit_after>"""
 Create a plot of today's estimated precipitation based on the Q3 data
"""

import datetime
import numpy as np
import os
import sys
sys.path.insert(0, '../mrms')
import util
import pytz
import gzip

from pyiem.plot import MapPlot

def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    # Start at 1 AM
    now = ts.replace(hour=1, minute=0)
    ets = now + datetime.timedelta(hours=24)
    interval = datetime.timedelta(hours=1)

    precip = np.zeros( (3500,7000) )
    lastts = None
    while now < ets:
        gmt = now.astimezone(pytz.timezone("UTC"))
        # Only need tile 1 and 2 to sufficiently do Iowa
        for tile in range(1,3):
            fn = util.get_fn('1hrad', gmt, tile)
            if os.path.isfile(fn):
                lastts = now
                tilemeta, val = util.reader(fn)
                ysz, xsz = np.shape(val)
                if tile == 1:
                    x0 = 0
                    y0 = 1750
                if tile == 2:
                    x0 = 3500
                    y0 = 1750
                precip[y0:(y0+ysz),x0:(x0+xsz)] += val
        now += interval

    lastts = lastts - datetime.timedelta(minutes=1)
    subtitle = "Total between 12:00 AM and %s" % (
                                            lastts.strftime("%I:%M %p %Z"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'
    pqstr = "plot %s %s00 iowa_q2_1d.png iowa_q2_1d.png png" % (routes,
            ts.strftime("%Y%m%d%H"), )
    m = MapPlot(title="%s NMQ Q3 Today's Precipitation" % (
                                                    ts.strftime("%-d %b %Y"),),
                subtitle=subtitle)

    clevs = np.arange(0,0.2,0.05)
    clevs = np.append(clevs, np.arange(0.2, 1.0, 0.1))
    clevs = np.append(clevs, np.arange(1.0, 5.0, 0.25))
    clevs = np.append(clevs, np.arange(5.0, 10.0, 1.0))
    clevs[0] = 0.01

    x,y = np.meshgrid(util.XAXIS, util.YAXIS)
    m.pcolormesh(x, y, precip / 24.5, clevs, units='inch')

    #map.drawstates(zorder=2)
    m.drawcounties()
    m.postprocess(pqstr=pqstr)

if __name__ == "__main__":
    ''' This is how we roll '''
    if len(sys.argv) == 4:
        date = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]),
                                 int(sys.argv[3]), 12, 0)
        realtime = False
    else:
        date = datetime.datetime.now()
        date = date - datetime.timedelta(minutes=60)
        date = date.replace(hour=12, minute=0, second=0, microsecond=0)
        realtime = True
    # Stupid pytz timezone dance
    date = date.replace(tzinfo=pytz.timezone("UTC"))
    date = date.astimezone(pytz.timezone("America/Chicago"))
    doday(date, realtime)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# SPDX-FileCopyrightText: 2013 Christopher Kerr
#
# SPDX-License-Identifier: GPL-3.0-or-later
"""
There is an option in the TNMR control software to save backup files at
regular intervals during data acquisition, so that in the event of power
failure or system crash only the data acquired since the last backup file is
lost. These backup files are saved in the same directory as the main file and
have the same name but with a suffix of _N (where N is a decimal number) on
the basename.

It is possible to use a standard find command to find files ending in _N.tnt,
but this will also catch files which just happen to have been given a name
ending in _N, which are not backup files. This script adds some sanity checks
to avoid finding files which look like backup files but actually aren't.
"""

from __future__ import print_function

import sys
import os
import os.path
import re
import time
import subprocess
from contextlib import contextmanager
import argparse

@contextmanager
def pushdir(dirpath):
    saved_dir = os.getcwd()
    try:
        os.chdir(dirpath)
        yield
    finally:
        os.chdir(saved_dir)

## Functions that define possible actions to take when a file is found

def print_filename(dirpath, fname):
    print(fname)

def print_filepath(dirpath, fname):
    print(os.path.join(dirpath, fname))

def delete(dirpath, fname):
    os.unlink(os.path.join(dirpath, fname))

def delete_svn(dirpath, fname):
    with pushdir(dirpath):
        subprocess.check_call(['svn', 'rm', fname])

def delete_git(dirpath, fname):
    with pushdir(dirpath):
        subprocess.check_call(['git', 'rm', fname])

## Handle command line arguments

parser = argparse.ArgumentParser(description='Find and process TNMR backup files')
parser.add_argument('path_to_search', type=str, default='.')
parser.add_argument('-print', action='append_const',
                    const=print_filepath, dest='actions',
                    help="Print the paths to the backup files found to stdout")
parser.add_argument('-print_basename', action='append_const',
                    const=print_filename, dest='actions',
                    help="Print the basenames of the backup files to stdout")
parser.add_argument('-delete', action='append_const',
                    const=delete, dest='actions',
                    help="Delete all backup files found")
parser.add_argument('-gitrm', action='append_const',
                    const=delete_git, dest='actions',
                    help="Delete backup files using git rm")
parser.add_argument('-svnrm', action='append_const',
                    const=delete_svn, dest='actions',
                    help="Delete backup files using svn rm")
parser.add_argument('--quiet', '-q', action='store_const', dest='log_std',
                    const=open('/dev/null', 'w'), default=sys.stderr,
                    help="Don't print reasons for omitting files")

args = parser.parse_args()

if args.actions is None:
    args.actions = [print_filepath]  # Default action

## Search the file system

for dirpath, dirnames, filenames in os.walk(args.path_to_search):
    for fname in filenames:
        if re.match('.*\.tnt_\d+\.tnt$', fname):
            for act in args.actions:
                act(dirpath, fname)
        elif re.match('.*_\d+\.tnt$', fname):
            base_fname = re.sub('_\d+\.tnt$', '.tnt', fname)
            if base_fname in filenames:
                my_stat = os.stat(os.path.join(dirpath, fname))
                base_stat = os.stat(os.path.join(dirpath, base_fname))
                if my_stat.st_size > base_stat.st_size:
                    print("%s is bigger than %s, keeping" % (fname, base_fname), file=args.log_std)
                elif my_stat.st_mtime > base_stat.st_mtime:
                    print("%s is newer than %s, keeping" % (fname, base_fname), file=args.log_std)

                ## I had the idea of checking for file age difference but the
                ## files I have all seem to have much bigger mtime differences
                ## than would make sense based on when they were acquired.
#                elif my_stat.st_mtime < (base_stat.st_mtime - 3600 * 24 * 7):
#                    my_mtime = time.gmtime(my_stat.st_mtime)
#                    base_mtime = time.gmtime(base_stat.st_mtime)
#                    print ("%s is more than a week older than %s, keeping" % (fname, base_fname), file=args.log_std)
#                    print ("%s was last modified at %s" % (fname, time.strftime('%c', my_mtime)), file=sys.stderr)
#                    print ("%s was last modified at %s" % (base_fname, time.strftime('%c', base_mtime)), file=sys.stderr)
                else:
                    for act in args.actions:
                        act(dirpath, fname)
            else:
                print('%s does not have a matching base file %s, keeping' % (fname, base_fname), file=args.log_std)
<commit_msg>Refactor find_TNMR_backup_files script to be importable<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# SPDX-FileCopyrightText: 2013 Christopher Kerr
#
# SPDX-License-Identifier: GPL-3.0-or-later
"""
There is an option in the TNMR control software to save backup files at
regular intervals during data acquisition, so that in the event of power
failure or system crash only the data acquired since the last backup file is
lost. These backup files are saved in the same directory as the main file and
have the same name but with a suffix of _N (where N is a decimal number) on
the basename.

It is possible to use a standard find command to find files ending in _N.tnt,
but this will also catch files which just happen to have been given a name
ending in _N, which are not backup files. This script adds some sanity checks
to avoid finding files which look like backup files but actually aren't.
"""

from __future__ import print_function

import sys
import os
import os.path
import re
import subprocess
from contextlib import contextmanager
import argparse

@contextmanager
def pushdir(dirpath):
    saved_dir = os.getcwd()
    try:
        os.chdir(dirpath)
        yield
    finally:
        os.chdir(saved_dir)

## Functions that define possible actions to take when a file is found

def print_filename(dirpath, fname):
    print(fname)

def print_filepath(dirpath, fname):
    print(os.path.join(dirpath, fname))

def delete(dirpath, fname):
    os.unlink(os.path.join(dirpath, fname))

def delete_svn(dirpath, fname):
    with pushdir(dirpath):
        subprocess.check_call(['svn', 'rm', fname])

def delete_git(dirpath, fname):
    with pushdir(dirpath):
        subprocess.check_call(['git', 'rm', fname])

## Handle command line arguments

parser = argparse.ArgumentParser(description='Find and process TNMR backup files')
parser.add_argument('path_to_search', type=str, default='.')
parser.add_argument('-print', action='append_const',
                    const=print_filepath, dest='actions',
                    help="Print the paths to the backup files found to stdout")
parser.add_argument('-print_basename', action='append_const',
                    const=print_filename, dest='actions',
                    help="Print the basenames of the backup files to stdout")
parser.add_argument('-delete', action='append_const',
                    const=delete, dest='actions',
                    help="Delete all backup files found")
parser.add_argument('-gitrm', action='append_const',
                    const=delete_git, dest='actions',
                    help="Delete backup files using git rm")
parser.add_argument('-svnrm', action='append_const',
                    const=delete_svn, dest='actions',
                    help="Delete backup files using svn rm")
parser.add_argument('--quiet', '-q', action='store_const', dest='log_std',
                    const=open('/dev/null', 'w'), default=sys.stderr,
                    help="Don't print reasons for omitting files")

def main():
    args = parser.parse_args()

    if args.actions is None:
        actions = [print_filepath]  # Default action
    else:
        actions = args.actions

    find_TNMR_backup_files(args.path_to_search, actions, args.log_std)

## Search the file system
def find_TNMR_backup_files(path_to_search, actions, log_std):
    for dirpath, dirnames, filenames in os.walk(path_to_search):
        for fname in filenames:
            if re.match('.*\.tnt_\d+\.tnt$', fname):
                for act in actions:
                    act(dirpath, fname)
            elif re.match('.*_\d+\.tnt$', fname):
                base_fname = re.sub('_\d+\.tnt$', '.tnt', fname)
                if base_fname in filenames:
                    my_stat = os.stat(os.path.join(dirpath, fname))
                    base_stat = os.stat(os.path.join(dirpath, base_fname))
                    if my_stat.st_size > base_stat.st_size:
                        print("%s is bigger than %s, keeping" % (fname, base_fname), file=log_std)
                    elif my_stat.st_mtime > base_stat.st_mtime:
                        print("%s is newer than %s, keeping" % (fname, base_fname), file=log_std)

                    ## I had the idea of checking for file age difference but the
                    ## files I have all seem to have much bigger mtime differences
                    ## than would make sense based on when they were acquired.
#                    elif my_stat.st_mtime < (base_stat.st_mtime - 3600 * 24 * 7):
#                        my_mtime = time.gmtime(my_stat.st_mtime)
#                        base_mtime = time.gmtime(base_stat.st_mtime)
#                        print ("%s is more than a week older than %s, keeping" % (fname, base_fname), file=log_std)
#                        print ("%s was last modified at %s" % (fname, time.strftime('%c', my_mtime)), file=sys.stderr)
#                        print ("%s was last modified at %s" % (base_fname, time.strftime('%c', base_mtime)), file=sys.stderr)
                    else:
                        for act in actions:
                            act(dirpath, fname)
                else:
                    print('%s does not have a matching base file %s, keeping' % (fname, base_fname), file=log_std)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""A simple script that attempts to directly download a single blob or stream from a given peer"""
import argparse
import logging
import sys
import tempfile
import time
import shutil
from pprint import pprint

from twisted.internet import defer, reactor, threads

from lbrynet import conf
from lbrynet.core import log_support, Wallet, Peer
from lbrynet.core.SinglePeerDownloader import SinglePeerDownloader
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorReader
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.database.storage import SQLiteStorage

log = logging.getLogger()

def main(args=None):
    conf.initialize_settings()
    parser = argparse.ArgumentParser()
    parser.add_argument('peer')
    parser.add_argument('blob_hash')
    parser.add_argument('--timeout', type=int, default=30)
    args = parser.parse_args(args)

    log_support.configure_console(level='DEBUG')
    log_support.configure_twisted()

    if ":" in str(args.peer):
        host, port = str(args.peer).strip().split(":")
    else:
        host = args.peer
        port = 3333

    d = download_it(Peer.Peer(host, int(port)), args.timeout, args.blob_hash)
    d.addErrback(log.exception)
    d.addBoth(lambda _: reactor.callLater(0, reactor.stop))
    reactor.run()

@defer.inlineCallbacks
def download_it(peer, timeout, blob_hash):
    tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
    storage = SQLiteStorage(tmp_dir, reactor)
    yield storage.setup()
    tmp_blob_manager = DiskBlobManager(tmp_dir, storage)

    config = {'auto_connect': True}
    if conf.settings['lbryum_wallet_dir']:
        config['lbryum_path'] = conf.settings['lbryum_wallet_dir']
    wallet = Wallet.LBRYumWallet(storage, config)

    downloader = SinglePeerDownloader()
    downloader.setup(wallet)

    try:
        blob_downloaded = yield downloader.download_blob_from_peer(peer, timeout, blob_hash,
                                                                   tmp_blob_manager)
        if blob_downloaded:
            log.info("SUCCESS!")
            blob = yield tmp_blob_manager.get_blob(blob_hash)
            pprint(blob)
            if not blob.verified:
                log.error("except that its not verified....")
            else:
                reader = BlobStreamDescriptorReader(blob)
                info = None
                for x in range(0, 3):
                    try:
                        info = yield reader.get_info()
                    except ValueError:
                        pass
                    if info:
                        break

                    # there's some kind of race condition where it sometimes doesnt write the blob to disk in time
                    time.sleep(0.1)

                if info is not None:
                    pprint(info)
                    for content_blob in info['blobs']:
                        if 'blob_hash' in content_blob:
                            yield download_it(peer, timeout, content_blob['blob_hash'])
        else:
            log.error("Download failed")
    finally:
        yield tmp_blob_manager.stop()
        yield threads.deferToThread(shutil.rmtree, tmp_dir)

    defer.returnValue(True)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>fix download_blob_from_peer script<commit_after>"""A simple script that attempts to directly download a single blob or stream from a given peer"""
import argparse
import logging
import sys
import tempfile
import time
import shutil
from pprint import pprint

from twisted.internet import defer, reactor, threads

from lbrynet import conf
from lbrynet.core import log_support, Peer
from lbrynet.core.SinglePeerDownloader import SinglePeerDownloader
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorReader
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.database.storage import SQLiteStorage
from lbrynet.wallet.manager import LbryWalletManager

log = logging.getLogger()

def main(args=None):
    conf.initialize_settings()
    parser = argparse.ArgumentParser()
    parser.add_argument('peer')
    parser.add_argument('blob_hash')
    parser.add_argument('--timeout', type=int, default=30)
    args = parser.parse_args(args)

    log_support.configure_console(level='DEBUG')
    log_support.configure_twisted()

    if ":" in str(args.peer):
        host, port = str(args.peer).strip().split(":")
    else:
        host = args.peer
        port = 3333

    d = download_it(Peer.Peer(host, int(port)), args.timeout, args.blob_hash)
    d.addErrback(log.exception)
    d.addBoth(lambda _: reactor.callLater(0, reactor.stop))
    reactor.run()

@defer.inlineCallbacks
def download_it(peer, timeout, blob_hash):
    tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
    storage = SQLiteStorage(tmp_dir, reactor)
    yield storage.setup()
    tmp_blob_manager = DiskBlobManager(tmp_dir, storage)

    config = {'auto_connect': True}
    if conf.settings['lbryum_wallet_dir']:
        config['lbryum_wallet_dir'] = conf.settings['lbryum_wallet_dir']
        config['use_keyring'] = False
        config['blockchain_name'] = conf.settings['blockchain_name']
        config['lbryum_servers'] = []
    wallet = LbryWalletManager.from_lbrynet_config(config, storage)

    downloader = SinglePeerDownloader()
    downloader.setup(wallet)

    try:
        blob_downloaded = yield downloader.download_blob_from_peer(peer, timeout, blob_hash,
                                                                   tmp_blob_manager)
        if blob_downloaded:
            log.info("SUCCESS!")
            blob = yield tmp_blob_manager.get_blob(blob_hash)
            pprint(blob)
            if not blob.verified:
                log.error("except that its not verified....")
            else:
                reader = BlobStreamDescriptorReader(blob)
                info = None
                for x in range(0, 3):
                    try:
                        info = yield reader.get_info()
                    except ValueError:
                        pass
                    if info:
                        break

                    # there's some kind of race condition where it sometimes doesnt write the blob to disk in time
                    time.sleep(0.1)

                if info is not None:
                    pprint(info)
                    for content_blob in info['blobs']:
                        if 'blob_hash' in content_blob:
                            yield download_it(peer, timeout, content_blob['blob_hash'])
        else:
            log.error("Download failed")
    finally:
        yield tmp_blob_manager.stop()
        yield threads.deferToThread(shutil.rmtree, tmp_dir)

    defer.returnValue(True)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!env/bin/python
# pylint: disable=no-member

import logging

from memegen.settings import ProdConfig
from memegen.app import create_app
from memegen.stores.image import MemeModel

def main():
    logging.info("Generating sample images...")

    app = create_app(ProdConfig)

    with app.app_context():

        for template in app.template_service.all():
            app.image_service.create(template, template.sample_text)
            MemeModel(key=template.key).save()

if __name__ == '__main__':
    main()
<commit_msg>Prerender blank images<commit_after>#!env/bin/python
# pylint: disable=no-member

import logging

from memegen.settings import ProdConfig
from memegen.app import create_app
from memegen.domain import Text
from memegen.stores.image import MemeModel

def main():
    logging.info("Generating sample images...")

    app = create_app(ProdConfig)

    with app.app_context():

        for template in app.template_service.all():
            app.image_service.create(template, Text("_"))
            app.image_service.create(template, Text("_/_"))
            app.image_service.create(template, template.sample_text)
            MemeModel(key=template.key).save()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging
import sys

from django.db import transaction

from website.app import setup_django
setup_django()
from osf.models import AbstractNode
from scripts import utils as script_utils

logger = logging.getLogger(__name__)

def main():
    dry = '--dry' in sys.argv
    if not dry:
        # If we're not running in dry mode log everything to a file
        script_utils.add_file_logger(logger, __file__)
    with transaction.atomic():
        nodes = AbstractNode.objects.filter(is_fork=True)

        for node in nodes:
            if not node.root == node.get_root():
                logger.info(
                    'Fixing {} (node): Replacing {} (wrong root) with {} (correct root)'.format(node._id, node.root._id, node.get_root()._id)
                )
                node.root = node.get_root()
                node.save()

        if dry:
            raise Exception('Abort Transaction - Dry Run')
    print('Done')

if __name__ == '__main__':
    main()
<commit_msg>remove the scripts<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy, os
import numpy as np
# Streaming
import optirx as rx
import socket, fcntl, struct
# Transformations
import tf
import tf.transformations as tr
from hrl_geom.pose_converter import PoseConv
# Messages
from geometry_msgs.msg import Pose, Point, Quaternion
from optitrack.msg import RigidBody, RigidBodyArray

def get_ip_address(ifname):
  """
  Get the ip address from the specified interface.

    >>> from denso_openrave.utils import get_ip_address
    >>> get_ip_address('eth0')
    '169.254.100.10'

  @type ifname: string
  @param ifname: The interface name. Typical names are C{'eth0'},
  C{'wlan0'}, etc.
  @rtype: string
  @return: The IP address of the specified interface.
  """
  s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  return socket.inet_ntoa(fcntl.ioctl(
      s.fileno(),
      0x8915,  # SIOCGIFADDR
      struct.pack('256s', ifname[:15]))[20:24])

def read_parameter(name, default):
  """
  Get a parameter from the ROS parameter server. If it's not found, a
  warn is printed.
  @type name: string
  @param name: Parameter name
  @param default: Default value for the parameter. The type should be
  the same as the one expected for the parameter.
  @return: The restulting parameter
  """
  if not rospy.has_param(name):
    rospy.logwarn('Parameter [%s] not found, using default: %s' % (name, default))
  return rospy.get_param(name, default)

class RigidBodiesPublisher(object):
  def __init__(self):
    # Read parameters to configure the calibration detector
    tf_publish_rate = read_parameter('~tf_publish_rate', 50.)
    parent_frame = read_parameter('~parent_frame', 'left/base_link')
    optitrack_frame = read_parameter('~optitrack_frame', 'optitrack')
    rigid_bodies = read_parameter('~rigid_bodies', dict())
    mapped_ids = rigid_bodies.values()
    # Setup Publishers
    pose_pub = rospy.Publisher('/optitrack/rigid_bodies', RigidBodyArray)
    # Setup TF listener and broadcaster
    tf_listener = tf.TransformListener()
    tf_broadcaster = tf.TransformBroadcaster()
    ## Connect to the optitrack system
    self._optitrack = None
    iface = read_parameter('~iface', 'eth1')
    version = (2, 7, 0, 0)  # the latest SDK version
    self._optitrack = rx.mkdatasock(ip_address=get_ip_address(iface))
    rospy.loginfo("Successfully connected to optitrack")
    start_time = rospy.get_time()
    prevtime = np.ones(100)*rospy.get_time()    # Track up to 100 rigid bodies
    while not rospy.is_shutdown():
      try:
        data = self._optitrack.recv(rx.MAX_PACKETSIZE)
      except socket.error:
        if not rospy.is_shutdown():
          rospy.logwarn('Failed to receive packet from optitrack')
      packet = rx.unpack(data, version=version)
      if type(packet) is rx.SenderData:
        version = packet.natnet_version
        rospy.loginfo('NatNet version received: ' + str(version))
      if type(packet) in [rx.SenderData, rx.ModelDefs, rx.FrameOfData]:
        rospy.logdebug('Timestamp: %f' % packet.timestamp)
        # Get transformation from optitrack to denso frame
        (parent, child) = (parent_frame, optitrack_frame)
        try:
          (pos,rot) = tf_listener.lookupTransform(parent, child, rospy.Time(0))
        except (tf.LookupException, tf.ConnectivityException):
          # Wait 2 seconds before showing this warning
          if rospy.get_time() - start_time > 2.0:
            rospy.logwarn('Failed lookupTransform with respect %s frame' % parent)
          continue
        dTo = PoseConv.to_homo_mat(pos, rot)
        # Optitrack gives the position of the centroid.
        array_msg = RigidBodyArray()
        for i, rigid_body in enumerate(packet.rigid_bodies):
          body_id = rigid_body.id
          pos_opt = np.array(rigid_body.position)
          rot_opt = np.array(rigid_body.orientation)
          oTobj = PoseConv.to_homo_mat(pos_opt, rot_opt)
          # Transformation between parent frame and the rigid body
          dTobj = np.dot(dTo, oTobj)
          array_msg.header.stamp = rospy.Time.now()
          array_msg.header.frame_id = parent_frame
          body_msg = RigidBody()
          pose = Pose()
          pose.position = Point(*dTobj[:3,3])
          pose.orientation = Quaternion(*tr.quaternion_from_matrix(dTobj))
          body_msg.id = body_id
          body_msg.tracking_valid = rigid_body.tracking_valid
          body_msg.mrk_mean_error = rigid_body.mrk_mean_error
          body_msg.pose = pose
          for marker in rigid_body.markers:
            body_msg.markers.append(Point(*marker))
          array_msg.bodies.append(body_msg)
          # TF broadcaster
          if rigid_body.tracking_valid and (rospy.get_time()-prevtime[body_id] >= (1./tf_publish_rate)):
            body_name = 'rigid_body_%d' % (body_id)
            if body_id in mapped_ids:
              idx = mapped_ids.index(body_id)
              body_name = rigid_bodies.keys()[idx]
            tf_broadcaster.sendTransform(pos_opt, rot_opt, rospy.Time.now(), body_name, optitrack_frame)
            prevtime[body_id] = rospy.get_time()
        pose_pub.publish(array_msg)

if __name__ == '__main__':
  node_name = os.path.splitext(os.path.basename(__file__))[0]
  rospy.init_node(node_name)
  rospy.loginfo('Starting [%s] node' % node_name)
  opti_node = RigidBodiesPublisher()
  rospy.loginfo('Shuting down [%s] node' % node_name)

<commit_msg>Minor changes<commit_after>#!/usr/bin/env python
import rospy, os
import numpy as np
# Streaming
import optirx as rx
import socket, fcntl, struct
# Transformations
import tf
import tf.transformations as tr
from hrl_geom.pose_converter import PoseConv
# Messages
from geometry_msgs.msg import Pose, Point, Quaternion
from optitrack.msg import RigidBody, RigidBodyArray

def get_ip_address(ifname):
  """
  Get the ip address from the specified interface.

    >>> get_ip_address('eth0')
    '192.168.0.7'

  @type ifname: string
  @param ifname: The interface name. Typical names are C{'eth0'},
  C{'wlan0'}, etc.
  @rtype: string
  @return: The IP address of the specified interface.
  """
  s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  return socket.inet_ntoa(fcntl.ioctl(
      s.fileno(),
      0x8915,  # SIOCGIFADDR
      struct.pack('256s', ifname[:15]))[20:24])

def read_parameter(name, default):
  """
  Get a parameter from the ROS parameter server. If it's not found, a
  warn is printed.
  @type name: string
  @param name: Parameter name
  @param default: Default value for the parameter. The type should be
  the same as the one expected for the parameter.
  @return: The restulting parameter
  """
  if not rospy.has_param(name):
    rospy.logwarn('Parameter [%s] not found, using default: %s' % (name, default))
  return rospy.get_param(name, default)

class RigidBodiesPublisher(object):
  def __init__(self):
    # Read parameters to configure the calibration detector
    tf_publish_rate = read_parameter('~tf_publish_rate', 50.)
    parent_frame = read_parameter('~parent_frame', 'left/base_link')
    optitrack_frame = read_parameter('~optitrack_frame', 'optitrack')
    rigid_bodies = read_parameter('~rigid_bodies', dict())
    mapped_ids = rigid_bodies.values()
    # Setup Publishers
    pose_pub = rospy.Publisher('/optitrack/rigid_bodies', RigidBodyArray)
    # Setup TF listener and broadcaster
    tf_listener = tf.TransformListener()
    tf_broadcaster = tf.TransformBroadcaster()
    ## Connect to the optitrack system
    self._optitrack = None
    iface = read_parameter('~iface', 'eth1')
    version = (2, 7, 0, 0)  # the latest SDK version
    self._optitrack = rx.mkdatasock(ip_address=get_ip_address(iface))
    rospy.loginfo("Successfully connected to optitrack")
    start_time = rospy.get_time()
    prevtime = np.ones(100)*rospy.get_time()    # Track up to 100 rigid bodies
    while not rospy.is_shutdown():
      try:
        data = self._optitrack.recv(rx.MAX_PACKETSIZE)
      except socket.error:
        if rospy.is_shutdown():
          return    # exit gracefully
        else:
          rospy.logwarn('Failed to receive packet from optitrack')
      packet = rx.unpack(data, version=version)
      if type(packet) is rx.SenderData:
        version = packet.natnet_version
        rospy.loginfo('NatNet version received: ' + str(version))
      if type(packet) in [rx.SenderData, rx.ModelDefs, rx.FrameOfData]:
        rospy.logdebug('Timestamp: %f' % packet.timestamp)
        # Get transformation from optitrack to denso frame
        (parent, child) = (parent_frame, optitrack_frame)
        try:
          (pos,rot) = tf_listener.lookupTransform(parent, child, rospy.Time(0))
        except (tf.LookupException, tf.ConnectivityException):
          # Wait 2 seconds before showing this warning
          if rospy.get_time() - start_time > 2.0:
            rospy.logwarn('Failed lookupTransform with respect %s frame' % parent)
          continue
        dTo = PoseConv.to_homo_mat(pos, rot)
        # Optitrack gives the position of the centroid.
        array_msg = RigidBodyArray()
        for i, rigid_body in enumerate(packet.rigid_bodies):
          body_id = rigid_body.id
          pos_opt = np.array(rigid_body.position)
          rot_opt = np.array(rigid_body.orientation)
          oTobj = PoseConv.to_homo_mat(pos_opt, rot_opt)
          # Transformation between parent frame and the rigid body
          dTobj = np.dot(dTo, oTobj)
          array_msg.header.stamp = rospy.Time.now()
          array_msg.header.frame_id = parent_frame
          body_msg = RigidBody()
          pose = Pose()
          pose.position = Point(*dTobj[:3,3])
          pose.orientation = Quaternion(*tr.quaternion_from_matrix(dTobj))
          body_msg.id = body_id
          body_msg.tracking_valid = rigid_body.tracking_valid
          body_msg.mrk_mean_error = rigid_body.mrk_mean_error
          body_msg.pose = pose
          for marker in rigid_body.markers:
            body_msg.markers.append(Point(*marker))
          array_msg.bodies.append(body_msg)
          # TF broadcaster
          if rigid_body.tracking_valid and (rospy.get_time()-prevtime[body_id] >= (1./tf_publish_rate)):
            body_name = 'rigid_body_%d' % (body_id)
            if body_id in mapped_ids:
              idx = mapped_ids.index(body_id)
              body_name = rigid_bodies.keys()[idx]
            tf_broadcaster.sendTransform(pos_opt, rot_opt, rospy.Time.now(), body_name, optitrack_frame)
            prevtime[body_id] = rospy.get_time()
        pose_pub.publish(array_msg)

if __name__ == '__main__':
  node_name = os.path.splitext(os.path.basename(__file__))[0]
  rospy.init_node(node_name)
  rospy.loginfo('Starting [%s] node' % node_name)
  opti_node = RigidBodiesPublisher()
  rospy.loginfo('Shuting down [%s] node' % node_name)
<|endoftext|>


<commit_before># !/usr/local/bin/python3.4.2
# ----Copyright (c) 2017 Carnegie Hall | The MIT License (MIT)----
# ----For the full license terms, please visit https://github.com/CarnegieHall/linked-data/blob/master/LICENSE----

## Argument[0] is script to run

import csv
import httplib2
import json
import os
import sys
from bs4 import BeautifulSoup

mbz_instDict = {}

h = httplib2.Http()
link = 'https://musicbrainz.org/instruments'
resp, html_doc = h.request(link, "GET")
soup = BeautifulSoup(html_doc, "lxml")

for result in soup.body('li'):
    uri = ''.join([link, result.a.attrs['href']])
    label = result.a.contents[0].string

    mbz_instDict[str(uri)] = label

mbz_instDict_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'source-files', 'mbz_instDict.json')

mbz_instList_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'source-files', 'mbz_instList.csv')

with open(mbz_instDict_path, 'w') as f1:
    json.dump(mbz_instDict, f1)

with open(mbz_instList_path, 'w', newline='') as csvfile:
    w = csv.writer(csvfile, dialect='excel', delimiter=',')
    for k,v in mbz_instDict.items():
        w.writerow([k,v])

print("Finished gathering MusicBrainz instrument URIs and labels")

<commit_msg>Added comment<commit_after># !/usr/local/bin/python3.4.2
# ----Copyright (c) 2017 Carnegie Hall | The MIT License (MIT)----
# ----For the full license terms, please visit https://github.com/CarnegieHall/linked-data/blob/master/LICENSE----

##needs further refinement to eliminate non-instrument link results

## Argument[0] is script to run

import csv
import httplib2
import json
import os
import sys
from bs4 import BeautifulSoup

mbz_instDict = {}

h = httplib2.Http()
link = 'https://musicbrainz.org/instruments'
resp, html_doc = h.request(link, "GET")
soup = BeautifulSoup(html_doc, "lxml")

for result in soup.body('li'):
    uri = ''.join([link, result.a.attrs['href']])
    label = result.a.contents[0].string

    mbz_instDict[str(uri)] = label

mbz_instDict_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'source-files', 'mbz_instDict.json')

mbz_instList_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'source-files', 'mbz_instList.csv')

with open(mbz_instDict_path, 'w') as f1:
    json.dump(mbz_instDict, f1)

with open(mbz_instList_path, 'w', newline='') as csvfile:
    w = csv.writer(csvfile, dialect='excel', delimiter=',')
    for k,v in mbz_instDict.items():
        w.writerow([k,v])

print("Finished gathering MusicBrainz instrument URIs and labels")

<|endoftext|>


<commit_before>import time

from peas.sensors import ArduinoSerialMonitor

def main(loop=True, delay=1., verbose=False):
    # Weather object
    monitor = ArduinoSerialMonitor(auto_detect=False)

    while True:
        data = monitor.capture()

        if verbose:
            print(data)

        if not args.loop:
            break

        time.sleep(args.delay)

if __name__ == '__main__':
    import argparse

    # Get the command line option
    parser = argparse.ArgumentParser(description="Read sensor data from arduinos")

    parser.add_argument('--loop', action='store_true', default=True,
                        help="If should keep reading, defaults to True")
    parser.add_argument("-d", "--delay", dest="delay", default=1.0, type=float,
                        help="Interval to read sensors")
    parser.add_argument('-v', '--verbose', action='store_true', default=False,
                        help="Print results to stdout")
    args = parser.parse_args()

    main(**vars(args))
<commit_msg>Don't print empty entries<commit_after>import time

from peas.sensors import ArduinoSerialMonitor

def main(loop=True, delay=1., verbose=False):
    # Weather object
    monitor = ArduinoSerialMonitor(auto_detect=False)

    while True:
        data = monitor.capture()

        if verbose and len(data.keys) > 0:
            print(data)

        if not args.loop:
            break

        time.sleep(args.delay)

if __name__ == '__main__':
    import argparse

    # Get the command line option
    parser = argparse.ArgumentParser(description="Read sensor data from arduinos")

    parser.add_argument('--loop', action='store_true', default=True,
                        help="If should keep reading, defaults to True")
    parser.add_argument("-d", "--delay", dest="delay", default=1.0, type=float,
                        help="Interval to read sensors")
    parser.add_argument('-v', '--verbose', action='store_true', default=False,
                        help="Print results to stdout")
    args = parser.parse_args()

    main(**vars(args))
<|endoftext|>


<commit_before>import datetime
import os
import pandas
import sys
import time

from plotly import graph_objs as plotly_go
from plotly import plotly
from plotly import tools as plotly_tools

sys.path.append(os.getenv('PEAS', '.'))  # Append the $PEAS dir

from peas import weather

names = [
    'date',
    'safe',
    'ambient_temp_C',
    'sky_temp_C',
    'rain_sensor_temp_C',
    'rain_frequency',
    'wind_speed_KPH',
    'ldr_resistance_Ohm',
    'pwm_value',
    'gust_condition',
    'wind_condition',
    'sky_condition',
    'rain_condition',
]

header = ','.join(names)

def get_plot(filename=None):
    stream_tokens = plotly_tools.get_credentials_file()['stream_ids']
    token_1 = stream_tokens[0]
    token_2 = stream_tokens[1]
    token_3 = stream_tokens[2]
    stream_id1 = dict(token=token_1, maxpoints=1500)
    stream_id2 = dict(token=token_2, maxpoints=1500)
    stream_id3 = dict(token=token_3, maxpoints=1500)

    # Get existing data
    x_data = {
        'time': [],
    }
    y_data = {
        'temp': [],
        'cloudiness': [],
        'rain': [],
    }

    if filename is not None:
        data = pandas.read_csv(filename, names=names)
        data.date = pandas.to_datetime(data.date)
        # Convert from UTC
        data.date = data.date + datetime.timedelta(hours=11)
        x_data['time'] = data.date
        y_data['temp'] = data.ambient_temp_C
        y_data['cloudiness'] = data.sky_temp_C
        y_data['rain'] = data.rain_frequency

    trace1 = plotly_go.Scatter(
        x=x_data['time'], y=y_data['temp'], name='Temperature', mode='lines', stream=stream_id1)
    trace2 = plotly_go.Scatter(
        x=x_data['time'], y=y_data['cloudiness'], name='Cloudiness', mode='lines', stream=stream_id2)
    trace3 = plotly_go.Scatter(
        x=x_data['time'], y=y_data['rain'], name='Rain', mode='lines', stream=stream_id3)

    fig = plotly_tools.make_subplots(rows=3, cols=1, shared_xaxes=True, shared_yaxes=False)
    fig.append_trace(trace1, 1, 1)
    fig.append_trace(trace2, 2, 1)
    fig.append_trace(trace3, 3, 1)

    fig['layout'].update(title="MQ Observatory Weather")

    fig['layout']['xaxis1'].update(title="Time [AEDT]")

    fig['layout']['yaxis1'].update(title="Temp [C]")
    fig['layout']['yaxis2'].update(title="Cloudiness")
    fig['layout']['yaxis3'].update(title="Rain Sensor")

    url = plotly.plot(fig, filename='MQObs Weather - Temp')
    print("Plot available at {}".format(url))

    stream_temp = plotly.Stream(stream_id=token_1)
    stream_temp.open()

    stream_cloudiness = plotly.Stream(stream_id=token_2)
    stream_cloudiness.open()

    stream_rain = plotly.Stream(stream_id=token_3)
    stream_rain.open()

    streams = {
        'temp': stream_temp,
        'cloudiness': stream_cloudiness,
        'rain': stream_rain,
    }

    return streams

def write_header(filename):
    # Write out the header to the CSV file
    with open(filename, 'w') as f:
        f.write(header)

def write_capture(filename=None, data=None):
    """ A function that reads the AAG weather can calls itself on a timer """
    entry = "{},{},{},{},{},{},{},{:0.5f},{:0.5f},{},{},{},{}\n".format(
        data['date'].strftime('%Y-%m-%d %H:%M:%S'),
        data['safe'],
        data['ambient_temp_C'],
        data['sky_temp_C'],
        data['rain_sensor_temp_C'],
        data['rain_frequency'],
        data['wind_speed_KPH'],
        data['ldr_resistance_Ohm'],
        data['pwm_value'],
        data['gust_condition'],
        data['wind_condition'],
        data['sky_condition'],
        data['rain_condition'],
    )

    if filename is not None:
        with open(filename, 'a') as f:
            f.write(entry)

if __name__ == '__main__':
    import argparse

    # Get the command line option
    parser = argparse.ArgumentParser(
        description="Make a plot of the weather for a give date.")

    parser.add_argument('--loop', action='store_true', default=True,
                        help="If should keep reading, defaults to True")
    parser.add_argument("-d", "--delay", dest="delay", default=30.0, type=float,
                        help="Interval to read weather")
    parser.add_argument("-f", "--filename", dest="filename", default=None,
                        help="Where to save results")
    parser.add_argument('--plotly-stream', action='store_true', default=False, help="Stream to plotly")
    parser.add_argument('--store-mongo', action='store_true', default=True, help="Save to mongo")
    args = parser.parse_args()

    # Weather object
    aag = weather.AAGCloudSensor(use_mongo=args.store_mongo)

    if args.plotly_stream:
        streams = None
        streams = get_plot(filename=args.filename)

    while True:
        data = aag.capture(use_mongo=args.store_mongo)

        # Save to file
        if args.filename is not None:
            write_capture(filename=args.filename, data=data)

        if args.plotly_stream:
            now = datetime.datetime.now()
            streams['temp'].write({'x': now, 'y': data['ambient_temp_C']})
            streams['cloudiness'].write({'x': now, 'y': data['sky_temp_C']})
            streams['rain'].write({'x': now, 'y': data['rain_frequency']})

        if not args.loop:
            break

        time.sleep(args.delay)
<commit_msg>Add custom port<commit_after>import datetime
import os
import pandas
import sys
import time

from plotly import graph_objs as plotly_go
from plotly import plotly
from plotly import tools as plotly_tools

sys.path.append(os.getenv('PEAS', '.'))  # Append the $PEAS dir

from peas import weather

names = [
    'date',
    'safe',
    'ambient_temp_C',
    'sky_temp_C',
    'rain_sensor_temp_C',
    'rain_frequency',
    'wind_speed_KPH',
    'ldr_resistance_Ohm',
    'pwm_value',
    'gust_condition',
    'wind_condition',
    'sky_condition',
    'rain_condition',
]

header = ','.join(names)

def get_plot(filename=None):
    stream_tokens = plotly_tools.get_credentials_file()['stream_ids']
    token_1 = stream_tokens[0]
    token_2 = stream_tokens[1]
    token_3 = stream_tokens[2]
    stream_id1 = dict(token=token_1, maxpoints=1500)
    stream_id2 = dict(token=token_2, maxpoints=1500)
    stream_id3 = dict(token=token_3, maxpoints=1500)

    # Get existing data
    x_data = {
        'time': [],
    }
    y_data = {
        'temp': [],
        'cloudiness': [],
        'rain': [],
    }

    if filename is not None:
        data = pandas.read_csv(filename, names=names)
        data.date = pandas.to_datetime(data.date)
        # Convert from UTC
        data.date = data.date + datetime.timedelta(hours=11)
        x_data['time'] = data.date
        y_data['temp'] = data.ambient_temp_C
        y_data['cloudiness'] = data.sky_temp_C
        y_data['rain'] = data.rain_frequency

    trace1 = plotly_go.Scatter(
        x=x_data['time'], y=y_data['temp'], name='Temperature', mode='lines', stream=stream_id1)
    trace2 = plotly_go.Scatter(
        x=x_data['time'], y=y_data['cloudiness'], name='Cloudiness', mode='lines', stream=stream_id2)
    trace3 = plotly_go.Scatter(
        x=x_data['time'], y=y_data['rain'], name='Rain', mode='lines', stream=stream_id3)

    fig = plotly_tools.make_subplots(rows=3, cols=1, shared_xaxes=True, shared_yaxes=False)
    fig.append_trace(trace1, 1, 1)
    fig.append_trace(trace2, 2, 1)
    fig.append_trace(trace3, 3, 1)

    fig['layout'].update(title="MQ Observatory Weather")

    fig['layout']['xaxis1'].update(title="Time [AEDT]")

    fig['layout']['yaxis1'].update(title="Temp [C]")
    fig['layout']['yaxis2'].update(title="Cloudiness")
    fig['layout']['yaxis3'].update(title="Rain Sensor")

    url = plotly.plot(fig, filename='MQObs Weather - Temp')
    print("Plot available at {}".format(url))

    stream_temp = plotly.Stream(stream_id=token_1)
    stream_temp.open()

    stream_cloudiness = plotly.Stream(stream_id=token_2)
    stream_cloudiness.open()

    stream_rain = plotly.Stream(stream_id=token_3)
    stream_rain.open()

    streams = {
        'temp': stream_temp,
        'cloudiness': stream_cloudiness,
        'rain': stream_rain,
    }

    return streams

def write_header(filename):
    # Write out the header to the CSV file
    with open(filename, 'w') as f:
        f.write(header)

def write_capture(filename=None, data=None):
    """ A function that reads the AAG weather can calls itself on a timer """
    entry = "{},{},{},{},{},{},{},{:0.5f},{:0.5f},{},{},{},{}\n".format(
        data['date'].strftime('%Y-%m-%d %H:%M:%S'),
        data['safe'],
        data['ambient_temp_C'],
        data['sky_temp_C'],
        data['rain_sensor_temp_C'],
        data['rain_frequency'],
        data['wind_speed_KPH'],
        data['ldr_resistance_Ohm'],
        data['pwm_value'],
        data['gust_condition'],
        data['wind_condition'],
        data['sky_condition'],
        data['rain_condition'],
    )

    if filename is not None:
        with open(filename, 'a') as f:
            f.write(entry)

if __name__ == '__main__':
    import argparse

    # Get the command line option
    parser = argparse.ArgumentParser(
        description="Make a plot of the weather for a give date.")

    parser.add_argument('--loop', action='store_true', default=True,
                        help="If should keep reading, defaults to True")
    parser.add_argument("-d", "--delay", dest="delay", default=30.0, type=float,
                        help="Interval to read weather")
    parser.add_argument("-f", "--filename", dest="filename", default=None,
                        help="Where to save results")
    parser.add_argument('--serial-port', dest='serial_port', default=None,
                        help='Serial port to connect')
    parser.add_argument('--plotly-stream', action='store_true', default=False, help="Stream to plotly")
    parser.add_argument('--store-mongo', action='store_true', default=True, help="Save to mongo")
    args = parser.parse_args()

    # Weather object
    aag = weather.AAGCloudSensor(serial_address=args.serial_port, use_mongo=args.store_mongo)

    if args.plotly_stream:
        streams = None
        streams = get_plot(filename=args.filename)

    while True:
        data = aag.capture(use_mongo=args.store_mongo)

        # Save to file
        if args.filename is not None:
            write_capture(filename=args.filename, data=data)

        if args.plotly_stream:
            now = datetime.datetime.now()
            streams['temp'].write({'x': now, 'y': data['ambient_temp_C']})
            streams['cloudiness'].write({'x': now, 'y': data['sky_temp_C']})
            streams['rain'].write({'x': now, 'y': data['rain_frequency']})

        if not args.loop:
            break

        time.sleep(args.delay)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""annotee_indenter parses annotated input and indents each step level.

Used by ../slave/recipes/infra/try_recipe.py to execute sub-recipes
and indent output automatically.
"""

import argparse
import collections
import errno
import os
import select
import subprocess
import sys
import threading
import Queue

QUEUE_TERMINATE_ITEM = object()

def main(argv):
  parser = argparse.ArgumentParser(os.path.basename(__file__))
  parser.add_argument('--cwd', help='The cwd for executing child process.',
                      default=os.getcwd())
  parser.add_argument('--base-level', help='The base level for indents.',
                      type=int, required=True)
  parser.add_argument('--use-python-executable', action='store_true',
                      help='execute python command using same executbale, '
                           'as running this one.')
  parser.add_argument('sub', nargs='+')
  opts = parser.parse_args(argv)
  cmd = opts.sub
  if opts.use_python_executable:
    cmd = [sys.executable] + cmd
  return run(cmd, opts.cwd, opts.base_level, process_output)

def run(args, cwd, base_level, target):
  proc = subprocess.Popen(args, cwd=cwd,
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  queue = Queue.Queue()
  t = threading.Thread(target=target, args=(queue, base_level))
  t.start()
  try:
    if sys.platform == "win32":
      communicate_win(proc, queue)
    else:
      communicate_posix(proc, queue)
  finally:
    # Ensure that on KeyboardInterrupt, the thread terminates gracefully.
    queue.put(QUEUE_TERMINATE_ITEM)
    t.join()
  return proc.wait()

def communicate_posix(proc, queue):
  """While process runs, puts tuples of (stream name, data) into queue."""

  def reader(rlist, stream_name):
    stream = getattr(proc, stream_name)
    if stream in rlist:
      data = os.read(stream.fileno(), 1024)
      if data == '':
        stream.close()
        read_set.remove(stream)
      queue.put((stream_name, data))

  read_set = [proc.stdout, proc.stderr]
  while read_set:
    try:
      rlist, _, _ = select.select(read_set, [], [])
    except select.error, e:
      if e.args[0] == errno.EINTR:
        continue
      raise
    reader(rlist, 'stdout')
    reader(rlist, 'stderr')

def communicate_win(proc, queue):
  def reader(stream_name):
    stream = getattr(proc, stream_name)
    while True:
      data = stream.read(1024)
      if data == '':
        break
      queue.put((stream_name, data))

  threads = []
  for stream_name in ['stdout', 'stderr']:
    t = threading.Thread(target=reader, args=(stream_name, ))
    t.set_daemon(True)
    t.start()
    threads.append(t)

  for t in threads:
    t.join()

def line_generator(queue, stderr_stream=None):
  if stderr_stream is None:
    stderr_stream = sys.stderr
  # remainder stores the line which started, but not finished.
  remainder = {}
  while True:
    item = queue.get()
    if item == QUEUE_TERMINATE_ITEM:
      break
    stream, data = item
    data = remainder.pop(stream, '') + data
    if data == '':
      continue
    lines = data.splitlines(True)  # Keep endings.
    if not lines[-1].endswith('\n'):
      # Last line isn't finished yet, put into remainder.
      remainder[stream] = lines.pop()
    for l in lines:
      if stream == 'stderr':
        stderr_stream.write(l)
      else:
        yield l
  # Empty remainder, if any, at the end.
  if 'stderr' in remainder:
    stderr_stream.write(remainder.pop('stderr'))
  if 'stdout' in remainder:
    yield remainder.pop('stdout')
  assert not remainder

def process_output(queue, base_level):
  return indent(base_level, line_generator(queue), sys.stdout)

def indent(base_level, input_lines, output_stream):
  for l in input_lines:
    if l.startswith('@@@STEP_NEST_LEVEL@'):
      # Example: @@@STEP_NEST_LEVEL@1@@@
      level = int(l[len('@@@STEP_NEST_LEVEL@'):].strip().strip('@'))
      l = l.replace(str(level), str(level + base_level))
    output_stream.write(l)
    if l.startswith('@@@STEP_STARTED@@@'):
      output_stream.write('@@@STEP_NEST_LEVEL@%d@@@\n' % base_level)
  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<commit_msg>try_recipe fix daemon threading.<commit_after>#!/usr/bin/env python
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""annotee_indenter parses annotated input and indents each step level.

Used by ../slave/recipes/infra/try_recipe.py to execute sub-recipes
and indent output automatically.
"""

import argparse
import collections
import errno
import os
import select
import subprocess
import sys
import threading
import Queue

QUEUE_TERMINATE_ITEM = object()

def main(argv):
  parser = argparse.ArgumentParser(os.path.basename(__file__))
  parser.add_argument('--cwd', help='The cwd for executing child process.',
                      default=os.getcwd())
  parser.add_argument('--base-level', help='The base level for indents.',
                      type=int, required=True)
  parser.add_argument('--use-python-executable', action='store_true',
                      help='execute python command using same executbale, '
                           'as running this one.')
  parser.add_argument('sub', nargs='+')
  opts = parser.parse_args(argv)
  cmd = opts.sub
  if opts.use_python_executable:
    cmd = [sys.executable] + cmd
  return run(cmd, opts.cwd, opts.base_level, process_output)

def run(args, cwd, base_level, target):
  proc = subprocess.Popen(args, cwd=cwd,
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  queue = Queue.Queue()
  t = threading.Thread(target=target, args=(queue, base_level))
  t.start()
  try:
    if sys.platform == "win32":
      communicate_win(proc, queue)
    else:
      communicate_posix(proc, queue)
  finally:
    # Ensure that on KeyboardInterrupt, the thread terminates gracefully.
    queue.put(QUEUE_TERMINATE_ITEM)
    t.join()
  return proc.wait()

def communicate_posix(proc, queue):
  """While process runs, puts tuples of (stream name, data) into queue."""

  def reader(rlist, stream_name):
    stream = getattr(proc, stream_name)
    if stream in rlist:
      data = os.read(stream.fileno(), 1024)
      if data == '':
        stream.close()
        read_set.remove(stream)
      queue.put((stream_name, data))

  read_set = [proc.stdout, proc.stderr]
  while read_set:
    try:
      rlist, _, _ = select.select(read_set, [], [])
    except select.error, e:
      if e.args[0] == errno.EINTR:
        continue
      raise
    reader(rlist, 'stdout')
    reader(rlist, 'stderr')

def communicate_win(proc, queue):
  def reader(stream_name):
    stream = getattr(proc, stream_name)
    while True:
      data = stream.read(1024)
      if data == '':
        break
      queue.put((stream_name, data))

  threads = []
  for stream_name in ['stdout', 'stderr']:
    t = threading.Thread(target=reader, args=(stream_name, ))
    t.daemon = True
    t.start()
    threads.append(t)

  for t in threads:
    t.join()

def line_generator(queue, stderr_stream=None):
  if stderr_stream is None:
    stderr_stream = sys.stderr
  # remainder stores the line which started, but not finished.
  remainder = {}
  while True:
    item = queue.get()
    if item == QUEUE_TERMINATE_ITEM:
      break
    stream, data = item
    data = remainder.pop(stream, '') + data
    if data == '':
      continue
    lines = data.splitlines(True)  # Keep endings.
    if not lines[-1].endswith('\n'):
      # Last line isn't finished yet, put into remainder.
      remainder[stream] = lines.pop()
    for l in lines:
      if stream == 'stderr':
        stderr_stream.write(l)
      else:
        yield l
  # Empty remainder, if any, at the end.
  if 'stderr' in remainder:
    stderr_stream.write(remainder.pop('stderr'))
  if 'stdout' in remainder:
    yield remainder.pop('stdout')
  assert not remainder

def process_output(queue, base_level):
  return indent(base_level, line_generator(queue), sys.stdout)

def indent(base_level, input_lines, output_stream):
  for l in input_lines:
    if l.startswith('@@@STEP_NEST_LEVEL@'):
      # Example: @@@STEP_NEST_LEVEL@1@@@
      level = int(l[len('@@@STEP_NEST_LEVEL@'):].strip().strip('@'))
      l = l.replace(str(level), str(level + base_level))
    output_stream.write(l)
    if l.startswith('@@@STEP_STARTED@@@'):
      output_stream.write('@@@STEP_NEST_LEVEL@%d@@@\n' % base_level)
  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>#!/usr/bin/env python2
import csv
import json
import re
from StringIO import StringIO
from urllib2 import urlopen

# The JSON is invalid, because of mixed encodings. The CSV also has
# data quality issues, but most can be ignored.
DB_URL = ('http://www.dmr-marc.net/cgi-bin/trbo-database/datadump.cgi'
          '?table=users&format=csv&header=1')

# BrandMeister has a list of groups in JavaScript format, not quite JSON
BM_GROUPS_JS = ('https://raw.githubusercontent.com/zarya'
                '/BrandMeister-Dashboard/master/js/groups.js')
JSON_DICT = re.compile(r'({.*})', re.DOTALL)

# The CS750 uses a 6-bit encoding for the Call Alias, using only letters,
# numbers, space, and period.
# Dash would have been a better choice than period, but the decision was made
# without looking closely at the data.
ALIAS_ILLEGAL = re.compile('[^a-zA-Z0-9\. ]')

# These are the field names as given in a CS750 contacts export.
FIELDNAMES = ('No', 'Call Alias', 'Call Type', 'Call ID', 'Receive Tone')

def alias_user(user):
    """Takes a user record as given in the DMR-MARC csv and returns a
    Call Alias composed of the Callsign and either the Nickname or Name.
    """
    callsign = user['Callsign']

    name = user['Nickname']
    if name:
        name = name.strip()

    names = user['Name']
    if names:
        names = names.split()

    # name is still Nickname here. If it's None or empty, use the first name.
    if not name and names:
        name = names[0]
        # Skip titles. Have not seen other titles in the data yet.
        if name.lower() in ('dr', 'dr.'):
            name = names[1]

    if name:
        alias = ' '.join((callsign, name))
    else:
        # Both the Nickname and Name fields could be empty.
        alias = callsign

    # Could do something more useful, like transliterating, but this will
    # require detecting the encoding, which varies from record to record.
    return ALIAS_ILLEGAL.sub('', alias)

def alias_group(group):
    """Takes a talkgroup record in an intermediate format and returns an alias
    with the timeslot number appended if not already represented.

    The intention is to make it easier to apply the correct timeslot when
    creating a channel in the CPS.

    Input:
    {
      "timeslot": 2,
      "name": "Utah",
    }

    Output:
    "Utah 2"
    """
    alias = None
    if isinstance(group, basestring):
        group = { 'name': group }
    if 'timeslot' not in group or not group['timeslot']:
        alias = group['name']
    else:
        timeslot = str(group['timeslot'])
        name = group['name']
        # Only if it ends in ' n', or we won't append to 'TAC 312'
        if name[-2:] == ' ' + timeslot:
            alias = name
        else:
            alias = ' '.join((name, timeslot))
    return ALIAS_ILLEGAL.sub('', alias)

def read_users_csv(users):
    """Reads DMR-MARC csv from the users file-like object and returns a list of
    dicts in CS750 export format."""
    csvr = csv.DictReader(users)
    result = []
    for row in csvr:
        try:
            result.append({
                'Call Alias': alias_user(row),
                'Call Type': 'Private Call',
                'Call ID': row['Radio ID'],
                'Receive Tone': 'No'})
        except TypeError:
            # For now, skip records that have problems. The most common is an
            # empty record, because of a newline in a field.
            pass
    return sorted(result, key=lambda k: k['Call ID'])

def read_groups_json(groups):
    """Reads json from the groups file-like object and returns a list of dicts
    in CS750 export format."""
    result = []
    for tgid, group in json.load(groups).items():
        result.append({
            'Call Alias': alias_group(group),
            'Call Type': 'Group Call',
            'Call ID': tgid,
            'Receive Tone': 'No'})
    return sorted(result, key=lambda k: k['Call Alias'])

def write_contacts_csv(contacts, csvo, fieldnames=FIELDNAMES):
    """Writes contacts (expected in CS750 format) to the csvo file-like object.
    """
    csvw = csv.DictWriter(csvo, fieldnames)
    csvw.writeheader()
    for row in contacts:
        csvw.writerow(row)

def write_contacts_xlsx(contacts, xlsxo,
                        fieldnames=FIELDNAMES, worksheet='DMR_contacts'):
    """Writes contacts (expected in CS750 format) to the xlsxo file-like
    object."""
    import xlsxwriter
    wb = xlsxwriter.Workbook(xlsxo, {'in_memory': True})
    ws = wb.add_worksheet(worksheet)
    col = 0
    for field in fieldnames:
        ws.write_string(0, col, field)
        col += 1
    row = 1
    for contact in contacts:
        col = 0
        for field in fieldnames:
            if field in contact:
                ws.write(row, col, contact[field])
            col += 1
        row += 1
    wb.close()

def get_users(db_url=DB_URL):
    db = urlopen(DB_URL)
    users = read_users_csv(db)
    db.close()
    return users

def js_json(js):
    """Unwraps JSON from the JavaScript containing it."""
    return JSON_DICT.search(js).groups()[0]

def get_groups():
    groups = []
    with open('dci-groups.json') as dci:
        groups.extend(read_groups_json(dci))
    bm = urlopen(BM_GROUPS_JS)
    bm_json = StringIO(js_json(bm.read()))
    groups.extend(read_groups_json(bm_json))
    bm.close()
    return groups

if __name__ == '__main__':
    users = get_users()
    groups = get_groups()

    with open('contacts-dci.xlsx', 'wb') as xlsxo:
        write_contacts_xlsx(groups + users, xlsxo)

    # In exported contacts, the sheet name is DMR_contacts. Naming the file
    # this way maintains that, though it seems to not be important.
    with open('DMR_contacts.csv', 'wb') as csvo:
        write_contacts_csv(groups + users, csvo)
<commit_msg>Use requests instead of urlopen<commit_after>#!/usr/bin/env python2
import csv
import json
import re
from StringIO import StringIO

import requests

# The JSON is invalid, because of mixed encodings. The CSV also has
# data quality issues, but most can be ignored.
DB_URL = ('http://www.dmr-marc.net/cgi-bin/trbo-database/datadump.cgi'
          '?table=users&format=csv&header=1')

# BrandMeister has a list of groups in JavaScript format, not quite JSON
BM_GROUPS_JS = ('https://raw.githubusercontent.com/zarya'
                '/BrandMeister-Dashboard/master/js/groups.js')
JSON_DICT = re.compile(r'({.*})', re.DOTALL)

# The CS750 uses a 6-bit encoding for the Call Alias, using only letters,
# numbers, space, and period.
# Dash would have been a better choice than period, but the decision was made
# without looking closely at the data.
ALIAS_ILLEGAL = re.compile('[^a-zA-Z0-9\. ]')

# These are the field names as given in a CS750 contacts export.
FIELDNAMES = ('No', 'Call Alias', 'Call Type', 'Call ID', 'Receive Tone')

def alias_user(user):
    """Takes a user record as given in the DMR-MARC csv and returns a
    Call Alias composed of the Callsign and either the Nickname or Name.
    """
    callsign = user['Callsign']

    name = user['Nickname']
    if name:
        name = name.strip()

    names = user['Name']
    if names:
        names = names.split()

    # name is still Nickname here. If it's None or empty, use the first name.
    if not name and names:
        name = names[0]
        # Skip titles. Have not seen other titles in the data yet.
        if name.lower() in ('dr', 'dr.'):
            name = names[1]

    if name:
        alias = ' '.join((callsign, name))
    else:
        # Both the Nickname and Name fields could be empty.
        alias = callsign

    # Could do something more useful, like transliterating, but this will
    # require detecting the encoding, which varies from record to record.
    return ALIAS_ILLEGAL.sub('', alias)

def alias_group(group):
    """Takes a talkgroup record in an intermediate format and returns an alias
    with the timeslot number appended if not already represented.

    The intention is to make it easier to apply the correct timeslot when
    creating a channel in the CPS.

    Input:
    {
      "timeslot": 2,
      "name": "Utah",
    }

    Output:
    "Utah 2"
    """
    alias = None
    if isinstance(group, basestring):
        group = { 'name': group }
    if 'timeslot' not in group or not group['timeslot']:
        alias = group['name']
    else:
        timeslot = str(group['timeslot'])
        name = group['name']
        # Only if it ends in ' n', or we won't append to 'TAC 312'
        if name[-2:] == ' ' + timeslot:
            alias = name
        else:
            alias = ' '.join((name, timeslot))
    return ALIAS_ILLEGAL.sub('', alias)

def read_users_csv(users):
    """Reads DMR-MARC csv from the users file-like object and returns a list of
    dicts in CS750 export format."""
    csvr = csv.DictReader(users)
    result = []
    for row in csvr:
        try:
            result.append({
                'Call Alias': alias_user(row),
                'Call Type': 'Private Call',
                'Call ID': row['Radio ID'],
                'Receive Tone': 'No'})
        except TypeError:
            # For now, skip records that have problems. The most common is an
            # empty record, because of a newline in a field.
            pass
    return sorted(result, key=lambda k: k['Call ID'])

def read_groups_json(groups):
    """Reads json from the groups file-like object and returns a list of dicts
    in CS750 export format."""
    result = []
    for tgid, group in json.load(groups).items():
        result.append({
            'Call Alias': alias_group(group),
            'Call Type': 'Group Call',
            'Call ID': tgid,
            'Receive Tone': 'No'})
    return sorted(result, key=lambda k: k['Call Alias'])

def write_contacts_csv(contacts, csvo, fieldnames=FIELDNAMES):
    """Writes contacts (expected in CS750 format) to the csvo file-like object.
    """
    csvw = csv.DictWriter(csvo, fieldnames)
    csvw.writeheader()
    for row in contacts:
        csvw.writerow(row)

def write_contacts_xlsx(contacts, xlsxo,
                        fieldnames=FIELDNAMES, worksheet='DMR_contacts'):
    """Writes contacts (expected in CS750 format) to the xlsxo file-like
    object."""
    import xlsxwriter
    wb = xlsxwriter.Workbook(xlsxo, {'in_memory': True})
    ws = wb.add_worksheet(worksheet)
    col = 0
    for field in fieldnames:
        ws.write_string(0, col, field)
        col += 1
    row = 1
    for contact in contacts:
        col = 0
        for field in fieldnames:
            if field in contact:
                ws.write(row, col, contact[field])
            col += 1
        row += 1
    wb.close()

def get_users(db_url=DB_URL):
    db = requests.get(DB_URL)
    db_io = StringIO(db.content)
    users = read_users_csv(db_io)
    db_io.close()
    return users

def js_json(js):
    """Unwraps JSON from the JavaScript containing it."""
    return JSON_DICT.search(js).groups()[0]

def get_groups():
    groups = []
    with open('dci-groups.json') as dci:
        groups.extend(read_groups_json(dci))
    bm = requests.get(BM_GROUPS_JS)
    bm_json = StringIO(js_json(bm.content))
    groups.extend(read_groups_json(bm_json))
    bm_json.close()
    return groups

if __name__ == '__main__':
    users = get_users()
    groups = get_groups()

    with open('contacts-dci.xlsx', 'wb') as xlsxo:
        write_contacts_xlsx(groups + users, xlsxo)

    # In exported contacts, the sheet name is DMR_contacts. Naming the file
    # this way maintains that, though it seems to not be important.
    with open('DMR_contacts.csv', 'wb') as csvo:
        write_contacts_csv(groups + users, csvo)
<|endoftext|>


<commit_before># Wolfram Alpha (Maths)
#
# @website     http://www.wolframalpha.com
# @provide-api yes (http://api.wolframalpha.com/v2/)
#
# @using-api   yes
# @results     XML
# @stable      yes
# @parse       result

from urllib import urlencode
from lxml import etree
from searx.engines.xpath import extract_text
from searx.utils import html_to_text

# search-url
base_url = 'http://api.wolframalpha.com/v2/query'
search_url = base_url + '?appid={api_key}&{query}&format=plaintext'
site_url = 'http://wolframalpha.com/input/?{query}'

#embedded_url = '<iframe width="540" height="304" ' +\
#    'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\
#    'frameborder="0" allowfullscreen></iframe>'

# do search-request
def request(query, params):
    params['url'] = search_url.format(query=urlencode({'input': query}),
                                      api_key=api_key)

    # need this for url in response
    global my_query
    my_query = query

    return params

# replace private user area characters to make text legible
def replace_pua_chars(text):
    pua_chars = { u'\uf74c': 'd',
                  u'\uf74d': u'\u212f',
                  u'\uf74e': 'i',
                  u'\uf7d9': '=' }

    for k, v in pua_chars.iteritems():
        text = text.replace(k, v)

    return text

# get response from search-request
def response(resp):
    results = []

    search_results = etree.XML(resp.content)

    # return empty array if there are no results
    if search_results.xpath('/queryresult[attribute::success="false"]'):
        return []

    # parse result
    result = search_results.xpath('//pod[attribute::primary="true"]/subpod/plaintext')[0].text
    result = replace_pua_chars(result)

    # bind url from site
    result_url = site_url.format(query=urlencode({'i': my_query}))

    # append result
    results.append({'url': result_url,
                    'title': result})

    # return results
    return results
<commit_msg>Returns answer rather than results.<commit_after># Wolfram Alpha (Maths)
#
# @website     http://www.wolframalpha.com
# @provide-api yes (http://api.wolframalpha.com/v2/)
#
# @using-api   yes
# @results     XML
# @stable      yes
# @parse       result

from urllib import urlencode
from lxml import etree
from searx.engines.xpath import extract_text
from searx.utils import html_to_text

# search-url
base_url = 'http://api.wolframalpha.com/v2/query'
search_url = base_url + '?appid={api_key}&{query}&format=plaintext'
site_url = 'http://wolframalpha.com/input/?{query}'

# do search-request
def request(query, params):
    params['url'] = search_url.format(query=urlencode({'input': query}),
                                      api_key=api_key)

     need this for url in response
    global my_query
    my_query = query

    return params

# replace private user area characters to make text legible
def replace_pua_chars(text):
    pua_chars = { u'\uf74c': 'd',
                  u'\uf74d': u'\u212f',
                  u'\uf74e': 'i',
                  u'\uf7d9': '=' }

    for k, v in pua_chars.iteritems():
        text = text.replace(k, v)

    return text

# get response from search-request
def response(resp):
    results = []

    search_results = etree.XML(resp.content)

    # return empty array if there are no results
    if search_results.xpath('/queryresult[attribute::success="false"]'):
        return []

    # parse result
    result = search_results.xpath('//pod[attribute::primary="true"]/subpod/plaintext')[0].text
    result = replace_pua_chars(result)

    # bind url from site
    result_url = site_url.format(query=urlencode({'i': my_query}))

    # append result
    # TODO: shouldn't it bind the source too?
    results.append({'url': result_url,
                    'answer': result})

    # return results
    return results
<|endoftext|>