<commit_before># I love writting data converters
# Daryl Herzmann 16 Jul 2003

import re, mx.DateTime, string
import sys
ST = sys.argv[1]
from pyIEM import stationTable, iemdb
i = iemdb.iemdb()
coopdb = i['coop']
coopdb.query("BEGIN;")

o = open("%s_dat.txt" % (ST,), 'r').readlines()
s = mx.DateTime.DateTime(2010,1,1)
e = mx.DateTime.DateTime(2011,1,1)
interval = mx.DateTime.RelativeDateTime(days=+1)

data = {}

for line in o[2:]:
  tokens = re.split(",", line)
  month = mx.DateTime.strptime(tokens[7], "%Y%m")
  var = tokens[5]
  coopid = "%s%s" % (ST.lower(), str(tokens[1][2:]))
  sid = coopid
  if (not data.has_key(sid)):
    data[sid] = {}
  for i in range(31):
    p = 9 + (i*4)
    val = tokens[p]
    ts = month + mx.DateTime.RelativeDateTime(day=(i+1))
    if (not data[sid].has_key(ts)):
      data[sid][ts] = {'TMAX':'Null', 'TMIN':'Null', 'SNOW':'Null', 'PRCP':'Null', 'SNWD': 'Null'}
    data[sid][ts][var] = val
    if (tokens[p+1] == "T"):
      #print "HERE"
      data[sid][ts][var] = 0.001

now = s
cnt = 0
while (now < e):
  for sid in data.keys():
    if not data[sid].has_key(now):
      continue
    if (data[sid][now]["TMAX"] != 'Null'):
      high = int(float(data[sid][now]["TMAX"]))
      if (high < -100 or high > 140):
        high = 'Null'
        cnt += 1
    else:
      high = data[sid][now]["TMAX"]
      cnt += 1

    if (data[sid][now]["TMIN"] != 'Null'):
      low = int(float(data[sid][now]["TMIN"]))
      if (low < -100 or low > 100):
        low = 'Null'
        cnt += 1
    else:
      low = data[sid][now]["TMIN"]
      cnt += 1

    if (data[sid][now]["PRCP"] != 'Null'):
      precip = int(float(data[sid][now]["PRCP"])) /100.0
      if (precip < 0 or precip > 100):
        precip = 'Null'
        cnt += 1
    else:
      precip =  data[sid][now]["PRCP"]
      cnt += 1

    if (data[sid][now]["SNOW"] != 'Null'):
      snow = int(float(data[sid][now]["SNOW"]))  /10.0
      if (snow < 0 or snow > 30):
        snow = 'Null'
        cnt += 1
    else:
      snow = data[sid][now]["SNOW"]
      cnt += 1

    if (data[sid][now]["SNWD"] != 'Null'):
      snowd = int(float(data[sid][now]["SNWD"]))
      if (snowd < 0 or snowd > 30):
        snowd = 'Null'
        cnt += 1
    else:
      snowd = 'Null'
      cnt += 1

    sql = "INSERT into alldata_%s (stationid, day, high, low, precip, snow, \
           sday, year, month,snowd) values ('%s','%s',%s,%s,%s,%s,'%s',%s,%s,%s) "\
           % (ST.lower(), sid.lower(), now.strftime("%Y-%m-%d"), high, low, precip, snow,\
           now.strftime("%m%d"), now.year, now.month, snowd)
    #sql = "UPDATE alldata SET snow = %s WHERE stationid = '%s' and day = '%s'" % (snow, sid.lower(), now.strftime("%Y-%m-%d"))
    #print sql
    #if (snow > 0):
    coopdb.query(sql)
    #print "%s,%s" % (now.strftime("%Y-%m-%d"), evap)

  now = now + interval

coopdb.query("COMMIT;")
print "Missing values", cnt
<commit_msg>more proper parsing, upstream<commit_after># I love writting data converters
# Daryl Herzmann 16 Jul 2003

import re, mx.DateTime, string
import sys
ST = sys.argv[1]
from pyIEM import stationTable, iemdb
i = iemdb.iemdb()
coopdb = i['coop']
coopdb.query("BEGIN;")

o = open("%s_dat.txt" % (ST,), 'r').readlines()
s = mx.DateTime.DateTime(1893,1,1)
e = mx.DateTime.DateTime(1951,1,1)
interval = mx.DateTime.RelativeDateTime(days=+1)

data = {}

for line in o[2:]:
  tokens = re.split(",", line)
  month = mx.DateTime.strptime(tokens[7], "%Y%m")
  var = tokens[5]
  coopid = "%s%s" % (ST.upper(), str(tokens[1][2:]))
  sid = coopid
  if (not data.has_key(sid)):
    data[sid] = {}
  for i in range(31):
    p = 9 + (i*4)
    val = tokens[p]
    ts = month + mx.DateTime.RelativeDateTime(day=(i+1))
    if (not data[sid].has_key(ts)):
      data[sid][ts] = {'TMAX':'Null', 'TMIN':'Null', 'SNOW':'Null', 'PRCP':'Null', 'SNWD': 'Null'}
    data[sid][ts][var] = val
    if (tokens[p+1] == "T"):
      #print "HERE"
      data[sid][ts][var] = 0.001

now = s
cnt = 0
while (now < e):
  for sid in data.keys():
    if not data[sid].has_key(now):
      continue
    if (data[sid][now]["TMAX"] != 'Null'):
      high = int(float(data[sid][now]["TMAX"]))
      if (high < -100 or high > 140):
        high = 'Null'
        cnt += 1
    else:
      high = data[sid][now]["TMAX"]
      cnt += 1

    if (data[sid][now]["TMIN"] != 'Null'):
      low = int(float(data[sid][now]["TMIN"]))
      if (low < -100 or low > 100):
        low = 'Null'
        cnt += 1
    else:
      low = data[sid][now]["TMIN"]
      cnt += 1

    if (data[sid][now]["PRCP"] != 'Null'):
      precip = int(float(data[sid][now]["PRCP"])) /100.0
      if (precip < 0 or precip > 100):
        precip = 'Null'
        cnt += 1
    else:
      precip =  data[sid][now]["PRCP"]
      cnt += 1

    if (data[sid][now]["SNOW"] not in ['Null','']):
      try:
        snow = int(float(data[sid][now]["SNOW"]))  /10.0
      except:
        print '|%s|' % (data[sid][now]["SNOW"],)
      if (snow < 0 or snow > 30):
        snow = 'Null'
        cnt += 1
    else:
      snow = data[sid][now]["SNOW"]
      cnt += 1

    if (data[sid][now]["SNWD"] != 'Null'):
      snowd = int(float(data[sid][now]["SNWD"]))
      if (snowd < 0 or snowd > 30):
        snowd = 'Null'
        cnt += 1
    else:
      snowd = 'Null'
      cnt += 1

    sql = """INSERT into alldata_%s (station, day, high, low, precip, snow,
           sday, year, month,snowd) values ('%s','%s',%s,%s,%s,%s,'%s',%s,%s,%s) """ % (ST.lower(), sid.upper(), now.strftime("%Y-%m-%d"), high, low, precip, snow,
           now.strftime("%m%d"), now.year, now.month, snowd)
    #sql = "UPDATE alldata SET snow = %s WHERE stationid = '%s' and day = '%s'" % (snow, sid.lower(), now.strftime("%Y-%m-%d"))
    #print sql
    #if (snow > 0):
    coopdb.query(sql)
    #print "%s,%s" % (now.strftime("%Y-%m-%d"), evap)

  now = now + interval

coopdb.query("COMMIT;")
print "Missing values", cnt
<|endoftext|>


<commit_before>import requests
import json
import bs4
import re
from ReallySecurePasswordModule import username, password
import argparse
import KijijiApi

def main():
   ##Start here
   #Takes: config(user/pass)
   #One of:
   #post adPostingFile
   #show
   #delete adId
   #showPresentAds adId
   #repost adPostingFile
   parser = argparse.ArgumentParser(
           description="Post ads on Kijiji")
   parser.add_argument('username', help='username of your kijiji account')
   parser.add_argument('password', help='password of your kijiji account')

   subparsers = parser.add_subparsers(help ='sub-command help')

   postParser = subparsers.add_parser('post', help='post a new ad')
   postParser.add_argument('inf_file', type=str, help='.inf file containing posting details')
   postParser.set_defaults(function=postAd)

   showParser = subparsers.add_parser('show', help='show currently listed ads')
   showParser.set_defaults(function=showAds)

   deleteParser = subparsers.add_parser('delete', help='delete a listed ad')
   deleteParser.add_argument('id',type=str, help='id of the ad you wish to delete')
   deleteParser.set_defaults(function=deleteAd)

   nukeParser = subparsers.add_parser('nuke', help='delete all ads')
   nukeParser.set_defaults(function=nuke)

   repostParser = subparsers.add_parser('repost', help='repost an existing ad')
   repostParser.add_argument('inf_file', type = str,help = '.inf file containing posting details')
   repostParser.set_defaults(function=repostAd)

   args = parser.parse_args()
   try:
       args.function(args)
   except AttributeError:
        parser.print_help()

def postAd(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    api.postAd(args.inf_file)

def showAds(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    [print(adId+","+adName) for adName, adId in api.getAllAds()]

def deleteAd(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    api.deleteAd(args.id)

#Try to delete ad with same name if possible
#post new ad
def repostAd(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    myAds = api.getAllAds()
    delAdName = ""
    for line in open(args.inf_file, 'rt'):
        [key, val] = line.strip().rstrip("\n").split("=")
        if key =='postAdForm.title':
            delAdName = val
    try:
        [api.deleteAd(adId) for adName, adId in api.getAllAds() if delAdName.strip() == adName]
    except DeleteAdException:
        pass
    api.postAd(args.inf_file)

def nuke(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    allAds = api.getAllAds()
    [api.deleteAd(adId) for adName, adId in allAds]

if __name__ == "__main__":
    main()
<commit_msg>Remove unused dependencies<commit_after>import argparse
import KijijiApi
import sys

if sys.version_info < (3, 0):
    raise Exception("This program requires Python 3.0 or greater")

def main():
   ##Start here
   #Takes: config(user/pass)
   #One of:
   #post adPostingFile
   #show
   #delete adId
   #showPresentAds adId
   #repost adPostingFile
   parser = argparse.ArgumentParser(
           description="Post ads on Kijiji")
   parser.add_argument('username', help='username of your kijiji account')
   parser.add_argument('password', help='password of your kijiji account')

   subparsers = parser.add_subparsers(help ='sub-command help')

   postParser = subparsers.add_parser('post', help='post a new ad')
   postParser.add_argument('inf_file', type=str, help='.inf file containing posting details')
   postParser.set_defaults(function=postAd)

   showParser = subparsers.add_parser('show', help='show currently listed ads')
   showParser.set_defaults(function=showAds)

   deleteParser = subparsers.add_parser('delete', help='delete a listed ad')
   deleteParser.add_argument('id',type=str, help='id of the ad you wish to delete')
   deleteParser.set_defaults(function=deleteAd)

   nukeParser = subparsers.add_parser('nuke', help='delete all ads')
   nukeParser.set_defaults(function=nuke)

   repostParser = subparsers.add_parser('repost', help='repost an existing ad')
   repostParser.add_argument('inf_file', type = str,help = '.inf file containing posting details')
   repostParser.set_defaults(function=repostAd)

   args = parser.parse_args()
   try:
       args.function(args)
   except AttributeError:
        parser.print_help()

def postAd(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    api.postAd(args.inf_file)

def showAds(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    [print(adId+","+adName) for adName, adId in api.getAllAds()]

def deleteAd(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    api.deleteAd(args.id)

#Try to delete ad with same name if possible
#post new ad
def repostAd(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    myAds = api.getAllAds()
    delAdName = ""
    for line in open(args.inf_file, 'rt'):
        [key, val] = line.strip().rstrip("\n").split("=")
        if key =='postAdForm.title':
            delAdName = val
    try:
        [api.deleteAd(adId) for adName, adId in api.getAllAds() if delAdName.strip() == adName]
    except DeleteAdException:
        pass
    api.postAd(args.inf_file)

def nuke(args):
    api = KijijiApi.KijijiApi()
    api.login(args.username, args.password)
    allAds = api.getAllAds()
    [api.deleteAd(adId) for adName, adId in allAds]

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # noqa pylint: disable=unused-import, redefined-builtin

import pytest
from jinja2 import Template

from flexget.plugins.parsers.parser_guessit import ParserGuessit
from flexget.plugins.parsers.parser_internal import ParserInternal
from flexget.utils.qualities import Quality

class TestUpgrade(object):
    _config = """
        templates:
          global:
            parsing:
              series: {{parser}}
              movie: {{parser}}
            mock:
              - {title: 'Smoke.1280x720'}
              - {title: 'Smoke.HDTV'}
              - {title: 'Smoke.cam'}
              - {title: 'Smoke.HR'}
            accept_all: yes
        tasks:
          qual:
            quality:
              - hdtv
              - 720p
          min:
            quality: HR+
          max:
            quality: "<=cam <HR"
          min_max:
            quality: HR-720i
    """

    @pytest.fixture(scope='class', params=['internal', 'guessit'], ids=['internal', 'guessit'])
    def config(self, request):
        """Override and parametrize default config fixture."""
        return Template(self._config).render({'parser': request.param})

    def test_quality(self, execute_task):
        task = execute_task('qual')
        entry = task.find_entry('rejected', title='Smoke.cam')
        assert entry, 'Smoke.cam should have been rejected'

        entry = task.find_entry(title='Smoke.1280x720')
        assert entry, 'entry not found?'
        assert entry in task.accepted, '720p should be accepted'
        assert len(task.rejected) == 2, 'wrong number of entries rejected'
        assert len(task.accepted) == 2, 'wrong number of entries accepted'

    def test_min(self, execute_task):
        task = execute_task('min')
        entry = task.find_entry('rejected', title='Smoke.HDTV')
        assert entry, 'Smoke.HDTV should have been rejected'

        entry = task.find_entry(title='Smoke.1280x720')
        assert entry, 'entry not found?'
        assert entry in task.accepted, '720p should be accepted'
        assert len(task.rejected) == 2, 'wrong number of entries rejected'
        assert len(task.accepted) == 2, 'wrong number of entries accepted'

    def test_max(self, execute_task):
        task = execute_task('max')
        entry = task.find_entry('rejected', title='Smoke.1280x720')
        assert entry, 'Smoke.1280x720 should have been rejected'

        entry = task.find_entry(title='Smoke.cam')
        assert entry, 'entry not found?'
        assert entry in task.accepted, 'cam should be accepted'
        assert len(task.rejected) == 3, 'wrong number of entries rejected'
        assert len(task.accepted) == 1, 'wrong number of entries accepted'

    def test_min_max(self, execute_task):
        task = execute_task('min_max')
        entry = task.find_entry('rejected', title='Smoke.1280x720')
        assert entry, 'Smoke.1280x720 should have been rejected'

        entry = task.find_entry(title='Smoke.HR')
        assert entry, 'entry not found?'
        assert entry in task.accepted, 'HR should be accepted'
        assert len(task.rejected) == 3, 'wrong number of entries rejected'
        assert len(task.accepted) == 1, 'wrong number of entries accepted'

class TestQualityAudio(object):
    config = """
        tasks:
          test_dd_audio_channels:
            quality: "dd+5.1"
            mock:
              - {title: 'My Show S01E05 720p HDTV DD+7.1'}
              - {title: 'My Show S01E05 720p HDTV DD+5.0'}
          test_dd_audio_min:
            quality: ">dd5.1"
            mock:
              - {title: 'My Show S01E05 720p HDTV DD5.1'}
              - {title: 'My Show S01E05 720p HDTV DD+2.0'}
          test_dd_audio_max:
            quality: "<=dd5.1"
            mock:
              - {title: 'My Show S01E05 720p HDTV DD5.1'}
              - {title: 'My Show S01E05 720p HDTV DD+5.1'}
              - {title: 'My Show S01E05 720p HDTV DD+7.1'}
    """

    def test_dd_audio_channels(self, execute_task):
        task = execute_task('test_dd_audio_channels')
        entry = task.find_entry('undecided', title='My Show S01E05 720p HDTV DD+7.1')
        assert entry, 'Entry "My Show S01E05 720p HDTV DD+7.1" should not have been rejected'
        assert entry['quality'].audio == 'dd+5.1', 'audio "dd+7.1" should have been parsed as dd+5.1'

        entry = task.find_entry('undecided', title='My Show S01E05 720p HDTV DD+5.0')
        assert entry['quality'].audio == 'dd+5.1', 'audio "dd+5.0" should have been parsed as dd+5.1'

    def test_dd_audio_min(self, execute_task):
        task = execute_task('test_dd_audio_min')
        assert len(task.rejected) == 1, 'should have rejected one'
        entry = task.find_entry('undecided', title='My Show S01E05 720p HDTV DD+2.0')
        assert entry, 'Entry "My Show S01E05 720p HDTV DD+2.0" should not have been rejected'
        assert entry['quality'].audio == 'dd+5.1', 'audio should have been parsed as dd+5.1'

    def test_dd_audio_max(self, execute_task):
        task = execute_task('test_dd_audio_max')
        assert len(task.rejected) == 2, 'should have rejected two'
        entry = task.find_entry('undecided', title='My Show S01E05 720p HDTV DD5.1')
        assert entry, 'Entry "My Show S01E05 720p HDTV DD5.1" should not have been rejected'
        assert entry['quality'].audio == 'dd5.1', 'audio should have been parsed as dd5.1'
<commit_msg>remove test_upgrade - commit by accident<commit_after><|endoftext|>


<commit_before>import unittest
import LiSE
import re
from collections import defaultdict
from LiSE.engine import crhandled_defaultdict
from examples import college as sim

class LiSETest(unittest.TestCase):
    maxDiff = None

    def setUp(self):
        """Start an engine, install the sim module, and run it a while.

        This gives us some world-state to test upon.

        """
        self.engine = LiSE.Engine(":memory:")
        sim.install(self.engine)
        for i in range(72):
            self.engine.next_tick()
        self.engine.commit()

    def tearDown(self):
        """Close my engine."""
        self.engine.close()

    def testCaches(self):
        rulebooks = defaultdict(list)
        for (rulebook, rule) in self.engine.rule.db.rulebooks_rules():
            rulebooks[rulebook].append(rule)
        self.assertEqual(rulebooks, self.engine._rulebooks_cache)
        charrb = {}
        for (
                character,
                character_rulebook,
                avatar_rulebook,
                character_thing_rulebook,
                character_place_rulebook,
                character_node_rulebook,
                character_portal_rulebook
        ) in self.engine.db.characters_rulebooks():
            charrb[character] = {
                'character': character_rulebook,
                'avatar': avatar_rulebook,
                'character_thing': character_thing_rulebook,
                'character_place': character_place_rulebook,
                'character_node': character_node_rulebook,
                'character_portal': character_portal_rulebook
            }
        self.assertEqual(
            charrb,
            self.engine._characters_rulebooks_cache
        )
        noderb = defaultdict(dict)
        for (character, node, rulebook) in self.engine.db.nodes_rulebooks():
            noderb[character][node] = rulebook
        self.assertEqual(
            self.engine._nodes_rulebooks_cache[character][node],
            noderb
        )
        portrb = defaultdict(
            lambda: defaultdict(dict)
        )
        for (character, nodeA, nodeB, rulebook) in self.engine.db.portals_rulebooks():
            portrb[character][nodeA][nodeB] = rulebook
        self.assertEqual(
            self.engine._portals_rulebooks_cache[character][nodeA][nodeB],
            portrb
        )
        db_avatarness = defaultdict(  # character:
            lambda: defaultdict(  # graph:
                lambda: defaultdict(  # node:
                    lambda: defaultdict(  # branch:
                        dict  # tick: is_avatar
                    )
                )
            )
        )
        user_avatarness = defaultdict(  # graph:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # character:
                    lambda: defaultdict(  # branch:
                        dict  # tick: is_avatar
                    )
                )
            )
        )
        for (character, graph, node, branch, tick, is_avatar) in self.engine.db.avatarness_dump():
            db_avatarness[character][graph][node][branch][tick] = is_avatar
            user_avatarness[graph][node][character][branch][tick] = is_avatar
        self.assertEqual(
            db_avatarness,
            self.engine._avatarness_cache.db_order
        )
        self.assertEqual(
            user_avatarness,
            self.engine._avatarness_cache.db_order
        )
        actrules = defaultdict(  # rulebook:
            lambda: defaultdict(  # rule:
                lambda: defaultdict(  # branch:
                    dict  # tick: active
                )
            )
        )
        for rulebook, rule, branch, tick, active in self.engine.db.dump_active_rules():
            actrules[rulebook][rule][branch][tick] = active
        self.assertEqual(
            actrules,
            self.engine._active_rules_cache
        )
        node_rules_handled_ticks = defaultdict(  # character:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # rulebook:
                    lambda: defaultdict(  # rule:
                        lambda: defaultdict(  # branch:
                            set  # ticks handled
                        )
                    )
                )
            )
        )
        for character, node, rulebook, rule, branch, tick in \
                self.engine.db.dump_node_rules_handled():
            node_rules_handled_ticks[
                character][node][rulebook][rule][branch].add(tick)
        self.assertEqual(
            self.engine._node_rules_handled_cache,
            node_rules_handled_ticks
        )
        portal_rules_handled_ticks = defaultdict(  # character:
            lambda: defaultdict(  # nodeA:
                lambda: defaultdict(  # nodeB:
                    lambda: defaultdict(  # rulebook:
                        lambda: defaultdict(  # rule:
                            lambda: defaultdict(  # branch:
                                set  # ticks handled
                            )
                        )
                    )
                )
            )
        )
        for (character, nodeA, nodeB, idx, rulebook, rule, branch, tick) \
                in self.engine.db.dump_portal_rules_handled():
            portal_rules_handled_ticks[
                character][nodeA][nodeB][rulebook][rule][branch].add(tick)
        self.assertEqual(
            self.engine._portal_rules_handled_cache,
            portal_rules_handled_ticks
        )
        for rulemap in [
                'character',
                'avatar',
                'character_thing',
                'character_place',
                'character_portal'
        ]:
            handled_ticks = crhandled_defaultdict()
            for character, rulebook, rule, branch, tick in getattr(
                    self.engine.db, 'handled_{}_rules'.format(rulemap)
            ):
                handled_ticks[character][rulebook][rule][branch].add(tick)
            self.assertEqual(
                handled_ticks,
                getattr(self.engine, '_{}_rules_handled_cache'.format(rulemap))
            )
        things = defaultdict(  # character:
            lambda: defaultdict(  # thing:
                lambda: defaultdict(  # branch:
                    dict  # tick: (location, next_location)
                )
            )
        )
        for (character, thing, branch, tick, loc, nextloc) in \
                self.engine.db.things_dump():
            things[character][thing][branch][tick] = (loc, nextloc)
        self.assertEqual(
            things,
            self.engine._things_cache
        )

    def testRoommateCollisions(self):
        """Test queries' ability to tell that all of the students that share rooms have been in the same place."""
        done = set()
        for chara in self.engine.character.values():
            if chara.name in done:
                continue
            match = re.match('dorm(\d)room(\d)student(\d)', chara.name)
            if not match:
                continue
            dorm, room, student = match.groups()
            other_student = 1 if student == 0 else 0
            student = chara
            other_student = self.engine.character['dorm{}room{}student{}'.format(dorm, room, other_student)]
            self.assertEqual(
                student.avatar.historical('location'),
                other_student.avatar.historical('location'),
                "Roommates don't seem to share a room: {}, {}".format(student, other_student)
            )
            done.add(student.name)
            done.add(other_student.name)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Split up the caches test a little<commit_after>import unittest
import LiSE
import re
from collections import defaultdict
from LiSE.engine import crhandled_defaultdict
from examples import college as sim

class LiSETest(unittest.TestCase):
    maxDiff = None

    def setUp(self):
        """Start an engine, install the sim module, and run it a while.

        This gives us some world-state to test upon.

        """
        self.engine = LiSE.Engine(":memory:")
        sim.install(self.engine)
        for i in range(72):
            self.engine.next_tick()
        self.engine.commit()

    def tearDown(self):
        """Close my engine."""
        self.engine.close()

    def testCharRulebooksCaches(self):
        rulebooks = defaultdict(list)
        for (rulebook, rule) in self.engine.rule.db.rulebooks_rules():
            rulebooks[rulebook].append(rule)
        self.assertEqual(rulebooks, self.engine._rulebooks_cache)
        charrb = {}
        for (
                character,
                character_rulebook,
                avatar_rulebook,
                character_thing_rulebook,
                character_place_rulebook,
                character_node_rulebook,
                character_portal_rulebook
        ) in self.engine.db.characters_rulebooks():
            charrb[character] = {
                'character': character_rulebook,
                'avatar': avatar_rulebook,
                'character_thing': character_thing_rulebook,
                'character_place': character_place_rulebook,
                'character_node': character_node_rulebook,
                'character_portal': character_portal_rulebook
            }
        self.assertEqual(
            charrb,
            self.engine._characters_rulebooks_cache
        )
        noderb = defaultdict(dict)
        for (character, node, rulebook) in self.engine.db.nodes_rulebooks():
            noderb[character][node] = rulebook
        self.assertEqual(
            self.engine._nodes_rulebooks_cache[character][node],
            noderb
        )
        portrb = defaultdict(
            lambda: defaultdict(dict)
        )
        for (character, nodeA, nodeB, rulebook) in self.engine.db.portals_rulebooks():
            portrb[character][nodeA][nodeB] = rulebook
        self.assertEqual(
            self.engine._portals_rulebooks_cache[character][nodeA][nodeB],
            portrb
        )

    def testAvatarnessCaches(self):
        db_avatarness = defaultdict(  # character:
            lambda: defaultdict(  # graph:
                lambda: defaultdict(  # node:
                    lambda: defaultdict(  # branch:
                        dict  # tick: is_avatar
                    )
                )
            )
        )
        user_avatarness = defaultdict(  # graph:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # character:
                    lambda: defaultdict(  # branch:
                        dict  # tick: is_avatar
                    )
                )
            )
        )
        for (character, graph, node, branch, tick, is_avatar) in self.engine.db.avatarness_dump():
            db_avatarness[character][graph][node][branch][tick] = is_avatar
            user_avatarness[graph][node][character][branch][tick] = is_avatar
        self.assertEqual(
            db_avatarness,
            self.engine._avatarness_cache.db_order
        )
        self.assertEqual(
            user_avatarness,
            self.engine._avatarness_cache.db_order
        )

    def testActiveRulesCache(self):
        actrules = defaultdict(  # rulebook:
            lambda: defaultdict(  # rule:
                lambda: defaultdict(  # branch:
                    dict  # tick: active
                )
            )
        )
        for rulebook, rule, branch, tick, active in self.engine.db.dump_active_rules():
            actrules[rulebook][rule][branch][tick] = active
        self.assertEqual(
            actrules,
            self.engine._active_rules_cache
        )

    def testRulesHandledCaches(self):
        node_rules_handled_ticks = defaultdict(  # character:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # rulebook:
                    lambda: defaultdict(  # rule:
                        lambda: defaultdict(  # branch:
                            set  # ticks handled
                        )
                    )
                )
            )
        )
        for character, node, rulebook, rule, branch, tick in \
                self.engine.db.dump_node_rules_handled():
            node_rules_handled_ticks[
                character][node][rulebook][rule][branch].add(tick)
        self.assertEqual(
            self.engine._node_rules_handled_cache,
            node_rules_handled_ticks
        )
        portal_rules_handled_ticks = defaultdict(  # character:
            lambda: defaultdict(  # nodeA:
                lambda: defaultdict(  # nodeB:
                    lambda: defaultdict(  # rulebook:
                        lambda: defaultdict(  # rule:
                            lambda: defaultdict(  # branch:
                                set  # ticks handled
                            )
                        )
                    )
                )
            )
        )
        for (character, nodeA, nodeB, idx, rulebook, rule, branch, tick) \
                in self.engine.db.dump_portal_rules_handled():
            portal_rules_handled_ticks[
                character][nodeA][nodeB][rulebook][rule][branch].add(tick)
        self.assertEqual(
            self.engine._portal_rules_handled_cache,
            portal_rules_handled_ticks
        )
        for rulemap in [
                'character',
                'avatar',
                'character_thing',
                'character_place',
                'character_portal'
        ]:
            handled_ticks = crhandled_defaultdict()
            for character, rulebook, rule, branch, tick in getattr(
                    self.engine.db, 'handled_{}_rules'.format(rulemap)
            ):
                handled_ticks[character][rulebook][rule][branch].add(tick)
            self.assertEqual(
                handled_ticks,
                getattr(self.engine, '_{}_rules_handled_cache'.format(rulemap))
            )
        things = defaultdict(  # character:
            lambda: defaultdict(  # thing:
                lambda: defaultdict(  # branch:
                    dict  # tick: (location, next_location)
                )
            )
        )
        for (character, thing, branch, tick, loc, nextloc) in \
                self.engine.db.things_dump():
            things[character][thing][branch][tick] = (loc, nextloc)
        self.assertEqual(
            things,
            self.engine._things_cache
        )

    def testRoommateCollisions(self):
        """Test queries' ability to tell that all of the students that share rooms have been in the same place."""
        done = set()
        for chara in self.engine.character.values():
            if chara.name in done:
                continue
            match = re.match('dorm(\d)room(\d)student(\d)', chara.name)
            if not match:
                continue
            dorm, room, student = match.groups()
            other_student = 1 if student == 0 else 0
            student = chara
            other_student = self.engine.character['dorm{}room{}student{}'.format(dorm, room, other_student)]
            self.assertEqual(
                student.avatar.historical('location'),
                other_student.avatar.historical('location'),
                "Roommates don't seem to share a room: {}, {}".format(student, other_student)
            )
            done.add(student.name)
            done.add(other_student.name)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#Requirements:
#cosmoutils
#
from utils import ioutils as io
import numpy as np
import matplotlib.pyplot as plt
import sys

#example input file for HACC
haccindat      =  "example_data/indat.params"
#haccdictfile   = io.builddict(haccindat , dictdelim = " " )
def haccredshiftsforstep(stepnumber ,
	zin = 200. ,
	zfinal  = 0. ,
	numsteps = 500) :

	"""Changed haccredshiftsforstep to match Adrian's time stepper doc"

	ain  =  1.0/(1.0 + zin)
	afinal  =  1.0/(1.0 + zfinal)
	astep =  (afinal - ain)/numsteps

	aatstep = astep * (stepnumber +1.)+ ain
	z = 1.0/aatstep - 1.0

	return z

def particlemass(
	numparticlescuberoot ,
	lengthcuberoot, # cube root of vol in h^{-1} Mpc
	Omega ,
	h = 1.0) :

	"""returns the mass assigned to N-Body particles of the particular
	species given the number of particles in the simulations, the length
	scale of simulation, and the relative density of the species wrt
	the critical density and the hubble parameter h = H0/100.
	"""

	args:
		numparticlescuberoot : float, mandatory
			cube root of the number of particles representing the
			species in the simulation,
			ie. numparticles= numparticlescuberoot^3

		lengthcuberoot : float, mandatory
			cube root of the length scale of the (assumed to be
			cubic) simulation in units of h^{-1} Mpc. Thus the
			vol of the simulation is (lengthcuberoot*h)^3 Mpc^3

		Omega : float, mandatory
			relative density of the species in units of critical
			density at z =0 . ie rho_{species} /rho_{crit}(z =0).

		h     : float, optional, defaults to 1.0
			H0/h used in simulation
	returns:
		mass assigned to particles of that species in the N-Body
		simulation in units of solar masses.

	example usage:
		>>> print particlemass (numparticlescuberoot = 256 ,
			lengthcuberoot = 512 ,
			Omega = 0.25 ,
			h = 0.7)
		>>> 93265221934.8

	status:
		Not tested,
	todo:
		Test with one known case in HACC

	"""
	from astropy import units as u
	from cosmoutils import critdensity

	#actual rhocrit0 = rhocrit0 * h^2
	rhocrit0 = critdensity(h = 1., unittype = 'solarmassperMpc3')
	#units of solar masses/ Mpc^3  / h^2
	#print "RHOCRIT", rhocrit0
	rho = Omega * rhocrit0

	numparticles = numparticlescuberoot**3.0
		#Get num density in m^{-3}
	#vol = (lengthcuberoot * h) **3.0  * (u.Mpc.to(u.m))**3.0
	#numdensity =  numparticles / vol

	#number density in number/ (h^{-1} Mpc) ^3
	numdensity = (numparticlescuberoot/lengthcuberoot)**3.0

		#particle mass in h^{-1} solar masses
	particlemass = rho / numdensity

		#particle mass in solar masses
	#convtosolmass =  u.kg.to(u.solMass)
	return particlemass #*convtosolmass

def simdict( haccinputdict) :

	simvariables = ['RL' , 'NP' , 'Z_IN', 'N_STEPS', 'Z_FIN']
	haccsim = {}
	for key in haccinputdict :
		if key in simvariables :
			#print key, float(haccinputdict[key])
			haccsim[key] = float( haccinputdict[key])

	return haccsim

def hacccosmologydict(haccinputdict ) :

	cosmologyvariables =  ['HUBBLE', 'OMEGA_CDM','DEUT','OMEGA_NU' ,'W_DE','WA_DE', 'SS8', 'NS']
	hacccosmologydict = {}
	for key in haccinputdict :
		if key in cosmologyvariables :
			hacccosmologydict[key] = float(haccinputdict[key])

	return hacccosmologydict

def _hacccosmo(inputfile ) :
		from utils import ioutils as io

		haccdict   = io.builddict(inputfile , dictdelim = " " )

		#print haccdict
		#print haccdict['OMEGA_NU']
		#sys.exit()
		hacc = hacccosmologydict (haccdict)

		h = hacc['HUBBLE']
		H0 = 100.0*h

		Omega_CDM= hacc['OMEGA_CDM']
		Omega_nu = hacc['OMEGA_NU']
		Omega_b = hacc['DEUT']/h/h
		Omega_m = Omega_CDM + Omega_b + Omega_nu

		w0 = hacc['W_DE']
		wa = hacc['WA_DE']
		sigma8  = hacc['SS8']
		ns = hacc['NS']

		return (H0 , Omega_m , Omega_nu, Omega_b , w0, wa , sigma8, ns) #check

class haccsim (object ) :

	def __init__(self, indatfile ,
		name  = None ,
		TKFile = None ) :

		if name == None:
			self.name = ""
		else:
			self.name = name
		haccdict = io.builddict(indatfile , dictdelim = " " )

		#set cosmo:
		from interfaces import FCPL
		H0 , Omega_m , Omega_nu , Omega_b, w0, wa, sigma8  , ns =  _hacccosmo(indatfile)
		sigma_mnu = Omega_nu*94.0
		#print sigma_mnu
		#sys.exit()

		self.cosmo = FCPL ( H0 = H0 , Om0 = Omega_m , Ob0 = Omega_b ,
			sigmamnu = sigma_mnu , w0 = w0, wa = wa,
			sigma8 = sigma8  , ns = ns)

		#set sim properties
		haccsimdict = simdict (haccdict )

			#set massresolution
		Omegacb0 = self.cosmo.Ob0 + self.cosmo.Oc0
		self._boxsize = haccsimdict ['RL']
		self._numparticlescuberoot = haccsimdict['NP']
		self._particlemass = particlemass(Omega = Omegacb0 ,
			lengthcuberoot = self._boxsize,
			numparticlescuberoot = self._numparticlescuberoot ,
			h = self.cosmo.h )

			#set time stepping info
		self._zin = haccsimdict["Z_IN"]
		self._zf  = haccsimdict["Z_FIN"]
		self._numtimesteps = haccsimdict["N_STEPS"]

	@property
	def numparticles (self) :
		npcuberoot = self._numparticlescuberoot
		return npcuberoot * npcuberoot * npcuberoot

	@property
	def particlemass (self) :
		return self._particlemass

	@property
	def boxsize (self) :
		return self._boxsize

	@property
	def simvolume (self) :
		l =  self._boxsize
		return l*l*l

	@property
	def initialredshift (self ) :
		return self._zin

	@property
	def numtimesteps (self ) :
		return self._numtimesteps

	@property
	def finalredshift (self ) :
		return self._zf

	#Methods

	def steptoredshift(self , step ) :

		zin =  self.initialredshift
		#print zin
		zf  =  self.finalredshift
		#print zf
		numsteps = self.numtimesteps

		return  haccredshiftsforstep(step ,
			zin = zin ,
			zfinal  = zf ,
			numsteps = numsteps)

	def summary (self , logfilename = None) :

		s  = "SUMMARY OF simulation, name = "+ self.name
		s += "\n\n========================================\n\n"
		s += "Cosmology\n====================\n"
		s += "massive neutrino energy density " + str(self.cosmo.sigmamnu/94.0) + "\n"
		s += "massive neutrino energy density " + str(self.cosmo.On0) + "\n"
		s += "Simulation Properties\n=====================\n\n"

		s += "Simulation Volume ("+ "{:.2e}".format(self._boxsize) +r'$)^3 h^{-3} Mpc^3$'+" \n"
		s += "Particle Mass = "+ "{:.2e}".format(self.particlemass)+"r'$h^{-1} M_\odot'$\n"
		if logfilename != None:

			f = open(logfilename, "w")
			f.write(s)
			f.close()

		return s

def tests() :

	if len(sys.argv) <2:
		haccindat = "example_data/indat.params"
		#"/media/star1/simulations/Neutrinos/M000n1/output"
	else:
		haccindat = sys.argv[1]

	hacc = haccsim ( haccindat, name = "M000")

	print hacc.summary()
	#print hacccosmologydict(haccindat)
	sys.exit()
	print hacc.cosmo.Om0
	print "particle mass in sim is ", hacc.particlemass
	print "mass res in sim is ", "{:.2e}".format(hacc.particlemass )
	print type(hacc.initialredshift)
	print hacc.finalredshift
	#print hacc.steptoredshift ( 499)
	print hacc.steptoredshift ( 161)

	print hacc.cosmo.growth(1.0)
	z  = hacc.steptoredshift ( np.arange(1,500,1))
	plt.plot ( 1.0/(1.0 + z) , hacc.cosmo.growth(z)[0] , 'r')
	plt.plot ( 1.0/(1.0 + z) ,1.0/(1.0 + z) , ls = 'dashed')
	plt.xlim(0.5,1.0)
	plt.show()

if __name__=="__main__":

	tests()

<commit_msg>fixed docstrings in hacc<commit_after>#!/usr/bin/env python
#Requirements:
#cosmoutils
#
from utils import ioutils as io
import numpy as np
import matplotlib.pyplot as plt
import sys

#example input file for HACC
haccindat      =  "example_data/indat.params"
#haccdictfile   = io.builddict(haccindat , dictdelim = " " )
def haccredshiftsforstep(stepnumber ,
	zin = 200. ,
	zfinal  = 0. ,
	numsteps = 500) :

	"""Changed haccredshiftsforstep to match Adrian's time stepper doc"""

	ain  =  1.0/(1.0 + zin)
	afinal  =  1.0/(1.0 + zfinal)
	astep =  (afinal - ain)/numsteps

	aatstep = astep * (stepnumber +1.)+ ain
	z = 1.0/aatstep - 1.0

	return z

def particlemass(
	numparticlescuberoot ,
	lengthcuberoot, # cube root of vol in h^{-1} Mpc
	Omega ,
	h = 1.0) :

	"""returns the mass assigned to N-Body particles of the particular
	species given the number of particles in the simulations, the length
	scale of simulation, and the relative density of the species wrt
	the critical density and the hubble parameter h = H0/100.
	"""

	args:
		numparticlescuberoot : float, mandatory
			cube root of the number of particles representing the
			species in the simulation,
			ie. numparticles= numparticlescuberoot^3

		lengthcuberoot : float, mandatory
			cube root of the length scale of the (assumed to be
			cubic) simulation in units of h^{-1} Mpc. Thus the
			vol of the simulation is (lengthcuberoot*h)^3 Mpc^3

		Omega : float, mandatory
			relative density of the species in units of critical
			density at z =0 . ie rho_{species} /rho_{crit}(z =0).

		h     : float, optional, defaults to 1.0
			H0/h used in simulation
	returns:
		mass assigned to particles of that species in the N-Body
		simulation in units of solar masses.

	example usage:
		>>> print particlemass (numparticlescuberoot = 256 ,
			lengthcuberoot = 512 ,
			Omega = 0.25 ,
			h = 0.7)
		>>> 93265221934.8

	status:
		Not tested,
	todo:
		Test with one known case in HACC

	"""
	from astropy import units as u
	from cosmoutils import critdensity

	#actual rhocrit0 = rhocrit0 * h^2
	rhocrit0 = critdensity(h = 1., unittype = 'solarmassperMpc3')
	#units of solar masses/ Mpc^3  / h^2
	#print "RHOCRIT", rhocrit0
	rho = Omega * rhocrit0

	numparticles = numparticlescuberoot**3.0
		#Get num density in m^{-3}
	#vol = (lengthcuberoot * h) **3.0  * (u.Mpc.to(u.m))**3.0
	#numdensity =  numparticles / vol

	#number density in number/ (h^{-1} Mpc) ^3
	numdensity = (numparticlescuberoot/lengthcuberoot)**3.0

		#particle mass in h^{-1} solar masses
	particlemass = rho / numdensity

		#particle mass in solar masses
	#convtosolmass =  u.kg.to(u.solMass)
	return particlemass #*convtosolmass

def simdict( haccinputdict) :

	simvariables = ['RL' , 'NP' , 'Z_IN', 'N_STEPS', 'Z_FIN']
	haccsim = {}
	for key in haccinputdict :
		if key in simvariables :
			#print key, float(haccinputdict[key])
			haccsim[key] = float( haccinputdict[key])

	return haccsim

def hacccosmologydict(haccinputdict ) :

	cosmologyvariables =  ['HUBBLE', 'OMEGA_CDM','DEUT','OMEGA_NU' ,'W_DE','WA_DE', 'SS8', 'NS']
	hacccosmologydict = {}
	for key in haccinputdict :
		if key in cosmologyvariables :
			hacccosmologydict[key] = float(haccinputdict[key])

	return hacccosmologydict

def _hacccosmo(inputfile ) :
		from utils import ioutils as io

		haccdict   = io.builddict(inputfile , dictdelim = " " )

		#print haccdict
		#print haccdict['OMEGA_NU']
		#sys.exit()
		hacc = hacccosmologydict (haccdict)

		h = hacc['HUBBLE']
		H0 = 100.0*h

		Omega_CDM= hacc['OMEGA_CDM']
		Omega_nu = hacc['OMEGA_NU']
		Omega_b = hacc['DEUT']/h/h
		Omega_m = Omega_CDM + Omega_b + Omega_nu

		w0 = hacc['W_DE']
		wa = hacc['WA_DE']
		sigma8  = hacc['SS8']
		ns = hacc['NS']

		return (H0 , Omega_m , Omega_nu, Omega_b , w0, wa , sigma8, ns) #check

class haccsim (object ) :

	def __init__(self, indatfile ,
		name  = None ,
		TKFile = None ) :

		if name == None:
			self.name = ""
		else:
			self.name = name
		haccdict = io.builddict(indatfile , dictdelim = " " )

		#set cosmo:
		from interfaces import FCPL
		H0 , Omega_m , Omega_nu , Omega_b, w0, wa, sigma8  , ns =  _hacccosmo(indatfile)
		sigma_mnu = Omega_nu*94.0
		#print sigma_mnu
		#sys.exit()

		self.cosmo = FCPL ( H0 = H0 , Om0 = Omega_m , Ob0 = Omega_b ,
			sigmamnu = sigma_mnu , w0 = w0, wa = wa,
			sigma8 = sigma8  , ns = ns)

		#set sim properties
		haccsimdict = simdict (haccdict )

			#set massresolution
		Omegacb0 = self.cosmo.Ob0 + self.cosmo.Oc0
		self._boxsize = haccsimdict ['RL']
		self._numparticlescuberoot = haccsimdict['NP']
		self._particlemass = particlemass(Omega = Omegacb0 ,
			lengthcuberoot = self._boxsize,
			numparticlescuberoot = self._numparticlescuberoot ,
			h = self.cosmo.h )

			#set time stepping info
		self._zin = haccsimdict["Z_IN"]
		self._zf  = haccsimdict["Z_FIN"]
		self._numtimesteps = haccsimdict["N_STEPS"]

	@property
	def numparticles (self) :
		npcuberoot = self._numparticlescuberoot
		return npcuberoot * npcuberoot * npcuberoot

	@property
	def particlemass (self) :
		return self._particlemass

	@property
	def boxsize (self) :
		return self._boxsize

	@property
	def simvolume (self) :
		l =  self._boxsize
		return l*l*l

	@property
	def initialredshift (self ) :
		return self._zin

	@property
	def numtimesteps (self ) :
		return self._numtimesteps

	@property
	def finalredshift (self ) :
		return self._zf

	#Methods

	def steptoredshift(self , step ) :

		zin =  self.initialredshift
		#print zin
		zf  =  self.finalredshift
		#print zf
		numsteps = self.numtimesteps

		return  haccredshiftsforstep(step ,
			zin = zin ,
			zfinal  = zf ,
			numsteps = numsteps)

	def summary (self , logfilename = None) :

		s  = "SUMMARY OF simulation, name = "+ self.name
		s += "\n\n========================================\n\n"
		s += "Cosmology\n====================\n"
		s += "massive neutrino energy density " + str(self.cosmo.sigmamnu/94.0) + "\n"
		s += "massive neutrino energy density " + str(self.cosmo.On0) + "\n"
		s += "Simulation Properties\n=====================\n\n"

		s += "Simulation Volume ("+ "{:.2e}".format(self._boxsize) +r'$)^3 h^{-3} Mpc^3$'+" \n"
		s += "Particle Mass = "+ "{:.2e}".format(self.particlemass)+"r'$h^{-1} M_\odot'$\n"
		if logfilename != None:

			f = open(logfilename, "w")
			f.write(s)
			f.close()

		return s

def tests() :

	if len(sys.argv) <2:
		haccindat = "example_data/indat.params"
		#"/media/star1/simulations/Neutrinos/M000n1/output"
	else:
		haccindat = sys.argv[1]

	hacc = haccsim ( haccindat, name = "M000")

	print hacc.summary()
	#print hacccosmologydict(haccindat)
	sys.exit()
	print hacc.cosmo.Om0
	print "particle mass in sim is ", hacc.particlemass
	print "mass res in sim is ", "{:.2e}".format(hacc.particlemass )
	print type(hacc.initialredshift)
	print hacc.finalredshift
	#print hacc.steptoredshift ( 499)
	print hacc.steptoredshift ( 161)

	print hacc.cosmo.growth(1.0)
	z  = hacc.steptoredshift ( np.arange(1,500,1))
	plt.plot ( 1.0/(1.0 + z) , hacc.cosmo.growth(z)[0] , 'r')
	plt.plot ( 1.0/(1.0 + z) ,1.0/(1.0 + z) , ls = 'dashed')
	plt.xlim(0.5,1.0)
	plt.show()

if __name__=="__main__":

	tests()

<|endoftext|>


<commit_before>import os
import sys
from collections import namedtuple

from conans import tools

from cpt import __version__ as package_tools_version
from cpt.printer import Printer
from cpt.profiles import load_profile, patch_default_base_profile

class CreateRunner(object):

    def __init__(self, profile_abs_path, reference, conan_api, uploader, args=None,
                 exclude_vcvars_precommand=False, build_policy=None, runner=None,
                 abs_folder=None, printer=None, upload=False):

        self.printer = printer or Printer()
        self._abs_folder = abs_folder or os.getcwd()
        self._uploader = uploader
        self._upload = upload
        self._conan_api = conan_api
        self._profile_abs_path = profile_abs_path
        self._reference = reference
        self._args = args
        self._exclude_vcvars_precommand = exclude_vcvars_precommand
        self._build_policy = build_policy
        self._runner = PrintRunner(runner or os.system, self.printer)
        self._uploader.remote_manager.add_remotes_to_conan()

        patch_default_base_profile(conan_api, profile_abs_path)
        self._profile = load_profile(profile_abs_path, self._conan_api._client_cache)

    @property
    def settings(self):
        return self._profile.settings

    def run(self):
        context = tools.no_op()
        compiler = self.settings.get("compiler", None)
        if not self._exclude_vcvars_precommand:
            if compiler == "Visual Studio" and "compiler.version" in self.settings:
                compiler_set = namedtuple("compiler", "version")(self.settings["compiler.version"])
                mock_sets = namedtuple("mock_settings",
                                       "arch compiler get_safe")(self.settings["arch"], compiler_set,
                                                                 lambda x: self.settings.get(x, None))
                context = tools.vcvars(mock_sets)
        with context:
            self.printer.print_rule()
            self.printer.print_profile(tools.load(self._profile_abs_path))

            with self.printer.foldable_output("conan_create"):
                name, version, user, channel = self._reference
                # print_dict
                # TODO: Get uploaded packages with Conan 1.3 from the ret json

                # FIXME: chdir Can be removed in 1.3, fixed issue about api changing curdir
                with tools.chdir(self._abs_folder):
                    if self._build_policy:
                        self._build_policy = [self._build_policy]
                    # https://github.com/conan-io/conan-package-tools/issues/184
                    with tools.environment_append({"_CONAN_CREATE_COMMAND_": "1"}):
                        params = {"name": name, "version": version, "user": user,
                                  "channel": channel, "build_modes": self._build_policy,
                                  "profile_name": self._profile_abs_path}
                        self.printer.print_message("Calling 'conan create'")
                        self.printer.print_dict(params)

                        self._conan_api.create(".", name=name, version=version,
                                               user=user, channel=channel,
                                               build_modes=self._build_policy,
                                               profile_name=self._profile_abs_path)

                self._uploader.upload_packages(self._reference, self._upload)

class DockerCreateRunner(object):
    def __init__(self, profile_text, base_profile_text, base_profile_name, reference,
                 args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=None,
                 sudo_pip_command=True,
                 docker_image_skip_update=False, build_policy=None,
                 docker_image_skip_pull=False,
                 always_update_conan_in_docker=False,
                 upload=False, runner=None,
                 docker_shell=None, docker_conan_home=None):

        self.printer = Printer()
        self._args = args
        self._upload = upload
        self._reference = reference
        self._conan_pip_package = conan_pip_package
        self._build_policy = build_policy
        self._docker_image = docker_image
        self._always_update_conan_in_docker = always_update_conan_in_docker
        self._docker_image_skip_update = docker_image_skip_update
        self._docker_image_skip_pull = docker_image_skip_pull
        self._sudo_docker_command = sudo_docker_command or ""
        self._sudo_pip_command = sudo_pip_command
        self._profile_text = profile_text
        self._base_profile_text = base_profile_text
        self._base_profile_name = base_profile_name
        self._docker_shell = docker_shell
        self._docker_conan_home = docker_conan_home
        self._runner = PrintRunner(runner, self.printer)

    def _pip_update_conan_command(self):
        commands = []
        # Hack for testing when retriving cpt from artifactory repo
        if "conan-package-tools" not in self._conan_pip_package:
            commands.append("%s pip install conan_package_tools==%s "
                            "--upgrade --no-cache" % (self._sudo_pip_command,
                                                      package_tools_version))

        if self._conan_pip_package:
            commands.append("%s pip install %s --no-cache" % (self._sudo_pip_command,
                                                              self._conan_pip_package))
        else:
            commands.append("%s pip install conan --upgrade --no-cache" % self._sudo_pip_command)

        command = " && ".join(commands)
        return command

    def run(self, pull_image=True, docker_entry_script=None):
        envs = self.get_env_vars()
        env_vars_text = " ".join(['-e %s="%s"' % (key, value)
                                        for key, value in envs.items() if value])

        env_vars_text += " -e PIP_INDEX_URL -e PIP_EXTRA_INDEX_URL"

        # Run the build
        if pull_image:
            if not self._docker_image_skip_pull:
                self.pull_image()
            if not self._docker_image_skip_update and not self._always_update_conan_in_docker:
                # Update the downloaded image
                with self.printer.foldable_output("update conan"):
                    try:
                        command = '%s docker run %s --name conan_runner ' \
                                  ' %s %s "%s"' % (self._sudo_docker_command,
                                                   env_vars_text,
                                                   self._docker_image,
                                                   self._docker_shell,
                                                   self._pip_update_conan_command())

                        ret = self._runner(command)
                        if ret != 0:
                            raise Exception("Error updating the image: %s" % command)
                        # Save the image with the updated installed
                        # packages and remove the intermediate container
                        command = "%s docker commit conan_runner %s" % (self._sudo_docker_command,
                                                                        self._docker_image)
                        ret = self._runner(command)
                        if ret != 0:
                            raise Exception("Error commiting the image: %s" % command)
                    finally:
                        command = "%s docker rm conan_runner" % self._sudo_docker_command
                        ret = self._runner(command)
                        if ret != 0:
                            raise Exception("Error removing the temp container: %s" % command)

        if self._always_update_conan_in_docker:
            update_command = self._pip_update_conan_command() + " && "
        else:
            update_command = ""
        command = ('%s docker run --rm -v %s:%s/project %s %s %s '
                   '"cd project && '
                   '%s run_create_in_docker "' % (self._sudo_docker_command,
                                                  os.getcwd(),
                                                  self._docker_conan_home,
                                                  env_vars_text,
                                                  self._docker_image,
                                                  self._docker_shell,
                                                  update_command))

        # Push entry command before to build
        if docker_entry_script:
            command = command.replace("run_create_in_docker",
                                      "%s && run_create_in_docker" % docker_entry_script)

        self.printer.print_in_docker(self._docker_image)
        ret = self._runner(command)
        if ret != 0:
            raise Exception("Error building: %s" % command)
        self.printer.print_message("Exiting docker...")

    def pull_image(self):
        with self.printer.foldable_output("docker pull"):
            ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image))
            if ret != 0:
                raise Exception("Error pulling the image: %s" % self._docker_image)

    def get_env_vars(self):
        ret = {key: value for key, value in os.environ.items() if key.startswith("CONAN_") and
               key != "CONAN_USER_HOME"}
        ret["CPT_ARGS"] = escape_env(self._args)
        ret["CONAN_REFERENCE"] = self._reference

        ret["CPT_PROFILE"] = escape_env(self._profile_text)
        ret["CPT_BASE_PROFILE"] = escape_env(self._base_profile_text)
        ret["CPT_BASE_PROFILE_NAME"] = escape_env(self._base_profile_name)

        ret["CONAN_USERNAME"] = escape_env(self._reference.user)
        ret["CONAN_TEMP_TEST_FOLDER"] = "1"  # test package folder to a temp one
        ret["CPT_UPLOAD_ENABLED"] = self._upload
        ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)

        return ret

def unscape_env(text):
    if not text:
        return text
    return text.replace("@@", "\n").replace('||', '"')

def escape_env(text):
    if not text:
        return text
    return text.replace("\n", "@@").replace('"', '||')

class PrintRunner(object):

    def __init__(self, runner, printer):
        self.runner = runner
        self.printer = printer

    def __call__(self, command):
        self.printer.print_command(command)
        sys.stderr.flush()
        sys.stdout.flush()
        return self.runner(command)
<commit_msg>propogate all PIP vars to container (#219)<commit_after>import os
import sys
from collections import namedtuple

from conans import tools

from cpt import __version__ as package_tools_version
from cpt.printer import Printer
from cpt.profiles import load_profile, patch_default_base_profile

class CreateRunner(object):

    def __init__(self, profile_abs_path, reference, conan_api, uploader, args=None,
                 exclude_vcvars_precommand=False, build_policy=None, runner=None,
                 abs_folder=None, printer=None, upload=False):

        self.printer = printer or Printer()
        self._abs_folder = abs_folder or os.getcwd()
        self._uploader = uploader
        self._upload = upload
        self._conan_api = conan_api
        self._profile_abs_path = profile_abs_path
        self._reference = reference
        self._args = args
        self._exclude_vcvars_precommand = exclude_vcvars_precommand
        self._build_policy = build_policy
        self._runner = PrintRunner(runner or os.system, self.printer)
        self._uploader.remote_manager.add_remotes_to_conan()

        patch_default_base_profile(conan_api, profile_abs_path)
        self._profile = load_profile(profile_abs_path, self._conan_api._client_cache)

    @property
    def settings(self):
        return self._profile.settings

    def run(self):
        context = tools.no_op()
        compiler = self.settings.get("compiler", None)
        if not self._exclude_vcvars_precommand:
            if compiler == "Visual Studio" and "compiler.version" in self.settings:
                compiler_set = namedtuple("compiler", "version")(self.settings["compiler.version"])
                mock_sets = namedtuple("mock_settings",
                                       "arch compiler get_safe")(self.settings["arch"], compiler_set,
                                                                 lambda x: self.settings.get(x, None))
                context = tools.vcvars(mock_sets)
        with context:
            self.printer.print_rule()
            self.printer.print_profile(tools.load(self._profile_abs_path))

            with self.printer.foldable_output("conan_create"):
                name, version, user, channel = self._reference
                # print_dict
                # TODO: Get uploaded packages with Conan 1.3 from the ret json

                # FIXME: chdir Can be removed in 1.3, fixed issue about api changing curdir
                with tools.chdir(self._abs_folder):
                    if self._build_policy:
                        self._build_policy = [self._build_policy]
                    # https://github.com/conan-io/conan-package-tools/issues/184
                    with tools.environment_append({"_CONAN_CREATE_COMMAND_": "1"}):
                        params = {"name": name, "version": version, "user": user,
                                  "channel": channel, "build_modes": self._build_policy,
                                  "profile_name": self._profile_abs_path}
                        self.printer.print_message("Calling 'conan create'")
                        self.printer.print_dict(params)

                        self._conan_api.create(".", name=name, version=version,
                                               user=user, channel=channel,
                                               build_modes=self._build_policy,
                                               profile_name=self._profile_abs_path)

                self._uploader.upload_packages(self._reference, self._upload)

class DockerCreateRunner(object):
    def __init__(self, profile_text, base_profile_text, base_profile_name, reference,
                 args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=None,
                 sudo_pip_command=True,
                 docker_image_skip_update=False, build_policy=None,
                 docker_image_skip_pull=False,
                 always_update_conan_in_docker=False,
                 upload=False, runner=None,
                 docker_shell=None, docker_conan_home=None):

        self.printer = Printer()
        self._args = args
        self._upload = upload
        self._reference = reference
        self._conan_pip_package = conan_pip_package
        self._build_policy = build_policy
        self._docker_image = docker_image
        self._always_update_conan_in_docker = always_update_conan_in_docker
        self._docker_image_skip_update = docker_image_skip_update
        self._docker_image_skip_pull = docker_image_skip_pull
        self._sudo_docker_command = sudo_docker_command or ""
        self._sudo_pip_command = sudo_pip_command
        self._profile_text = profile_text
        self._base_profile_text = base_profile_text
        self._base_profile_name = base_profile_name
        self._docker_shell = docker_shell
        self._docker_conan_home = docker_conan_home
        self._runner = PrintRunner(runner, self.printer)

    def _pip_update_conan_command(self):
        commands = []
        # Hack for testing when retriving cpt from artifactory repo
        if "conan-package-tools" not in self._conan_pip_package:
            commands.append("%s pip install conan_package_tools==%s "
                            "--upgrade --no-cache" % (self._sudo_pip_command,
                                                      package_tools_version))

        if self._conan_pip_package:
            commands.append("%s pip install %s --no-cache" % (self._sudo_pip_command,
                                                              self._conan_pip_package))
        else:
            commands.append("%s pip install conan --upgrade --no-cache" % self._sudo_pip_command)

        command = " && ".join(commands)
        return command

    def run(self, pull_image=True, docker_entry_script=None):
        envs = self.get_env_vars()
        env_vars_text = " ".join(['-e %s="%s"' % (key, value)
                                 for key, value in envs.items() if value])

        # Run the build
        if pull_image:
            if not self._docker_image_skip_pull:
                self.pull_image()
            if not self._docker_image_skip_update and not self._always_update_conan_in_docker:
                # Update the downloaded image
                with self.printer.foldable_output("update conan"):
                    try:
                        command = '%s docker run %s --name conan_runner ' \
                                  ' %s %s "%s"' % (self._sudo_docker_command,
                                                   env_vars_text,
                                                   self._docker_image,
                                                   self._docker_shell,
                                                   self._pip_update_conan_command())

                        ret = self._runner(command)
                        if ret != 0:
                            raise Exception("Error updating the image: %s" % command)
                        # Save the image with the updated installed
                        # packages and remove the intermediate container
                        command = "%s docker commit conan_runner %s" % (self._sudo_docker_command,
                                                                        self._docker_image)
                        ret = self._runner(command)
                        if ret != 0:
                            raise Exception("Error commiting the image: %s" % command)
                    finally:
                        command = "%s docker rm conan_runner" % self._sudo_docker_command
                        ret = self._runner(command)
                        if ret != 0:
                            raise Exception("Error removing the temp container: %s" % command)

        if self._always_update_conan_in_docker:
            update_command = self._pip_update_conan_command() + " && "
        else:
            update_command = ""
        command = ('%s docker run --rm -v %s:%s/project %s %s %s '
                   '"cd project && '
                   '%s run_create_in_docker "' % (self._sudo_docker_command,
                                                  os.getcwd(),
                                                  self._docker_conan_home,
                                                  env_vars_text,
                                                  self._docker_image,
                                                  self._docker_shell,
                                                  update_command))

        # Push entry command before to build
        if docker_entry_script:
            command = command.replace("run_create_in_docker",
                                      "%s && run_create_in_docker" % docker_entry_script)

        self.printer.print_in_docker(self._docker_image)
        ret = self._runner(command)
        if ret != 0:
            raise Exception("Error building: %s" % command)
        self.printer.print_message("Exiting docker...")

    def pull_image(self):
        with self.printer.foldable_output("docker pull"):
            ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image))
            if ret != 0:
                raise Exception("Error pulling the image: %s" % self._docker_image)

    def get_env_vars(self):
        ret = {key: value for key, value in os.environ.items() if key.startswith("CONAN_") and
               key != "CONAN_USER_HOME"}
        ret["CPT_ARGS"] = escape_env(self._args)
        ret["CONAN_REFERENCE"] = self._reference

        ret["CPT_PROFILE"] = escape_env(self._profile_text)
        ret["CPT_BASE_PROFILE"] = escape_env(self._base_profile_text)
        ret["CPT_BASE_PROFILE_NAME"] = escape_env(self._base_profile_name)

        ret["CONAN_USERNAME"] = escape_env(self._reference.user)
        ret["CONAN_TEMP_TEST_FOLDER"] = "1"  # test package folder to a temp one
        ret["CPT_UPLOAD_ENABLED"] = self._upload
        ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)

        ret.update({key: value for key, value in os.environ.items() if key.startswith("PIP_")})

        return ret

def unscape_env(text):
    if not text:
        return text
    return text.replace("@@", "\n").replace('||', '"')

def escape_env(text):
    if not text:
        return text
    return text.replace("\n", "@@").replace('"', '||')

class PrintRunner(object):

    def __init__(self, runner, printer):
        self.runner = runner
        self.printer = printer

    def __call__(self, command):
        self.printer.print_command(command)
        sys.stderr.flush()
        sys.stdout.flush()
        return self.runner(command)
<|endoftext|>


<commit_before># -*- test-case-name: flocker.node.agents.functional.test_cinder,flocker.node.agents.functional.test_cinder_behaviour -*- # noqa
# Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
A Cinder implementation of the ``IBlockDeviceAPI``.
"""
import time
from uuid import UUID
from subprocess import check_output

from bitmath import Byte, GB

from eliot import Message, start_action

from pyrsistent import PRecord, field

from keystoneclient.openstack.common.apiclient.exceptions import (
    NotFound as CinderNotFound,
)
from novaclient.exceptions import NotFound as NovaNotFound

from twisted.python.filepath import FilePath

from zope.interface import implementer, Interface

from ...common import auto_openstack_logging
from .blockdevice import (
    IBlockDeviceAPI, BlockDeviceVolume, UnknownVolume, AlreadyAttachedVolume,
    UnattachedVolume, get_blockdevice_volume,
)

# The key name used for identifying the Flocker cluster_id in the metadata for
# a volume.
CLUSTER_ID_LABEL = u'flocker-cluster-id'

# The key name used for identifying the Flocker dataset_id in the metadata for
# a volume.
DATASET_ID_LABEL = u'flocker-dataset-id'

class ICinderVolumeManager(Interface):
    """
    The parts of ``cinderclient.v1.volumes.VolumeManager`` that we use.
    See:
    https://github.com/openstack/python-cinderclient/blob/master/cinderclient/v1/volumes.py#L135
    """
    def create(size, metadata=None):
        """
        Creates a volume.

        :param size: Size of volume in GB
        :param metadata: Optional metadata to set on volume creation
        :rtype: :class:`Volume`
        """

    def list():
        """
        Lists all volumes.

        :rtype: list of :class:`Volume`
        """

    def set_metadata(volume, metadata):
        """
        Update/Set a volumes metadata.

        :param volume: The :class:`Volume`.
        :param metadata: A list of keys to be set.
        """

class INovaVolumeManager(Interface):
    """
    The parts of ``novaclient.v2.volumes.VolumeManager`` that we use.
    See:
    https://github.com/openstack/python-novaclient/blob/master/novaclient/v2/volumes.py
    """
    def create_server_volume(server_id, volume_id, device):
        """
        Attach a volume identified by the volume ID to the given server ID.

        :param server_id: The ID of the server
        :param volume_id: The ID of the volume to attach.
        :param device: The device name
        :rtype: :class:`Volume`
        """

    def delete_server_volume(server_id, attachment_id):
        """
        Detach the volume identified by the volume ID from the given server ID.

        :param server_id: The ID of the server
        :param volume_id: The ID of the volume to detach.
        """

    def get(volume_id):
        """
        Retrieve information about an existing volume.

        :param volume_id: The ID of the volume about which to retrieve
            information.

        :return: A ``Volume`` instance describing the identified volume.
        :rtype: :class:`Volume`
        """

def wait_for_volume(volume_manager, expected_volume,
                    expected_status=u'available',
                    time_limit=60):
    """
    Wait for a ``Volume`` with the same ``id`` as ``expected_volume`` to be
    listed and to have a ``status`` value of ``expected_status``.

    :param ICinderVolumeManager volume_manager: An API for listing volumes.
    :param Volume expected_volume: The ``Volume`` to wait for.
    :param unicode expected_status: The ``Volume.status`` to wait for.
    :param int time_limit: The maximum time, in seconds, to wait for the
        ``expected_volume`` to have ``expected_status``.
    :raises Exception: If ``expected_volume`` with ``expected_status`` is not
        listed within ``time_limit``.
    :returns: The listed ``Volume`` that matches ``expected_volume``.
    """
    start_time = time.time()
    # Log stuff happening in this loop.  FLOC-1833.
    while True:
        # Could be more efficient.  FLOC-1831
        for listed_volume in volume_manager.list():
            if listed_volume.id == expected_volume.id:
                # Could miss the expected status because race conditions.
                # FLOC-1832
                if listed_volume.status == expected_status:
                    return listed_volume

        elapsed_time = time.time() - start_time
        if elapsed_time < time_limit:
            time.sleep(0.1)
        else:
            raise Exception(
                'Timed out while waiting for volume. '
                'Expected Volume: {!r}, '
                'Expected Status: {!r}, '
                'Elapsed Time: {!r}, '
                'Time Limit: {!r}.'.format(
                    expected_volume, expected_status, elapsed_time, time_limit
                )
            )

@implementer(IBlockDeviceAPI)
class CinderBlockDeviceAPI(object):
    """
    A cinder implementation of ``IBlockDeviceAPI`` which creates block devices
    in an OpenStack cluster using Cinder APIs.
    """
    def __init__(self, cinder_volume_manager, nova_volume_manager, cluster_id):
        """
        :param ICinderVolumeManager cinder_volume_manager: A client for
            interacting with Cinder API.
        :param INovaVolumeManager nova_volume_manager: A client for interacting
            with Nova volume API.
        :param UUID cluster_id: An ID that will be included in the names of
            Cinder block devices in order to associate them with a particular
            Flocker cluster.
        """
        self.cinder_volume_manager = cinder_volume_manager
        self.nova_volume_manager = nova_volume_manager
        self.cluster_id = cluster_id

    def compute_instance_id(self):
        """
        Look up the Xen instance ID for this node.
        """
        # See http://wiki.christophchamp.com/index.php/Xenstore
        # $ sudo xenstore-read name
        # instance-6ddfb6c0-d264-4e77-846a-aa67e4fe89df
        prefix = u"instance-"
        command = [b"xenstore-read", b"name"]
        return check_output(command).strip().decode("ascii")[len(prefix):]

    def create_volume(self, dataset_id, size):
        """
        Create a block device using the ICinderVolumeManager.
        The cluster_id and dataset_id are stored as metadata on the volume.

        See:

        http://docs.rackspace.com/cbs/api/v1.0/cbs-devguide/content/POST_createVolume_v1__tenant_id__volumes_volumes.html

        TODO:
         * Assign a Human readable name and description?
        """
        metadata = {
            CLUSTER_ID_LABEL: unicode(self.cluster_id),
            DATASET_ID_LABEL: unicode(dataset_id),
        }
        action_type = u"blockdevice:cinder:create_volume"
        with start_action(action_type=action_type):
            requested_volume = self.cinder_volume_manager.create(
                size=Byte(size).to_GB().value,
                metadata=metadata,
            )
            Message.new(blockdevice_id=requested_volume.id).write()
            created_volume = wait_for_volume(
                volume_manager=self.cinder_volume_manager,
                expected_volume=requested_volume,
            )
        return _blockdevicevolume_from_cinder_volume(
            cinder_volume=created_volume,
        )

    def list_volumes(self):
        """
        Return ``BlockDeviceVolume`` instances for all the Cinder Volumes that
        have the expected ``cluster_id`` in their metadata.

        See:

        http://docs.rackspace.com/cbs/api/v1.0/cbs-devguide/content/GET_getVolumesDetail_v1__tenant_id__volumes_detail_volumes.html
        """
        flocker_volumes = []
        for cinder_volume in self.cinder_volume_manager.list():
            if _is_cluster_volume(self.cluster_id, cinder_volume):
                flocker_volume = _blockdevicevolume_from_cinder_volume(
                    cinder_volume
                )
                flocker_volumes.append(flocker_volume)
        return flocker_volumes

    def resize_volume(self, blockdevice_id, size):
        pass

    def attach_volume(self, blockdevice_id, attach_to):
        """
        Attach a volume to an instance using the Nova volume manager.
        """
        # The Cinder volume manager has an API for attaching volumes too.
        # However, it doesn't actually attach the volume: it only updates
        # internal state to indicate that the volume is attached!  Basically,
        # it is an implementation detail of how Nova attached volumes work and
        # no one outside of Nova has any business calling it.
        #
        # See
        # http://www.florentflament.com/blog/openstack-volume-in-use-although-vm-doesnt-exist.html
        unattached_volume = get_blockdevice_volume(blockdevice_id)
        if unattached_volume.attached_to is not None:
            raise AlreadyAttachedVolume(blockdevice_id)

        nova_volume = self.nova_volume_manager.create_server_volume(
            # Nova API expects an ID string not UUID.
            server_id=attach_to,
            volume_id=unattached_volume.blockdevice_id,
            # Have Nova assign a device file for us.
            device=None,
        )
        attached_volume = wait_for_volume(
            volume_manager=self.cinder_volume_manager,
            expected_volume=nova_volume,
            expected_status=u'in-use',
        )

        attached_volume = unattached_volume.set('attached_to', attach_to)

        return attached_volume

    def detach_volume(self, blockdevice_id):
        our_id = self.compute_instance_id()
        try:
            nova_volume = self.nova_volume_manager.get(blockdevice_id)
        except NovaNotFound:
            raise UnknownVolume(blockdevice_id)

        try:
            self.nova_volume_manager.delete_server_volume(
                server_id=our_id,
                attachment_id=blockdevice_id
            )
        except NovaNotFound:
            raise UnattachedVolume(blockdevice_id)

        # TODO This'll blow up if the volume is deleted from elsewhere.
        wait_for_volume(
            volume_manager=self.nova_volume_manager,
            expected_volume=nova_volume,
            expected_status=u'available',
        )

    def destroy_volume(self, blockdevice_id):
        try:
            self.cinder_volume_manager.delete(blockdevice_id)
        except CinderNotFound:
            raise UnknownVolume(blockdevice_id)

        while True:
            # Don't loop forever here.  FLOC-1853
            try:
                self.cinder_volume_manager.get(blockdevice_id)
            except CinderNotFound:
                break

    def get_device_path(self, blockdevice_id):
        try:
            cinder_volume = self.cinder_volume_manager.get(blockdevice_id)
        except CinderNotFound:
            raise UnknownVolume(blockdevice_id)

        # As far as we know you can not have more than one attachment,
        # but, perhaps we're wrong and there should be a test for the
        # multiple attachment case.  FLOC-1854.
        try:
            [attachment] = cinder_volume.attachments
        except ValueError:
            raise UnattachedVolume(blockdevice_id)

        # It could be attached somewher else...
        # https://clusterhq.atlassian.net/browse/FLOC-1830
        return FilePath(attachment['device'])

def _is_cluster_volume(cluster_id, cinder_volume):
    """
    :param UUID cluster_id: The uuid4 of a Flocker cluster.
    :param Volume cinder_volume: The Volume with metadata to examine.
    :return: ``True`` if ``cinder_volume`` metadata has a
        ``CLUSTER_ID_LABEL`` value matching ``cluster_id`` else ``False``.
    """
    actual_cluster_id = cinder_volume.metadata.get(CLUSTER_ID_LABEL)
    if actual_cluster_id is not None:
        actual_cluster_id = UUID(actual_cluster_id)
        if actual_cluster_id == cluster_id:
            return True
    return False

def _blockdevicevolume_from_cinder_volume(cinder_volume):
    """
    :param Volume cinder_volume: The ``cinderclient.v1.volumes.Volume`` to
        convert.
    :returns: A ``BlockDeviceVolume`` based on values found in the supplied
        cinder Volume.
    """
    if cinder_volume.attachments:
        # There should only be one.  FLOC-1854.
        [attachment_info] = cinder_volume.attachments
        # Nova and Cinder APIs return ID strings. Convert to unicode.
        server_id = attachment_info['server_id'].decode("ascii")
    else:
        server_id = None

    return BlockDeviceVolume(
        blockdevice_id=unicode(cinder_volume.id),
        size=int(GB(cinder_volume.size).to_Byte().value),
        attached_to=server_id,
        dataset_id=UUID(cinder_volume.metadata[DATASET_ID_LABEL])
    )

@auto_openstack_logging(ICinderVolumeManager, "_cinder_volumes")
class _LoggingCinderVolumeManager(PRecord):
    _cinder_volumes = field(mandatory=True)

@auto_openstack_logging(INovaVolumeManager, "_nova_volumes")
class _LoggingNovaVolumeManager(PRecord):
    _nova_volumes = field(mandatory=True)

def cinder_api(cinder_client, nova_client, cluster_id):
    """
    :param cinderclient.v1.client.Client cinder_client: The Cinder API client
        whose ``volumes`` attribute will be supplied as the
        ``cinder_volume_manager`` parameter of ``CinderBlockDeviceAPI``.
    :param novaclient.v2.client.Client nova_client: The Nova API client whose
        ``volumes`` attribute will be supplied as the ``nova_volume_manager``
        parameter of ``CinderBlockDeviceAPI``.
    :param UUID cluster_id: A Flocker cluster ID.

    :returns: A ``CinderBlockDeviceAPI``.
    """
    logging_cinder = _LoggingCinderVolumeManager(
        _cinder_volumes=cinder_client.volumes
    )
    logging_nova = _LoggingNovaVolumeManager(
        _nova_volumes=nova_client.volumes
    )
    return CinderBlockDeviceAPI(
        cinder_volume_manager=logging_cinder,
        nova_volume_manager=logging_nova,
        cluster_id=cluster_id,
    )
<commit_msg>Finish the other interface too.<commit_after># -*- test-case-name: flocker.node.agents.functional.test_cinder,flocker.node.agents.functional.test_cinder_behaviour -*- # noqa
# Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
A Cinder implementation of the ``IBlockDeviceAPI``.
"""
import time
from uuid import UUID
from subprocess import check_output

from bitmath import Byte, GB

from eliot import Message, start_action

from pyrsistent import PRecord, field

from keystoneclient.openstack.common.apiclient.exceptions import (
    NotFound as CinderNotFound,
)
from novaclient.exceptions import NotFound as NovaNotFound

from twisted.python.filepath import FilePath

from zope.interface import implementer, Interface

from ...common import auto_openstack_logging
from .blockdevice import (
    IBlockDeviceAPI, BlockDeviceVolume, UnknownVolume, AlreadyAttachedVolume,
    UnattachedVolume, get_blockdevice_volume,
)

# The key name used for identifying the Flocker cluster_id in the metadata for
# a volume.
CLUSTER_ID_LABEL = u'flocker-cluster-id'

# The key name used for identifying the Flocker dataset_id in the metadata for
# a volume.
DATASET_ID_LABEL = u'flocker-dataset-id'

class ICinderVolumeManager(Interface):
    """
    The parts of ``cinderclient.v1.volumes.VolumeManager`` that we use.
    See:
    https://github.com/openstack/python-cinderclient/blob/master/cinderclient/v1/volumes.py#L135
    """
    def create(size, metadata=None):
        """
        Creates a volume.

        :param size: Size of volume in GB
        :param metadata: Optional metadata to set on volume creation
        :rtype: :class:`Volume`
        """

    def list():
        """
        Lists all volumes.

        :rtype: list of :class:`Volume`
        """

    def delete(volume_id):
        """
        Delete a volume.

        :param volume_id: The ID of the volume to delete.

        :raise CinderNotFound: If no volume with the specified ID exists.

        :return: ``None``
        """

    def get(volume_id):
        """
        Retrieve information about an existing volume.

        :param volume_id: The ID of the volume about which to retrieve
            information.

        :return: A ``Volume`` instance describing the identified volume.
        :rtype: :class:`Volume`
        """

    def set_metadata(volume, metadata):
        """
        Update/Set a volumes metadata.

        :param volume: The :class:`Volume`.
        :param metadata: A list of keys to be set.
        """

class INovaVolumeManager(Interface):
    """
    The parts of ``novaclient.v2.volumes.VolumeManager`` that we use.
    See:
    https://github.com/openstack/python-novaclient/blob/master/novaclient/v2/volumes.py
    """
    def create_server_volume(server_id, volume_id, device):
        """
        Attach a volume identified by the volume ID to the given server ID.

        :param server_id: The ID of the server
        :param volume_id: The ID of the volume to attach.
        :param device: The device name
        :rtype: :class:`Volume`
        """

    def delete_server_volume(server_id, attachment_id):
        """
        Detach the volume identified by the volume ID from the given server ID.

        :param server_id: The ID of the server
        :param volume_id: The ID of the volume to detach.
        """

    def get(volume_id):
        """
        Retrieve information about an existing volume.

        :param volume_id: The ID of the volume about which to retrieve
            information.

        :return: A ``Volume`` instance describing the identified volume.
        :rtype: :class:`Volume`
        """

def wait_for_volume(volume_manager, expected_volume,
                    expected_status=u'available',
                    time_limit=60):
    """
    Wait for a ``Volume`` with the same ``id`` as ``expected_volume`` to be
    listed and to have a ``status`` value of ``expected_status``.

    :param ICinderVolumeManager volume_manager: An API for listing volumes.
    :param Volume expected_volume: The ``Volume`` to wait for.
    :param unicode expected_status: The ``Volume.status`` to wait for.
    :param int time_limit: The maximum time, in seconds, to wait for the
        ``expected_volume`` to have ``expected_status``.
    :raises Exception: If ``expected_volume`` with ``expected_status`` is not
        listed within ``time_limit``.
    :returns: The listed ``Volume`` that matches ``expected_volume``.
    """
    start_time = time.time()
    # Log stuff happening in this loop.  FLOC-1833.
    while True:
        # Could be more efficient.  FLOC-1831
        for listed_volume in volume_manager.list():
            if listed_volume.id == expected_volume.id:
                # Could miss the expected status because race conditions.
                # FLOC-1832
                if listed_volume.status == expected_status:
                    return listed_volume

        elapsed_time = time.time() - start_time
        if elapsed_time < time_limit:
            time.sleep(0.1)
        else:
            raise Exception(
                'Timed out while waiting for volume. '
                'Expected Volume: {!r}, '
                'Expected Status: {!r}, '
                'Elapsed Time: {!r}, '
                'Time Limit: {!r}.'.format(
                    expected_volume, expected_status, elapsed_time, time_limit
                )
            )

@implementer(IBlockDeviceAPI)
class CinderBlockDeviceAPI(object):
    """
    A cinder implementation of ``IBlockDeviceAPI`` which creates block devices
    in an OpenStack cluster using Cinder APIs.
    """
    def __init__(self, cinder_volume_manager, nova_volume_manager, cluster_id):
        """
        :param ICinderVolumeManager cinder_volume_manager: A client for
            interacting with Cinder API.
        :param INovaVolumeManager nova_volume_manager: A client for interacting
            with Nova volume API.
        :param UUID cluster_id: An ID that will be included in the names of
            Cinder block devices in order to associate them with a particular
            Flocker cluster.
        """
        self.cinder_volume_manager = cinder_volume_manager
        self.nova_volume_manager = nova_volume_manager
        self.cluster_id = cluster_id

    def compute_instance_id(self):
        """
        Look up the Xen instance ID for this node.
        """
        # See http://wiki.christophchamp.com/index.php/Xenstore
        # $ sudo xenstore-read name
        # instance-6ddfb6c0-d264-4e77-846a-aa67e4fe89df
        prefix = u"instance-"
        command = [b"xenstore-read", b"name"]
        return check_output(command).strip().decode("ascii")[len(prefix):]

    def create_volume(self, dataset_id, size):
        """
        Create a block device using the ICinderVolumeManager.
        The cluster_id and dataset_id are stored as metadata on the volume.

        See:

        http://docs.rackspace.com/cbs/api/v1.0/cbs-devguide/content/POST_createVolume_v1__tenant_id__volumes_volumes.html

        TODO:
         * Assign a Human readable name and description?
        """
        metadata = {
            CLUSTER_ID_LABEL: unicode(self.cluster_id),
            DATASET_ID_LABEL: unicode(dataset_id),
        }
        action_type = u"blockdevice:cinder:create_volume"
        with start_action(action_type=action_type):
            requested_volume = self.cinder_volume_manager.create(
                size=Byte(size).to_GB().value,
                metadata=metadata,
            )
            Message.new(blockdevice_id=requested_volume.id).write()
            created_volume = wait_for_volume(
                volume_manager=self.cinder_volume_manager,
                expected_volume=requested_volume,
            )
        return _blockdevicevolume_from_cinder_volume(
            cinder_volume=created_volume,
        )

    def list_volumes(self):
        """
        Return ``BlockDeviceVolume`` instances for all the Cinder Volumes that
        have the expected ``cluster_id`` in their metadata.

        See:

        http://docs.rackspace.com/cbs/api/v1.0/cbs-devguide/content/GET_getVolumesDetail_v1__tenant_id__volumes_detail_volumes.html
        """
        flocker_volumes = []
        for cinder_volume in self.cinder_volume_manager.list():
            if _is_cluster_volume(self.cluster_id, cinder_volume):
                flocker_volume = _blockdevicevolume_from_cinder_volume(
                    cinder_volume
                )
                flocker_volumes.append(flocker_volume)
        return flocker_volumes

    def resize_volume(self, blockdevice_id, size):
        pass

    def attach_volume(self, blockdevice_id, attach_to):
        """
        Attach a volume to an instance using the Nova volume manager.
        """
        # The Cinder volume manager has an API for attaching volumes too.
        # However, it doesn't actually attach the volume: it only updates
        # internal state to indicate that the volume is attached!  Basically,
        # it is an implementation detail of how Nova attached volumes work and
        # no one outside of Nova has any business calling it.
        #
        # See
        # http://www.florentflament.com/blog/openstack-volume-in-use-although-vm-doesnt-exist.html
        unattached_volume = get_blockdevice_volume(blockdevice_id)
        if unattached_volume.attached_to is not None:
            raise AlreadyAttachedVolume(blockdevice_id)

        nova_volume = self.nova_volume_manager.create_server_volume(
            # Nova API expects an ID string not UUID.
            server_id=attach_to,
            volume_id=unattached_volume.blockdevice_id,
            # Have Nova assign a device file for us.
            device=None,
        )
        attached_volume = wait_for_volume(
            volume_manager=self.cinder_volume_manager,
            expected_volume=nova_volume,
            expected_status=u'in-use',
        )

        attached_volume = unattached_volume.set('attached_to', attach_to)

        return attached_volume

    def detach_volume(self, blockdevice_id):
        our_id = self.compute_instance_id()
        try:
            nova_volume = self.nova_volume_manager.get(blockdevice_id)
        except NovaNotFound:
            raise UnknownVolume(blockdevice_id)

        try:
            self.nova_volume_manager.delete_server_volume(
                server_id=our_id,
                attachment_id=blockdevice_id
            )
        except NovaNotFound:
            raise UnattachedVolume(blockdevice_id)

        # TODO This'll blow up if the volume is deleted from elsewhere.
        wait_for_volume(
            volume_manager=self.nova_volume_manager,
            expected_volume=nova_volume,
            expected_status=u'available',
        )

    def destroy_volume(self, blockdevice_id):
        try:
            self.cinder_volume_manager.delete(blockdevice_id)
        except CinderNotFound:
            raise UnknownVolume(blockdevice_id)

        while True:
            # Don't loop forever here.  FLOC-1853
            try:
                self.cinder_volume_manager.get(blockdevice_id)
            except CinderNotFound:
                break

    def get_device_path(self, blockdevice_id):
        try:
            cinder_volume = self.cinder_volume_manager.get(blockdevice_id)
        except CinderNotFound:
            raise UnknownVolume(blockdevice_id)

        # As far as we know you can not have more than one attachment,
        # but, perhaps we're wrong and there should be a test for the
        # multiple attachment case.  FLOC-1854.
        try:
            [attachment] = cinder_volume.attachments
        except ValueError:
            raise UnattachedVolume(blockdevice_id)

        # It could be attached somewher else...
        # https://clusterhq.atlassian.net/browse/FLOC-1830
        return FilePath(attachment['device'])

def _is_cluster_volume(cluster_id, cinder_volume):
    """
    :param UUID cluster_id: The uuid4 of a Flocker cluster.
    :param Volume cinder_volume: The Volume with metadata to examine.
    :return: ``True`` if ``cinder_volume`` metadata has a
        ``CLUSTER_ID_LABEL`` value matching ``cluster_id`` else ``False``.
    """
    actual_cluster_id = cinder_volume.metadata.get(CLUSTER_ID_LABEL)
    if actual_cluster_id is not None:
        actual_cluster_id = UUID(actual_cluster_id)
        if actual_cluster_id == cluster_id:
            return True
    return False

def _blockdevicevolume_from_cinder_volume(cinder_volume):
    """
    :param Volume cinder_volume: The ``cinderclient.v1.volumes.Volume`` to
        convert.
    :returns: A ``BlockDeviceVolume`` based on values found in the supplied
        cinder Volume.
    """
    if cinder_volume.attachments:
        # There should only be one.  FLOC-1854.
        [attachment_info] = cinder_volume.attachments
        # Nova and Cinder APIs return ID strings. Convert to unicode.
        server_id = attachment_info['server_id'].decode("ascii")
    else:
        server_id = None

    return BlockDeviceVolume(
        blockdevice_id=unicode(cinder_volume.id),
        size=int(GB(cinder_volume.size).to_Byte().value),
        attached_to=server_id,
        dataset_id=UUID(cinder_volume.metadata[DATASET_ID_LABEL])
    )

@auto_openstack_logging(ICinderVolumeManager, "_cinder_volumes")
class _LoggingCinderVolumeManager(PRecord):
    _cinder_volumes = field(mandatory=True)

@auto_openstack_logging(INovaVolumeManager, "_nova_volumes")
class _LoggingNovaVolumeManager(PRecord):
    _nova_volumes = field(mandatory=True)

def cinder_api(cinder_client, nova_client, cluster_id):
    """
    :param cinderclient.v1.client.Client cinder_client: The Cinder API client
        whose ``volumes`` attribute will be supplied as the
        ``cinder_volume_manager`` parameter of ``CinderBlockDeviceAPI``.
    :param novaclient.v2.client.Client nova_client: The Nova API client whose
        ``volumes`` attribute will be supplied as the ``nova_volume_manager``
        parameter of ``CinderBlockDeviceAPI``.
    :param UUID cluster_id: A Flocker cluster ID.

    :returns: A ``CinderBlockDeviceAPI``.
    """
    logging_cinder = _LoggingCinderVolumeManager(
        _cinder_volumes=cinder_client.volumes
    )
    logging_nova = _LoggingNovaVolumeManager(
        _nova_volumes=nova_client.volumes
    )
    return CinderBlockDeviceAPI(
        cinder_volume_manager=logging_cinder,
        nova_volume_manager=logging_nova,
        cluster_id=cluster_id,
    )
<|endoftext|>