<commit_before># -*-coding:utf-8-*-

"""

            #######################################################################
            #    _                     _                                          #
            #   | |                   | |                                         #
            #   | | __   __ _   _ __  | |   ___     ___    _ __     ___   _ __    #
            #   | |/ /  / _` | | '__| | |  / _ \   / _ \  | '_ \   / _ \ | '__|   #
            #   |   <  | (_| | | |    | | | (_) | | (_) | | |_) | |  __/ | |      #
            #   |_|\_\  \__,_| |_|    |_|  \___/   \___/  | .__/   \___| |_|      #
            #                                             | |                     #
            #                                             |_|                     #
            #                                                                     #
            #######################################################################

       _                            ___   _                     _  _           ___
    _ | |  ___   ___  _  _   ___   | _ ) | |  ___   ___  ___   | \| |  ___    | _ )  _  _   __ _
   | || | / -_) (_-< | || | (_-<   | _ \ | | / -_) (_-< (_-<   | .` | / _ \   | _ \ | || | / _` |
    \__/  \___| /__/  \_,_| /__/   |___/ |_| \___| /__/ /__/   |_|\_| \___/   |___/  \_,_| \__, |
                                                                                |___/

"""

__author__ = 'karlvorndoenitz@gmail.com'
__version__ = '0.6.5'
__packages__ = [
    "autoreload",
    "config",
    "coroutine",
    "escape",
    "http_parser",
    "logger",
    "router",
    "template",
    "utils",
    "web"
]
VERSION = tuple(map(int, __version__.split('.')))
__logo__ = """

                                     .
                              .:tC0088880Gf;..
          ..18@@t.      ,t0@8Li.  . ...,,...:f0@8L:.  .,0808@i
          18i.  i@i.;G801,      .8@@888@@888@8:  .;C80f8:   .CG.
         :@: .Lf,t@0;.                ,@@.            :088@@G 0L
         :@: ;@@@f               f8@@@@@8@@88i          .L@@8 CC
          ,0@081.                     i@G.                .800G.
         .  00.                       i@G.                 .L0.
          :8t                   fCCLfttttttfCGC:            .L0.
         ;@;.                                                .LG
      . .81                                                   .0G
        GG.              .f0                        ;0i        ,8i
       .8;                .  .                  .L8L,           L0
       :@.                .;@@t                   ,Cf.          i8
       i@.                .1@8L.       :tti..     G@@f          18.
       :@Gt:.               .         ;@@@@8.      ::     .:f8@@@0
        8@@@@@@@8;                    . fG.               iG8800@1
        i8,..                     ..   .fL   iC             .,iG0
         L@888Gf1.                Cf...t8Gi;t8t .        ,8@@@@G.
          t@@@@8C,                 .:;.    .  .             :0L..
           .G8,                                           ;80.
             .L8L,                                    .iG8t.
               .,f08C;,.                       .,iL0808C.
                 ..Gf,iL0@@@888808000088888@@8Gfi,.   .GC
                  i8.  ,..                          i,..0f
                  G@Gi.8t               .   ..      i@,.08i
                 ;@t1::@;           i0@Gt1L@8i       G8GLC0.
                 L0;:,;8.         18t    .   t@i     18. :@1
                .8@@01t0.        0G.          ,8f    :8t@@@G
                ;@,  .GL        t0. .          ,8;   .@;  t8

"""
<commit_msg>release 0.7.0 beta<commit_after># -*-coding:utf-8-*-

"""

            #######################################################################
            #    _                     _                                          #
            #   | |                   | |                                         #
            #   | | __   __ _   _ __  | |   ___     ___    _ __     ___   _ __    #
            #   | |/ /  / _` | | '__| | |  / _ \   / _ \  | '_ \   / _ \ | '__|   #
            #   |   <  | (_| | | |    | | | (_) | | (_) | | |_) | |  __/ | |      #
            #   |_|\_\  \__,_| |_|    |_|  \___/   \___/  | .__/   \___| |_|      #
            #                                             | |                     #
            #                                             |_|                     #
            #                                                                     #
            #######################################################################

       _                            ___   _                     _  _           ___
    _ | |  ___   ___  _  _   ___   | _ ) | |  ___   ___  ___   | \| |  ___    | _ )  _  _   __ _
   | || | / -_) (_-< | || | (_-<   | _ \ | | / -_) (_-< (_-<   | .` | / _ \   | _ \ | || | / _` |
    \__/  \___| /__/  \_,_| /__/   |___/ |_| \___| /__/ /__/   |_|\_| \___/   |___/  \_,_| \__, |
                                                                                |___/

"""

__author__ = 'karlvorndoenitz@gmail.com'
__version__ = '0.7.0'
__packages__ = [
    "autoreload",
    "config",
    "coroutine",
    "escape",
    "http_parser",
    "logger",
    "router",
    "template",
    "utils",
    "web"
]
VERSION = tuple(map(int, __version__.split('.')))
__logo__ = """

                                     .
                              .:tC0088880Gf;..
          ..18@@t.      ,t0@8Li.  . ...,,...:f0@8L:.  .,0808@i
          18i.  i@i.;G801,      .8@@888@@888@8:  .;C80f8:   .CG.
         :@: .Lf,t@0;.                ,@@.            :088@@G 0L
         :@: ;@@@f               f8@@@@@8@@88i          .L@@8 CC
          ,0@081.                     i@G.                .800G.
         .  00.                       i@G.                 .L0.
          :8t                   fCCLfttttttfCGC:            .L0.
         ;@;.                                                .LG
      . .81                                                   .0G
        GG.              .f0                        ;0i        ,8i
       .8;                .  .                  .L8L,           L0
       :@.                .;@@t                   ,Cf.          i8
       i@.                .1@8L.       :tti..     G@@f          18.
       :@Gt:.               .         ;@@@@8.      ::     .:f8@@@0
        8@@@@@@@8;                    . fG.               iG8800@1
        i8,..                     ..   .fL   iC             .,iG0
         L@888Gf1.                Cf...t8Gi;t8t .        ,8@@@@G.
          t@@@@8C,                 .:;.    .  .             :0L..
           .G8,                                           ;80.
             .L8L,                                    .iG8t.
               .,f08C;,.                       .,iL0808C.
                 ..Gf,iL0@@@888808000088888@@8Gfi,.   .GC
                  i8.  ,..                          i,..0f
                  G@Gi.8t               .   ..      i@,.08i
                 ;@t1::@;           i0@Gt1L@8i       G8GLC0.
                 L0;:,;8.         18t    .   t@i     18. :@1
                .8@@01t0.        0G.          ,8f    :8t@@@G
                ;@,  .GL        t0. .          ,8;   .@;  t8

"""
<|endoftext|>


<commit_before>
from discord.ext import commands

import asyncio

import pydealer
import treys

from utilities import checks

def setup(bot):
	bot.add_cog(Poker())

class Poker(commands.Cog):

	def __init__(self):
		self.status = None
		self.players = []
		self.deck = None
		self.hands = {}
		self.turn = None
		self.bets = {}
		self.current_bet = None
		self.pot = None
		self.community_cards = None
		self.folded = []
		# check default values

	@commands.group(invoke_without_command = True, case_insensitive = True)
	@checks.not_forbidden()
	async def poker(self, ctx):
		'''WIP'''
		await ctx.send_help(ctx.command)

	@poker.command()
	async def start(self, ctx):
		# TODO: Handle folds
		if self.status not in (None, "started"):
			await ctx.embed_reply("There's already a round of poker in progress")
		elif self.status is None:
			self.status = "started"
			self.players = []
			self.hands = {}
			# reset other
			self.deck = pydealer.Deck()
			self.deck.shuffle()
			self.pot = 0
			await ctx.embed_reply("has started a round of poker\n"
									f"`{ctx.prefix}poker join` to join\n"
									f"`{ctx.prefix}poker start` again to start")
		elif self.players:
			self.status = "pre-flop"
			await ctx.embed_reply("The poker round has started\n"
									f"Players: {' '.join(player.mention for player in self.players)}")
			for player in self.players:
				cards_string = self.cards_to_string(self.hands[player.id].cards)
				await ctx.bot.send_embed(player, f"Your poker hand: {cards_string}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")
			self.community_cards = self.deck.deal(3)
			await ctx.embed_send(f"The flop: {self.cards_to_string(self.community_cards)}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")
			self.community_cards.add(self.deck.deal(1))
			await ctx.embed_send(f"The turn: {self.cards_to_string(self.community_cards)}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")
			self.community_cards.add(self.deck.deal(1))
			await ctx.embed_send(f"The river: {self.cards_to_string(self.community_cards)}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")

			evaluator = treys.Evaluator()
			board = []
			for card in self.community_cards.cards:
				abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
				board.append(treys.Card.new(abbreviation))
			best_hand_value = 7462
			best_player = None
			for player, hand in self.hands.items():
				hand_stack = []
				for card in hand:
					abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
					hand_stack.append(treys.Card.new(abbreviation))
				value = evaluator.evaluate(board, hand_stack)
				if value < best_hand_value:
					best_hand_value = value
					best_player = player
			player = await ctx.bot.fetch_user(best_player)
			hand_name = evaluator.class_to_string(evaluator.get_rank_class(best_hand_value))
			await ctx.embed_send(f"{player.mention} is the winner with a {hand_name}")

	@poker.command()
	async def join(self, ctx):
		if self.status is None:
			return await ctx.embed_reply("There's not currently a round of poker going on\n"
											f"Use `{ctx.prefix}poker start` to start one")
		if self.status != "started":
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} The current round of poker already started")
		self.players.append(ctx.author)
		self.hands[ctx.author.id] = self.deck.deal(2)
		await ctx.embed_reply("has joined the poker match")

	@poker.command(name = "raise")
	async def poker_raise(self, ctx, points: int):
		if not self.turn or self.turn.id == ctx.author.id
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		if points < self.current_bet:
			return await ctx.embed_reply("The current bet is more than that")
		self.bets[self.turn.id] = points
		if points > self.current_bet:
			self.current_bet = points
			await ctx.embed_reply(f"{ctx.author.display_name} has raised to {points}")
		else:
			await ctx.embed_reply("has called")
		self.turn = None

	@poker.command()
	async def call(self, ctx):
		if not self.turn or self.turn.id != ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		if self.current_bet == 0 or (self.turn.id in self.bets and self.bets[self.turn.id] == self.current_bet):
			await ctx.embed_reply("You can't call\nYou have checked instead")
			await ctx.embed_reply("has checked")
		else:
			self.bets[self.turn.id] = self.current_bet
			await ctx.embed_reply("has called")
		self.turn = None

	@poker.command()
	async def check(self, ctx):
		if not self.turn or self.turn.id != ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		if self.current_bet != 0 and (self.turn.id not in self.bets or self.bets[self.turn.id] < self.current_bet):
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't check")
		self.bets[self.turn.id] = self.current_bet
		await ctx.embed_reply("has checked")
		self.turn = None

	@poker.command()
	async def fold(self, ctx):
		if not self.turn or self.turn.id != ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		self.bets[self.turn.id] = -1
		self.folded.append(self.turn)
		await ctx.embed_reply("has folded")
		self.turn = None

	async def betting(self, ctx):
		self.status = "betting"
		self.current_bet = 0
		while True:
			for player in self.players:
				self.turn = player
				if player in self.folded:
					continue
				await ctx.embed_send(f"{player.mention}'s turn")
				while self.turn:
					await asyncio.sleep(1)
			if all([bet == -1 or bet == self.current_bet for bet in self.bets.values()]):
				break
		for bet in self.bets.values():
			if bet != -1:
				self.pot += bet
		self.status = None

	# Utility Functions

	def cards_to_string(self, cards):
		return "".join(f":{card.suit.lower()}: {card.value} " for card in cards)

<commit_msg>[Discord] Fix missing colon in poker raise command<commit_after>
from discord.ext import commands

import asyncio

import pydealer
import treys

from utilities import checks

def setup(bot):
	bot.add_cog(Poker())

class Poker(commands.Cog):

	def __init__(self):
		self.status = None
		self.players = []
		self.deck = None
		self.hands = {}
		self.turn = None
		self.bets = {}
		self.current_bet = None
		self.pot = None
		self.community_cards = None
		self.folded = []
		# check default values

	@commands.group(invoke_without_command = True, case_insensitive = True)
	@checks.not_forbidden()
	async def poker(self, ctx):
		'''WIP'''
		await ctx.send_help(ctx.command)

	@poker.command()
	async def start(self, ctx):
		# TODO: Handle folds
		if self.status not in (None, "started"):
			await ctx.embed_reply("There's already a round of poker in progress")
		elif self.status is None:
			self.status = "started"
			self.players = []
			self.hands = {}
			# reset other
			self.deck = pydealer.Deck()
			self.deck.shuffle()
			self.pot = 0
			await ctx.embed_reply("has started a round of poker\n"
									f"`{ctx.prefix}poker join` to join\n"
									f"`{ctx.prefix}poker start` again to start")
		elif self.players:
			self.status = "pre-flop"
			await ctx.embed_reply("The poker round has started\n"
									f"Players: {' '.join(player.mention for player in self.players)}")
			for player in self.players:
				cards_string = self.cards_to_string(self.hands[player.id].cards)
				await ctx.bot.send_embed(player, f"Your poker hand: {cards_string}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")
			self.community_cards = self.deck.deal(3)
			await ctx.embed_send(f"The flop: {self.cards_to_string(self.community_cards)}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")
			self.community_cards.add(self.deck.deal(1))
			await ctx.embed_send(f"The turn: {self.cards_to_string(self.community_cards)}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")
			self.community_cards.add(self.deck.deal(1))
			await ctx.embed_send(f"The river: {self.cards_to_string(self.community_cards)}")
			await self.betting(ctx)
			while self.status:
				await asyncio.sleep(1)
			await ctx.embed_send(f"The pot: {self.pot}")

			evaluator = treys.Evaluator()
			board = []
			for card in self.community_cards.cards:
				abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
				board.append(treys.Card.new(abbreviation))
			best_hand_value = 7462
			best_player = None
			for player, hand in self.hands.items():
				hand_stack = []
				for card in hand:
					abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
					hand_stack.append(treys.Card.new(abbreviation))
				value = evaluator.evaluate(board, hand_stack)
				if value < best_hand_value:
					best_hand_value = value
					best_player = player
			player = await ctx.bot.fetch_user(best_player)
			hand_name = evaluator.class_to_string(evaluator.get_rank_class(best_hand_value))
			await ctx.embed_send(f"{player.mention} is the winner with a {hand_name}")

	@poker.command()
	async def join(self, ctx):
		if self.status is None:
			return await ctx.embed_reply("There's not currently a round of poker going on\n"
											f"Use `{ctx.prefix}poker start` to start one")
		if self.status != "started":
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} The current round of poker already started")
		self.players.append(ctx.author)
		self.hands[ctx.author.id] = self.deck.deal(2)
		await ctx.embed_reply("has joined the poker match")

	@poker.command(name = "raise")
	async def poker_raise(self, ctx, points: int):
		if not self.turn or self.turn.id == ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		if points < self.current_bet:
			return await ctx.embed_reply("The current bet is more than that")
		self.bets[self.turn.id] = points
		if points > self.current_bet:
			self.current_bet = points
			await ctx.embed_reply(f"{ctx.author.display_name} has raised to {points}")
		else:
			await ctx.embed_reply("has called")
		self.turn = None

	@poker.command()
	async def call(self, ctx):
		if not self.turn or self.turn.id != ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		if self.current_bet == 0 or (self.turn.id in self.bets and self.bets[self.turn.id] == self.current_bet):
			await ctx.embed_reply("You can't call\nYou have checked instead")
			await ctx.embed_reply("has checked")
		else:
			self.bets[self.turn.id] = self.current_bet
			await ctx.embed_reply("has called")
		self.turn = None

	@poker.command()
	async def check(self, ctx):
		if not self.turn or self.turn.id != ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		if self.current_bet != 0 and (self.turn.id not in self.bets or self.bets[self.turn.id] < self.current_bet):
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't check")
		self.bets[self.turn.id] = self.current_bet
		await ctx.embed_reply("has checked")
		self.turn = None

	@poker.command()
	async def fold(self, ctx):
		if not self.turn or self.turn.id != ctx.author.id:
			return await ctx.embed_reply(f"{ctx.bot.error_emoji} You can't do that right now")
		self.bets[self.turn.id] = -1
		self.folded.append(self.turn)
		await ctx.embed_reply("has folded")
		self.turn = None

	async def betting(self, ctx):
		self.status = "betting"
		self.current_bet = 0
		while True:
			for player in self.players:
				self.turn = player
				if player in self.folded:
					continue
				await ctx.embed_send(f"{player.mention}'s turn")
				while self.turn:
					await asyncio.sleep(1)
			if all([bet == -1 or bet == self.current_bet for bet in self.bets.values()]):
				break
		for bet in self.bets.values():
			if bet != -1:
				self.pot += bet
		self.status = None

	# Utility Functions

	def cards_to_string(self, cards):
		return "".join(f":{card.suit.lower()}: {card.value} " for card in cards)

<|endoftext|>


<commit_before>import time
from datetime import datetime
from zipfile import ZipFile, ZIP_DEFLATED

import requests
from btrc import CouchbaseClient, StatsReporter
from couchbase import Couchbase
from logger import logger

from perfrunner.helpers.misc import uhex, pretty_dict
from perfrunner.settings import CBMONITOR_HOST, SF_STORAGE

class BtrcReporter(object):

    def __init__(self, test):
        self.test = test

    def reset_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Resetting utilization stats from {}/{}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            cb.reset_utilization_stats()

    def save_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving utilization stats from {}/{}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('util_stats')

    def save_btree_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving B-tree stats from {}/{}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('btree_stats')

class Comparator(object):

    def get_snapshots(self, benckmark):
        # Get all snapshots order by build version for given benchmark
        self.snapshots_by_build = dict()
        for row in self.cbb.query('benchmarks', 'build_and_snapshots_by_metric',
                                  key=benckmark['metric'], stale='false'):
            self.snapshots_by_build[row.value[0]] = row.value[1]

    def find_previous(self, new_build):
        # Find previous build within current release and latest build from
        # previous release
        all_builds = sorted(self.snapshots_by_build.keys(), reverse=True)

        self.prev_release = None
        self.prev_build = None
        for build in all_builds[all_builds.index(new_build) + 1:]:
            if build.startswith(new_build.split('-')[0]) and not self.prev_build:
                self.prev_build = build
            else:
                self.prev_release = build
                break

    def compare(self, new_build):
        # Compare snapshots if possible
        api = 'http://{}/reports/compare/'.format(CBMONITOR_HOST)
        snapshot_api = 'http://{}/reports/html/?snapshot={{}}&snapshot={{}}'\
            .format(CBMONITOR_HOST)
        for build in filter(lambda _: _, (self.prev_build, self.prev_release)):
            baselines = self.snapshots_by_build[build]
            targets = self.snapshots_by_build[new_build]
            if baselines and targets and len(baselines) == len(targets):
                changes = {}
                reports = {}
                for baseline, target in zip(baselines, targets):
                    params = {'baseline': baseline, 'target': target}
                    comparison = requests.get(url=api, params=params).json()
                    changes[build] = tuple({
                        m for m, confidence in comparison if confidence > 50
                    })
                    reports[build] = snapshot_api.format(baseline, target)
                for report in reports.values():
                    requests.get(url=report)
                yield {
                    'datetime': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                    'changes': changes,
                    'reports': reports,
                }

    def __call__(self, test, benckmark):
        try:
            self.cbb = Couchbase.connect(bucket='benchmarks', **SF_STORAGE)
            self.cbf = Couchbase.connect(bucket='feed', **SF_STORAGE)
        except Exception, e:
            logger.warn('Failed to connect to database, {}'.format(e))
            return

        self.get_snapshots(benckmark)
        self.find_previous(new_build=benckmark['build'])

        # Base feed record
        base_feed = {
            'build': benckmark['build'],
            'cluster': test.cluster_spec.name,
            'summary': test.test_config.test_summary
        }
        for changes in self.compare(new_build=benckmark['build']):
            feed = dict(base_feed, **changes)
            _id = str(int(time.time() * 10 ** 6))
            self.cbf.set(_id, feed)
            logger.info('Snapshot comparison: {}'.format(pretty_dict(feed)))

class SFReporter(object):

    def __init__(self, test):
        self.test = test

    def _add_cluster(self):
        cluster = self.test.cluster_spec.name
        params = self.test.cluster_spec.parameters
        try:
            cb = Couchbase.connect(bucket='clusters', **SF_STORAGE)
            cb.set(cluster, params)
        except Exception, e:
            logger.warn('Failed to add cluster, {}'.format(e))
        else:
            logger.info('Successfully posted: {}, {}'.format(
                cluster, pretty_dict(params)
            ))

    def _add_metric(self, metric, metric_info):
        if metric_info is None:
            metric_info = {
                'title': self.test.test_config.metric_title,
                'cluster': self.test.cluster_spec.name,
                'larger_is_better': self.test.test_config.regression_criterion,
                'level': self.test.test_config.level,
            }
        try:
            cb = Couchbase.connect(bucket='metrics', **SF_STORAGE)
            cb.set(metric, metric_info)
        except Exception, e:
            logger.warn('Failed to add cluster, {}'.format(e))
        else:
            logger.info('Successfully posted: {}, {}'.format(
                metric, pretty_dict(metric_info)
            ))

    def _prepare_data(self, metric, value):
        key = uhex()
        master = self.test.cluster_spec.yield_masters().next()
        build = self.test.rest.get_version(master)
        data = {'build': build, 'metric': metric, 'value': value,
                'snapshots': self.test.snapshots}
        return key, data

    @staticmethod
    def _mark_previous_as_obsolete(cb, benckmark):
        for row in cb.query('benchmarks', 'values_by_build_and_metric',
                            key=[benckmark['metric'], benckmark['build']]):
            doc = cb.get(row.docid)
            doc.value.update({'obsolete': True})
            cb.set(row.docid, doc.value)

    def _log_benchmark(self, metric, value):
        _, benckmark = self._prepare_data(metric, value)
        logger.info('Dry run stats: {}'.format(
            pretty_dict(benckmark)
        ))

    def _post_benckmark(self, metric, value):
        key, benckmark = self._prepare_data(metric, value)
        try:
            cb = Couchbase.connect(bucket='benchmarks', **SF_STORAGE)
            self._mark_previous_as_obsolete(cb, benckmark)
            cb.set(key, benckmark)
            Comparator()(test=self.test, benckmark=benckmark)
        except Exception, e:
            logger.warn('Failed to post results, {}'.format(e))
        else:
            logger.info('Successfully posted: {}'.format(
                pretty_dict(benckmark)
            ))

    def post_to_sf(self, value, metric=None, metric_info=None):
        if metric is None:
            metric = '{}_{}'.format(self.test.test_config.name,
                                    self.test.cluster_spec.name)

        stats_settings = self.test.test_config.stats_settings

        if stats_settings.post_to_sf:
            self._add_metric(metric, metric_info)
            self._add_cluster()
            self._post_benckmark(metric, value)
        else:
            self._log_benchmark(metric, value)
        return value

class LogReporter(object):

    def __init__(self, test):
        self.test = test

    def save_web_logs(self):
        for master in self.test.cluster_spec.yield_masters():
            logs = self.test.rest.get_logs(master)
            fname = 'web_log_{}.json'.format(master.split(':')[0])
            with open(fname, 'w') as fh:
                fh.write(pretty_dict(logs))

    def save_master_events(self):
        with ZipFile('master_events.zip', 'w', ZIP_DEFLATED) as zh:
            for master in self.test.cluster_spec.yield_masters():
                master_events = self.test.rest.get_master_events(master)
                fname = 'master_events_{}.log'.format(master.split(':')[0])
                zh.writestr(zinfo_or_arcname=fname, bytes=master_events)

class Reporter(BtrcReporter, SFReporter, LogReporter):

    def start(self):
        self.ts = time.time()

    def finish(self, action, time_elapsed=None):
        time_elapsed = time_elapsed or (time.time() - self.ts)
        time_elapsed = round(time_elapsed / 60, 1)
        logger.info(
            'Time taken to perform "{}": {} min'.format(action, time_elapsed)
        )
        return time_elapsed
<commit_msg>fix iteration over build changes<commit_after>import time
from datetime import datetime
from zipfile import ZipFile, ZIP_DEFLATED

import requests
from btrc import CouchbaseClient, StatsReporter
from couchbase import Couchbase
from logger import logger

from perfrunner.helpers.misc import uhex, pretty_dict
from perfrunner.settings import CBMONITOR_HOST, SF_STORAGE

class BtrcReporter(object):

    def __init__(self, test):
        self.test = test

    def reset_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Resetting utilization stats from {}/{}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            cb.reset_utilization_stats()

    def save_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving utilization stats from {}/{}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('util_stats')

    def save_btree_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving B-tree stats from {}/{}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('btree_stats')

class Comparator(object):

    CONFIDENCE_THRESHOLD = 50

    def get_snapshots(self, benckmark):
        """Get all snapshots ordered by build version for given benchmark"""
        self.snapshots_by_build = dict()
        for row in self.cbb.query('benchmarks', 'build_and_snapshots_by_metric',
                                  key=benckmark['metric'], stale='false'):
            self.snapshots_by_build[row.value[0]] = row.value[1]

    def find_previous(self, new_build):
        """Find previous build within current release and latest build from
        previous release"""
        all_builds = sorted(self.snapshots_by_build.keys(), reverse=True)

        self.prev_release = None
        self.prev_build = None
        for build in all_builds[all_builds.index(new_build) + 1:]:
            if build.startswith(new_build.split('-')[0]) and not self.prev_build:
                self.prev_build = build
            else:
                self.prev_release = build
                break

    def compare(self, new_build):
        """Compare snapshots if possible"""
        api = 'http://{}/reports/compare/'.format(CBMONITOR_HOST)
        snapshot_api = 'http://{}/reports/html/?snapshot={{}}&snapshot={{}}'\
            .format(CBMONITOR_HOST)

        changes = []
        reports = []
        for build in filter(lambda _: _, (self.prev_build, self.prev_release)):
            baselines = self.snapshots_by_build[build]
            targets = self.snapshots_by_build[new_build]
            if baselines and targets and len(baselines) == len(targets):
                for baseline, target in zip(baselines, targets):
                    params = {'baseline': baseline, 'target': target}
                    comparison = requests.get(url=api, params=params).json()
                    diff = tuple({
                        m for m, confidence in comparison
                        if confidence > self.CONFIDENCE_THRESHOLD
                    })
                    changes.append((build, diff))
                    snapshots_url = snapshot_api.format(baseline, target)
                    reports.append((build, snapshots_url))

        # Prefetch (trigger) HTML reports
        for _, url in reports:
            requests.get(url=url)

        return {'changes': changes, 'reports': reports}

    def __call__(self, test, benckmark):
        try:
            self.cbb = Couchbase.connect(bucket='benchmarks', **SF_STORAGE)
            self.cbf = Couchbase.connect(bucket='feed', **SF_STORAGE)
        except Exception, e:
            logger.warn('Failed to connect to database, {}'.format(e))
            return

        self.get_snapshots(benckmark)
        self.find_previous(new_build=benckmark['build'])

        # Feed record
        _id = str(int(time.time() * 10 ** 6))
        base_feed = {
            'build': benckmark['build'],
            'cluster': test.cluster_spec.name,
            'summary': test.test_config.test_summary,
            'datetime': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        }
        changes = self.compare(new_build=benckmark['build'])
        feed = dict(base_feed, **changes)
        self.cbf.set(_id, feed)
        logger.info('Snapshot comparison: {}'.format(pretty_dict(feed)))

class SFReporter(object):

    def __init__(self, test):
        self.test = test

    def _add_cluster(self):
        cluster = self.test.cluster_spec.name
        params = self.test.cluster_spec.parameters
        try:
            cb = Couchbase.connect(bucket='clusters', **SF_STORAGE)
            cb.set(cluster, params)
        except Exception, e:
            logger.warn('Failed to add cluster, {}'.format(e))
        else:
            logger.info('Successfully posted: {}, {}'.format(
                cluster, pretty_dict(params)
            ))

    def _add_metric(self, metric, metric_info):
        if metric_info is None:
            metric_info = {
                'title': self.test.test_config.metric_title,
                'cluster': self.test.cluster_spec.name,
                'larger_is_better': self.test.test_config.regression_criterion,
                'level': self.test.test_config.level,
            }
        try:
            cb = Couchbase.connect(bucket='metrics', **SF_STORAGE)
            cb.set(metric, metric_info)
        except Exception, e:
            logger.warn('Failed to add cluster, {}'.format(e))
        else:
            logger.info('Successfully posted: {}, {}'.format(
                metric, pretty_dict(metric_info)
            ))

    def _prepare_data(self, metric, value):
        key = uhex()
        master = self.test.cluster_spec.yield_masters().next()
        build = self.test.rest.get_version(master)
        data = {'build': build, 'metric': metric, 'value': value,
                'snapshots': self.test.snapshots}
        return key, data

    @staticmethod
    def _mark_previous_as_obsolete(cb, benckmark):
        for row in cb.query('benchmarks', 'values_by_build_and_metric',
                            key=[benckmark['metric'], benckmark['build']]):
            doc = cb.get(row.docid)
            doc.value.update({'obsolete': True})
            cb.set(row.docid, doc.value)

    def _log_benchmark(self, metric, value):
        _, benckmark = self._prepare_data(metric, value)
        logger.info('Dry run stats: {}'.format(
            pretty_dict(benckmark)
        ))

    def _post_benckmark(self, metric, value):
        key, benckmark = self._prepare_data(metric, value)
        try:
            cb = Couchbase.connect(bucket='benchmarks', **SF_STORAGE)
            self._mark_previous_as_obsolete(cb, benckmark)
            cb.set(key, benckmark)
            Comparator()(test=self.test, benckmark=benckmark)
        except Exception, e:
            logger.warn('Failed to post results, {}'.format(e))
        else:
            logger.info('Successfully posted: {}'.format(
                pretty_dict(benckmark)
            ))

    def post_to_sf(self, value, metric=None, metric_info=None):
        if metric is None:
            metric = '{}_{}'.format(self.test.test_config.name,
                                    self.test.cluster_spec.name)

        stats_settings = self.test.test_config.stats_settings

        if stats_settings.post_to_sf:
            self._add_metric(metric, metric_info)
            self._add_cluster()
            self._post_benckmark(metric, value)
        else:
            self._log_benchmark(metric, value)
        return value

class LogReporter(object):

    def __init__(self, test):
        self.test = test

    def save_web_logs(self):
        for master in self.test.cluster_spec.yield_masters():
            logs = self.test.rest.get_logs(master)
            fname = 'web_log_{}.json'.format(master.split(':')[0])
            with open(fname, 'w') as fh:
                fh.write(pretty_dict(logs))

    def save_master_events(self):
        with ZipFile('master_events.zip', 'w', ZIP_DEFLATED) as zh:
            for master in self.test.cluster_spec.yield_masters():
                master_events = self.test.rest.get_master_events(master)
                fname = 'master_events_{}.log'.format(master.split(':')[0])
                zh.writestr(zinfo_or_arcname=fname, bytes=master_events)

class Reporter(BtrcReporter, SFReporter, LogReporter):

    def start(self):
        self.ts = time.time()

    def finish(self, action, time_elapsed=None):
        time_elapsed = time_elapsed or (time.time() - self.ts)
        time_elapsed = round(time_elapsed / 60, 1)
        logger.info(
            'Time taken to perform "{}": {} min'.format(action, time_elapsed)
        )
        return time_elapsed
<|endoftext|>


<commit_before>"""Example analysis using somewhat realistic data.

This exampla analysis shows how to do a Bayesian analysis of (somewhat)
realistic neutrino interaction data. The data consists of the true and
reconstructed muon momentum and angle in a given selection and the true
values of all simulated events.
"""

from __future__ import print_function

import numpy as np
from matplotlib import pyplot as plt
from pymc.Matplot import plot as mcplot
from pymc.Matplot import summary_plot

# Set working directory and sys.path
import sys, os
pathname = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
sys.path.append(pathname)
os.chdir(os.path.dirname(sys.argv[0]))

import binning
import migration
import likelihood

# Parallelization
from multiprocessing import Pool

# We split the MC into parts for training and testing
N_toy = 4
toy_div = 100

if __name__ == '__main__':

    print("Loading the truth and reco binnings...")

    with open("neutrino_data/truth-binning.yml", 'r') as f:
        truth_binning = binning.yaml.load(f)

    with open("neutrino_data/reco-binning.yml", 'r') as f:
        reco_binning = binning.yaml.load(f)

    print("Creating a response matrix object...")
    response = migration.ResponseMatrix(reco_binning, truth_binning)

    print("Filling the response matrix...")
    response.fill_from_csv_file("neutrino_data/selectedEvents.csv")
    # HACK: Re-fill the truth binning to simulate the detector efficiency
    response._truth_binning.reset()
    response._truth_binning.fill_from_csv_file("neutrino_data/trueEvents.csv")

    print("Getting normalised response matrix...")
    resp = response.get_response_matrix_as_ndarray()
    # Fake a 10% efficiency uncertainty
    resp = np.array([resp * x for x in np.linspace(0.9, 1.1, 10)])

    print("Creating test data...")
    truth = response.get_truth_values_as_ndarray() / toy_div
    data = likelihood.LikelihoodMachine.generate_random_data_sample(resp[0], truth, N_toy)

    print("Creating likelihood machines...")
    lm = [ likelihood.LikelihoodMachine(x, resp) for x in data ]

    print("Calculating absolute maximum likelihood...")
    def lm_maxlog(x):
        return x.absolute_max_log_likelihood(kwargs={'niter':10})
    pool = Pool()
    ret = pool.map(lm_maxlog, lm)
    del pool

    print("Calculating posterior probabilities...")
    def prior(value=1.):
        return -np.inf if value < 0. else -0.5*np.log(value) # P = 1/sqrt(lambda), Jeffreys prior for Poisson expectation values
    # Free floating hypothesis
    H = [ likelihood.CompositeHypothesis(lambda x: x, parameter_priors=[prior]*len(truth)) for i in range(N_toy) ]
    M = [ lm[i].MCMC(H[i]) for i in range(N_toy) ]
    def f_MCMC(i):
        # Do the actual MCMC sampling
        M[i].sample(5000, burn=0, thin=100, tune_interval=200, tune_throughout=True, progress_bar=False)
        # Debug plots to check convergence of MCMC
        mcplot(M[i])
        summary_plot(M[i])
        # Get all traces and save them in an array
        # Must be done here, because MCMC object are not pickleable
        ret = {}
        for par in M[i].stochastics:
            ret[str(par)] = M[i].trace(par)[:]
        # Parameters are namedd 'par_i'
        arr = np.array([ ret['par_%d'%(j,)] for j in range(len(truth)) ])
        toy_trace = np.array(ret['toy_index'])
        return arr, toy_trace
    pool = Pool()
    trace, toy_index = zip(*pool.map(f_MCMC, range(N_toy)))
    trace = np.array(trace)
    toy_index = np.array(toy_index)
    del pool
    median = np.median(trace, axis=-1)
    total = np.sum(trace, axis=-2)

    print("Plotting results...")
    print("- response")
    response.plot_values('response.png', variables=(None, None))

    print("- migration")
    response.plot_values('migration.png', variables=(['muMomRec','muCosThetaRec'], ['muMomTrue','muCosThetaTrue']))

    print("- max likelihood")
    col = ['r', 'g', 'b', 'm']
    figax = None
    figax = truth_binning.plot_ndarray('maxL-truth.png', truth, figax=figax,
                kwargs1d={'linestyle': 'solid', 'color': 'k', 'label': "Truth"})
    for i in range(N_toy):
        figax = truth_binning.plot_ndarray('maxL-truth.png', H[i].translate(ret[i].x), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$L_\mathrm{max,%02d}$"%(i,)})
    figax = None
    for i in range(N_toy):
        figax = reco_binning.plot_ndarray('maxL-data.png', data[i], figax=figax,
                    kwargs1d={'linestyle': 'solid', 'color': col[i], 'label': "Data$_{%02d}$"%(i,)})
        figax = reco_binning.plot_ndarray('maxL-data.png', resp[ret[i].i].dot(H[i].translate(ret[i].x)), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$L_\mathrm{max,%02d}$"%(i,)})

    print("- median posterior")
    figax = None
    figax = truth_binning.plot_ndarray('posterior-truth.png', truth, figax=figax,
                kwargs1d={'linestyle': 'solid', 'color': 'k', 'label': "Truth"})
    for i in range(N_toy):
        figax = truth_binning.plot_ndarray('posterior-truth.png', H[i].translate(median[i]), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "median $P_\mathrm{post,%02d}$"%(i,)})
    figax = None
    for i in range(N_toy):
        figax = reco_binning.plot_ndarray('posterior-data.png', data[i], figax=figax,
                    kwargs1d={'linestyle': 'solid', 'color': col[i], 'label': "Data$_{%02d}$"%(i,)})
        figax = reco_binning.plot_ndarray('posterior-data.png', resp[0].dot(H[i].translate(median[i])), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "median $P_\mathrm{post,%02d}$"%(i,)})

    fig, ax = plt.subplots()
    ax.set_xlabel("total true events")
    ax.hist(total.T, histtype='step', color=col, label=["$P_\mathrm{post,%02d}$"%(i,) for i in range(N_toy)])
    ax.axvline(np.sum(truth), color='k', linewidth=2., linestyle='dotted', label="Truth")
    ax.legend(loc='best', framealpha=0.5)
    fig.savefig('total.png')
<commit_msg>Update neutrino example.<commit_after>"""Example analysis using somewhat realistic data.

This exampla analysis shows how to do a Bayesian analysis of (somewhat)
realistic neutrino interaction data. The data consists of the true and
reconstructed muon momentum and angle in a given selection and the true
values of all simulated events.
"""

from __future__ import print_function

import numpy as np
from matplotlib import pyplot as plt
from pymc.Matplot import plot as mcplot
from pymc.Matplot import summary_plot

# Set working directory and sys.path
import sys, os
pathname = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
sys.path.append(pathname)
os.chdir(os.path.dirname(sys.argv[0]))

import argparse

import binning
import migration
import likelihood

# Parallelization
from multiprocessing import Pool

# We split the MC into parts for training and testing
N_toy = 5
toy_div = 100

if __name__ == '__main__':
    # Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("--test", help="do a quicker test-run", action='store_true')
    args = parser.parse_args()

    print("Loading the truth and reco binnings...")

    with open("neutrino_data/truth-binning.yml", 'r') as f:
        truth_binning = binning.yaml.load(f)

    with open("neutrino_data/reco-binning.yml", 'r') as f:
        reco_binning = binning.yaml.load(f)

    print("Creating a response matrix object...")
    response = migration.ResponseMatrix(reco_binning, truth_binning)

    print("Filling the response matrix...")
    response.fill_from_csv_file("neutrino_data/selectedEvents.csv")
    # HACK: Re-fill the truth binning to simulate the detector efficiency
    response._truth_binning.reset()
    response._truth_binning.fill_from_csv_file("neutrino_data/trueEvents.csv")

    print("Getting normalised response matrix...")
    true_resp = response.get_response_matrix_as_ndarray()
    # Fake a 10% efficiency uncertainty
    resp = np.array([true_resp * x for x in np.linspace(0.9, 1.1, 10)])
    eff = resp.sum(axis=1)
    max_eff = eff.max(axis=0)
    min_eff = eff.min(axis=0)
    i_eff = (min_eff > 0.2)

    print("Creating test data...")
    truth = response.get_truth_values_as_ndarray() / toy_div
    eff_truth = truth[i_eff]
    data = likelihood.LikelihoodMachine.generate_random_data_sample(true_resp, truth, N_toy)

    print("Creating likelihood machines...")
    lm = [ likelihood.LikelihoodMachine(x, resp) for x in data ]

    print("Calculating absolute maximum likelihood...")
    def lm_maxlog(x):
        return x.absolute_max_log_likelihood(kwargs={'niter':10})
    pool = Pool()
    ret = pool.map(lm_maxlog, lm)
    del pool

    print("Calculating posterior probabilities...")
    def prior(value=1.):
        return -np.inf if value < 0. else -0.5*np.log(value) # P = 1/sqrt(lambda), Jeffreys prior for Poisson expectation values
    # Free floating hypothesis
    prior = likelihood.JeffreysPrior(resp, lambda x: x, [(0,None)]*len(truth), [100.]*len(truth))
    H = [ likelihood.CompositeHypothesis(lambda x: x, parameter_priors=[prior]) for i in range(N_toy) ]
    M = [ lm[i].MCMC(H[i]) for i in range(N_toy) ]
    def f_MCMC(i):
        # Do the actual MCMC sampling
        if args.test:
            M[i].sample(100*1000, burn=10*1000, thin=1000, tune_interval=1000, tune_throughout=True, progress_bar=True)
        else:
            M[i].sample(1000*1000, burn=100*1000, thin=1000, tune_interval=1000, tune_throughout=True, progress_bar=True)
        # Get all traces and save them in an array
        # Must be done here, because MCMC object are not pickleable
        ret = {}
        for par in M[i].stochastics:
            ret[str(par)] = M[i].trace(par)[:]
        # Parameters are namedd 'par_i'
        arr = np.array( ret['par_0'].T )
        toy_trace = np.array(ret['toy_index'])
        return arr, toy_trace
    pool = Pool()
    trace, toy_index = zip(*pool.map(f_MCMC, range(N_toy)))
    del pool
    #trace, toy_index = zip(*map(f_MCMC, range(N_toy)))
    trace = np.array(trace)
    toy_index = np.array(toy_index)
    median = np.median(trace, axis=-1)
    mean = np.mean(trace, axis=-1)
    total = np.sum(trace, axis=-2)
    eff_total = np.sum(trace[:,i_eff,:], axis=-2)

    print("Plotting results...")
    print("- response")
    response.plot_values('response.png', variables=(None, None))

    print("- migration")
    response.plot_values('migration.png', variables=(['muMomRec','muCosThetaRec'], ['muMomTrue','muCosThetaTrue']))

    print("- efficiencies")
    i = range(len(min_eff))
    fig, ax = plt.subplots(1)
    ax.set_xlabel("Bin #")
    ax.set_ylabel("Efficiency")
    ax.fill_between(i, min_eff, max_eff, step='mid')
    fig.savefig('efficiencies.png')

    print("- parameters")
    # Debug plots to check convergence of MCMC
    for i, t in enumerate(trace[0]):
        mcplot(t, 'par_%d'%(i,), last=False)
        fig = plt.gcf()
        ax = fig.axes[-1]
        ax.axvline(truth[i], linewidth=2, linestyle='dashed', color='r')
        fig.savefig('par_%d.png'%(i,))
    mcplot(toy_index[0], 'toy_index')

    col = ['r', 'g', 'b', 'm', 'c'] * 10
    print("- posterior median")
    figax = None
    figax = truth_binning.plot_ndarray('posterior-median-truth.png', truth, figax=figax,
                kwargs1d={'linestyle': 'solid', 'color': 'k', 'label': "Truth"})
    for i in range(N_toy):
        figax = truth_binning.plot_ndarray('posterior-median-truth.png', H[i].translate(median[i]), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$P_\mathrm{post,%02d}$ median"%(i,)})
    figax = None
    for i in range(N_toy):
        figax = reco_binning.plot_ndarray('posterior-median-data.png', data[i], figax=figax,
                    kwargs1d={'linestyle': 'solid', 'color': col[i], 'label': "Data$_{%02d}$"%(i,)})
        figax = reco_binning.plot_ndarray('posterior-median-data.png', true_resp.dot(H[i].translate(median[i])), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$P_\mathrm{post,%02d}$ median"%(i,)})

    print("- posterior mean")
    figax = None
    figax = truth_binning.plot_ndarray('posterior-mean-truth.png', truth, figax=figax,
                kwargs1d={'linestyle': 'solid', 'color': 'k', 'label': "Truth"})
    for i in range(N_toy):
        figax = truth_binning.plot_ndarray('posterior-mean-truth.png', H[i].translate(mean[i]), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$P_\mathrm{post,%02d}$ mean"%(i,)})
    figax = None
    for i in range(N_toy):
        figax = reco_binning.plot_ndarray('posterior-mean-data.png', data[i], figax=figax,
                    kwargs1d={'linestyle': 'solid', 'color': col[i], 'label': "Data$_{%02d}$"%(i,)})
        figax = reco_binning.plot_ndarray('posterior-mean-data.png', true_resp.dot(H[i].translate(mean[i])), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$P_\mathrm{post,%02d}$ mean"%(i,)})

    print("- total")
    fig, ax = plt.subplots()
    ax.set_xlabel("total true events")
    ax.hist(total.T, histtype='step', color=col[0:len(total)], label=["$P_\mathrm{post,%02d}$"%(i,) for i in range(N_toy)])
    ax.axvline(np.sum(truth), color='k', linewidth=2., linestyle='dotted', label="Truth")
    ax.legend(loc='best', framealpha=0.5)
    fig.savefig('total.png')

    print("- eff total")
    fig, ax = plt.subplots()
    ax.set_xlabel("total true efficient events")
    ax.hist(eff_total.T, histtype='step', color=col[0:len(total)], label=["$P_\mathrm{post,%02d}$"%(i,) for i in range(N_toy)])
    ax.axvline(np.sum(eff_truth), color='k', linewidth=2., linestyle='dotted', label="Truth")
    ax.legend(loc='best', framealpha=0.5)
    fig.savefig('eff_total.png')

    print("- max likelihood")
    figax = None
    figax = truth_binning.plot_ndarray('maxL-truth.png', truth, figax=figax,
                kwargs1d={'linestyle': 'solid', 'color': 'k', 'label': "Truth"})
    for i in range(N_toy):
        figax = truth_binning.plot_ndarray('maxL-truth.png', H[i].translate(ret[i].x), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$L_\mathrm{max,%02d}$"%(i,)})
    figax = None
    for i in range(N_toy):
        figax = reco_binning.plot_ndarray('maxL-data.png', data[i], figax=figax,
                    kwargs1d={'linestyle': 'solid', 'color': col[i], 'label': "Data$_{%02d}$"%(i,)})
        figax = reco_binning.plot_ndarray('maxL-data.png', resp[ret[i].i].dot(H[i].translate(ret[i].x)), figax=figax,
                    kwargs1d={'linestyle': 'dashed', 'linewidth': 2.0, 'color': col[i], 'label': "$L_\mathrm{max,%02d}$"%(i,)})
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of ckanext-ngsipreview.
#
# Ckanext-ngsipreview is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# FlaskContextProvider is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.

from logging import getLogger
import urlparse
import requests
import json
import ckan.logic as logic
import ckan.lib.base as base
import ckan.plugins as p

log = getLogger(__name__)

MAX_FILE_SIZE = 1024 * 1024  # 1MB
CHUNK_SIZE = 512

def proxy_ngsi_resource(context, data_dict):
    # Chunked proxy for ngsi resources.
    resource_id = data_dict['resource_id']
    log.info('Proxify resource {id}'.format(id=resource_id))
    resource = logic.get_action('resource_show')(context, {'id': resource_id})
    url = resource['url']
    if 'oauth_req' in resource and resource['oauth_req'] == 'true':
        token = p.toolkit.c.usertoken['access_token']
        headers = {'X-Auth-Token': token, 'Content-Type': 'application/json', 'Accept': 'application/json'}
    else:
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}

    parts = urlparse.urlsplit(url)
    if not parts.scheme or not parts.netloc:
        base.abort(409, detail='Invalid URL.')

    try:
        for i in range(2):
            if url.lower().find('/querycontext') != -1:
                resource['payload'] = resource['payload'].replace("'", '"')
                resource['payload'] = resource['payload'].replace(" ", "")

                payload = json.dumps(json.loads(resource['payload']))

                r = requests.post(url, headers=headers, data=payload, stream=True)
            else:
                r = requests.get(url, headers=headers, stream=True)

            if r.status_code == 401 and 'oauth_req' in resource and resource['oauth_req'] == 'true':
                details = 'ERROR 401 token expired. Retrieving new token and retrying...'
                log.info(details)
                p.toolkit.c.usertoken_refresh()
            else:
                break

        if r.status_code == 401:
            if 'oauth_req' not in resource or resource['oauth_req'] == 'false':
                details = 'This query may need Oauth-token, please check if the token field on resource_edit is correct.'
                log.info(details)
                base.abort(409, detail=details)
            else:
                base.abort(409, detail='Cannot retrieve a new token.')

        else:
            r.raise_for_status()
            base.response.content_type = r.headers['content-type']
            base.response.charset = r.encoding

        length = 0
        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
            base.response.body_file.write(chunk)
            length += len(chunk)
            if length >= MAX_FILE_SIZE:
                details = 'Content is too large to be proxied. Complete the Context Broker query with pagination parameters to resolve this issue.'
                base.abort(409, headers={'content-encoding': ''}, detail=details)

    except ValueError:
        details = 'There is a problem with the payload, please check if the query is properly parsed.'
        base.abort(409, detail=details)
    except requests.HTTPError:
        details = 'Could not proxy ngsi_resource. We are working to resolve this issue as quickly as possible'
        base.abort(409, detail=details)
    except requests.ConnectionError:
        details = 'Could not proxy ngsi_resource because a connection error occurred.'
        base.abort(502, detail=details)
    except requests.Timeout:
        details = 'Could not proxy ngsi_resource because the connection timed out.'
        base.abort(504, detail=details)

class ProxyNGSIController(base.BaseController):
    def proxy_ngsi_resource(self, resource_id):
        data_dict = {'resource_id': resource_id}
        context = {'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author}
        return proxy_ngsi_resource(context, data_dict)
<commit_msg>Fiware-Service (tenant) and Fiware-ServicePath (service path) headers added to the request<commit_after>#!/usr/bin/env python
# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of ckanext-ngsipreview.
#
# Ckanext-ngsipreview is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# FlaskContextProvider is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.

from logging import getLogger
import urlparse
import requests
import json
import ckan.logic as logic
import ckan.lib.base as base
import ckan.plugins as p

log = getLogger(__name__)

MAX_FILE_SIZE = 1024 * 1024  # 1MB
CHUNK_SIZE = 512

def proxy_ngsi_resource(context, data_dict):
    # Chunked proxy for ngsi resources.
    resource_id = data_dict['resource_id']
    log.info('Proxify resource {id}'.format(id=resource_id))
    resource = logic.get_action('resource_show')(context, {'id': resource_id})

    if 'oauth_req' in resource and resource['oauth_req'] == 'true':
        token = p.toolkit.c.usertoken['access_token']
        headers = {'X-Auth-Token': token, 'Content-Type': 'application/json', 'Accept': 'application/json'}
    else:
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}

    if 'tenant' in resource:
        headers['Fiware-Service'] = resource['tenant']
    if 'service_path' in resource:
        headers['Fiware-ServicePath'] = resource['service_path']

    url = resource['url']
    parts = urlparse.urlsplit(url)

    if not parts.scheme or not parts.netloc:
        base.abort(409, detail='Invalid URL.')

    try:
        for i in range(2):
            if url.lower().find('/querycontext') != -1:
                resource['payload'] = resource['payload'].replace("'", '"')
                resource['payload'] = resource['payload'].replace(" ", "")

                payload = json.dumps(json.loads(resource['payload']))

                r = requests.post(url, headers=headers, data=payload, stream=True)
            else:
                r = requests.get(url, headers=headers, stream=True)

            if r.status_code == 401 and 'oauth_req' in resource and resource['oauth_req'] == 'true':
                details = 'ERROR 401 token expired. Retrieving new token and retrying...'
                log.info(details)
                p.toolkit.c.usertoken_refresh()
            else:
                break

        if r.status_code == 401:
            if 'oauth_req' not in resource or resource['oauth_req'] == 'false':
                details = 'This query may need Oauth-token, please check if the token field on resource_edit is correct.'
                log.info(details)
                base.abort(409, detail=details)
            else:
                base.abort(409, detail='Cannot retrieve a new token.')

        else:
            r.raise_for_status()
            base.response.content_type = r.headers['content-type']
            base.response.charset = r.encoding

        length = 0
        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
            base.response.body_file.write(chunk)
            length += len(chunk)
            if length >= MAX_FILE_SIZE:
                details = 'Content is too large to be proxied. Complete the Context Broker query with pagination parameters to resolve this issue.'
                base.abort(409, headers={'content-encoding': ''}, detail=details)

    except ValueError:
        details = 'There is a problem with the payload, please check if the query is properly parsed.'
        base.abort(409, detail=details)
    except requests.HTTPError:
        details = 'Could not proxy ngsi_resource. We are working to resolve this issue as quickly as possible'
        base.abort(409, detail=details)
    except requests.ConnectionError:
        details = 'Could not proxy ngsi_resource because a connection error occurred.'
        base.abort(502, detail=details)
    except requests.Timeout:
        details = 'Could not proxy ngsi_resource because the connection timed out.'
        base.abort(504, detail=details)

class ProxyNGSIController(base.BaseController):
    def proxy_ngsi_resource(self, resource_id):
        data_dict = {'resource_id': resource_id}
        context = {'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author}
        return proxy_ngsi_resource(context, data_dict)
<|endoftext|>


<commit_before>"""
Represents tables.
"""
import typing
from types import MappingProxyType

from cached_property import cached_property

from katagawa.exceptions import OperationalException
from katagawa.orm import column as cl

class Table(object):
    """
    A table corresponds to an ORM table.
    """

    def __init__(self,
                 name: str,
                 *columns: 'cl.Column'):
        """
        :param name:
            The table name in the database this table corresponds to.

        :param columns:
            The columns that this table contains.
        """
        #: The name of the table.
        self.name = name

        #: The list of columns.
        self._columns = columns
        if not all(isinstance(i, cl.Column) for i in self._columns):
            raise TypeError("Columns arguments must be instances of Column")

        # register a field mapping
        self._field_mapping = {}

        for column in self._columns:
            self._field_mapping[column.name] = column
            column.register_table(self)

    @property
    def column_mapping(self) -> 'typing.Mapping[str, cl.Column]':
        """
        :return: A read-only mapping of column_name -> column.
        """
        return MappingProxyType(self._field_mapping)

    @property
    def columns(self) -> 'typing.ValuesView[cl.Column]':
        """
        :return: A list of columns for this table.
        """
        return self.column_mapping.values()

    @cached_property
    def primary_key(self):
        """
        :return: The :class:`~.Column` that represents the primary key for this table.
        """
        try:
            return next(filter(lambda column: column.primary_key is True, self.columns))
        except StopIteration:
            raise OperationalException("Table `{}` has no primary key".format(self.name))

    # magic method overrides
    def __getattr__(self, item):
        try:
            return self._field_mapping[item]
        except KeyError:
            raise AttributeError(item) from None

class TableRow(object):
    """
    Represents a row in a table. This is created when a table is called, or from query results.
    """
    def __init__(self, table: 'Table'):
        """
        :param table: The :class:`~.Table` instance that this row is associated with.
        """

        #: The :class:`~.Table` that this row is associated with.
        self.table = table

        # internal mappings

        #: A dict of Column: Previous value.
        #: Used when updating a column, it will store the previous value to know an update happened.
        self._previous_values = {}

        #: A dict of Column: Current value.
        #: Used when updating a column, it will store the current values.
        #: When created, this is initialized to the current values of all columns.
        self._values = {}

    # workhouse methods
    def get_value(self, col_name: str) -> typing.Any:
        """
        Gets the current value of the column by name ``col_name``.

        :param col_name: The name of the column to get.
        :return: The value of the column.
        """
        col = self.table.column_mapping[col_name]
        val = self._values.get(col.name)  # col.default)
        return val

    def update_value(self, col_name: str, new_val: typing.Any):
        """
        Updates or inserts the value into a result.

        :param col_name: The name of the column to update.
        :param new_val: The new value of the data to insert or update.
        """
        if col_name not in self._previous_values and col_name in self._values:
            self._previous_values[col_name] = self._values[col_name]

        self._values[col_name] = new_val

    def __iter__(self):
        """
        Alias for ``iter(row._values)``.
        """
        return iter(self._values)

    def __getattr__(self, item):
        # called when all else failed
        # this means we need to return the column value
        try:
            return self.get_value(item)
        except KeyError as e:
            raise AttributeError(item) from e
<commit_msg>Return `TableRow`s from calling tables.<commit_after>"""
Represents tables.
"""
import typing
from types import MappingProxyType

from cached_property import cached_property

from katagawa.exceptions import OperationalException
from katagawa.orm import column as cl

class Table(object):
    """
    A table corresponds to an ORM table.
    """

    def __init__(self,
                 name: str,
                 *columns: 'cl.Column'):
        """
        :param name:
            The table name in the database this table corresponds to.

        :param columns:
            The columns that this table contains.
        """
        #: The name of the table.
        self.name = name

        #: The list of columns.
        self._columns = columns
        if not all(isinstance(i, cl.Column) for i in self._columns):
            raise TypeError("Columns arguments must be instances of Column")

        # register a field mapping
        self._column_mapping = {}

        for column in self._columns:
            self._column_mapping[column.name] = column
            column.register_table(self)

    @cached_property
    def column_mapping(self) -> 'typing.Mapping[str, cl.Column]':
        """
        :return: A read-only mapping of column_name -> column.
        """
        return MappingProxyType(self._column_mapping)

    @cached_property
    def columns(self) -> 'typing.ValuesView[cl.Column]':
        """
        :return: A list of columns for this table.
        """
        return self.column_mapping.values()

    @cached_property
    def primary_key(self):
        """
        :return: The :class:`~.Column` that represents the primary key for this table.
        """
        try:
            return next(filter(lambda column: column.primary_key is True, self.columns))
        except StopIteration:
            raise OperationalException("Table `{}` has no primary key".format(self.name))

    # magic method overrides
    def __getattr__(self, item):
        try:
            return self._column_mapping[item]
        except KeyError:
            raise AttributeError(item) from None

    def __call__(self, *args, **kwargs) -> 'TableRow':
        row = TableRow(self)
        # update values
        for name, val in kwargs.items():
            if name not in self._column_mapping:
                raise ValueError("{} is not a column in this table".format(name))

            row.update_value(name, val)

        return row

class TableRow(object):
    """
    Represents a row in a table. This is created when a table is called, or from query results.
    """

    def __init__(self, table: 'Table'):
        """
        :param table: The :class:`~.Table` instance that this row is associated with.
        """

        #: The :class:`~.Table` that this row is associated with.
        self._table = table

        # internal mappings

        #: A dict of Column: Previous value.
        #: Used when updating a column, it will store the previous value to know an update happened.
        self._previous_values = {}

        #: A dict of Column: Current value.
        #: Used when updating a column, it will store the current values.
        #: When created, this is initialized to the current values of all columns.
        self._values = {}

    # workhouse methods
    def get_value(self, col_name: str) -> typing.Any:
        """
        Gets the current value of the column by name ``col_name``.

        :param col_name: The name of the column to get.
        :return: The value of the column.
        """
        col = self._table.column_mapping[col_name]
        val = self._values.get(col.name)  # col.default)
        return val

    def update_value(self, col_name: str, new_val: typing.Any):
        """
        Updates or inserts the value into a result.

        :param col_name: The name of the column to update.
        :param new_val: The new value of the data to insert or update.
        """
        if col_name not in self._previous_values and col_name in self._values:
            self._previous_values[col_name] = self._values[col_name]

        self._values[col_name] = new_val

    def __repr__(self):
        fmt = " "
        for col in self._table.columns:
            val = self.get_value(col.name)
            fmt += "{}='{}'".format(col.name, val)

        return "<{.name}{}>".format(self._table, fmt)

    def __iter__(self):
        """
        Alias for ``iter(row._values)``.
        """
        return iter(self._values)

    def __getattr__(self, item):
        # called when all else failed
        # this means we need to return the column value
        try:
            return self.get_value(item)
        except KeyError as e:
            raise AttributeError(item) from e

    def __setattr__(self, key: str, value: typing.Any):
        if key.startswith("_"):
            super().__setattr__(key, value)

        if key in self._table.column_mapping:
            self.update_value(key, new_val=value)
        else:
            super().__setattr__(key, value)
<|endoftext|>


<commit_before>from django.conf import settings
from persistent_messages.models import Message
from persistent_messages.constants import PERSISTENT_MESSAGE_LEVELS
from django.contrib.auth.models import AnonymousUser
from django.contrib.messages.storage.fallback import FallbackStorage
from django.db.models import Q
import datetime

def get_user(request):
    if hasattr(request, 'user') and request.user.__class__ != AnonymousUser:
        return request.user
    else:
        return AnonymousUser()

"""
Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
Hence, they need to be stored when being added. You can disable this, but then you'll only be able to
close a message when it is displayed for the second time.
"""
STORE_WHEN_ADDING = getattr(settings, 'MESSAGES_STORE_WHEN_ADDING', True)

#@TODO USE FALLBACK
class PersistentMessageStorage(FallbackStorage):
    def __init__(self, *args, **kwargs):
        super(PersistentMessageStorage, self).__init__(*args, **kwargs)
        self.non_persistent_messages = []
        self.is_anonymous = not get_user(self.request).is_authenticated()

    def _message_queryset(self, exclude_unread=True):
        qs = Message.objects.filter(user=get_user(self.request)).filter(
            Q(expires=None) | Q(expires__gt=datetime.datetime.now()))
        if exclude_unread:
            qs = qs.exclude(read=True)
        return qs

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of stored messages. Returns a tuple of the messages
        and a flag indicating whether or not all the messages originally
        intended to be stored in this storage were, in fact, stored and
        retrieved; e.g., ``(messages, all_retrieved)``.
        """
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._get(*args, **kwargs)
        messages = []
        for message in self._message_queryset():
            if not message.is_persistent():
                self.non_persistent_messages.append(message)
            messages.append(message)
        return (messages, True)

    def get_persistent(self):
        return self._message_queryset(exclude_unread=False).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def get_persistent_unread(self):
        return self._message_queryset(exclude_unread=True).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def count_unread(self):
        return self._message_queryset(exclude_unread=True).count()

    def count_persistent_unread(self):
        return self.get_persistent_unread().count()

    def _delete_non_persistent(self):
        for message in self.non_persistent_messages:
            message.delete()
        self.non_persistent_messages = []
        self.non_persistent_messages = []

    def __iter__(self):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).__iter__()
        self.used = True
        messages = []
        messages.extend(self._loaded_messages)
        if self._queued_messages:
            messages.extend(self._queued_messages)
        return iter(messages)

    def _prepare_messages(self, messages):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._prepare_messages(messages)
        """
        Obsolete method since model takes care of this.
        """
        pass

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores a list of messages, returning a list of any messages which could
        not be stored.

        If STORE_WHEN_ADDING is True, messages are already stored at this time and won't be
        saved again.
        """

        messages = [message for message in messages if not message.is_sticky()]

        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._store(messages, response, *args, **kwargs)
        for message in messages:
            if not self.used or message.is_persistent():
                if not message.pk:
                    message.save()
        return []

    def update(self, response):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).update(response)
        """
        Deletes all non-persistent, read messages. Saves all unstored messages.
        """
        if self.used:
            self._delete_non_persistent()
        return super(PersistentMessageStorage, self).update(response)

    def add(self, level, message, extra_tags='', subject='', user=None, from_user=None, expires=None,
            close_timeout=None):
        """
        Queues a message to be stored.

        The message is only queued if it contained something and its level is
        not less than the recording level (``self.level``).
        """
        to_user = user or get_user(self.request)
        if not to_user.is_authenticated() or Message(
            level=level).is_sticky():# si es sticky también dejar que lo procese la session
            if Message(level=level).is_persistent():
                raise NotImplementedError('Persistent message levels cannot be used for anonymous users.')
            else:
                return super(PersistentMessageStorage, self).add(level, message, extra_tags)
        if not message:
            return
            # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
            # Add the message.
        message = Message(user=to_user, level=level, message=message, extra_tags=extra_tags, subject=subject,
            from_user=from_user, expires=expires, close_timeout=close_timeout)
        # Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
        # Hence, save it now instead of adding it to queue:
        if STORE_WHEN_ADDING:
            message.save()
        else:
            self.added_new = True
            self._queued_messages.append(message)
<commit_msg>Fix some problems with the characters in python<commit_after>from django.conf import settings
from persistent_messages.models import Message
from persistent_messages.constants import PERSISTENT_MESSAGE_LEVELS
from django.contrib.auth.models import AnonymousUser
from django.contrib.messages.storage.fallback import FallbackStorage
from django.db.models import Q
import datetime

def get_user(request):
    if hasattr(request, 'user') and request.user.__class__ != AnonymousUser:
        return request.user
    else:
        return AnonymousUser()

"""
Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
Hence, they need to be stored when being added. You can disable this, but then you'll only be able to
close a message when it is displayed for the second time.
"""
STORE_WHEN_ADDING = getattr(settings, 'MESSAGES_STORE_WHEN_ADDING', True)

#@TODO USE FALLBACK
class PersistentMessageStorage(FallbackStorage):
    def __init__(self, *args, **kwargs):
        super(PersistentMessageStorage, self).__init__(*args, **kwargs)
        self.non_persistent_messages = []
        self.is_anonymous = not get_user(self.request).is_authenticated()

    def _message_queryset(self, exclude_unread=True):
        qs = Message.objects.filter(user=get_user(self.request)).filter(
            Q(expires=None) | Q(expires__gt=datetime.datetime.now()))
        if exclude_unread:
            qs = qs.exclude(read=True)
        return qs

    def _get(self, *args, **kwargs):
        """
        Retrieves a list of stored messages. Returns a tuple of the messages
        and a flag indicating whether or not all the messages originally
        intended to be stored in this storage were, in fact, stored and
        retrieved; e.g., ``(messages, all_retrieved)``.
        """
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._get(*args, **kwargs)
        messages = []
        for message in self._message_queryset():
            if not message.is_persistent():
                self.non_persistent_messages.append(message)
            messages.append(message)
        return (messages, True)

    def get_persistent(self):
        return self._message_queryset(exclude_unread=False).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def get_persistent_unread(self):
        return self._message_queryset(exclude_unread=True).filter(level__in=PERSISTENT_MESSAGE_LEVELS)

    def count_unread(self):
        return self._message_queryset(exclude_unread=True).count()

    def count_persistent_unread(self):
        return self.get_persistent_unread().count()

    def _delete_non_persistent(self):
        for message in self.non_persistent_messages:
            message.delete()
        self.non_persistent_messages = []
        self.non_persistent_messages = []

    def __iter__(self):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).__iter__()
        self.used = True
        messages = []
        messages.extend(self._loaded_messages)
        if self._queued_messages:
            messages.extend(self._queued_messages)
        return iter(messages)

    def _prepare_messages(self, messages):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._prepare_messages(messages)
        """
        Obsolete method since model takes care of this.
        """
        pass

    def _store(self, messages, response, *args, **kwargs):
        """
        Stores a list of messages, returning a list of any messages which could
        not be stored.

        If STORE_WHEN_ADDING is True, messages are already stored at this time and won't be
        saved again.
        """

        messages = [message for message in messages if not message.is_sticky()]

        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self)._store(messages, response, *args, **kwargs)
        for message in messages:
            if not self.used or message.is_persistent():
                if not message.pk:
                    message.save()
        return []

    def update(self, response):
        if not get_user(self.request).is_authenticated():
            return super(PersistentMessageStorage, self).update(response)
        """
        Deletes all non-persistent, read messages. Saves all unstored messages.
        """
        if self.used:
            self._delete_non_persistent()
        return super(PersistentMessageStorage, self).update(response)

    def add(self, level, message, extra_tags='', subject='', user=None, from_user=None, expires=None,
            close_timeout=None):
        """
        Queues a message to be stored.

        The message is only queued if it contained something and its level is
        not less than the recording level (``self.level``).
        """
        to_user = user or get_user(self.request)
        if not to_user.is_authenticated() or Message(level=level).is_sticky():
            if Message(level=level).is_persistent():
                raise NotImplementedError('Persistent message levels cannot be used for anonymous users.')
            else:
                #if user is not logged or message is sticky
                return super(PersistentMessageStorage, self).add(level, message, extra_tags)
        if not message:
            return
            # Check that the message level is not less than the recording level.
        level = int(level)
        if level < self.level:
            return
            # Add the message.
        message = Message(user=to_user, level=level, message=message, extra_tags=extra_tags, subject=subject,
            from_user=from_user, expires=expires, close_timeout=close_timeout)
        # Messages need a primary key when being displayed so that they can be closed/marked as read by the user.
        # Hence, save it now instead of adding it to queue:
        if STORE_WHEN_ADDING:
            message.save()
        else:
            self.added_new = True
            self._queued_messages.append(message)
<|endoftext|>


<commit_before>from pylons import config
import ckan.logic as logic
import ckan.lib.base as base
import ckan.model as model
import ckan.plugins as p
from ckan.common import _, c
import logging
import json
from ckanext.nhm.lib.helpers import get_datastore_fields
from collections import OrderedDict

log = logging.getLogger(__name__)

render = base.render
abort = base.abort
redirect = base.redirect

NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
ValidationError = logic.ValidationError
get_action = logic.get_action

TILED_MAP_TYPE = 'tiledmap'  # The view type for the tiledmap

class RecordController(base.BaseController):
    """
    Controller for displaying an individual record
    """

    def _load_data(self, package_name, resource_id, record_id):
        """
        Load the data for dataset, resource and record (into C var)
        @param package_name:
        @param resource_id:
        @param record_id:
        @return:
        """
        context = {'model': model, 'session': model.Session, 'user': c.user or c.author}

        # Try & get the resource
        try:
            c.resource = get_action('resource_show')(context, {'id': resource_id})
            c.package = get_action('package_show')(context, {'id': package_name})
            # required for nav menu
            c.pkg = context['package']
            c.pkg_dict = c.package
            c.record_dict = get_action('record_get')(context, {'resource_id': resource_id, 'record_id': record_id})
        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % package_name)

        image_field = c.resource.get('_image_field', None)

        if image_field:
            try:
                # Pop the image field so it won't be output as part of the record_dict / field_data dict (see self.view())
                # Also thumbnail it - there is a thumbnail=yes option, but that seems a bit small
                c.images = [image.strip() for image in c.record_dict.pop(image_field).split(';')]
            except (KeyError, AttributeError):
                # Skip errors - there are no images
                pass

        # Loop through all the views - if we have a tiled map view with lat/lon fields
        # We'll use those fields to add the map
        views = p.toolkit.get_action('resource_view_list')(context, {'id': resource_id})

        for view in views:
            if view['view_type'] == TILED_MAP_TYPE:
                latitude, longitude = c.record_dict.get(view[u'latitude_field']), c.record_dict.get(view[u'longitude_field'])

                if latitude and longitude:
                    c.record_map = json.dumps({
                        'type': 'Point',
                        'coordinates': [longitude, latitude]
                    })

                break

    def view(self, package_name, resource_id, record_id):

        """
        View an individual record
        :param id:
        :param resource_id:
        :param record_id:
        :return: html
        """
        self._load_data(package_name, resource_id, record_id)

        # The record_dict does not have field sin the correct order
        # So load the fields, and create an OrderedDict with field: value
        c.field_data = OrderedDict()
        for field in get_datastore_fields(resource_id):
            if not field['id'].startswith('_'):
                c.field_data[field['id']] = c.record_dict.get(field['id'], None)

        # Image and map should be a block in main record view
        return p.toolkit.render('record/view.html')<commit_msg>Updated image view to use gallery<commit_after>from pylons import config
import ckan.logic as logic
import ckan.lib.base as base
import ckan.model as model
import ckan.plugins as p
from ckan.common import _, c
import logging
import json
from ckanext.nhm.lib.helpers import get_datastore_fields, record_display_name
from collections import OrderedDict

log = logging.getLogger(__name__)

render = base.render
abort = base.abort
redirect = base.redirect

NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
ValidationError = logic.ValidationError
get_action = logic.get_action

TILED_MAP_TYPE = 'tiledmap'  # The view type for the tiledmap

class RecordController(base.BaseController):
    """
    Controller for displaying an individual record
    """

    def _load_data(self, package_name, resource_id, record_id):
        """
        Load the data for dataset, resource and record (into C var)
        @param package_name:
        @param resource_id:
        @param record_id:
        @return:
        """
        context = {'model': model, 'session': model.Session, 'user': c.user or c.author}

        # Try & get the resource
        try:
            c.resource = get_action('resource_show')(context, {'id': resource_id})
            c.package = get_action('package_show')(context, {'id': package_name})
            # required for nav menu
            c.pkg = context['package']
            c.pkg_dict = c.package
            c.record_dict = get_action('record_get')(context, {'resource_id': resource_id, 'record_id': record_id})
        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % package_name)

        image_field = c.resource.get('_image_field', None)
        if image_field:
            try:
                # Pop the image field so it won't be output as part of the record_dict / field_data dict (see self.view())
                images = [image.strip() for image in c.record_dict.pop(image_field).split(';')]
            except (KeyError, AttributeError):
                # Skip errors - there are no images
                pass
            else:
                title = record_display_name(c.resource, c.record_dict)
                c.images = [{'title': title, 'url': image} for image in images]

        # Loop through all the views - if we have a tiled map view with lat/lon fields
        # We'll use those fields to add the map
        views = p.toolkit.get_action('resource_view_list')(context, {'id': resource_id})

        for view in views:
            if view['view_type'] == TILED_MAP_TYPE:
                latitude, longitude = c.record_dict.get(view[u'latitude_field']), c.record_dict.get(view[u'longitude_field'])

                if latitude and longitude:
                    c.record_map = json.dumps({
                        'type': 'Point',
                        'coordinates': [longitude, latitude]
                    })

                break

    def view(self, package_name, resource_id, record_id):

        """
        View an individual record
        :param id:
        :param resource_id:
        :param record_id:
        :return: html
        """
        self._load_data(package_name, resource_id, record_id)

        # The record_dict does not have field sin the correct order
        # So load the fields, and create an OrderedDict with field: value
        c.field_data = OrderedDict()
        for field in get_datastore_fields(resource_id):
            if not field['id'].startswith('_'):
                c.field_data[field['id']] = c.record_dict.get(field['id'], None)

        # Image and map should be a block in main record view
        return p.toolkit.render('record/view.html')<|endoftext|>