<commit_before>#!/bin/python

from datetime import datetime
from elasticsearch import Elasticsearch
import sqlite3
import requests
import json

res = requests.get('http://elastic.int.cemm.at:9200')
con = sqlite3.connect("/mnt/fhgfsMgmt/admon/fhgfs-admon-data.db")
es = Elasticsearch([{'host': 'elastic.int.cemm.at', 'port': 9200}])

print(res.content)

def dict_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
                d[col[0]] = row[idx]
    return d

#con.row_factory = dict_factory

#cur = con.cursor()

cursor = con.execute("select time, is_responding, workRequests, queuedRequests from metaNormal order by time desc limit 0,1")

#one_row = str(cur.fetchone())
#print (one_row.replace("'",'"'))

for row in cursor:
        print row[0]
        print row[1]
        print row[2]
        print row[3]

#while one_row is not None:
#       print (one_row)
#       one_row = cur.fetchone()

con.close()
<commit_msg>Update sqlite2elastic.py<commit_after>#!/usr/bin/env python

# compressandmove.py Code
#
# Copyright (c) Jose M. Molero
#
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

"""
Code sample to compress and move folders
Notes:
-
TODO list:
"""

# stdlib imports
import argparse
import os
import tarfile
import shutil
# import errno
# import string

# global variables
_SCRIPT_VERSION = '1.0'

from datetime import datetime
from elasticsearch import Elasticsearch
import sqlite3
import requests
import json

res = requests.get('http://elastic.int.cemm.at:9200')
con = sqlite3.connect("/mnt/fhgfsMgmt/admon/fhgfs-admon-data.db")
es = Elasticsearch([{'host': 'elastic.int.cemm.at', 'port': 9200}])

print(res.content)

def dict_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
                d[col[0]] = row[idx]
    return d

#con.row_factory = dict_factory

#cur = con.cursor()

cursor = con.execute("select time, is_responding, workRequests, queuedRequests from metaNormal order by time desc limit 0,1")

#one_row = str(cur.fetchone())
#print (one_row.replace("'",'"'))

for row in cursor:
        print row[0]
        print row[1]
        print row[2]
        print row[3]

#while one_row is not None:
#       print (one_row)
#       one_row = cur.fetchone()

con.close()
<|endoftext|>


<commit_before>import discord
import requests
import json
import requests_cache
from discord.ext import commands
from pathlib import Path
from riot_observer import RiotObserver as ro
from cogs.utils import LeagueUtils as lu
from cogs.utils import UserFileManip as ufm
from cogs.utils import DictManip as dm

class League():
    def __init__(self, bot):
        self.bot = bot

    # Get API keys
    with open("data/apikeys.json", "r") as f:
        api_keys = json.load(f)

    # Riot API key
    riot_api_key = api_keys["riot"]

    # Champion.gg API key
    champion_gg_api_key = api_keys["champion.gg"]

    # Init RiotObserver
    rito = ro(riot_api_key)

    @commands.bot.command(pass_context=True, aliases=['aln', 'addl', 'addleague'])
    async def addLeagueName(self, ctx, *args):
        member = str(ctx.message.author)
        summoner_name = " ".join(args)

        if not ufm.foundUserFile():
            await self.bot.say("No user file found...\nCreated user file and added `{}'s`' summoner name as `{}`.".format(member, summoner_name))
            ufm.createUserFile(member, "summoner_name",
                               summoner_name)
            return
        else:
            await self.bot.say("Added `{}` as summoner name for `{}`.".format(summoner_name, member))
            ufm.updateUserInfo(member, "summoner_name",
                               summoner_name)
            return

    @commands.bot.command(aliases=['champ', 'ci'])
    async def getChampInfo(self, *args):
        """ Return play, ban, and win rate for a champ """
        uri = "http://api.champion.gg/v2/champions/{}?api_key={}"
        champ = " ".join(args)
        champID = lu.getChampID(champ)
        res = requests.get(uri.format(
            champID, League.champion_gg_api_key)).json()
        role = res[-1]["role"]
        role_rate = res[-1]["percentRolePlayed"]
        play_rate = res[-1]["playRate"]
        win_rate = res[-1]["winRate"]
        ban_rate = res[-1]["banRate"]

        await self.bot.say("Champion.gg stats for `{} - {} ({:.2%} in role)`:\nPlay Rate: `{:.2%}`\nWin Rate: `{:.2%}`\nBan Rate: `{:.2%}`".format(champ.title(), role.title(), role_rate, play_rate, win_rate, ban_rate))

    # @commands.bot.command(pass_context=True, aliases=['matches'])
    # async def matchHistory(self, ctx):
    #     """ do not use """
    #     member = str(ctx.message.author)
    #     summoner_name = ufm.getUserInfo(member, "summoner_name")
    #     summoner_obj = League.getSummonerObj(summoner_name)
    #     return await self.bot.say("Latest match information for {} ({})\n{}".format(member, summoner_name, League.getLastTenMatches(summoner_obj)))

    @commands.bot.command(aliases=['ucf'])
    async def update_champion_file(self):
        """ Creates / updates a json file containing champion IDs, names, etc """

        # Case where champ data found
        if lu.foundChampFile():
            with open("data/champ_data.json", "r") as f:
                file_champ_list = json.load(f)

            # Get champ list from Riot's API
            champ_list = League.rito.static_get_champion_list()

            # If file is up to date, don't update
            if len(champ_list) == len(file_champ_list):
                return await self.bot.say("Champion information file already up to date.")

            # File needs updating
            else:
                with open("data/champ_data.json", "w") as f:
                    json.dump(champ_list, f)
                return

        # Create champion data file if not found
        else:
            champ_list = League.rito.static_get_champion_list()
            await self.bot.say("Creating chamption information file.")
            with open("data/champ_data.json", "w") as f:
                json.dump(champ_list, f)
            return

    @commands.bot.command(pass_context=True, aliases=['elo', 'mmr'])
    async def get_elo(self, ctx, summoner=""):
        """ Get League of Legends elo / mmr from na.whatismymmr.com """

        # WhatIsMyMMR API licensed under Creative Commons Attribution 2.0 Generic
        # More information here: https://creativecommons.org/licenses/by/2.0

        # Requests call information
        uri = "https://na.whatismymmr.com/api/v1/summoner?name={}"
        header = {'user-agent': 'qtbot/1.0'}

        # Get who's calling the function
        member = str(ctx.message.author)

        # Cache results for 2hrs
        requests_cache.install_cache(expire_after=3600)

        # Try to read summoner from file if none supplied
        if (summoner == ""):
            try:
                summoner = ufm.getUserInfo(member, "summoner_name")
            except KeyError:
                return await self.bot.say("Sorry you're not in my file. Use `aln` or `addl` to add your League of Legends summoner name, or supply the name to this command.")

        # Store results from call
        result_json = requests.get(
            uri.format(summoner), headers=header).json()

        # No MMR results found
        if result_json["ranked"]["avg"] == None:
            return await self.bot.say("Sorry, I found no ranked data for `{}`".format(summoner))

        estimated_rank = result_json["ranked"]["summary"].split('<b>')[
            1].split('</b')[0]

        return await self.bot.say("Average MMR for `{}`: `{}±{}`\nApproximate ranking: `{}`".format(summoner, result_json["ranked"]["avg"], result_json["ranked"]["err"], estimated_rank))

def setup(bot):
    bot.add_cog(League(bot))
<commit_msg>champ_info update<commit_after>import discord
import requests
import json
import requests_cache
from discord.ext import commands
from pathlib import Path
from riot_observer import RiotObserver as ro
from cogs.utils import LeagueUtils as lu
from cogs.utils import UserFileManip as ufm
from cogs.utils import DictManip as dm

class League():
    def __init__(self, bot):
        self.bot = bot

    # Get API keys
    with open("data/apikeys.json", "r") as f:
        api_keys = json.load(f)

    # Riot API key
    riot_api_key = api_keys["riot"]

    # Champion.gg API key
    champion_gg_api_key = api_keys["champion.gg"]

    # Init RiotObserver
    rito = ro(riot_api_key)

    @commands.bot.command(pass_context=True, aliases=['aln', 'addl', 'addleague'])
    async def addLeagueName(self, ctx, *args):
        member = str(ctx.message.author)
        summoner_name = " ".join(args)

        if not ufm.foundUserFile():
            await self.bot.say("No user file found...\nCreated user file and added `{}'s`' summoner name as `{}`.".format(member, summoner_name))
            ufm.createUserFile(member, "summoner_name",
                               summoner_name)
            return
        else:
            await self.bot.say("Added `{}` as summoner name for `{}`.".format(summoner_name, member))
            ufm.updateUserInfo(member, "summoner_name",
                               summoner_name)
            return

    @commands.bot.command(aliases=['champ', 'ci'])
    async def getChampInfo(self, *args):
        """ Return play, ban, and win rate for a champ """

        uri = "http://api.champion.gg/v2/champions/{}?api_key={}"
        champ = " ".join(args)
        champID = lu.getChampID(champ)
        res = requests.get(uri.format(
            champID, League.champion_gg_api_key)).json()

        role = []
        role_rate = []
        play_rate = []
        win_rate = []
        ban_rate = []

        format_str = ""

        for x in range(len(res)):
            role[x] = res[x]["role"]
            role_rate[x] = res[x]["percentRolePlayed"]
            play_rate[x] = res[x]["playRate"]
            win_rate[x] = res[x]["winRate"]
            ban_rate[x] = res[x]["banRate"]

        for x in range(len(res)):
            format_str + "```{} ({:.2%})\nPlay Rate: {:.2%}\nWin Rate: {:.2%}\nBan Rate: {:.2%}\n\n".format(role[x], role_rate[x], play_rate[x], win_rate[x], ban_rate[x])

        return await self.bot.say("**{}**\n{}".format(champ, format_str))

    # @commands.bot.command(pass_context=True, aliases=['matches'])
    # async def matchHistory(self, ctx):
    #     """ do not use """
    #     member = str(ctx.message.author)
    #     summoner_name = ufm.getUserInfo(member, "summoner_name")
    #     summoner_obj = League.getSummonerObj(summoner_name)
    #     return await self.bot.say("Latest match information for {} ({})\n{}".format(member, summoner_name, League.getLastTenMatches(summoner_obj)))

    @commands.bot.command(aliases=['ucf'])
    async def update_champion_file(self):
        """ Creates / updates a json file containing champion IDs, names, etc """

        # Case where champ data found
        if lu.foundChampFile():
            with open("data/champ_data.json", "r") as f:
                file_champ_list = json.load(f)

            # Get champ list from Riot's API
            champ_list = League.rito.static_get_champion_list()

            # If file is up to date, don't update
            if len(champ_list) == len(file_champ_list):
                return await self.bot.say("Champion information file already up to date.")

            # File needs updating
            else:
                with open("data/champ_data.json", "w") as f:
                    json.dump(champ_list, f)
                return

        # Create champion data file if not found
        else:
            champ_list = League.rito.static_get_champion_list()
            await self.bot.say("Creating chamption information file.")
            with open("data/champ_data.json", "w") as f:
                json.dump(champ_list, f)
            return

    @commands.bot.command(pass_context=True, aliases=['elo', 'mmr'])
    async def get_elo(self, ctx, summoner=""):
        """ Get League of Legends elo / mmr from na.whatismymmr.com """

        # WhatIsMyMMR API licensed under Creative Commons Attribution 2.0 Generic
        # More information here: https://creativecommons.org/licenses/by/2.0

        # Requests call information
        uri = "https://na.whatismymmr.com/api/v1/summoner?name={}"
        header = {'user-agent': 'qtbot/1.0'}

        # Get who's calling the function
        member = str(ctx.message.author)

        # Cache results for 2hrs
        requests_cache.install_cache(expire_after=3600)

        # Try to read summoner from file if none supplied
        if (summoner == ""):
            try:
                summoner = ufm.getUserInfo(member, "summoner_name")
            except KeyError:
                return await self.bot.say("Sorry you're not in my file. Use `aln` or `addl` to add your League of Legends summoner name, or supply the name to this command.")

        # Store results from call
        result_json = requests.get(
            uri.format(summoner), headers=header).json()

        # No MMR results found
        if result_json["ranked"]["avg"] == None:
            return await self.bot.say("Sorry, I found no ranked data for `{}`".format(summoner))

        estimated_rank = result_json["ranked"]["summary"].split('<b>')[
            1].split('</b')[0]

        return await self.bot.say("Average MMR for `{}`: `{}±{}`\nApproximate ranking: `{}`".format(summoner, result_json["ranked"]["avg"], result_json["ranked"]["err"], estimated_rank))

def setup(bot):
    bot.add_cog(League(bot))
<|endoftext|>


<commit_before>from borg.logger import setup_logging

# Ensure that the loggers exist for all tests
setup_logging()

from borg.testsuite import has_lchflags, no_lchlfags_because, has_llfuse
from borg.testsuite.platform import fakeroot_detected
from borg import xattr

def pytest_report_header(config, startdir):
    yesno = ['no', 'yes']
    flags = 'Testing BSD-style flags: %s %s' % (yesno[has_lchflags], no_lchlfags_because)
    fakeroot = 'fakeroot: %s (>=1.20.2: %s)' % (
        yesno[fakeroot_detected()],
        yesno[xattr.XATTR_FAKEROOT])
    llfuse = 'Testing fuse: %s' % yesno[has_llfuse]
    return '\n'.join((flags, llfuse, fakeroot))
<commit_msg>Fix tox<commit_after>import sys

# This is a hack to fix path problems because "borg" (the package) is in the source root.
# When importing the conftest an "import borg" can incorrectly import the borg from the
# source root instead of the one installed in the environment.
# The workaround is to remove entries pointing there from the path and check whether "borg"
# is still importable. If it is not, then it has not been installed in the environment
# and the entries are put back.
#
# TODO: After moving the package to src/: remove this.

original_path = list(sys.path)
for entry in original_path:
    if entry == '' or entry.endswith('/borg'):
        sys.path.remove(entry)

try:
    import borg
except ImportError:
    sys.path = original_path

from borg.logger import setup_logging

# Ensure that the loggers exist for all tests
setup_logging()

from borg.testsuite import has_lchflags, no_lchlfags_because, has_llfuse
from borg.testsuite.platform import fakeroot_detected
from borg import xattr

def pytest_report_header(config, startdir):
    yesno = ['no', 'yes']
    flags = 'Testing BSD-style flags: %s %s' % (yesno[has_lchflags], no_lchlfags_because)
    fakeroot = 'fakeroot: %s (>=1.20.2: %s)' % (
        yesno[fakeroot_detected()],
        yesno[xattr.XATTR_FAKEROOT])
    llfuse = 'Testing fuse: %s' % yesno[has_llfuse]
    return '\n'.join((flags, llfuse, fakeroot))
<|endoftext|>


<commit_before>import asyncio
import datetime
import logging

import aiohttp
import discord
import discord.ext.commands as commands

import paths
from .util import config, utils

log = logging.getLogger(__name__)

def setup(bot):
    cog = Twitch(bot)
    bot.add_cog(cog)
    cog.start()

class TwitchConfig(config.ConfigElement):
    def __init__(self, client_id, **kwargs):
        self.client_id = client_id
        self.follows = kwargs.get('follows', {})

    def remove_channels(self, *channels):
        """Unregister the given channels from every followed channel, and
        removes any followed channel that end up without any channel.
        """
        channels = set(c.id for c in channels)
        conf_to_remove = set()

        # Check every followed channel
        for stream_id, destinations in self.follows.items():
            # Remove the given channels from this followed channel
            for channel in channels:
                try:
                    del destinations.channels[channel]
                except KeyError:
                    pass
            if not destinations.channels:
                conf_to_remove.add(stream_id)

        # Cleanup the followed channels
        for stream_id in conf_to_remove:
            del self.follows[stream_id]

class FollowConfig(config.ConfigElement):
    def __init__(self, stream_id, live=False, preview_url=None, preview_updates=0, **kwargs):
        self.stream_id = stream_id
        self.live = live
        self.preview_url = preview_url
        self.preview_updates = preview_updates
        self.channels = utils.dict_keys_to_int(kwargs.pop('channels', {}))

    async def remove_previews(self, bot):
        for channel_conf in self.channels.values():
            await channel_conf.remove_preview(bot)
        self.preview_updates = 0

class ChannelConfig(config.ConfigElement):
    def __init__(self, id, content, message_id=None, **kwargs):
        self.id = id
        self.content = content
        self.message_id = message_id
        self._message = None

    async def get_message(self, bot):
        if self._message is not None:
            return self._message
        else:
            return await bot.get_message(bot.get_channel(self.id), self.message_id)

    async def remove_preview(self, bot):
        if self.message_id is None:
            return

        message = await self.get_message(bot)
        embed = message.embeds[0]
        del embed._image
        await message.edit(embed=embed)
        self.message_id = None
        self._message = None

class Twitch:
    """Follow Twitch channel and display alerts in Discord when one goes live."""
    api_base = 'https://api.twitch.tv/kraken'

    def __init__(self, bot):
        self.bot = bot
        self.conf = config.Config(paths.TWITCH_CONFIG, encoding='utf-8')
        default_headers = {
            'Client-ID': self.conf.client_id,
            'Accept': 'application/vnd.twitchtv.v5+json'
        }
        self.session = aiohttp.ClientSession(loop=bot.loop, headers=default_headers)
        self.daemon = None

    def __unload(self):
        self.session.close()
        self.daemon.cancel()

    async def on_guild_channel_delete(self, channel):
        self.conf.remove_channels(channel)
        self.conf.save()

    async def on_guild_remove(self, guild):
        self.conf.remove_channels(*guild.channels)
        self.conf.save()

    async def on_ready(self):
        self.start()

    def start(self):
        if self.daemon is None:
            self.daemon = self.bot.loop.create_task(self._daemon())

    async def _daemon(self):
        # ERMAHGERD ! MAH FRAVRIT LERP !
        while True:
            try:
                streams = await self.poll_streams()
            except Exception as e:
                log.info(f'Polling error: {e}')
            else:
                await self.bot.wait_until_ready()
                for stream_id, follow_conf in self.conf.follows.copy().items():
                    if follow_conf.live:
                        if stream_id not in streams:
                            try:
                                # Stream went offline, remove the preview images
                                await follow_conf.remove_previews(self.bot)
                            except Exception as e:
                                log.error(f'Preview removal error: {e}')
                            else:
                                follow_conf.live = False
                        else:
                            try:
                                # Stream is still online, update its info
                                await self.update(streams[stream_id])
                            except Exception as e:
                                log.error(f'Update error: {e}')
                    elif stream_id in streams:
                        # Stream came online, save the preview url and send notifications
                        stream_info = streams[stream_id]
                        follow_conf.preview_url = stream_info['preview']['template'].format(width=640, height=360)
                        try:
                            await self.notify(stream_info)
                        except Exception as e:
                            log.error(f'Notification error: {e}')
                        else:
                            follow_conf.live = True
                    self.conf.save()
            finally:
                await asyncio.sleep(60)

    async def poll_streams(self):
        streams = {}

        # Get the streams statuses in chunks of 100
        channels = list(self.conf.follows.keys())
        for i in range(0, len(channels), 100):
            chunk = channels[i:i + 100]
            try:
                streams_chunk = await utils.fetch_page(f'{self.api_base}/streams', session=self.session, params={'channel': ', '.join(chunk), 'limit': 100})
            except utils.HTTPError as e:
                raise Exception(f'HTTP error when fetching streams chunk #{i / 100 + 1} : {e}') from e

            # Extract the newly live streams
            for stream_info in streams_chunk['streams']:
                stream_id = str(stream_info['channel']['_id'])
                streams[stream_id] = stream_info

        return streams

    async def notify(self, stream_info):
        # Build the embed
        embed = discord.Embed(title='Click here to join the fun !', url=stream_info['channel']['url'], colour=discord.Colour.blurple())
        embed.set_author(name=stream_info['channel']['display_name'])
        embed.set_thumbnail(url=stream_info['channel']['logo'])
        embed.set_image(url=stream_info['preview']['large'])
        embed.add_field(name=stream_info['channel']['status'], value=f'Playing {stream_info["game"]}')
        embed.timestamp = datetime.datetime.strptime(stream_info['created_at'], '%Y-%m-%dT%H:%M:%SZ')

        # Send the notification to every interested channel
        for channel_id, chan_conf in self.conf.follows[str(stream_info['channel']['_id'])].channels.items():
            destination = self.bot.get_channel(channel_id)
            chan_conf._message = await destination.send(chan_conf.content, embed=embed)
            chan_conf.message_id = chan_conf._message.id
        self.conf.save()

    async def update(self, stream_info):
        follow_conf = self.conf.follows[str(stream_info['channel']['_id'])]
        for channel_id, chan_conf in follow_conf.channels.items():
            message = await chan_conf.get_message(self.bot)
            embed = message.embeds[0]

            # Update the stream preview
            url = f'{follow_conf.preview_url}?v={str(follow_conf.preview_updates + 1)}'
            embed.set_image(url=url)

            # Update the stream title and played game
            del embed._fields
            embed.add_field(name=stream_info['channel']['status'], value=f'Playing {stream_info["game"]}')

            await message.edit(embed=embed)
            follow_conf.preview_updates += 1

    async def get_user(self, channel):
        try:
            data = await utils.fetch_page(f'{self.api_base}/users', session=self.session, params={'login': channel})
        except utils.HTTPError as e:
            if e.code == 400:
                raise commands.BadArgument(e.resp_msg)
            else:
                raise e
        count = data['_total']
        if count == 0:
            raise commands.BadArgument('Channel not found.')
        elif count > 1:
            raise commands.BadArgument('More than one channel found.')
        return data['users'][0]

    @commands.group(name='twitch')
    @commands.has_permissions(manage_guild=True)
    async def twitch_group(self, ctx):
        pass

    @twitch_group.command(name='follow')
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def twitch_follow(self, ctx, channel, *, message=''):
        """Follows a Twitch channel.

        When the given Twitch channel goes online, a notification will be sent
        in the Discord channel this command was used in.

        The message will be sent along the stream notification.
        """
        user = await self.get_user(channel)
        user_id = user['_id']
        channel_id = ctx.channel.id

        if channel not in self.conf.follows.keys():
            self.conf.follows[user_id] = FollowConfig(user_id)
        elif channel_id in self.conf.follows[user_id].channels:
            raise commands.BadArgument(f'Already following "{channel}" on this channel.')

        self.conf.follows[user_id].channels[channel_id] = ChannelConfig(channel_id, message)
        self.conf.save()

        # Check if the channel is live
        streams = await utils.fetch_page(f'{self.api_base}/streams', session=self.session, params={'channel': user_id})
        if streams['_total'] == 1:
            self.conf.follows[user_id].preview_url = streams['streams'][0]['preview']['template'].format(width=640, height=360)
            await self.notify(streams['streams'][0])
            self.conf.follows[user_id].live = True

        await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    @twitch_group.command(name='unfollow')
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def twitch_unfollow(self, ctx, channel):
        """Unfollows a twitch channel.

        The notifications about the given channel will not be displayed in
        the Discord channel this command was used in anymore.
        """
        user = await self.get_user(channel)
        user_id = user['_id']
        if user_id not in self.conf.follows.keys() or ctx.channel.id not in self.conf.follows[user_id].channels:
            raise commands.BadArgument(f'Not following "{channel}" on this channel.')

        # Remove the discord channel from the conf and clean it up if no one else follow that stream
        await self.conf.follows[user_id].channels[ctx.channel.id].remove_preview(self.bot)
        del self.conf.follows[user_id].channels[ctx.channel.id]
        if not self.conf.follows[user_id].channels:
            del self.conf.follows[user_id]
        self.conf.save()

        await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')
<commit_msg>Twitch: Clearer offline stream display.<commit_after>import asyncio
import datetime
import logging

import aiohttp
import discord
import discord.ext.commands as commands

import paths
from .util import config, utils

log = logging.getLogger(__name__)

def setup(bot):
    cog = Twitch(bot)
    bot.add_cog(cog)
    cog.start()

class TwitchConfig(config.ConfigElement):
    def __init__(self, client_id, **kwargs):
        self.client_id = client_id
        self.follows = kwargs.get('follows', {})

    def remove_channels(self, *channels):
        """Unregister the given channels from every followed channel, and
        removes any followed channel that end up without any channel.
        """
        channels = set(c.id for c in channels)
        conf_to_remove = set()

        # Check every followed channel
        for stream_id, destinations in self.follows.items():
            # Remove the given channels from this followed channel
            for channel in channels:
                try:
                    del destinations.channels[channel]
                except KeyError:
                    pass
            if not destinations.channels:
                conf_to_remove.add(stream_id)

        # Cleanup the followed channels
        for stream_id in conf_to_remove:
            del self.follows[stream_id]

class FollowConfig(config.ConfigElement):
    def __init__(self, stream_id, live=False, preview_url=None, preview_updates=0, **kwargs):
        self.stream_id = stream_id
        self.live = live
        self.preview_url = preview_url
        self.preview_updates = preview_updates
        self.channels = utils.dict_keys_to_int(kwargs.pop('channels', {}))

    async def put_offline(self, bot):
        for channel_conf in self.channels.values():
            await channel_conf.put_offline(bot)
        self.preview_updates = 0
        self.live = False

class ChannelConfig(config.ConfigElement):
    def __init__(self, id, content, message_id=None, **kwargs):
        self.id = id
        self.content = content
        self.message_id = message_id
        self._message = None

    async def get_message(self, bot):
        if self._message is not None:
            return self._message
        else:
            return await bot.get_message(bot.get_channel(self.id), self.message_id)

    async def put_offline(self, bot):
        if self.message_id is None:
            return

        # Get the notification's embed
        message = await self.get_message(bot)
        embed = message.embeds[0]

        # Remove the image preview, modify the title and timestamp
        del embed._image
        embed.title = 'Offline'
        embed.description = None
        embed.timestamp = datetime.datetime.utcnow()

        # Send the edit and remove the message's reference in the conf
        await message.edit(embed=embed)
        self.message_id = None
        self._message = None

class Twitch:
    """Follow Twitch channel and display alerts in Discord when one goes live."""
    api_base = 'https://api.twitch.tv/kraken'

    def __init__(self, bot):
        self.bot = bot
        self.conf = config.Config(paths.TWITCH_CONFIG, encoding='utf-8')
        default_headers = {
            'Client-ID': self.conf.client_id,
            'Accept': 'application/vnd.twitchtv.v5+json'
        }
        self.session = aiohttp.ClientSession(loop=bot.loop, headers=default_headers)
        self.daemon = None

    def __unload(self):
        self.session.close()
        self.daemon.cancel()

    async def on_guild_channel_delete(self, channel):
        self.conf.remove_channels(channel)
        self.conf.save()

    async def on_guild_remove(self, guild):
        self.conf.remove_channels(*guild.channels)
        self.conf.save()

    async def on_ready(self):
        self.start()

    def start(self):
        if self.daemon is None:
            self.daemon = self.bot.loop.create_task(self._daemon())

    async def _daemon(self):
        # ERMAHGERD ! MAH FRAVRIT LERP !
        while True:
            try:
                streams = await self.poll_streams()
            except Exception as e:
                log.info(f'Polling error: {e}')
            else:
                await self.bot.wait_until_ready()
                for stream_id, follow_conf in self.conf.follows.copy().items():
                    if follow_conf.live:
                        if stream_id not in streams:
                            try:
                                # Stream went offline, remove the preview images
                                await follow_conf.put_offline(self.bot)
                            except Exception as e:
                                log.error(f'Preview removal error: {e}')
                        else:
                            try:
                                # Stream is still online, update its info
                                await self.update(streams[stream_id])
                            except Exception as e:
                                log.error(f'Update error: {e}')
                    elif stream_id in streams:
                        # Stream came online, save the preview url and send notifications
                        stream_info = streams[stream_id]
                        follow_conf.preview_url = stream_info['preview']['template'].format(width=640, height=360)
                        try:
                            await self.notify(stream_info)
                        except Exception as e:
                            log.error(f'Notification error: {e}')
                        else:
                            follow_conf.live = True
                    self.conf.save()
            finally:
                await asyncio.sleep(60)

    async def poll_streams(self):
        streams = {}

        # Get the streams statuses in chunks of 100
        channels = list(self.conf.follows.keys())
        for i in range(0, len(channels), 100):
            chunk = channels[i:i + 100]
            try:
                streams_chunk = await utils.fetch_page(f'{self.api_base}/streams', session=self.session, params={'channel': ', '.join(chunk), 'limit': 100})
            except utils.HTTPError as e:
                raise Exception(f'HTTP error when fetching streams chunk #{i / 100 + 1} : {e}') from e

            # Extract the newly live streams
            for stream_info in streams_chunk['streams']:
                stream_id = str(stream_info['channel']['_id'])
                streams[stream_id] = stream_info

        return streams

    async def notify(self, stream_info):
        # Build the embed
        embed = discord.Embed(url=stream_info['channel']['url'],
                              colour=discord.Colour.blurple(),
                              title=stream_info['channel']['status'],
                              description=f'Playing [{stream_info["game"]}](https://www.twitch.tv/directory/game/{stream_info["game"]})')
        embed.set_author(name=stream_info['channel']['display_name'])
        embed.set_thumbnail(url=stream_info['channel']['logo'])
        embed.set_image(url=stream_info['preview']['large'])
        embed.timestamp = datetime.datetime.strptime(stream_info['created_at'], '%Y-%m-%dT%H:%M:%SZ')

        # Send the notification to every interested channel
        for channel_id, chan_conf in self.conf.follows[str(stream_info['channel']['_id'])].channels.items():
            destination = self.bot.get_channel(channel_id)
            chan_conf._message = await destination.send(chan_conf.content, embed=embed)
            chan_conf.message_id = chan_conf._message.id
        self.conf.save()

    async def update(self, stream_info):
        follow_conf = self.conf.follows[str(stream_info['channel']['_id'])]
        for channel_id, chan_conf in follow_conf.channels.items():
            message = await chan_conf.get_message(self.bot)
            embed = message.embeds[0]

            # Update the stream preview
            url = f'{follow_conf.preview_url}?v={str(follow_conf.preview_updates + 1)}'
            embed.set_image(url=url)

            # Update the stream title and played game
            del embed._fields
            embed.add_field(name=stream_info['channel']['status'], value=f'Playing {stream_info["game"]}')

            await message.edit(embed=embed)
            follow_conf.preview_updates += 1

    async def get_user(self, channel):
        try:
            data = await utils.fetch_page(f'{self.api_base}/users', session=self.session, params={'login': channel})
        except utils.HTTPError as e:
            if e.code == 400:
                raise commands.BadArgument(e.resp_msg)
            else:
                raise e
        count = data['_total']
        if count == 0:
            raise commands.BadArgument('Channel not found.')
        elif count > 1:
            raise commands.BadArgument('More than one channel found.')
        return data['users'][0]

    @commands.group(name='twitch')
    @commands.has_permissions(manage_guild=True)
    async def twitch_group(self, ctx):
        pass

    @twitch_group.command(name='follow')
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def twitch_follow(self, ctx, channel, *, message=''):
        """Follows a Twitch channel.

        When the given Twitch channel goes online, a notification will be sent
        in the Discord channel this command was used in.

        The message will be sent along the stream notification.
        """
        user = await self.get_user(channel)
        user_id = user['_id']
        channel_id = ctx.channel.id

        if channel not in self.conf.follows.keys():
            self.conf.follows[user_id] = FollowConfig(user_id)
        elif channel_id in self.conf.follows[user_id].channels:
            raise commands.BadArgument(f'Already following "{channel}" on this channel.')

        self.conf.follows[user_id].channels[channel_id] = ChannelConfig(channel_id, message)
        self.conf.save()

        # Check if the channel is live
        streams = await utils.fetch_page(f'{self.api_base}/streams', session=self.session, params={'channel': user_id})
        if streams['_total'] == 1:
            self.conf.follows[user_id].preview_url = streams['streams'][0]['preview']['template'].format(width=640, height=360)
            await self.notify(streams['streams'][0])
            self.conf.follows[user_id].live = True

        await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    @twitch_group.command(name='unfollow')
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def twitch_unfollow(self, ctx, channel):
        """Unfollows a twitch channel.

        The notifications about the given channel will not be displayed in
        the Discord channel this command was used in anymore.
        """
        user = await self.get_user(channel)
        user_id = user['_id']
        if user_id not in self.conf.follows.keys() or ctx.channel.id not in self.conf.follows[user_id].channels:
            raise commands.BadArgument(f'Not following "{channel}" on this channel.')

        # Remove the discord channel from the conf and clean it up if no one else follow that stream
        await self.conf.follows[user_id].channels[ctx.channel.id].put_offline(self.bot)
        del self.conf.follows[user_id].channels[ctx.channel.id]
        if not self.conf.follows[user_id].channels:
            del self.conf.follows[user_id]
        self.conf.save()

        await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')
<|endoftext|>


<commit_before># Copyright  2015, Province of British Columbia
# License: https://github.com/bcgov/ckanext-bcgov/blob/master/license

#Author : Khalegh Mamakani, Highway Three Solutions  Inc.
#Date created : January 29 2014
#Last modified on January 29 2014

from ckan import model
from ckan.lib.cli import CkanCommand
from ckan.logic import get_action, NotFound
from ckan.lib.datapreview import (get_default_view_plugins,
                                  add_views_to_dataset_resources)

import logging
import os
import sys
import json
import pprint
import ckanext.bcgov
import ckan.lib.base as base
_ = base._

import ckan.plugins as plugins

log = logging.getLogger()

class EdcCommand(CkanCommand):
    '''
    This class contains commnds for creating special vocabularies and tags
    required for EDC-Dataset manipulation

    Commands:
        create-all-vocabs   Create all vocabularies
        delete-all-vocabs   Delete all vocabularies
        create-all-orgs     Create all organizations
        delete-all-orgs     Delete all organizations
        purge-records       Purge all already-deleted datasets
        purge-revisions     Purge all already-deleted revisions
        create-views         Create CKAN 2.3 resource views
    '''
    summary = __doc__.split('\n')[0]
    usage = __doc__

    default_data_dir = os.path.join(
        os.path.dirname(ckanext.bcgov.__file__),
        'scripts/data'
        )
    default_vocab_file = os.path.join(default_data_dir, 'edc-vocabs.json')
    default_org_file =   os.path.join(default_data_dir, 'orgs.json')

    def command(self):
        '''
        Parse command line arguments and call appropriate method.
        '''
        if not self.args or self.args[0] in ['--help', '-h', 'help']:
            print EdcCommand.__doc__
            return

        cmd = self.args[0]
        self._load_config()

        user = get_action('get_site_user')(
            {'model': model, 'ignore_auth': True}, {}
        )
        self.user_name = user['name']

        # file_path is used by create-all-vocabs command
        vocab_file = self.args[1] if len(self.args) >= 2 else None

        if cmd == 'create-all-vocabs' :
            self.create_all_vocabs()
        elif cmd == 'delete-all-vocabs' :
            self.delete_all_vocabs()
        elif cmd == 'create-all-orgs' :
            self.create_orgs()
        elif cmd == 'delete-all-orgs' :
            self.delete_orgs()
        elif cmd == 'purge-records' :
            self.purge_records()
        elif cmd == 'purge-revisions':
            self.purge_revisions()
        elif cmd == 'create-views':
            self.create_views()
        else:
            log.error('Command "%s" not defined' % (cmd,))

    def _create_vocab(self, context, vocab_name, tags):
        print "Vocab ========> ", vocab_name, "tags ===> ", tags
        vocab = get_action('vocabulary_create')(context, {'name': vocab_name})
        for tag in tags:
            get_action('tag_create')(context, {'name': tag,
                                               'vocabulary_id': vocab['id']})

    def create_all_vocabs(self, vocab_file=None):

        if not vocab_file:
            vocab_file = self.default_vocab_file

        if not os.path.exists(vocab_file):
            log.error('File {0} does not exists'.format(vocab_file))
            sys.exit(1)

        context = {'model': model, 'session':model.Session, 'user':self.user_name}
#
        #Read vocabularies json file
        with open(vocab_file) as json_file:
            vocabs = json.loads(json_file.read())

        #Get the list of available vocabularies
        available_vocabs = get_action('vocabulary_list')(context)

        available_vocab_names = [vocab['name'] for vocab in available_vocabs]
        #for each vocabulary defined in json file
        for vocab_item in vocabs :

            vocab_name = vocab_item['name']
            vocab_tags = vocab_item['tags']

            #check if the vocabulary exists :
            if not vocab_name in available_vocab_names:
                #Create the vocabulary
                self._create_vocab(context, vocab_name, vocab_tags)
            else:
                #Get the vocab from the list of available vocabularies
                vocab = get_action('vocabulary_show')(context, {'id' : vocab_name})
                #Get the list of available tags for this vocab
                available_tags = get_action('tag_list')(context, {'vocabulary_id' : vocab['id']})

                #Add each tag that is not in the list of available tags
                for tag in vocab_tags:
                    if not tag in available_tags:
                        new_tag = {'name': tag, 'vocabulary_id': vocab['id']}
                        get_action('tag_create')(context, new_tag)

    def delete_all_vocabs(self):

        context = {'model': model, 'session':model.Session, 'user':self.user_name}
        #Get all available vocabularies
        all_vocabs = get_action('vocabulary_list')(context)

        for vocab in all_vocabs:
            log.info('Deleting vocabulary "{0}.'.format(vocab['name']))
            #Deleting all tags of the current vocab
            for tag in vocab.get('tags') :
                log.info('Deleting tag "%s".' % tag['name'])
                get_action('tag_delete')(context, {'id': tag['id']})

            #Deleting the vocabulary
            get_action('vocabulary_delete')(context, {'id': vocab['id']})

    def delete_orgs(self):

        context = {'model': model, 'session':model.Session, 'user':self.user_name}

        org_model = context['model']
        top_level_orgs = org_model.Group.get_top_level_groups(type="organization")

        for orgg in top_level_orgs :
            org = get_action('organization_show')(context, {'id': orgg.id, 'include_datasets' : False})
            print "Organization ------------> ", org['name']
            group = org_model.Group.get(orgg.id)
            branches = group.get_children_groups(type="organization")
            for branch in branches:
                branch_org = get_action('organization_show')(context, {'id': branch.id,'include_datasets' : False})
                print "\t Branch ------------> ", branch_org['name']
#                get_action('memeber_delete')(context, {'id' : branch_org['id'], 'object': org['id'], 'object_type' : 'group'})
                get_action('organization_purge')(context, {'id' : branch_org['id']})
            get_action('organization_purge')(context, {'id' : org['id']})

    def create_orgs(self, org_file=None):
        # TODO: This command is broken - the data that feeds into this command
        # is much different than the format expected by this command
        # - @deniszgonjanin

        if not org_file:
            org_file = self.default_org_file

        if not os.path.exists(org_file):
            log.error('File {0} does not exists'.format(org_file))
            sys.exit(1)

        context = {'model': model, 'session':model.Session, 'user':self.user_name}

        #Read the organizations json file
        with open(org_file) as json_file:
            orgs = json.loads(json_file.read())

        #Create each organization and all of its branches if it is not in the list of available organizations
        for org_item in orgs:
            org_name = org_item['name']
            branches = org_item['branches']
            org_title = org_item['title']

#            print "Organization ------> " ,org_name
            try :
                org = get_action('organization_show')(context, {'id' : org_name })
            except NotFound:
                org = get_action('organization_create')(context, {'name' : org_name, 'title' : org_title })

            group = model.Group.get(org['id'])

#            pprint.pprint(branches)
            for branch in branches:
                branch_name = branch['name']
                branch_title = branch['title']

#                print "Branch --------> ", branch_name
                try:
                    sub_org = get_action('organization_show')(context, {'id' : branch_name })
                except NotFound:
                    sub_org = get_action('organization_create')(context, {'name' : branch_name, 'title' : branch_title })

                sub_group = model.Group.get(sub_org['id'])

                #Add the sub_org as a member of the org
                get_action('member_create')(context, {'id' : sub_org['id'], 'object' : org['id'], 'object_type' : 'group', 'capacity' : 'admin'})

        org_model = context['model']
        top_level_org = org_model.Group.get_top_level_groups(type="organization")

        for org in top_level_org:
            print "Organization +---------> ", org.name
            group = org_model.Group.get(org.id)
            branches = group.get_children_groups(type = 'organization')
            for branch in branches:
                print '\t Branch +-------------> ', branch.name

    def purge_records(self):

        #Get the list of all deleted records
        deleted_packages = model.Session.query(model.Package).filter_by(state=model.State.DELETED)

        msgs = []

        revs_to_purge = []
        count = 0
        for pkg in deleted_packages:
            count += 1
            #pprint.pprint(pkg)
            revisions = [x[0] for x in pkg.all_related_revisions]
                        # ensure no accidental purging of other(non-deleted)
                        # packages initially just avoided purging revisions
                        # where non-deleted packages were affected
                        # however this lead to confusing outcomes e.g.
                        # we succesfully deleted revision in which package
                        # was deleted (so package now active again) but no
                        # other revisions
            problem = False
            for r in revisions:
                affected_pkgs = set(r.packages).difference(set(deleted_packages))
                if affected_pkgs:
                    msg = _('Cannot purge package %s as '
                            'associated revision %s includes '
                            'non-deleted packages %s')
                    msg = msg % (pkg.id, r.id, [pkg.id for r in affected_pkgs])
                    msgs.append(msg)
                    problem = True
                    break
            if not problem:
                revs_to_purge += [r.id for r in revisions]
            print '.'
        model.Session.remove()

        revs_to_purge = list(set(revs_to_purge))
        for id in revs_to_purge:
            revision = model.Session.query(model.Revision).get(id)
            try:
                        # TODO deleting the head revision corrupts the edit
                        # page Ensure that whatever 'head' pointer is used
                        # gets moved down to the next revision
                model.repo.purge_revision(revision, leave_record=False)
            except Exception, inst:
                msg = _('Problem purging revision %s: %s') % (id, inst)
                msgs.append(msg)

        print '------------------------------- Purge complete -------------------------------'

        print 'Total number of records : ' + str(count)
        print 'Purge errors : '
        for msg in msgs:
            print msg

    def purge_revisions(self):
        deleted_revisions = model.Session.query(model.Revision).filter_by(state=model.State.DELETED)

        msgs = []
        revs_to_purge = [rev.id for rev in deleted_revisions]
        revs_to_purge = list(set(revs_to_purge))
        for id in revs_to_purge:
            print id
            revision = model.Session.query(model.Revision).get(id)
            try:
                        # TODO deleting the head revision corrupts the edit
                        # page Ensure that whatever 'head' pointer is used
                        # gets moved down to the next revision
                model.repo.purge_revision(revision, leave_record=False)
            except Exception, inst:
                msg = _('Problem purging revision %s: %s') % (id, inst)
                msgs.append(msg)
        print '------------------------------- Purge complete -------------------------------'

        print 'Purge errors : '
        for msg in msgs:
            print msg

    def create_views(self):
        #TODO: we need this method because of this bug - @deniszgonjanin
        #https://github.com/ckan/ckan/issues/2532
        view_plugins = [view_plugin.info()['name']
                        for view_plugin in get_default_view_plugins()]

        for p in get_action('package_list')({},{}):
            package = get_action('package_show')({}, {'id':p})

            add_views_to_dataset_resources({}, package, view_types=view_plugins)
<commit_msg>fix file path<commit_after># Copyright  2015, Province of British Columbia
# License: https://github.com/bcgov/ckanext-bcgov/blob/master/license

#Author : Khalegh Mamakani, Highway Three Solutions  Inc.
#Date created : January 29 2014
#Last modified on January 29 2014

from ckan import model
from ckan.lib.cli import CkanCommand
from ckan.logic import get_action, NotFound
from ckan.lib.datapreview import (get_default_view_plugins,
                                  add_views_to_dataset_resources)

import logging
import os
import sys
import json
import pprint
import ckanext.bcgov
import ckan.lib.base as base
_ = base._

import ckan.plugins as plugins

log = logging.getLogger()

class EdcCommand(CkanCommand):
    '''
    This class contains commnds for creating special vocabularies and tags
    required for EDC-Dataset manipulation

    Commands:
        create-all-vocabs   Create all vocabularies
        delete-all-vocabs   Delete all vocabularies
        create-all-orgs     Create all organizations
        delete-all-orgs     Delete all organizations
        purge-records       Purge all already-deleted datasets
        purge-revisions     Purge all already-deleted revisions
        create-views         Create CKAN 2.3 resource views
    '''
    summary = __doc__.split('\n')[0]
    usage = __doc__

    default_data_dir = os.path.join(
        ckanext.bcgov.__path__[0],
        'scripts/data'
        )
    default_vocab_file = os.path.join(default_data_dir, 'edc-vocabs.json')
    default_org_file =   os.path.join(default_data_dir, 'orgs.json')

    def command(self):
        '''
        Parse command line arguments and call appropriate method.
        '''
        if not self.args or self.args[0] in ['--help', '-h', 'help']:
            print EdcCommand.__doc__
            return

        cmd = self.args[0]
        self._load_config()

        user = get_action('get_site_user')(
            {'model': model, 'ignore_auth': True}, {}
        )
        self.user_name = user['name']

        # file_path is used by create-all-vocabs command
        vocab_file = self.args[1] if len(self.args) >= 2 else None

        if cmd == 'create-all-vocabs' :
            self.create_all_vocabs()
        elif cmd == 'delete-all-vocabs' :
            self.delete_all_vocabs()
        elif cmd == 'create-all-orgs' :
            self.create_orgs()
        elif cmd == 'delete-all-orgs' :
            self.delete_orgs()
        elif cmd == 'purge-records' :
            self.purge_records()
        elif cmd == 'purge-revisions':
            self.purge_revisions()
        elif cmd == 'create-views':
            self.create_views()
        else:
            log.error('Command "%s" not defined' % (cmd,))

    def _create_vocab(self, context, vocab_name, tags):
        print "Vocab ========> ", vocab_name, "tags ===> ", tags
        vocab = get_action('vocabulary_create')(context, {'name': vocab_name})
        for tag in tags:
            get_action('tag_create')(context, {'name': tag,
                                               'vocabulary_id': vocab['id']})

    def create_all_vocabs(self, vocab_file=None):

        if not vocab_file:
            vocab_file = self.default_vocab_file

        if not os.path.exists(vocab_file):
            log.error('File {0} does not exists'.format(vocab_file))
            sys.exit(1)

        context = {'model': model, 'session':model.Session, 'user':self.user_name}
#
        #Read vocabularies json file
        with open(vocab_file) as json_file:
            vocabs = json.loads(json_file.read())

        #Get the list of available vocabularies
        available_vocabs = get_action('vocabulary_list')(context)

        available_vocab_names = [vocab['name'] for vocab in available_vocabs]
        #for each vocabulary defined in json file
        for vocab_item in vocabs :

            vocab_name = vocab_item['name']
            vocab_tags = vocab_item['tags']

            #check if the vocabulary exists :
            if not vocab_name in available_vocab_names:
                #Create the vocabulary
                self._create_vocab(context, vocab_name, vocab_tags)
            else:
                #Get the vocab from the list of available vocabularies
                vocab = get_action('vocabulary_show')(context, {'id' : vocab_name})
                #Get the list of available tags for this vocab
                available_tags = get_action('tag_list')(context, {'vocabulary_id' : vocab['id']})

                #Add each tag that is not in the list of available tags
                for tag in vocab_tags:
                    if not tag in available_tags:
                        new_tag = {'name': tag, 'vocabulary_id': vocab['id']}
                        get_action('tag_create')(context, new_tag)

    def delete_all_vocabs(self):

        context = {'model': model, 'session':model.Session, 'user':self.user_name}
        #Get all available vocabularies
        all_vocabs = get_action('vocabulary_list')(context)

        for vocab in all_vocabs:
            log.info('Deleting vocabulary "{0}.'.format(vocab['name']))
            #Deleting all tags of the current vocab
            for tag in vocab.get('tags') :
                log.info('Deleting tag "%s".' % tag['name'])
                get_action('tag_delete')(context, {'id': tag['id']})

            #Deleting the vocabulary
            get_action('vocabulary_delete')(context, {'id': vocab['id']})

    def delete_orgs(self):

        context = {'model': model, 'session':model.Session, 'user':self.user_name}

        org_model = context['model']
        top_level_orgs = org_model.Group.get_top_level_groups(type="organization")

        for orgg in top_level_orgs :
            org = get_action('organization_show')(context, {'id': orgg.id, 'include_datasets' : False})
            print "Organization ------------> ", org['name']
            group = org_model.Group.get(orgg.id)
            branches = group.get_children_groups(type="organization")
            for branch in branches:
                branch_org = get_action('organization_show')(context, {'id': branch.id,'include_datasets' : False})
                print "\t Branch ------------> ", branch_org['name']
#                get_action('memeber_delete')(context, {'id' : branch_org['id'], 'object': org['id'], 'object_type' : 'group'})
                get_action('organization_purge')(context, {'id' : branch_org['id']})
            get_action('organization_purge')(context, {'id' : org['id']})

    def create_orgs(self, org_file=None):
        # TODO: This command is broken - the data that feeds into this command
        # is much different than the format expected by this command
        # - @deniszgonjanin

        if not org_file:
            org_file = self.default_org_file

        if not os.path.exists(org_file):
            log.error('File {0} does not exists'.format(org_file))
            sys.exit(1)

        context = {'model': model, 'session':model.Session, 'user':self.user_name}

        #Read the organizations json file
        with open(org_file) as json_file:
            orgs = json.loads(json_file.read())

        #Create each organization and all of its branches if it is not in the list of available organizations
        for org_item in orgs:
            org_name = org_item['name']
            branches = org_item['branches']
            org_title = org_item['title']

#            print "Organization ------> " ,org_name
            try :
                org = get_action('organization_show')(context, {'id' : org_name })
            except NotFound:
                org = get_action('organization_create')(context, {'name' : org_name, 'title' : org_title })

            group = model.Group.get(org['id'])

#            pprint.pprint(branches)
            for branch in branches:
                branch_name = branch['name']
                branch_title = branch['title']

#                print "Branch --------> ", branch_name
                try:
                    sub_org = get_action('organization_show')(context, {'id' : branch_name })
                except NotFound:
                    sub_org = get_action('organization_create')(context, {'name' : branch_name, 'title' : branch_title })

                sub_group = model.Group.get(sub_org['id'])

                #Add the sub_org as a member of the org
                get_action('member_create')(context, {'id' : sub_org['id'], 'object' : org['id'], 'object_type' : 'group', 'capacity' : 'admin'})

        org_model = context['model']
        top_level_org = org_model.Group.get_top_level_groups(type="organization")

        for org in top_level_org:
            print "Organization +---------> ", org.name
            group = org_model.Group.get(org.id)
            branches = group.get_children_groups(type = 'organization')
            for branch in branches:
                print '\t Branch +-------------> ', branch.name

    def purge_records(self):

        #Get the list of all deleted records
        deleted_packages = model.Session.query(model.Package).filter_by(state=model.State.DELETED)

        msgs = []

        revs_to_purge = []
        count = 0
        for pkg in deleted_packages:
            count += 1
            #pprint.pprint(pkg)
            revisions = [x[0] for x in pkg.all_related_revisions]
                        # ensure no accidental purging of other(non-deleted)
                        # packages initially just avoided purging revisions
                        # where non-deleted packages were affected
                        # however this lead to confusing outcomes e.g.
                        # we succesfully deleted revision in which package
                        # was deleted (so package now active again) but no
                        # other revisions
            problem = False
            for r in revisions:
                affected_pkgs = set(r.packages).difference(set(deleted_packages))
                if affected_pkgs:
                    msg = _('Cannot purge package %s as '
                            'associated revision %s includes '
                            'non-deleted packages %s')
                    msg = msg % (pkg.id, r.id, [pkg.id for r in affected_pkgs])
                    msgs.append(msg)
                    problem = True
                    break
            if not problem:
                revs_to_purge += [r.id for r in revisions]
            print '.'
        model.Session.remove()

        revs_to_purge = list(set(revs_to_purge))
        for id in revs_to_purge:
            revision = model.Session.query(model.Revision).get(id)
            try:
                        # TODO deleting the head revision corrupts the edit
                        # page Ensure that whatever 'head' pointer is used
                        # gets moved down to the next revision
                model.repo.purge_revision(revision, leave_record=False)
            except Exception, inst:
                msg = _('Problem purging revision %s: %s') % (id, inst)
                msgs.append(msg)

        print '------------------------------- Purge complete -------------------------------'

        print 'Total number of records : ' + str(count)
        print 'Purge errors : '
        for msg in msgs:
            print msg

    def purge_revisions(self):
        deleted_revisions = model.Session.query(model.Revision).filter_by(state=model.State.DELETED)

        msgs = []
        revs_to_purge = [rev.id for rev in deleted_revisions]
        revs_to_purge = list(set(revs_to_purge))
        for id in revs_to_purge:
            print id
            revision = model.Session.query(model.Revision).get(id)
            try:
                        # TODO deleting the head revision corrupts the edit
                        # page Ensure that whatever 'head' pointer is used
                        # gets moved down to the next revision
                model.repo.purge_revision(revision, leave_record=False)
            except Exception, inst:
                msg = _('Problem purging revision %s: %s') % (id, inst)
                msgs.append(msg)
        print '------------------------------- Purge complete -------------------------------'

        print 'Purge errors : '
        for msg in msgs:
            print msg

    def create_views(self):
        #TODO: we need this method because of this bug - @deniszgonjanin
        #https://github.com/ckan/ckan/issues/2532
        view_plugins = [view_plugin.info()['name']
                        for view_plugin in get_default_view_plugins()]

        for p in get_action('package_list')({},{}):
            package = get_action('package_show')({}, {'id':p})

            add_views_to_dataset_resources({}, package, view_types=view_plugins)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from collections import defaultdict
import csv
import datetime
from django.core.management.base import BaseCommand

from dimagi.utils.decorators.memoized import memoized

from corehq.apps.case_search.models import CLAIM_CASE_TYPE
from corehq.apps.locations.models import SQLLocation
from corehq.apps.hqcase.utils import bulk_update_cases
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.util.log import with_progress_bar

from custom.enikshay.case_utils import (
    CASE_TYPE_EPISODE,
    CASE_TYPE_INVESTIGATION,
    CASE_TYPE_OCCURRENCE,
    CASE_TYPE_PERSON,
    CASE_TYPE_REFERRAL,
    CASE_TYPE_SECONDARY_OWNER,
    CASE_TYPE_TEST,
    CASE_TYPE_TRAIL,
    get_all_occurrence_cases_from_person,
)
from custom.enikshay.duplicate_ids import get_cases_with_duplicate_ids, get_new_readable_id
from custom.enikshay.user_setup import compress_nikshay_id, join_chunked

class Command(BaseCommand):
    help = """
    Finds cases with duplicate IDs and marks all but one of each ID as a duplicate
    """
    # TODO what are the headers we need?
    logfile_fields = ['name', 'dto_name', 'phi_name', 'owner_id', 'dob', 'phone_number']

    def add_arguments(self, parser):
        parser.add_argument('domain')
        parser.add_argument(
            '--commit',
            action='store_true',
            dest='commit',
            default=False,
        )

    def handle(self, domain, **options):
        self.domain = domain
        self.accessor = CaseAccessors(domain)
        commit = options['commit']

        filename = '{}-{}.csv'.format(self.__module__.split('.')[-1],
                                      datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S'))
        print("Logging actions to {}".format(filename))
        with open(filename, 'w') as f:
            self.logfile = csv.DictWriter(f, self.logfile_fields, extrasaction='ignore')
            self.logfile.writeheader()

            print("Finding duplicates")
            bad_cases = get_cases_with_duplicate_ids(self.domain, CASE_TYPE_PERSON)

            print("Processing duplicate cases")
            for person_case in with_progress_bar(bad_cases):
                self.log_person_case_info(person_case)
                updates = self.get_updates(person_case)
                bulk_update_cases(self.domain, updates, self.__module__)

    @property
    @memoized
    def districts_by_id(self):
        locs = SQLLocation.objects.filter(domain=self.domain, location_type__code='dto')
        return defaultdict(lambda: '', (
            (loc.location_id, loc.loc_name) for loc in locs
        ))

    def log_person_case_info(self, person_case):
        """Pull info that we want to log but not update"""
        person = person_case.dynamic_case_properties()
        self.logfile.writerow({
            'person_case_id': person_case.case_id,
            'name': ' '.join(filter(None, [person.get('first_name'), person.get('last_name')])),
            'dto_name': self.districts_by_id(person.get('current_address_district_choice')),
            'phi_name': person.get('phi'),
            'owner_id': person_case.owner_id,
            'dob': person.get('dob'),
            'phone_number': person.get('contact_phone_number'),
        })

    def get_reverse_indexed(self, case, case_type):
        all_cases = self.accessor.get_reverse_indexed_cases([case.case_id])
        return [case for case in all_cases if case.type == case_type]

    def get_updates(self, person_case):
        """ Get updates for a person case and a bunch of its children
        https://docs.google.com/document/d/1NS5ozgk7w-2AADsrdTtjgqODkgWIEaCw6eqQ0cLg138/edit#
        """
        old_id = person_case.get_case_property('person_id')
        new_id = get_new_readable_id()  # TODO uppercase
        yield get_case_update(person_case, {
            'person_id_deprecated': old_id,
            'person_id_flat_deprecated': person_case.get_case_property('person_id_flat'),
            'person_id': new_id,
            'person_id_flat': new_id.replace('-', ''),
        })

        for person_child_case in self.accessor.get_reverse_indexed_cases([person_case.case_id]):

            # Update occurrence and claim names
            if person_child_case.type in (CASE_TYPE_OCCURRENCE, CLAIM_CASE_TYPE):
                yield get_name_update(person_child_case, old_id, new_id)

            # look at all extensions of the occurrence cases
            if person_child_case.type == CASE_TYPE_OCCURRENCE:
                for case in self.accessor.get_reverse_indexed_cases([person_child_case.case_id]):

                    # update the names of episode, secondary_owner, trail, and referral cases
                    if case.type in (CASE_TYPE_EPISODE, CASE_TYPE_SECONDARY_OWNER,
                                     CASE_TYPE_TRAIL, CASE_TYPE_REFERRAL):
                        yield get_name_update(case, old_id, new_id)

                    if case.type == CASE_TYPE_TEST:
                        yield get_case_update(case, {
                            'person_id_at_request': new_id,
                            'person_id_flat_at_request': new_id.replace('-', '')
                        })

                    if case.type == CASE_TYPE_EPISODE:
                        for investigation_case in self.accessor.get_reverse_indexed_cases([case.case_id]):
                            if investigation_case.type == CASE_TYPE_INVESTIGATION:
                                yield get_name_update(investigation_case, old_id, new_id)

def get_case_update(case, update):
    # check that this is actually an update, else return None
    if any(case.get_case_property(k) != v for k, v in update.items()):
        return (case.case_id, update, False)

def get_name_update(case, old_id, new_id):
    new_name = case.get_case_property('name').replace(old_id, new_id)
    return get_case_update(case, {'name': new_name})
<commit_msg>Add private sector case updates<commit_after>from __future__ import absolute_import
from __future__ import print_function
from collections import defaultdict
import csv
import datetime
from django.core.management.base import BaseCommand

from dimagi.utils.decorators.memoized import memoized

from corehq.apps.case_search.models import CLAIM_CASE_TYPE
from corehq.apps.locations.models import SQLLocation
from corehq.apps.hqcase.utils import bulk_update_cases
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.util.log import with_progress_bar

from custom.enikshay.case_utils import (
    CASE_TYPE_EPISODE,
    CASE_TYPE_INVESTIGATION,
    CASE_TYPE_OCCURRENCE,
    CASE_TYPE_PERSON,
    CASE_TYPE_PRESCRIPTION,
    CASE_TYPE_REFERRAL,
    CASE_TYPE_SECONDARY_OWNER,
    CASE_TYPE_TEST,
    CASE_TYPE_TRAIL,
    CASE_TYPE_VOUCHER,
    get_all_occurrence_cases_from_person,
)
from custom.enikshay.duplicate_ids import get_cases_with_duplicate_ids, get_new_readable_id
from custom.enikshay.user_setup import compress_nikshay_id, join_chunked

class Command(BaseCommand):
    help = """
    Finds cases with duplicate IDs and marks all but one of each ID as a duplicate
    """
    # TODO what are the headers we need?
    logfile_fields = ['name', 'dto_name', 'phi_name', 'owner_id', 'dob',
                      'phone_number', 'enrolled_in_private']

    def add_arguments(self, parser):
        parser.add_argument('domain')
        parser.add_argument(
            '--commit',
            action='store_true',
            dest='commit',
            default=False,
        )

    def handle(self, domain, **options):
        self.domain = domain
        self.accessor = CaseAccessors(domain)
        commit = options['commit']

        filename = '{}-{}.csv'.format(self.__module__.split('.')[-1],
                                      datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S'))
        print("Logging actions to {}".format(filename))
        with open(filename, 'w') as f:
            self.logfile = csv.DictWriter(f, self.logfile_fields, extrasaction='ignore')
            self.logfile.writeheader()

            print("Finding duplicates")
            bad_cases = get_cases_with_duplicate_ids(self.domain, CASE_TYPE_PERSON)

            print("Processing duplicate cases")
            for person_case in with_progress_bar(bad_cases):
                self.log_person_case_info(person_case)
                if person_case.get_case_property('enrolled_in_private') = 'true':
                    updates = self.get_private_updates(person_case)
                else:
                    updates = self.get_public_updates(person_case)
                bulk_update_cases(self.domain, updates, self.__module__)

    @property
    @memoized
    def districts_by_id(self):
        locs = SQLLocation.objects.filter(domain=self.domain, location_type__code='dto')
        return defaultdict(lambda: '', (
            (loc.location_id, loc.loc_name) for loc in locs
        ))

    def log_person_case_info(self, person_case):
        """Pull info that we want to log but not update"""
        person = person_case.dynamic_case_properties()
        self.logfile.writerow({
            'person_case_id': person_case.case_id,
            'name': ' '.join(filter(None, [person.get('first_name'), person.get('last_name')])),
            'enrolled_in_private': person.get('enrolled_in_private'),
            'dto_name': self.districts_by_id(person.get('current_address_district_choice')),
            'phi_name': person.get('phi'),
            'owner_id': person_case.owner_id,
            'dob': person.get('dob'),
            'phone_number': person.get('contact_phone_number'),
        })

    def get_public_updates(self, person_case):
        """ Get updates for a public sector person case and a bunch of its children
        https://docs.google.com/document/d/1NS5ozgk7w-2AADsrdTtjgqODkgWIEaCw6eqQ0cLg138/edit#
        """
        old_id = person_case.get_case_property('person_id')
        new_id = get_new_readable_id()  # TODO uppercase
        yield get_case_update(person_case, {
            'person_id_deprecated': old_id,
            'person_id_flat_deprecated': person_case.get_case_property('person_id_flat'),
            'person_id': new_id,
            'person_id_flat': new_id.replace('-', ''),
        })

        for person_child_case in self.accessor.get_reverse_indexed_cases([person_case.case_id]):

            # Update occurrence and claim names
            if person_child_case.type in (CASE_TYPE_OCCURRENCE, CLAIM_CASE_TYPE):
                yield get_name_update(person_child_case, old_id, new_id)

            # look at all extensions of the occurrence cases
            if person_child_case.type == CASE_TYPE_OCCURRENCE:
                for case in self.accessor.get_reverse_indexed_cases([person_child_case.case_id]):

                    # update the names of episode, secondary_owner, trail, and referral cases
                    if case.type in (CASE_TYPE_EPISODE, CASE_TYPE_SECONDARY_OWNER,
                                     CASE_TYPE_TRAIL, CASE_TYPE_REFERRAL):
                        yield get_name_update(case, old_id, new_id)

                    if case.type == CASE_TYPE_TEST:
                        yield get_case_update(case, {
                            'person_id_at_request': new_id,
                            'person_id_flat_at_request': new_id.replace('-', '')
                        })

                    if case.type == CASE_TYPE_EPISODE:
                        for investigation_case in self.accessor.get_reverse_indexed_cases([case.case_id]):
                            if investigation_case.type == CASE_TYPE_INVESTIGATION:
                                yield get_name_update(investigation_case, old_id, new_id)

    def get_private_updates(self, person_case):
        """ Get updates for a private sector person case and a bunch of its children
        https://docs.google.com/document/d/1NS5ozgk7w-2AADsrdTtjgqODkgWIEaCw6eqQ0cLg138/edit#
        """
        old_id = person_case.get_case_property('person_id')
        new_id = get_new_readable_id()  # TODO
        yield get_case_update(person_case, {
            'person_id_deprecated': old_id,
            'person_id_flat_deprecated': person_case.get_case_property('person_id_flat'),
            'person_id': new_id,
            'person_id_flat': new_id.replace('-', ''),
        })

        for voucher_case in get_all_vouchers_from_person(self.domain, person_case):
            yield get_case_update(voucher_case, {
                'person_id': new_id,
                'person_id_flat': new_id.replace('-', ''),
            })

def get_case_update(case, update):
    # check that this is actually an update, else return None
    if any(case.get_case_property(k) != v for k, v in update.items()):
        return (case.case_id, update, False)

def get_name_update(case, old_id, new_id):
    new_name = case.get_case_property('name').replace(old_id, new_id)
    return get_case_update(case, {'name': new_name})

def get_all_vouchers_from_person(person_case):
    """Returns all voucher cases under tests or prescriptions"""
    accessor = CaseAccessors(domain)
    for occurrence_case in accessor.get_reverse_indexed_cases([person_case.case_id]):
        if occurrence_case.type == CASE_TYPE_OCCURRENCE:
            for case in accessor.get_reverse_indexed_cases([occurrence_case.case_id]):
                if case.type == CASE_TYPE_TEST:
                    for voucher_case in accessor.get_reverse_indexed_cases([case.case_id]):
                        if voucher_case.type == CASE_TYPE_VOUCHER:
                            yield voucher_case
                if case.type == CASE_TYPE_EPISODE:
                    for prescription_case in accessor.get_reverse_indexed_cases([case.case_id]):
                        if prescription_case.type == CASE_TYPE_PRESCRIPTION:
                            for voucher_case in accessor.get_reverse_indexed_cases([prescription_case.case_id]):
                                if voucher_case.type == CASE_TYPE_VOUCHER:
                                    yield voucher_case
<|endoftext|>