<commit_before>import asyncio
import re

import discord
from discord_slash import cog_ext
from discord_slash.context import SlashContext
from discord_slash.model import SlashCommandOptionType
from discord_slash.utils.manage_components import (create_actionrow,
                                                   create_select,
                                                   create_select_option,
                                                   wait_for_component)

from .exceptions import APIError, APIInactiveError

class KeyMixin:
    @cog_ext.cog_subcommand(
        base="key",
        name="add",
        base_description="Key related commands",
        options=[{
            "name": "key",
            "description":
            "Generate at https://account.arena.net under Applications tab",
            "type": SlashCommandOptionType.STRING,
            "required": True,
        }])
    async def key_add(self, ctx: SlashContext, key):
        """Adds a key and associates it with your discord account"""
        doc = await self.bot.database.get(ctx.author, self)
        await ctx.defer(hidden=True)
        try:
            endpoints = ["tokeninfo", "account"]
            token, acc = await self.call_multiple(endpoints, key=key)
        except APIInactiveError:
            return await ctx.send("The API is currently down. "
                                  "Try again later. ")
        except APIError:
            return await ctx.send("The key is invalid.")
        key_doc = {
            "key": key,
            "account_name": acc["name"],
            "name": token["name"],
            "permissions": token["permissions"]
        }
        # at this point we know the key is valid
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys and key:
            # user already had a key but it isn't in keys
            # (existing user) so add it
            keys.append(key)
        if key_doc["key"] in [k["key"] for k in keys]:
            return await ctx.send("You have already added this key before.")
        if len(keys) >= 15:
            return await ctx.send(
                "You've reached the maximum limit of "
                "15 API keys, please remove one before adding "
                "another")
        keys.append(key_doc)
        await self.bot.database.set(ctx.author, {
            "key": key_doc,
            "keys": keys
        }, self)
        if len(keys) > 1:
            output = ("Your key was verified and "
                      "added to your list of keys, you can swap between "
                      "them at any time using /key switch.")
        else:
            output = ("Your key was verified and "
                      "associated with your account.")
        all_permissions = ("account", "builds", "characters", "guilds",
                           "inventories", "progression", "pvp", "tradingpost",
                           "unlocks", "wallet")
        missing = [
            x for x in all_permissions if x not in key_doc["permissions"]
        ]
        if missing:
            output += ("\nPlease note that your API key doesn't have the "
                       "following permissions checked: "
                       f"```{', '.join(missing)}```\nSome commands "
                       "will not work. Consider adding a new key with "
                       "those permissions checked.")

        await ctx.send(output)
        try:
            if ctx.guild:
                await self.worldsync_on_member_join(ctx.author)
                await self.guildsync_on_member_join(ctx.author)
                return
            for guild in self.bot.guilds:
                try:
                    if len(guild.members) > 5000:
                        continue
                    if ctx.author not in guild.members:
                        continue
                    doc = await self.bot.database.get(guild, self)
                    worldsync = doc.get("worldsync", {})
                    worldsync_enabled = worldsync.get("enabled", False)
                    if worldsync_enabled:
                        member = guild.get_member(ctx.author.id)
                        await self.worldsync_on_member_join(member)
                    guildsync = doc.get("sync", {})
                    if guildsync.get("on", False) and guildsync.get(
                            "setupdone", False):
                        member = guild.get_member(ctx.author.id)
                        await self.guildsync_on_member_join(member)
                except Exception:
                    pass
        except Exception:
            pass

    async def key_dropdown(self, ctx, keys, placeholder, max_values=None):
        if not max_values:
            max_values = len(keys)
        options = []
        for i, key in enumerate(keys):
            options.append(
                create_select_option(key["account_name"],
                                     description=key["name"],
                                     value=i))
        select = create_select(min_values=1,
                               max_values=max_values,
                               options=options,
                               placeholder=placeholder)
        components = [create_actionrow(select)]
        msg = await ctx.send("** **", components=components, hidden=True)
        try:
            answer = await wait_for_component(self.bot,
                                              components=components,
                                              timeout=120)
            return answer
        except asyncio.TimeoutError:
            await msg.edit(content="No response in time.", components=None)
            return None

    @cog_ext.cog_subcommand(base="key",
                            name="remove",
                            base_description="Key related commands")
    async def key_remove(self, ctx: SlashContext):
        """Remove selected keys from the bot"""
        doc = await self.bot.database.get(ctx.author, self)
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys and not key:
            return await ctx.send(
                "You have no keys added, you can add one with /key add.")
        if not keys and key:
            keys.append(key)
        answer = await self.key_dropdown(ctx, keys,
                                         "Select the keys you want to remove")
        if not answer:
            return
        choices = [int(ans) for ans in answer.selected_options]
        for choice in choices:
            if keys[choice] == key:
                key = {}
            del keys[choice]
        await self.bot.database.set(ctx.author, {
            "key": key,
            "keys": keys
        }, self)
        await answer.edit_origin(content="Successfuly removed selected keys.",
                                 components=None)

    @cog_ext.cog_subcommand(base="key",
                            name="info",
                            base_description="Key related commands")
    async def key_info(self, ctx: SlashContext):
        """Information about your active api keys"""
        doc = await self.bot.database.get(ctx.author, self)
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys and not key:
            return await ctx.send(
                "You have no keys added, you can add one with /key add.",
                hidden=True)
        embed = await self.display_keys(ctx,
                                        doc,
                                        display_active=True,
                                        show_tokens=True,
                                        reveal_tokens=True)
        await ctx.send(embed=embed, hidden=True)

    @cog_ext.cog_subcommand(
        base="key",
        name="switch",
        base_description="Key related commands",
        options=[{
            "name": "index",
            "description":
            "Key index to switch to. Skip to get a list instead.",
            "type": SlashCommandOptionType.INTEGER,
            "required": False,
        }])
    async def key_switch(self, ctx, index: int = 0):
        """Swaps between multiple stored API keys."""
        doc = await self.bot.database.get(ctx.author, self)
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys:
            return await ctx.send(
                "You need to add additional API keys first using /key "
                "add first.",
                hidden=True)
        answer = None
        if not index:
            answer = await self.key_dropdown(
                ctx,
                keys,
                "Select the key you want to switch to",
                max_values=1)
            if not answer:
                return
            index = int(answer.selected_options[0])
        try:
            key = keys[index]
        except IndexError:
            text = ("You don't have a key with this ID, remember you can "
                    "check your list of keys by using this command without "
                    "a number.")
            if answer:
                await answer.edit_origin(content=text, components=None)
            else:
                await ctx.send(text, hidden=True)
            await answer.edit_origin(content=text, components=None)
        await self.bot.database.set(ctx.author, {"key": key}, self)
        msg = "Swapped to selected key."
        if key["name"]:
            msg += " Name : `{}`".format(key["name"])
        await answer.edit_origin(content=msg, components=None)
        try:
            if ctx.guild:
                await self.worldsync_on_member_join(ctx.author)
            for guild in self.bot.guilds:
                try:
                    if len(guild.members) > 5000:
                        continue
                    if ctx.author not in guild.members:
                        continue
                    doc = await self.bot.database.get(guild, self)
                    worldsync = doc.get("worldsync", {})
                    worldsync_enabled = worldsync.get("enabled", False)
                    if worldsync_enabled:
                        member = guild.get_member(ctx.author.id)
                        await self.worldsync_on_member_join(member)
                except Exception:
                    pass
        except Exception:
            pass

    async def display_keys(self,
                           ctx,
                           doc,
                           *,
                           display_active=False,
                           display_permissions=True,
                           show_tokens=False,
                           reveal_tokens=False):
        def get_value(key):
            lines = []
            if display_permissions:
                lines.append("Permissions: " + ", ".join(key["permissions"]))
            if show_tokens:
                token = key["key"]
                if not reveal_tokens:
                    token = token[:7] + re.sub("[a-zA-Z0-9]", r"\*", token[8:])
                else:
                    token = f"||{token}||"
                lines.append(token)
            return "\n".join(lines)

        keys = doc.get("keys", [])
        embed = discord.Embed(title="Your keys",
                              color=await self.get_embed_color(ctx))
        embed.set_author(name=ctx.author.name, icon_url=ctx.author.avatar_url)
        if display_active:
            active_key = doc.get("key", {})
            if active_key:
                name = "**Active key**: {}".format(active_key["account_name"])
                token_name = active_key["name"]
                if token_name:
                    name += " - " + token_name
                embed.add_field(name=name, value=get_value(active_key))
        for counter, key in enumerate(keys, start=1):
            name = "**{}**: {}".format(counter, key["account_name"])
            token_name = key["name"]
            if token_name:
                name += " - " + token_name
            embed.add_field(name=name, value=get_value(key), inline=False)
        if show_tokens and not reveal_tokens:
            embed.set_footer(text="Use {}key info to see full API keys".format(
                ctx.prefix),
                             icon_url=self.bot.user.avatar_url)
        else:
            embed.set_footer(text=self.bot.user.name,
                             icon_url=self.bot.user.avatar_url)
        return embed
<commit_msg>Fix index error in key<commit_after>import asyncio
import re

import discord
from discord_slash import cog_ext
from discord_slash.context import SlashContext
from discord_slash.model import SlashCommandOptionType
from discord_slash.utils.manage_components import (create_actionrow,
                                                   create_select,
                                                   create_select_option,
                                                   wait_for_component)

from .exceptions import APIError, APIInactiveError

class KeyMixin:
    @cog_ext.cog_subcommand(
        base="key",
        name="add",
        base_description="Key related commands",
        options=[{
            "name": "key",
            "description":
            "Generate at https://account.arena.net under Applications tab",
            "type": SlashCommandOptionType.STRING,
            "required": True,
        }])
    async def key_add(self, ctx: SlashContext, key):
        """Adds a key and associates it with your discord account"""
        doc = await self.bot.database.get(ctx.author, self)
        await ctx.defer(hidden=True)
        try:
            endpoints = ["tokeninfo", "account"]
            token, acc = await self.call_multiple(endpoints, key=key)
        except APIInactiveError:
            return await ctx.send("The API is currently down. "
                                  "Try again later. ")
        except APIError:
            return await ctx.send("The key is invalid.")
        key_doc = {
            "key": key,
            "account_name": acc["name"],
            "name": token["name"],
            "permissions": token["permissions"]
        }
        # at this point we know the key is valid
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys and key:
            # user already had a key but it isn't in keys
            # (existing user) so add it
            keys.append(key)
        if key_doc["key"] in [k["key"] for k in keys]:
            return await ctx.send("You have already added this key before.")
        if len(keys) >= 15:
            return await ctx.send(
                "You've reached the maximum limit of "
                "15 API keys, please remove one before adding "
                "another")
        keys.append(key_doc)
        await self.bot.database.set(ctx.author, {
            "key": key_doc,
            "keys": keys
        }, self)
        if len(keys) > 1:
            output = ("Your key was verified and "
                      "added to your list of keys, you can swap between "
                      "them at any time using /key switch.")
        else:
            output = ("Your key was verified and "
                      "associated with your account.")
        all_permissions = ("account", "builds", "characters", "guilds",
                           "inventories", "progression", "pvp", "tradingpost",
                           "unlocks", "wallet")
        missing = [
            x for x in all_permissions if x not in key_doc["permissions"]
        ]
        if missing:
            output += ("\nPlease note that your API key doesn't have the "
                       "following permissions checked: "
                       f"```{', '.join(missing)}```\nSome commands "
                       "will not work. Consider adding a new key with "
                       "those permissions checked.")

        await ctx.send(output)
        try:
            if ctx.guild:
                await self.worldsync_on_member_join(ctx.author)
                await self.guildsync_on_member_join(ctx.author)
                return
            for guild in self.bot.guilds:
                try:
                    if len(guild.members) > 5000:
                        continue
                    if ctx.author not in guild.members:
                        continue
                    doc = await self.bot.database.get(guild, self)
                    worldsync = doc.get("worldsync", {})
                    worldsync_enabled = worldsync.get("enabled", False)
                    if worldsync_enabled:
                        member = guild.get_member(ctx.author.id)
                        await self.worldsync_on_member_join(member)
                    guildsync = doc.get("sync", {})
                    if guildsync.get("on", False) and guildsync.get(
                            "setupdone", False):
                        member = guild.get_member(ctx.author.id)
                        await self.guildsync_on_member_join(member)
                except Exception:
                    pass
        except Exception:
            pass

    async def key_dropdown(self, ctx, keys, placeholder, max_values=None):
        if not max_values:
            max_values = len(keys)
        options = []
        for i, key in enumerate(keys):
            options.append(
                create_select_option(key["account_name"],
                                     description=key["name"],
                                     value=str(i)))
        select = create_select(min_values=1,
                               max_values=max_values,
                               options=options,
                               placeholder=placeholder)
        components = [create_actionrow(select)]
        msg = await ctx.send("** **", components=components, hidden=True)
        try:
            answer = await wait_for_component(self.bot,
                                              components=components,
                                              timeout=120)
            return answer
        except asyncio.TimeoutError:
            await msg.edit(content="No response in time.", components=None)
            return None

    @cog_ext.cog_subcommand(base="key",
                            name="remove",
                            base_description="Key related commands")
    async def key_remove(self, ctx: SlashContext):
        """Remove selected keys from the bot"""
        doc = await self.bot.database.get(ctx.author, self)
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys and not key:
            return await ctx.send(
                "You have no keys added, you can add one with /key add.")
        if not keys and key:
            keys.append(key)
        answer = await self.key_dropdown(ctx, keys,
                                         "Select the keys you want to remove")
        if not answer:
            return
        choices = sorted((int(ans) for ans in answer.selected_options),
                         reverse=True)
        for choice in choices:
            if keys[choice] == key:
                key = {}
            del keys[choice]
        await self.bot.database.set(ctx.author, {
            "key": key,
            "keys": keys
        }, self)
        await answer.edit_origin(content="Successfuly removed selected keys.",
                                 components=None)

    @cog_ext.cog_subcommand(base="key",
                            name="info",
                            base_description="Key related commands")
    async def key_info(self, ctx: SlashContext):
        """Information about your active api keys"""
        doc = await self.bot.database.get(ctx.author, self)
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys and not key:
            return await ctx.send(
                "You have no keys added, you can add one with /key add.",
                hidden=True)
        embed = await self.display_keys(ctx,
                                        doc,
                                        display_active=True,
                                        show_tokens=True,
                                        reveal_tokens=True)
        await ctx.send(embed=embed, hidden=True)

    @cog_ext.cog_subcommand(
        base="key",
        name="switch",
        base_description="Key related commands",
        options=[{
            "name": "index",
            "description":
            "Key index to switch to. Skip to get a list instead.",
            "type": SlashCommandOptionType.INTEGER,
            "required": False,
        }])
    async def key_switch(self, ctx, index: int = 0):
        """Swaps between multiple stored API keys."""
        doc = await self.bot.database.get(ctx.author, self)
        keys = doc.get("keys", [])
        key = doc.get("key", {})
        if not keys:
            return await ctx.send(
                "You need to add additional API keys first using /key "
                "add first.",
                hidden=True)
        answer = None
        if not index:
            answer = await self.key_dropdown(
                ctx,
                keys,
                "Select the key you want to switch to",
                max_values=1)
            if not answer:
                return
            index = int(answer.selected_options[0])
        try:
            key = keys[index]
        except IndexError:
            text = ("You don't have a key with this ID, remember you can "
                    "check your list of keys by using this command without "
                    "a number.")
            if answer:
                await answer.edit_origin(content=text, components=None)
            else:
                await ctx.send(text, hidden=True)
            await answer.edit_origin(content=text, components=None)
        await self.bot.database.set(ctx.author, {"key": key}, self)
        msg = "Swapped to selected key."
        if key["name"]:
            msg += " Name : `{}`".format(key["name"])
        if answer:
            await answer.edit_origin(content=msg, components=None)
        else:
            await ctx.send(msg, hidden=True)
        try:
            if ctx.guild:
                await self.worldsync_on_member_join(ctx.author)
            for guild in self.bot.guilds:
                try:
                    if len(guild.members) > 5000:
                        continue
                    if ctx.author not in guild.members:
                        continue
                    doc = await self.bot.database.get(guild, self)
                    worldsync = doc.get("worldsync", {})
                    worldsync_enabled = worldsync.get("enabled", False)
                    if worldsync_enabled:
                        member = guild.get_member(ctx.author.id)
                        await self.worldsync_on_member_join(member)
                except Exception:
                    pass
        except Exception:
            pass

    async def display_keys(self,
                           ctx,
                           doc,
                           *,
                           display_active=False,
                           display_permissions=True,
                           show_tokens=False,
                           reveal_tokens=False):
        def get_value(key):
            lines = []
            if display_permissions:
                lines.append("Permissions: " + ", ".join(key["permissions"]))
            if show_tokens:
                token = key["key"]
                if not reveal_tokens:
                    token = token[:7] + re.sub("[a-zA-Z0-9]", r"\*", token[8:])
                else:
                    token = f"||{token}||"
                lines.append(token)
            return "\n".join(lines)

        keys = doc.get("keys", [])
        embed = discord.Embed(title="Your keys",
                              color=await self.get_embed_color(ctx))
        embed.set_author(name=ctx.author.name, icon_url=ctx.author.avatar_url)
        if display_active:
            active_key = doc.get("key", {})
            if active_key:
                name = "**Active key**: {}".format(active_key["account_name"])
                token_name = active_key["name"]
                if token_name:
                    name += " - " + token_name
                embed.add_field(name=name, value=get_value(active_key))
        for counter, key in enumerate(keys, start=1):
            name = "**{}**: {}".format(counter, key["account_name"])
            token_name = key["name"]
            if token_name:
                name += " - " + token_name
            embed.add_field(name=name, value=get_value(key), inline=False)
        if show_tokens and not reveal_tokens:
            embed.set_footer(text="Use {}key info to see full API keys".format(
                ctx.prefix),
                             icon_url=self.bot.user.avatar_url)
        else:
            embed.set_footer(text=self.bot.user.name,
                             icon_url=self.bot.user.avatar_url)
        return embed
<|endoftext|>


<commit_before>"""
    filebrowser.views

"""
import logging
import re
import json
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.template.loader import get_template
from django.conf import settings
from .utils import get_rucio_file, get_rucio_pfns_from_guids, fetch_file, get_filebrowser_vo, \
    remove_folder, get_fullpath_filebrowser_directory, list_file_directory

from core.common.models import Filestable4, FilestableArch
from core.views import DateTimeEncoder, initSelfMonitor
from datetime import datetime

_logger = logging.getLogger('bigpandamon-filebrowser')
filebrowserDateTimeFormat = "%Y %b %d %H:%M:%S"
hostname = "bigpanda.cern.ch"

def index(request):
    """
        index -- filebrowser's default page

        :param request: Django's HTTP request
        :type request: django.http.HttpRequest

    """

    try:
        initSelfMonitor(request)
    except:
        _logger.exception('Failed to init self monitor')

    errors = {}

    _logger.debug("index started - " + datetime.now().strftime("%H:%M:%S") + "  ")

    ### check that all expected parameters are in URL
    expectedFields = ['guid', 'site', 'scope', 'lfn']
    for expectedField in expectedFields:
        try:
            request.GET[expectedField]
        except:
            msg = 'Missing expected GET parameter %s. ' % expectedField
            _logger.error(msg)
            if 'missingparameter' not in errors.keys():
                errors['missingparameter'] = ''
            errors['missingparameter'] += msg

    ### if all expected GET parameters are present, execute file lookup
    pfns = []
    scope = ''
    lfn = ''
    guid = ''
    site = ''
    pattern_string='^[a-zA-Z0-9.\-_]+$'
    pattern_site = '^[a-zA-Z0-9.,\-_\/]+$'
    pattern_guid='^(\{){0,1}[0-9a-zA-Z]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}(\}){0,1}$'
    try:
        guid = request.GET['guid']
        if re.match(pattern_guid, guid) is None:
            guid = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'guid: %s ' % (request.GET['guid'])
    except:
        pass
    try:
        site = request.GET['site']
        if re.match(pattern_site, site) is None:
            site = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'site: %s ' % (request.GET['site'])
    except:
        pass
    try:
        lfn = request.GET['lfn']
        if re.match(pattern_string, lfn) is None:
            lfn = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'lfn: %s ' % (request.GET['lfn'])
    except:
        pass
    try:
        scope = request.GET['scope']
        if re.match(pattern_string, scope) is None:
            scope = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'scope: %s ' % (request.GET['scope'])
    except:
        pass

    # check if size of logfile is too big return to user error message containing rucio cli command to download it locally
    max_sizemb = 1000
    sizemb = None
    try:
        fileid = int(request.GET['fileid'])
    except:
        fileid = -1
    lquery = {'type': 'log'}
    if lfn and len(lfn) > 0:
        lquery['lfn'] = lfn
        fsize = Filestable4.objects.filter(**lquery).values('fsize', 'fileid')
        if len(fsize) == 0:
            fsize = FilestableArch.objects.filter(**lquery).values('fsize', 'fileid')
        if len(fsize) > 0:
            try:
                if fileid > 0:
                    sizemb = round(int([f['fsize'] for f in fsize if f['fileid'] == fileid][0])/1000/1000)
                else:
                    sizemb = round(int([f['fsize'] for f in fsize][0])/1000/1000)
            except:
                _logger.warning("ERROR!!! Failed to calculate log tarball size in MB")

    _logger.debug("index step1 - " + datetime.now().strftime("%H:%M:%S") + "  ")

    ### download the file
    files = []
    dirprefix = ''
    tardir = ''
    if sizemb and sizemb > max_sizemb:
        _logger.warning('Size of the requested log is {} MB which is more than limit {} MB'.format(sizemb, max_sizemb))
        errormessage = """The size of requested log is too big ({}MB).
                            Please try to download it locally using Rucio CLI by the next command:
                            rucio download {}:{}""".format(sizemb, scope, lfn)
        data = {
            'errormessage': errormessage
        }
        return render_to_response('errorPage.html', data, content_type='text/html')
    if not (guid is None or lfn is None or scope is None):
        files, errtxt, dirprefix, tardir = get_rucio_file(scope,lfn, guid, 100)
    else:
        errormessage = ''
        if guid is None:
            errormessage = 'No guid provided.'
        elif lfn is None:
            errormessage = 'No lfn provided.'
        elif scope is None:
            errormessage = 'No scope provided.'
        _logger.warning(errormessage)
        data = {
            'errormessage': errormessage
        }
        return render_to_response('errorPage.html', data, content_type='text/html')
    if not len(files):
        msg = 'Something went wrong while the log file downloading. [guid=%s, site=%s, scope=%s, lfn=%s] \n' % \
              (guid, site, scope, lfn)
        _logger.warning(msg)
        errors['download'] = msg
    if len(errtxt):
        if 'download' not in errors:
            errors['download'] = ''
        errors['download'] += errtxt

    _logger.debug("index step2 - " + datetime.now().strftime("%H:%M:%S") + "  ")

    totalLogSize = 0
    if type(files) is list and len(files) > 0:
        for file in files:
            totalLogSize += file['size'] if 'size' in file and file['size'] > 0 else 0

    # from B to MB
    if totalLogSize > 0:
        totalLogSize = round(totalLogSize*1.0/1024/1024, 2)

    ### return the file page

    ### set request response data
    data = {
        'request': request,
        'errors': errors,
        'pfns': pfns,
        'files': files,
        'dirprefix': dirprefix,
        'tardir': tardir,
        'scope': scope,
        'lfn': lfn,
        'site': site,
        'guid': guid,
        'MEDIA_URL': settings.MEDIA_URL,
        'viewParams' : {'MON_VO': str(get_filebrowser_vo()).upper()},
        'HOSTNAME': hostname,
        'totalLogSize': totalLogSize,
        'nfiles': len(files),
    }

    _logger.debug("index step3 - " + datetime.now().strftime("%H:%M:%S") + "  ")
    if 'json' not in request.GET:
        return render_to_response('filebrowser/filebrowser_index.html', data, RequestContext(request))
    else:
        resp = HttpResponse(json.dumps(data, cls=DateTimeEncoder), content_type='application/json')
        _logger.debug("index step4 - " + datetime.now().strftime("%H:%M:%S") + "  ")
        return resp

def api_single_pandaid(request):
    """
        api_single_pandaid -- return log file URL for a single PanDA job

        :param request: Django's HTTP request
        :type request: django.http.HttpRequest

    """
    errors = {}

    ### check that all expected parameters are in URL
#    expectedFields = ['guid', 'site', 'scope', 'lfn']
    expectedFields = ['pandaid']
    for expectedField in expectedFields:
        try:
            if len(request.GET[expectedField]) < 1:
                msg = 'Missing expected GET parameter %s. ' % expectedField
                if 'missingparameter' not in errors.keys():
                    errors['missingparameter'] = ''
                errors['missingparameter'] += msg
        except:
            msg = 'Missing expected GET parameter %s. ' % expectedField
            _logger.error(msg)
            if 'missingparameter' not in errors.keys():
                errors['missingparameter'] = ''
            errors['missingparameter'] += msg

    ### if all expected GET parameters are present, execute file lookup
    pfns = []
    scope = ''
    lfn = ''
    guid = ''
    site = ''
    pandaid = None
    status = ''
    query = {}
    query['type'] = 'log'
    try:
        pandaid = int(request.GET['pandaid'])
    except:
        pass
    query['pandaid'] = pandaid
    file_properties = []
    try:
        file_properties = Filestable4.objects.filter(**query).values('pandaid', 'guid', \
                                'scope', 'lfn', 'destinationse', 'status')
    except:
        pass

    if len(file_properties):
        file_properties = file_properties[0]
        try:
            guid = file_properties['guid']
        except:
            pass
        try:
            site = file_properties['destinationse']
        except:
            pass
        try:
            lfn = file_properties['lfn']
        except:
            pass
        try:
            scope = file_properties['scope']
        except:
            pass
        try:
            status = file_properties['status']
        except:
            pass

        if 'missingparameter' not in errors.keys():
            pfns, errtxt = get_rucio_pfns_from_guids(guids=[guid], site=[site], \
                        lfns=[lfn], scopes=[scope])
            if len(errtxt):
                if 'lookup' not in errors:
                    errors['lookup'] = ''
                errors['lookup'] += errtxt

        ### download the file
        files = []
        dirprefix = ''
        tardir = ''
        if len(pfns):
            pfn = pfns[0]
            files, errtxt, dirprefix, tardir = fetch_file(pfn, guid, unpack=False, listfiles=False)
            if not len(pfns):
                msg = 'File download failed. [pfn=%s guid=%s, site=%s, scope=%s, lfn=%s]' % \
                    (pfn, guid, site, scope, lfn)
                _logger.warning(msg)
                errors['download'] = msg
            if len(errtxt):
                if 'download' in errors:
                    errors['download'] += errtxt
    else: # file not found in DB
        if 'lookup' not in errors:
            errors['lookup'] = ''
        errors['lookup'] += 'Log file for this job has not been found. '
    ### return the file page

    url = None
    data = { \
        'pandaid': pandaid, \
        'url': url, \
        'errors': errors, \
        'pfns': pfns, \
        'scope': scope, \
        'lfn': lfn, \
        'site': site, \
        'guid': guid, \
        'status': status, \
        'timestamp': datetime.utcnow().isoformat() \
    }
    if not len(errors):
        url = 'http://' + hostname + \
                    settings.MEDIA_URL + dirprefix + '/' + lfn
        data['url'] = url
        ### set request response data
        return render_to_response('filebrowser/filebrowser_api_single_pandaid.html', {'data': data}, RequestContext(request))
    elif 'pandaid' not in request.GET.keys() or pandaid == None:
        t = get_template('filebrowser/filebrowser_api_single_pandaid.html')
        context = RequestContext(request, {'data':data})
        return HttpResponse(t.render(context), status=400)
    elif not len(file_properties):
        t = get_template('filebrowser/filebrowser_api_single_pandaid.html')
        context = RequestContext(request, {'data':data})
        return HttpResponse(t.render(context), status=404)
    else:
        t = get_template('filebrowser/filebrowser_api_single_pandaid.html')
        context = RequestContext(request, {'data':data})
        return HttpResponse(t.render(context), status=400)

def get_job_memory_monitor_output(pandaid):
    """
    Download log tarball of a job and return path to a local copy of memory_monitor_output.txt file
    :param pandaid:
    :return: mmo_path: str
    """
    mmo_path = None
    files = []
    scope = ''
    lfn = ''
    guid = ''
    dirprefix = ''
    tardir = ''
    query = {}
    query['type'] = 'log'
    query['pandaid'] = int(pandaid)
    values = ['pandaid', 'guid', 'scope', 'lfn']
    file_properties = []

    file_properties.extend(Filestable4.objects.filter(**query).values(*values))
    if len(file_properties) == 0:
        file_properties.extend(FilestableArch.objects.filter(**query).values(*values))

    if len(file_properties):
        file_properties = file_properties[0]
        try:
            guid = file_properties['guid']
        except:
            pass
        try:
            lfn = file_properties['lfn']
        except:
            pass
        try:
            scope = file_properties['scope']
        except:
            pass

        if guid and lfn and scope:
            # check if files are already available in common CEPH storage
            tarball_path = get_fullpath_filebrowser_directory() + '/' + guid.lower() + '/' + scope + '/' + lfn
            files, err, tardir = list_file_directory(tarball_path, 100)
            _logger.debug('tarball path is {} \nError message is {} \nGot tardir: {}'.format(tarball_path, err, tardir))
            if len(files) == 0 and len(err) > 0:
                # download tarball
                _logger.debug('log tarball has not been downloaded, so downloading it now')
                files, errtxt, dirprefix, tardir = get_rucio_file(scope, lfn, guid)
                _logger.debug('Got files for dir: {} and tardir: {}. Error message: {}'.format(dirprefix, tardir, errtxt))
            if type(files) is list and len(files) > 0:
                for f in files:
                    if f['name'] == 'memory_monitor_output.txt':
                        mmo_path = tarball_path + '/' + tardir + '/' + 'memory_monitor_output.txt'
    _logger.debug('Final mmo_path: {}'.format(mmo_path))
    return mmo_path

def delete_files(request):
    """
    Clear subfolder containing log files
    :param request:
    :return:
    """

    ### check that path to logs is provided
    guid = None
    try:
        guid = request.GET['guid']
    except:
        msg = 'Missing guid GET parameter'
        _logger.error(msg)

    ### clean folder if guid provided
    if guid is not None:
        logdir = remove_folder(guid)
        data = {'message':'The folder was cleaned ' + logdir}
        return HttpResponse(json.dumps(data), content_type='application/json')
    else:
        return HttpResponse(status=404)

<commit_msg>Added a few debug messages to new filebrowser function got memory monitor<commit_after>"""
    filebrowser.views

"""
import logging
import re
import json
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.template.loader import get_template
from django.conf import settings
from .utils import get_rucio_file, get_rucio_pfns_from_guids, fetch_file, get_filebrowser_vo, \
    remove_folder, get_fullpath_filebrowser_directory, list_file_directory

from core.common.models import Filestable4, FilestableArch
from core.views import DateTimeEncoder, initSelfMonitor
from datetime import datetime

_logger = logging.getLogger('bigpandamon-filebrowser')
filebrowserDateTimeFormat = "%Y %b %d %H:%M:%S"
hostname = "bigpanda.cern.ch"

def index(request):
    """
        index -- filebrowser's default page

        :param request: Django's HTTP request
        :type request: django.http.HttpRequest

    """

    try:
        initSelfMonitor(request)
    except:
        _logger.exception('Failed to init self monitor')

    errors = {}

    _logger.debug("index started - " + datetime.now().strftime("%H:%M:%S") + "  ")

    ### check that all expected parameters are in URL
    expectedFields = ['guid', 'site', 'scope', 'lfn']
    for expectedField in expectedFields:
        try:
            request.GET[expectedField]
        except:
            msg = 'Missing expected GET parameter %s. ' % expectedField
            _logger.error(msg)
            if 'missingparameter' not in errors.keys():
                errors['missingparameter'] = ''
            errors['missingparameter'] += msg

    ### if all expected GET parameters are present, execute file lookup
    pfns = []
    scope = ''
    lfn = ''
    guid = ''
    site = ''
    pattern_string='^[a-zA-Z0-9.\-_]+$'
    pattern_site = '^[a-zA-Z0-9.,\-_\/]+$'
    pattern_guid='^(\{){0,1}[0-9a-zA-Z]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}(\}){0,1}$'
    try:
        guid = request.GET['guid']
        if re.match(pattern_guid, guid) is None:
            guid = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'guid: %s ' % (request.GET['guid'])
    except:
        pass
    try:
        site = request.GET['site']
        if re.match(pattern_site, site) is None:
            site = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'site: %s ' % (request.GET['site'])
    except:
        pass
    try:
        lfn = request.GET['lfn']
        if re.match(pattern_string, lfn) is None:
            lfn = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'lfn: %s ' % (request.GET['lfn'])
    except:
        pass
    try:
        scope = request.GET['scope']
        if re.match(pattern_string, scope) is None:
            scope = None
            if 'improperformat' not in errors.keys():
                errors['improperformat'] = ''
            errors['improperformat'] += 'scope: %s ' % (request.GET['scope'])
    except:
        pass

    # check if size of logfile is too big return to user error message containing rucio cli command to download it locally
    max_sizemb = 1000
    sizemb = None
    try:
        fileid = int(request.GET['fileid'])
    except:
        fileid = -1
    lquery = {'type': 'log'}
    if lfn and len(lfn) > 0:
        lquery['lfn'] = lfn
        fsize = Filestable4.objects.filter(**lquery).values('fsize', 'fileid')
        if len(fsize) == 0:
            fsize = FilestableArch.objects.filter(**lquery).values('fsize', 'fileid')
        if len(fsize) > 0:
            try:
                if fileid > 0:
                    sizemb = round(int([f['fsize'] for f in fsize if f['fileid'] == fileid][0])/1000/1000)
                else:
                    sizemb = round(int([f['fsize'] for f in fsize][0])/1000/1000)
            except:
                _logger.warning("ERROR!!! Failed to calculate log tarball size in MB")

    _logger.debug("index step1 - " + datetime.now().strftime("%H:%M:%S") + "  ")

    ### download the file
    files = []
    dirprefix = ''
    tardir = ''
    if sizemb and sizemb > max_sizemb:
        _logger.warning('Size of the requested log is {} MB which is more than limit {} MB'.format(sizemb, max_sizemb))
        errormessage = """The size of requested log is too big ({}MB).
                            Please try to download it locally using Rucio CLI by the next command:
                            rucio download {}:{}""".format(sizemb, scope, lfn)
        data = {
            'errormessage': errormessage
        }
        return render_to_response('errorPage.html', data, content_type='text/html')
    if not (guid is None or lfn is None or scope is None):
        files, errtxt, dirprefix, tardir = get_rucio_file(scope,lfn, guid, 100)
    else:
        errormessage = ''
        if guid is None:
            errormessage = 'No guid provided.'
        elif lfn is None:
            errormessage = 'No lfn provided.'
        elif scope is None:
            errormessage = 'No scope provided.'
        _logger.warning(errormessage)
        data = {
            'errormessage': errormessage
        }
        return render_to_response('errorPage.html', data, content_type='text/html')
    if not len(files):
        msg = 'Something went wrong while the log file downloading. [guid=%s, site=%s, scope=%s, lfn=%s] \n' % \
              (guid, site, scope, lfn)
        _logger.warning(msg)
        errors['download'] = msg
    if len(errtxt):
        if 'download' not in errors:
            errors['download'] = ''
        errors['download'] += errtxt

    _logger.debug("index step2 - " + datetime.now().strftime("%H:%M:%S") + "  ")

    totalLogSize = 0
    if type(files) is list and len(files) > 0:
        for file in files:
            totalLogSize += file['size'] if 'size' in file and file['size'] > 0 else 0

    # from B to MB
    if totalLogSize > 0:
        totalLogSize = round(totalLogSize*1.0/1024/1024, 2)

    ### return the file page

    ### set request response data
    data = {
        'request': request,
        'errors': errors,
        'pfns': pfns,
        'files': files,
        'dirprefix': dirprefix,
        'tardir': tardir,
        'scope': scope,
        'lfn': lfn,
        'site': site,
        'guid': guid,
        'MEDIA_URL': settings.MEDIA_URL,
        'viewParams' : {'MON_VO': str(get_filebrowser_vo()).upper()},
        'HOSTNAME': hostname,
        'totalLogSize': totalLogSize,
        'nfiles': len(files),
    }

    _logger.debug("index step3 - " + datetime.now().strftime("%H:%M:%S") + "  ")
    if 'json' not in request.GET:
        return render_to_response('filebrowser/filebrowser_index.html', data, RequestContext(request))
    else:
        resp = HttpResponse(json.dumps(data, cls=DateTimeEncoder), content_type='application/json')
        _logger.debug("index step4 - " + datetime.now().strftime("%H:%M:%S") + "  ")
        return resp

def api_single_pandaid(request):
    """
        api_single_pandaid -- return log file URL for a single PanDA job

        :param request: Django's HTTP request
        :type request: django.http.HttpRequest

    """
    errors = {}

    ### check that all expected parameters are in URL
#    expectedFields = ['guid', 'site', 'scope', 'lfn']
    expectedFields = ['pandaid']
    for expectedField in expectedFields:
        try:
            if len(request.GET[expectedField]) < 1:
                msg = 'Missing expected GET parameter %s. ' % expectedField
                if 'missingparameter' not in errors.keys():
                    errors['missingparameter'] = ''
                errors['missingparameter'] += msg
        except:
            msg = 'Missing expected GET parameter %s. ' % expectedField
            _logger.error(msg)
            if 'missingparameter' not in errors.keys():
                errors['missingparameter'] = ''
            errors['missingparameter'] += msg

    ### if all expected GET parameters are present, execute file lookup
    pfns = []
    scope = ''
    lfn = ''
    guid = ''
    site = ''
    pandaid = None
    status = ''
    query = {}
    query['type'] = 'log'
    try:
        pandaid = int(request.GET['pandaid'])
    except:
        pass
    query['pandaid'] = pandaid
    file_properties = []
    try:
        file_properties = Filestable4.objects.filter(**query).values('pandaid', 'guid', \
                                'scope', 'lfn', 'destinationse', 'status')
    except:
        pass

    if len(file_properties):
        file_properties = file_properties[0]
        try:
            guid = file_properties['guid']
        except:
            pass
        try:
            site = file_properties['destinationse']
        except:
            pass
        try:
            lfn = file_properties['lfn']
        except:
            pass
        try:
            scope = file_properties['scope']
        except:
            pass
        try:
            status = file_properties['status']
        except:
            pass

        if 'missingparameter' not in errors.keys():
            pfns, errtxt = get_rucio_pfns_from_guids(guids=[guid], site=[site], \
                        lfns=[lfn], scopes=[scope])
            if len(errtxt):
                if 'lookup' not in errors:
                    errors['lookup'] = ''
                errors['lookup'] += errtxt

        ### download the file
        files = []
        dirprefix = ''
        tardir = ''
        if len(pfns):
            pfn = pfns[0]
            files, errtxt, dirprefix, tardir = fetch_file(pfn, guid, unpack=False, listfiles=False)
            if not len(pfns):
                msg = 'File download failed. [pfn=%s guid=%s, site=%s, scope=%s, lfn=%s]' % \
                    (pfn, guid, site, scope, lfn)
                _logger.warning(msg)
                errors['download'] = msg
            if len(errtxt):
                if 'download' in errors:
                    errors['download'] += errtxt
    else: # file not found in DB
        if 'lookup' not in errors:
            errors['lookup'] = ''
        errors['lookup'] += 'Log file for this job has not been found. '
    ### return the file page

    url = None
    data = { \
        'pandaid': pandaid, \
        'url': url, \
        'errors': errors, \
        'pfns': pfns, \
        'scope': scope, \
        'lfn': lfn, \
        'site': site, \
        'guid': guid, \
        'status': status, \
        'timestamp': datetime.utcnow().isoformat() \
    }
    if not len(errors):
        url = 'http://' + hostname + \
                    settings.MEDIA_URL + dirprefix + '/' + lfn
        data['url'] = url
        ### set request response data
        return render_to_response('filebrowser/filebrowser_api_single_pandaid.html', {'data': data}, RequestContext(request))
    elif 'pandaid' not in request.GET.keys() or pandaid == None:
        t = get_template('filebrowser/filebrowser_api_single_pandaid.html')
        context = RequestContext(request, {'data':data})
        return HttpResponse(t.render(context), status=400)
    elif not len(file_properties):
        t = get_template('filebrowser/filebrowser_api_single_pandaid.html')
        context = RequestContext(request, {'data':data})
        return HttpResponse(t.render(context), status=404)
    else:
        t = get_template('filebrowser/filebrowser_api_single_pandaid.html')
        context = RequestContext(request, {'data':data})
        return HttpResponse(t.render(context), status=400)

def get_job_memory_monitor_output(pandaid):
    """
    Download log tarball of a job and return path to a local copy of memory_monitor_output.txt file
    :param pandaid:
    :return: mmo_path: str
    """
    mmo_path = None
    files = []
    scope = ''
    lfn = ''
    guid = ''
    dirprefix = ''
    tardir = ''
    query = {}
    query['type'] = 'log'
    query['pandaid'] = int(pandaid)
    values = ['pandaid', 'guid', 'scope', 'lfn']
    file_properties = []

    file_properties.extend(Filestable4.objects.filter(**query).values(*values))
    if len(file_properties) == 0:
        file_properties.extend(FilestableArch.objects.filter(**query).values(*values))

    if len(file_properties):
        file_properties = file_properties[0]
        try:
            guid = file_properties['guid']
        except:
            pass
        try:
            lfn = file_properties['lfn']
        except:
            pass
        try:
            scope = file_properties['scope']
        except:
            pass

        if guid and lfn and scope:
            # check if files are already available in common CEPH storage
            tarball_path = get_fullpath_filebrowser_directory() + '/' + guid.lower() + '/' + scope + '/'
            files, err, tardir = list_file_directory(tarball_path, 100)
            _logger.debug('tarball path is {} \nError message is {} \nGot tardir: {}'.format(tarball_path, err, tardir))
            if len(files) == 0 and len(err) > 0:
                # download tarball
                _logger.debug('log tarball has not been downloaded, so downloading it now')
                files, errtxt, dirprefix, tardir = get_rucio_file(scope, lfn, guid)
                _logger.debug('Got files for dir: {} and tardir: {}. Error message: {}'.format(dirprefix, tardir, errtxt))
            if type(files) is list and len(files) > 0:
                for f in files:
                    if f['name'] == 'memory_monitor_output.txt':
                        mmo_path = tarball_path + '/' + tardir + '/' + 'memory_monitor_output.txt'
    _logger.debug('Final mmo_path: {}'.format(mmo_path))
    return mmo_path

def delete_files(request):
    """
    Clear subfolder containing log files
    :param request:
    :return:
    """

    ### check that path to logs is provided
    guid = None
    try:
        guid = request.GET['guid']
    except:
        msg = 'Missing guid GET parameter'
        _logger.error(msg)

    ### clean folder if guid provided
    if guid is not None:
        logdir = remove_folder(guid)
        data = {'message':'The folder was cleaned ' + logdir}
        return HttpResponse(json.dumps(data), content_type='application/json')
    else:
        return HttpResponse(status=404)

<|endoftext|>


<commit_before>"""Handlers for KMLSync server.

Overview:
- GE loads 'myplaces.kml' after it starts (lg_earth should create it)
- myplaces.kml contains networklinks to master.kml and to regular
  updates
- master.kml is a container for content
- network_link_update.kml returns commands to change the contents of
  the container

Example scenario:
- GE loads myplaces (contains networklinks to master and networklinkupdate.kml with viewport name)
- master.kml returns ampty document
- networklinkupdate will return list of commands with empty cookiestring

<director message loads smthn>

- earth hits networklinkupdate.kml - we see the viewport - it should
  have something loaded now - we check the cookiestring (which is empty)
- since it doesnt have anything loaded, we tell GE to load all the assets
- we have a list of slugs and URLs from the director that we want to load
- networklinkupdate will return a different cookie string with a list of the slugs e.g.
    asset_slug=foo,bar,baz
- for every asset not loaded on GE we need to create it by "<Create></Create>" section and delete by "Delete"
  section - those elements need to be children of "Update" element
- example networklinkupdate: https://github.com/EndPointCorp/lg_ros_nodes/issues/2#issuecomment-116727990

Special case - tours:
- when a tour KML is loaded - GE will try to reach KMLSyncServer's URL to send "playtour" command to itself (weird xD)
- Application needs to listen on /query.html link and accept GE's requests like:
    - /query.html?query=playtour=taipei (http://pastebin.com/3M5xHj0g)

    the query argument "playtour=taipei" needs to be forwarded to queryfile ROS node so GE gets a command to go
    to certain position

asset example: 'http://lg-head:8060/media/blah.kml'
cookie example: asset_slug=http___lg-head_8060_media_blah_kml

State data structure:
{ "right_one" : { "cookie": '', "assets": ['http://lg-head:8060/media/blah.kml] }
"""

import json
import rospy

import time
import urllib2
import threading
import xml.etree.ElementTree as ET

from xml.dom import minidom
from lg_earth.srv import KmlState, PlaytourQuery
from xml.sax.saxutils import unescape, escape
from lg_common.helpers import escape_asset_url, generate_cookie
from std_msgs.msg import String
from Queue import Queue
import tornado.web
import collections
from tornado import gen

def get_kml_root():
    """Get headers of KML file - shared by all kml generation methods."""
    kml_root = ET.Element('kml', attrib={})
    kml_root.attrib['xmlns'] = 'http://www.opengis.net/kml/2.2'
    kml_root.attrib['xmlns:gx'] = 'http://www.google.com/kml/ext/2.2'
    kml_root.attrib['xmlns:kml'] = 'http://www.opengis.net/kml/2.2'
    kml_root.attrib['xmlns:atom'] = 'http://www.w3.org/2005/Atom'
    return kml_root

class KmlMasterHandler(tornado.web.RequestHandler):

    def get(self):
        """Serve the master.kml which is updated by NLC."""
        rospy.loginfo("Got master.kml GET request")
        kml_root = get_kml_root()
        kml_document = ET.SubElement(kml_root, 'Document')
        kml_document.attrib['id'] = 'master'
        kml_reparsed = minidom.parseString(ET.tostring(kml_root))
        kml_content = kml_reparsed.toprettyxml(indent='\t')
        self.finish(kml_content)

class KmlUpdateHandler(tornado.web.RequestHandler):
    counter = 0
    timeout = 10
    global_dict = {}
    lock = threading.Lock()

    @classmethod
    def add_to_global_dict(cls, reference, unique_id):
        cls.global_dict[unique_id] = reference

    @classmethod
    def get_unique_id(cls):
        with cls.lock:
            unique_id = cls.counter
            cls.counter += 1
        return unique_id

    @classmethod
    def finish_all_requests(cls):
        for req in cls.global_dict.itervalues():
            req.get(True)
        cls.global_dict = {}

    @classmethod
    def get_scene_msg(cls, msg):
        try:
            cls.finish_all_requests()
        except Exception as e:
            rospy.loginfo("Exception getting scene changes"+str(e))
            pass

    def initialize(self):
        self.asset_service = self.application.asset_service

    @gen.coroutine
    def get(self, second_time=False):
        """
        Return XML with latest list of assets for specific window_slug
            - get window slug and calculate difference between loaded assets and the desired state
            - create the KML and return it only if cookie was different and the window slug came in the request
        """
        rospy.loginfo("Got network_link_update.kml GET request with params: %s" % self.request.query_arguments)
        window_slug = self.get_query_argument('window_slug', default=None)
        incoming_cookie_string = ''

        incoming_cookie_string = self.get_query_argument('asset_slug', default='')

        rospy.loginfo("Got network_link_update GET request for slug: %s with cookie: %s" % (window_slug, incoming_cookie_string))

        if window_slug:
            try:
                assets = self._get_assets(window_slug)
            except Exception as e:
                rospy.logerr('Failed to get assets for {}: {}'.format(
                    window_slug,
                    e.message
                ))
                # Always return a valid KML or Earth will stop requesting updates
                self.finish(get_kml_root())
                return
            assets_to_delete = self._get_assets_to_delete(incoming_cookie_string, assets)
            assets_to_create = self._get_assets_to_create(incoming_cookie_string, assets)
            if (assets_to_delete or assets_to_create) or second_time:
                self.finish(self._get_kml_for_networklink_update(assets_to_delete, assets_to_create, assets))
            else:
                self.unique_id = unique_id = KmlUpdateHandler.get_unique_id()
                rospy.loginfo("Request Counter Value {}".format(self.unique_id))
                rospy.loginfo("Global Dictionary Value {}".format(KmlUpdateHandler.global_dict))
                KmlUpdateHandler.add_to_global_dict(self, self.unique_id)
                yield gen.sleep(KmlUpdateHandler.timeout)
                if self.unique_id in KmlUpdateHandler.global_dict:
                    del KmlUpdateHandler.global_dict[self.unique_id]
                    assets_to_delete = self._get_assets_to_delete(incoming_cookie_string, assets)
                    assets_to_create = self._get_assets_to_create(incoming_cookie_string, assets)
                    self.finish(self._get_kml_for_networklink_update(assets_to_delete, assets_to_create, assets))
        else:
            self.set_status(400, "No window slug provided")
            self.finish("400 Bad Request: No window slug provided")

    def _get_kml_for_networklink_update(self, assets_to_delete, assets_to_create, assets):
        """ Generate static part of NetworkLinkUpdate xml"""
        kml_root = get_kml_root()
        kml_networklink = ET.SubElement(kml_root, 'NetworkLinkControl')
        kml_min_refresh_period = ET.SubElement(kml_networklink, 'minRefreshPeriod').text = '1'
        kml_max_session_length = ET.SubElement(kml_networklink, 'maxSessionLength').text = '-1'
        cookie_cdata_string = "<![CDATA[%s]]>" % self._get_full_cookie(assets)
        kml_cookies = ET.SubElement(kml_networklink, 'cookie').text = escape(cookie_cdata_string)
        kml_update = ET.SubElement(kml_networklink, 'Update')
        kml_target_href = ET.SubElement(kml_update, 'targetHref').text = self.request.protocol + '://' + self.request.host + '/master.kml'
        kml_create_assets = self._get_kml_for_create_assets(assets_to_create, kml_update)
        kml_delete_assets = self._get_kml_for_delete_assets(assets_to_delete, kml_update)

        kml_reparsed = minidom.parseString(unescape(ET.tostring(kml_root)))
        kml_content = kml_reparsed.toprettyxml(indent='\t')
        return unescape(kml_content)

    def _get_assets_to_delete(self, incoming_cookie_string, assets):
        """
        Calculate the difference between assets loaded on GE client and those expected to be loaded by server.
        Return a list of slugs to be deleted from client e.g. ['blah_kml', 'zomg_kml']
        param incoming_cookie_string: str
            e.g. 'blah_kml,zomg_kml'
        param assets: list
            list of assets for the request window_slug
        rtype: list
            e.g. ['blah_kml']
        """
        server_slugs_list = self._get_server_slugs_state(assets)
        client_slugs_list = self._get_client_slugs_state(incoming_cookie_string)
        ret = list(set(client_slugs_list) - set(server_slugs_list))
        rospy.logdebug("Got the assets to delete as: %s" % ret)
        return ret

    def _get_assets(self, window_slug):
        return self.asset_service(window_slug).assets

    def _get_assets_to_create(self, incoming_cookie_string, assets):
        """
        Calculate the difference between assets loaded on GE client and those expected to be loaded by server.
        Return a list of slugs to be created in GE client e.g. ['blah_kml', 'zomg_kml']
        param incoming_cookie_string: str
            e.g. 'blah_kml,zomg_kml'
        param assets: list
            list of assets for the request window_slug
        rtype: list
            e.g. ['http://lg-head:8060/blah.kml', 'http://lg-head:8060/zomg.kml']
        """

        urls_to_create = []
        client_state_slugs = self._get_client_slugs_state(incoming_cookie_string)

        for url in assets:
            if escape_asset_url(url) in client_state_slugs:
                continue
            urls_to_create.append(url)

        return urls_to_create

    def _get_client_slugs_state(self, incoming_cookie_string):
        """
        Return list of slugs submitted by client in a cookie string
        param incoming_cookie_string: str
            (e.g. 'blah_kml,zomg_foo_bar_kml')
        rtype: list
            e.g. ['blah_kml', 'zomg_foo_bar_kml']
        """
        if not incoming_cookie_string:
            return []
        return [z for z in incoming_cookie_string.split(',')]

    def _get_cookie(self, assets):
        """
        Return comma separated list of slugs e.g. 'asd_kml,blah_kml'
        rtype: str
        """
        return generate_cookie(assets)

    def _get_server_slugs_state(self, assets):
        """
        Return a list of slugs that server expects to be loaded
        """
        try:
            cookie = self._get_cookie(assets)
            return [z for z in cookie.split(',')]
        except AttributeError, e:
            return []

    def _get_full_cookie(self, assets):
        """return the cookie prepended by asset_slug= only if cookie is not blank"""
        cookie = self._get_cookie(assets)
        if cookie != '':
            cookie = 'asset_slug=' + cookie
        return cookie

    def _get_kml_for_create_assets(self, assets_to_create, parent):
        """
        Generate a networklinkupdate xml 'Create' section with assets_to_create (if any)
        """
        if assets_to_create:
            kml_create = ET.SubElement(parent, 'Create')
            kml_document = ET.SubElement(kml_create, 'Document', {'targetId': 'master'})
            for asset in assets_to_create:
                new_asset = ET.SubElement(kml_document, 'NetworkLink', {'id': escape_asset_url(asset)})
                ET.SubElement(new_asset, 'name').text = escape_asset_url(asset)
                link = ET.SubElement(new_asset, 'Link')
                ET.SubElement(link, 'href').text = asset
            return kml_create #not sure if this is needed since we're already building on the xml tree...
        return parent

    def _get_kml_for_delete_assets(self, assets_to_delete, parent):
        """
        Generate a networklinkupdate xml 'Delete' section with assets_to_delete (if any)
        """
        if assets_to_delete:
            kml_delete = ET.SubElement(parent, 'Delete')
            for asset in assets_to_delete:
                ET.SubElement(kml_delete, 'NetworkLink', {'targetId': escape_asset_url(asset)})
            return kml_delete

class KmlQueryHandler(tornado.web.RequestHandler):
    def initialize(self):
        self.playtour = self.application.playtour
        self.playtour_service = self.application.playtour_service

    def get(self):
        """
        Publish play tour message on the topic /earth/query/tour
        """
        query_string = self.get_query_argument('query', default='')
        try:
            while not rospy.is_shutdown():
                tour_string = query_string.split('=')[1]
                tour_string = urllib2.unquote(tour_string)
                self.playtour.tourname = str(tour_string)
                self.playtour_service(tour_string)
                self.finish("OK")
            self.set_status(503, "Server shutting down")
            self.finish("Server shutting down")
        except IndexError as e:
            rospy.logerr("Got the wrong query string %s" % e)
            self.set_status(400, "Got the wrong query string")
            self.finish("Bad Request: Got a bad query string")

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>Removed unnecessary imports<commit_after>"""Handlers for KMLSync server.

Overview:
- GE loads 'myplaces.kml' after it starts (lg_earth should create it)
- myplaces.kml contains networklinks to master.kml and to regular
  updates
- master.kml is a container for content
- network_link_update.kml returns commands to change the contents of
  the container

Example scenario:
- GE loads myplaces (contains networklinks to master and networklinkupdate.kml with viewport name)
- master.kml returns ampty document
- networklinkupdate will return list of commands with empty cookiestring

<director message loads smthn>

- earth hits networklinkupdate.kml - we see the viewport - it should
  have something loaded now - we check the cookiestring (which is empty)
- since it doesnt have anything loaded, we tell GE to load all the assets
- we have a list of slugs and URLs from the director that we want to load
- networklinkupdate will return a different cookie string with a list of the slugs e.g.
    asset_slug=foo,bar,baz
- for every asset not loaded on GE we need to create it by "<Create></Create>" section and delete by "Delete"
  section - those elements need to be children of "Update" element
- example networklinkupdate: https://github.com/EndPointCorp/lg_ros_nodes/issues/2#issuecomment-116727990

Special case - tours:
- when a tour KML is loaded - GE will try to reach KMLSyncServer's URL to send "playtour" command to itself (weird xD)
- Application needs to listen on /query.html link and accept GE's requests like:
    - /query.html?query=playtour=taipei (http://pastebin.com/3M5xHj0g)

    the query argument "playtour=taipei" needs to be forwarded to queryfile ROS node so GE gets a command to go
    to certain position

asset example: 'http://lg-head:8060/media/blah.kml'
cookie example: asset_slug=http___lg-head_8060_media_blah_kml

State data structure:
{ "right_one" : { "cookie": '', "assets": ['http://lg-head:8060/media/blah.kml] }
"""

import json
import rospy

import urllib2
import threading
import xml.etree.ElementTree as ET

from xml.dom import minidom
from lg_earth.srv import KmlState, PlaytourQuery
from xml.sax.saxutils import unescape, escape
from lg_common.helpers import escape_asset_url, generate_cookie
from std_msgs.msg import String
import tornado.web
from tornado import gen

def get_kml_root():
    """Get headers of KML file - shared by all kml generation methods."""
    kml_root = ET.Element('kml', attrib={})
    kml_root.attrib['xmlns'] = 'http://www.opengis.net/kml/2.2'
    kml_root.attrib['xmlns:gx'] = 'http://www.google.com/kml/ext/2.2'
    kml_root.attrib['xmlns:kml'] = 'http://www.opengis.net/kml/2.2'
    kml_root.attrib['xmlns:atom'] = 'http://www.w3.org/2005/Atom'
    return kml_root

class KmlMasterHandler(tornado.web.RequestHandler):

    def get(self):
        """Serve the master.kml which is updated by NLC."""
        rospy.loginfo("Got master.kml GET request")
        kml_root = get_kml_root()
        kml_document = ET.SubElement(kml_root, 'Document')
        kml_document.attrib['id'] = 'master'
        kml_reparsed = minidom.parseString(ET.tostring(kml_root))
        kml_content = kml_reparsed.toprettyxml(indent='\t')
        self.finish(kml_content)

class KmlUpdateHandler(tornado.web.RequestHandler):
    counter = 0
    timeout = 10
    global_dict = {}
    lock = threading.Lock()

    @classmethod
    def add_to_global_dict(cls, reference, unique_id):
        cls.global_dict[unique_id] = reference

    @classmethod
    def get_unique_id(cls):
        with cls.lock:
            unique_id = cls.counter
            cls.counter += 1
        return unique_id

    @classmethod
    def finish_all_requests(cls):
        for req in cls.global_dict.itervalues():
            req.get(True)
        cls.global_dict = {}

    @classmethod
    def get_scene_msg(cls, msg):
        try:
            cls.finish_all_requests()
        except Exception as e:
            rospy.loginfo("Exception getting scene changes"+str(e))
            pass

    def initialize(self):
        self.asset_service = self.application.asset_service

    @gen.coroutine
    def get(self, second_time=False):
        """
        Return XML with latest list of assets for specific window_slug
            - get window slug and calculate difference between loaded assets and the desired state
            - create the KML and return it only if cookie was different and the window slug came in the request
        """
        rospy.loginfo("Got network_link_update.kml GET request with params: %s" % self.request.query_arguments)
        window_slug = self.get_query_argument('window_slug', default=None)
        incoming_cookie_string = ''

        incoming_cookie_string = self.get_query_argument('asset_slug', default='')

        rospy.loginfo("Got network_link_update GET request for slug: %s with cookie: %s" % (window_slug, incoming_cookie_string))

        if window_slug:
            try:
                assets = self._get_assets(window_slug)
            except Exception as e:
                rospy.logerr('Failed to get assets for {}: {}'.format(
                    window_slug,
                    e.message
                ))
                # Always return a valid KML or Earth will stop requesting updates
                self.finish(get_kml_root())
                return
            assets_to_delete = self._get_assets_to_delete(incoming_cookie_string, assets)
            assets_to_create = self._get_assets_to_create(incoming_cookie_string, assets)
            if (assets_to_delete or assets_to_create) or second_time:
                self.finish(self._get_kml_for_networklink_update(assets_to_delete, assets_to_create, assets))
            else:
                self.unique_id = unique_id = KmlUpdateHandler.get_unique_id()
                rospy.loginfo("Request Counter Value {}".format(self.unique_id))
                rospy.loginfo("Global Dictionary Value {}".format(KmlUpdateHandler.global_dict))
                KmlUpdateHandler.add_to_global_dict(self, self.unique_id)
                yield gen.sleep(KmlUpdateHandler.timeout)
                if self.unique_id in KmlUpdateHandler.global_dict:
                    del KmlUpdateHandler.global_dict[self.unique_id]
                    assets_to_delete = self._get_assets_to_delete(incoming_cookie_string, assets)
                    assets_to_create = self._get_assets_to_create(incoming_cookie_string, assets)
                    self.finish(self._get_kml_for_networklink_update(assets_to_delete, assets_to_create, assets))
        else:
            self.set_status(400, "No window slug provided")
            self.finish("400 Bad Request: No window slug provided")

    def _get_kml_for_networklink_update(self, assets_to_delete, assets_to_create, assets):
        """ Generate static part of NetworkLinkUpdate xml"""
        kml_root = get_kml_root()
        kml_networklink = ET.SubElement(kml_root, 'NetworkLinkControl')
        kml_min_refresh_period = ET.SubElement(kml_networklink, 'minRefreshPeriod').text = '1'
        kml_max_session_length = ET.SubElement(kml_networklink, 'maxSessionLength').text = '-1'
        cookie_cdata_string = "<![CDATA[%s]]>" % self._get_full_cookie(assets)
        kml_cookies = ET.SubElement(kml_networklink, 'cookie').text = escape(cookie_cdata_string)
        kml_update = ET.SubElement(kml_networklink, 'Update')
        kml_target_href = ET.SubElement(kml_update, 'targetHref').text = self.request.protocol + '://' + self.request.host + '/master.kml'
        kml_create_assets = self._get_kml_for_create_assets(assets_to_create, kml_update)
        kml_delete_assets = self._get_kml_for_delete_assets(assets_to_delete, kml_update)

        kml_reparsed = minidom.parseString(unescape(ET.tostring(kml_root)))
        kml_content = kml_reparsed.toprettyxml(indent='\t')
        return unescape(kml_content)

    def _get_assets_to_delete(self, incoming_cookie_string, assets):
        """
        Calculate the difference between assets loaded on GE client and those expected to be loaded by server.
        Return a list of slugs to be deleted from client e.g. ['blah_kml', 'zomg_kml']
        param incoming_cookie_string: str
            e.g. 'blah_kml,zomg_kml'
        param assets: list
            list of assets for the request window_slug
        rtype: list
            e.g. ['blah_kml']
        """
        server_slugs_list = self._get_server_slugs_state(assets)
        client_slugs_list = self._get_client_slugs_state(incoming_cookie_string)
        ret = list(set(client_slugs_list) - set(server_slugs_list))
        rospy.logdebug("Got the assets to delete as: %s" % ret)
        return ret

    def _get_assets(self, window_slug):
        return self.asset_service(window_slug).assets

    def _get_assets_to_create(self, incoming_cookie_string, assets):
        """
        Calculate the difference between assets loaded on GE client and those expected to be loaded by server.
        Return a list of slugs to be created in GE client e.g. ['blah_kml', 'zomg_kml']
        param incoming_cookie_string: str
            e.g. 'blah_kml,zomg_kml'
        param assets: list
            list of assets for the request window_slug
        rtype: list
            e.g. ['http://lg-head:8060/blah.kml', 'http://lg-head:8060/zomg.kml']
        """

        urls_to_create = []
        client_state_slugs = self._get_client_slugs_state(incoming_cookie_string)

        for url in assets:
            if escape_asset_url(url) in client_state_slugs:
                continue
            urls_to_create.append(url)

        return urls_to_create

    def _get_client_slugs_state(self, incoming_cookie_string):
        """
        Return list of slugs submitted by client in a cookie string
        param incoming_cookie_string: str
            (e.g. 'blah_kml,zomg_foo_bar_kml')
        rtype: list
            e.g. ['blah_kml', 'zomg_foo_bar_kml']
        """
        if not incoming_cookie_string:
            return []
        return [z for z in incoming_cookie_string.split(',')]

    def _get_cookie(self, assets):
        """
        Return comma separated list of slugs e.g. 'asd_kml,blah_kml'
        rtype: str
        """
        return generate_cookie(assets)

    def _get_server_slugs_state(self, assets):
        """
        Return a list of slugs that server expects to be loaded
        """
        try:
            cookie = self._get_cookie(assets)
            return [z for z in cookie.split(',')]
        except AttributeError, e:
            return []

    def _get_full_cookie(self, assets):
        """return the cookie prepended by asset_slug= only if cookie is not blank"""
        cookie = self._get_cookie(assets)
        if cookie != '':
            cookie = 'asset_slug=' + cookie
        return cookie

    def _get_kml_for_create_assets(self, assets_to_create, parent):
        """
        Generate a networklinkupdate xml 'Create' section with assets_to_create (if any)
        """
        if assets_to_create:
            kml_create = ET.SubElement(parent, 'Create')
            kml_document = ET.SubElement(kml_create, 'Document', {'targetId': 'master'})
            for asset in assets_to_create:
                new_asset = ET.SubElement(kml_document, 'NetworkLink', {'id': escape_asset_url(asset)})
                ET.SubElement(new_asset, 'name').text = escape_asset_url(asset)
                link = ET.SubElement(new_asset, 'Link')
                ET.SubElement(link, 'href').text = asset
            return kml_create #not sure if this is needed since we're already building on the xml tree...
        return parent

    def _get_kml_for_delete_assets(self, assets_to_delete, parent):
        """
        Generate a networklinkupdate xml 'Delete' section with assets_to_delete (if any)
        """
        if assets_to_delete:
            kml_delete = ET.SubElement(parent, 'Delete')
            for asset in assets_to_delete:
                ET.SubElement(kml_delete, 'NetworkLink', {'targetId': escape_asset_url(asset)})
            return kml_delete

class KmlQueryHandler(tornado.web.RequestHandler):
    def initialize(self):
        self.playtour = self.application.playtour
        self.playtour_service = self.application.playtour_service

    def get(self):
        """
        Publish play tour message on the topic /earth/query/tour
        """
        query_string = self.get_query_argument('query', default='')
        try:
            while not rospy.is_shutdown():
                tour_string = query_string.split('=')[1]
                tour_string = urllib2.unquote(tour_string)
                self.playtour.tourname = str(tour_string)
                self.playtour_service(tour_string)
                self.finish("OK")
            self.set_status(503, "Server shutting down")
            self.finish("Server shutting down")
        except IndexError as e:
            rospy.logerr("Got the wrong query string %s" % e)
            self.set_status(400, "Got the wrong query string")
            self.finish("Bad Request: Got a bad query string")

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>""" Manage the queue for the log messages """

import config
from log import Log

from azure.storage.queue import QueueService

import os

class Queue:
    def __init__(self, account_name, account_key, queue_name="logqueue"):
        """Initialiaze a queue. The type is set by the
        'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
        'AzureStorageQueue' then values must be provided for
        'account_name' and 'account_key' which are values associated
        with the Azure Storage account. 'queue_name' is optional and
        defaults to 'logqueue'.

        """
        self.log = Log()
        self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
        self.queue_name = queue_name
        # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)

        if self.queue_type == "AzureStorageQueue":
            self.createAzureQueues(account_name, account_key)
        elif self.queue_type == "LocalFile":
            self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+')
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def createAzureQueues(self, account_name, account_key):
        """
        Create a queue for unprocessed log messages. Entries in the queue
        will be dequeued, processed and deleted upon success.
        """
        self.queue_service = QueueService(account_name, account_key)
        self.queue_service.create_queue(self.queue_name)

    def close(self):
        """Perform any necessary clearnup on the queue
           at the end of a run.
        """
        if self.queue_type == "AzureStorageQueue":
            pass
        elif self.queue_type == "LocalFile":
            self.file_queue.close()
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def enqueue(self, msg):
        if self.queue_type == "LocalFile":
            file_queue.write(msg + '\n')
        elif self.queue_type == "AzureStorageQueue":
            self.queue_service.put_message(self.queue_name, msg)
        else:
            self.log.error("We don't know how to handle queues of type " + self.queue_type)
        self.log.debug(msg)

    def dequeue(self):
        messages = []
        if self.queue_type == "LocalFile":
            with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
                messages = f.readlines()[1]
        elif self.queue_type == "AzureStorageQueue":
            messages = self.queue_service.get_messages(self.queue_name)
        return messages

    def delete(self, message):
        self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
        #  with open(config.PROCESSED_LOG_FILE, 'a') as processed:
        #    processed.write(log)
        #  os.remove(config.UNPROCESSED_LOG_FILE)

    def delete_queue(self, queue_name):
        self.queue_service.delete_queue(queue_name, False)

    def getLength(self):
        """
        Get the approximate length of the queue
        """
        queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
        count = queue_metadata['x-ms-approximate-messages-count']
        return int(count)

    def peek_messages(self, num_messages):
        """
        Peek at the top messages in the queue. This method does not remove the
        messages from the queue.
        """
        return self.queue_service.peek_messages(self.queue_name, num_messages)
<commit_msg>add a getname method to the messageQueueclass<commit_after>""" Manage the queue for the log messages """

import config
from log import Log

from azure.storage.queue import QueueService

import os

class Queue:
    def __init__(self, account_name, account_key, queue_name="logqueue"):
        """Initialiaze a queue. The type is set by the
        'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
        'AzureStorageQueue' then values must be provided for
        'account_name' and 'account_key' which are values associated
        with the Azure Storage account. 'queue_name' is optional and
        defaults to 'logqueue'.

        """
        self.log = Log()
        self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
        self.queue_name = queue_name
        # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)

        if self.queue_type == "AzureStorageQueue":
            self.createAzureQueues(account_name, account_key)
        elif self.queue_type == "LocalFile":
            self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+')
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def getName(self):
        return self.queue_name

    def createAzureQueues(self, account_name, account_key):
        """
        Create a queue for unprocessed log messages. Entries in the queue
        will be dequeued, processed and deleted upon success.
        """
        self.queue_service = QueueService(account_name, account_key)
        self.queue_service.create_queue(self.queue_name)

    def close(self):
        """Perform any necessary clearnup on the queue
           at the end of a run.
        """
        if self.queue_type == "AzureStorageQueue":
            pass
        elif self.queue_type == "LocalFile":
            self.file_queue.close()
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def enqueue(self, msg):
        if self.queue_type == "LocalFile":
            file_queue.write(msg + '\n')
        elif self.queue_type == "AzureStorageQueue":
            self.queue_service.put_message(self.queue_name, msg)
        else:
            self.log.error("We don't know how to handle queues of type " + self.queue_type)
        self.log.debug(msg)

    def dequeue(self):
        messages = []
        if self.queue_type == "LocalFile":
            with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
                messages = f.readlines()[1]
        elif self.queue_type == "AzureStorageQueue":
            messages = self.queue_service.get_messages(self.queue_name)
        return messages

    def delete(self, message):
        self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
        #  with open(config.PROCESSED_LOG_FILE, 'a') as processed:
        #    processed.write(log)
        #  os.remove(config.UNPROCESSED_LOG_FILE)

    def delete_queue(self, queue_name):
        self.queue_service.delete_queue(queue_name, False)

    def getLength(self):
        """
        Get the approximate length of the queue
        """
        queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
        count = queue_metadata['x-ms-approximate-messages-count']
        return int(count)

    def peek_messages(self, num_messages):
        """
        Peek at the top messages in the queue. This method does not remove the
        messages from the queue.
        """
        return self.queue_service.peek_messages(self.queue_name, num_messages)
<|endoftext|>