<commit_before>import random
import string
import urllib.parse
import asyncio

import discord
import pymongo

from discord.ext import commands
from .common import Cog, WIDE_MAP

MAX_MEME_NAME = 30

RI_TABLE = {
    '0': ':zero:',
    '1': ':one:',
    '2': ':two:',
    '3': ':three:',
    '4': ':four:',
    '5': ':five:',
    '6': ':six:',
    '7': ':seven:',
    '8': ':eight:',
    '9': ':nine:',

    '.': ':record_button:',
    '!': ':exclamation:',
    '?': ':question:',
    '+': ':heavy_plus_sign:',
    '-': ':heavy_minus_sign:',
}

# implement letters
RI_STR = '🇦🇧🇨🇩🇪🇫🇬🇭🇮🇯🇰🇱🇲🇳🇴🇵🇶🇷🇸🇹🇺🇻🇼🇽🇾🇿'
RI_TABLE.update({letter: RI_STR[string.ascii_lowercase.find(letter)] for
                 letter in string.ascii_lowercase})

class Memes(Cog):
    def __init__(self, bot):
        super().__init__(bot)
        self.memes_coll = self.config.jose_db['memes']
        self.urban_cache = {}

    async def add_meme(self, name, value, author_id, uses=0, nsfw=None):
        """Add a meme to the database."""

        meme = {
            'name': name,
            'value': value,
            'uses': uses,
            'author_id': author_id,
            'nsfw': nsfw,
        }

        await self.memes_coll.insert_one(meme)

    async def get_meme(self, name):
        """Get a meme from the database."""
        return await self.memes_coll.find_one({'name': name})

    async def delete_meme(self, name):
        """Remove a meme from the database."""
        return await self.memes_coll.delete_many({'name': name})

    async def update_meme(self, new_meme):
        """Update a new meme into the database."""

        await self.memes_coll.update_one({'name': new_meme['name']},
                                         {'$set': new_meme})

    async def search_memes(self, name):
        """Get a list of memes that have names that are close to
        the name given."""

        cur = self.memes_coll.find({
            'name': {
                '$regex': name,
            }
        })

        return [m['name'] for m in await cur.to_list(length=None)]

    @commands.group(aliases=['m'], invoke_without_command=True)
    async def meme(self, ctx):
        """The meme database."""
        await ctx.send("`j!help m`")

    @meme.command(name='add')
    async def add(self, ctx, name: str, *, value: str):
        """Add a meme."""
        name, value = name.strip(), value.strip()

        if len(name) > MAX_MEME_NAME:
            await ctx.send('2 long 5 me')
            return

        meme = await self.get_meme(name)
        if meme is not None:
            await ctx.send('meme already exists')
            return

        await self.add_meme(name, value, ctx.author.id, 0,
                            ctx.channel.is_nsfw())
        await ctx.ok()

    @meme.command(name='get')
    async def get(self, ctx, *, name: str):
        """Retrieve a meme.

        Doesn't allow NSFW memes(memes created in NSFW channels) to be shown
        in non-NSFW channels
        """
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            probables = await self.search_memes(name)
            if len(probables) > 0:
                await ctx.send(f'Didn\'t you mean `{",".join(probables)}`')
                return
            await ctx.send('Meme not found.')
            return

        nsfw_meme = meme.get('nsfw', False)
        if nsfw_meme and not ctx.channel.is_nsfw():
            await ctx.send('Can\'t show NSFW memes in a non-NSFW channel')
            return

        meme['uses'] += 1
        await self.update_meme(meme)
        await ctx.send(meme['value'])

    @meme.command()
    async def rm(self, ctx, name: str):
        """Remove a meme."""
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        owner = (await self.bot.application_info()).owner
        authorized = (meme.get('author_id') == ctx.author.id) or \
                     (ctx.author == owner)
        if not authorized:
            await ctx.send('Unauthorized')
            return

        await self.delete_meme(name)
        await ctx.ok()

    @meme.command()
    async def rename(self, ctx, name: str, new_name: str):
        """Rename a meme's name."""
        name, new_name = name.strip(), new_name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        new_name_meme = await self.get_meme(new_name)
        if new_name_meme is not None:
            await ctx.send('New name already used.')
            return

        await self.delete_meme(meme['name'])
        await self.add_meme(new_name, meme['value'], meme.get('author_id'),
                            meme['uses'], meme['nsfw'])
        await ctx.send(f'Renamed {meme["name"]!r} to {new_name!r}')

    @meme.command()
    async def owner(self, ctx, name: str):
        """Get a meme's owner (who created it)."""
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        await ctx.send(f'`{name}` was made by '
                       f'{self.get_name(meme.get("author_id"))}')

    @meme.command()
    async def count(self, ctx):
        """Amount of memes in the database."""
        amnt = await self.memes_coll.count()
        await ctx.send(f'amount: {amnt}')

    @meme.command()
    async def top(self, ctx):
        """Shows the top 15 most used memes."""
        res = []
        cur = self.memes_coll.find({}).sort('uses', pymongo.DESCENDING)

        for (idx, meme) in enumerate(await cur.to_list(length=15)):
            res.append(f'[{idx}] {meme["name"]} used {meme["uses"]} times')

        _joined = "\n".join(res)
        await ctx.send(f'```\n{_joined}\n```')

    @meme.command()
    async def used(self, ctx, name: str):
        """Shows the amount of times this meme was used."""
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        await ctx.send(f'`{meme["name"]}` => {meme["uses"]} times')

    @meme.command()
    async def see(self, ctx, owner: discord.User, page: int = 0):
        """See memes made by someone."""
        cur = self.memes_coll.find({'author_id': owner.id})

        from_owner = [m['name'] for m in await cur.to_list(length=None)]

        if len(from_owner) < 1:
            await ctx.send("No memes found.")
            return

        page_slice = from_owner[page * 50:(page + 1) * 50]

        res = len(from_owner), len(page_slice), ', '.join(page_slice)
        report = f'Showing {res[0]}[{res[1]} in page],\nMemes: {res[2]}'
        await ctx.send(report)

    @meme.command()
    async def rand(self, ctx):
        """Get a random meme."""
        total = await self.memes_coll.count()
        rand_idx = random.randint(0, total - 1)

        cur = self.memes_coll.find()
        async for meme in cur.limit(-1).skip(rand_idx):
            await ctx.send(f'{meme["name"]}: {meme["value"]}')

    @meme.command()
    async def search(self, ctx, term: str):
        """Search for memes"""
        term = term.strip()
        if len(term) < 1:
            await ctx.send('lul')
            return

        memes = await self.search_memes(term)
        if len(memes) < 1:
            await ctx.send('No memes found')
            return

        await ctx.send(f'```\n{",".join(memes)}\n```')

    @commands.command(aliases=['fw'])
    async def fullwidth(self, ctx, *, text: str):
        """Convert text to full width."""
        if len(text.strip()) <= 0:
            return

        await ctx.send(text.translate(WIDE_MAP))

    @commands.command()
    async def ri(self, ctx, *, inputstr: str):
        """Convert text to Regional Indicators."""
        inputstr = inputstr.strip().lower()
        l_inputstr = len(inputstr)
        if l_inputstr < 1 or l_inputstr > 1995:
            await ctx.send('lul')
            return

        inputstr = list(inputstr)

        for (index, char) in enumerate(inputstr):
            if char in RI_TABLE:
                inputstr[index] = f'{RI_TABLE[char]}\u200b'

        res = ''.join(inputstr)
        await ctx.send(res)

    @commands.command(hidden=True)
    async def pupper(self, ctx):
        await ctx.send('http://i.imgur.com/9Le8rW7.jpg :sob:')

    @commands.command(name='8ball')
    async def _8ball(self, ctx):
        """8ball lul"""
        answer = random.choice([
            'Yes',
            'No',
            'Maybe',
            'Potentially',
            'Answer hazy',
            'Only in your dreams',
            'Ask later',
        ])

        await ctx.send(f'**{ctx.author.name}**, :8ball: said {answer}.')

    @commands.command(hidden=True)
    async def br(self, ctx):
        """br?"""
        await ctx.send('br?')

    @commands.command()
    async def urban(self, ctx, *, term: str):
        """Search for a word in the Urban Dictionary."""

        # cache :DDDDDDDDD
        if term in self.urban_cache:
            definition = self.urban_cache[term][0]
            example = self.urban_cache[term][1]
            return await ctx.send(f'```\n{term!r}:\n'
                                  f'{definition}\n{example}```')

        await self.jcoin.pricing(ctx, self.prices['API'])

        urban_url = f'https://api.urbandictionary.com/v0/define?term={urllib.parse.quote(term)}'
        content = await self.get_json(urban_url)
        c_list = content['list']

        if len(c_list) < 1:
            raise self.SayException('No results found')

        definition = c_list[0]['definition']
        example = ''
        if len(definition) < 1940:
            example = c_list[0]['example']

        self.urban_cache[term] = [definition, example]
        await ctx.send(f'```\n{term!r}:\n{definition}\n{example}```')

    @commands.command(hidden=True)
    async def ejaculate(self, ctx):
        """PUMPE THE MOOSCLES YIIIS"""
        await ctx.send('https://www.youtube.com/watch?v=S6UqgjaBt4w')

    @commands.command()
    @commands.is_owner()
    async def blink(self, ctx, *, text: str):
        m = await ctx.send(text)
        for i in range(10):
            if i % 2 == 0:
                await m.edit(content=f'**{text}**')
            else:
                await m.edit(content=f'{text}')
            await asyncio.sleep(2)

    @commands.command()
    async def neko(self, ctx):
        """Posts a random neko picture."""
        api_url = 'http://nekos.life/api/neko'

        response = await self.get_json(api_url)
        image_url = response['neko']

        embed = discord.Embed(color=discord.Color(0xf84a6e))
        embed.set_image(url=image_url)

        await ctx.send(embed=embed)

def setup(bot):
    bot.add_cog(Memes(bot))
    bot.remove_command('meme')
<commit_msg>memes: make neko cmd nsfw<commit_after>import random
import string
import urllib.parse
import asyncio

import discord
import pymongo

from discord.ext import commands
from .common import Cog, WIDE_MAP

MAX_MEME_NAME = 30

RI_TABLE = {
    '0': ':zero:',
    '1': ':one:',
    '2': ':two:',
    '3': ':three:',
    '4': ':four:',
    '5': ':five:',
    '6': ':six:',
    '7': ':seven:',
    '8': ':eight:',
    '9': ':nine:',

    '.': ':record_button:',
    '!': ':exclamation:',
    '?': ':question:',
    '+': ':heavy_plus_sign:',
    '-': ':heavy_minus_sign:',
}

# implement letters
RI_STR = '🇦🇧🇨🇩🇪🇫🇬🇭🇮🇯🇰🇱🇲🇳🇴🇵🇶🇷🇸🇹🇺🇻🇼🇽🇾🇿'
RI_TABLE.update({letter: RI_STR[string.ascii_lowercase.find(letter)] for
                 letter in string.ascii_lowercase})

class Memes(Cog):
    def __init__(self, bot):
        super().__init__(bot)
        self.memes_coll = self.config.jose_db['memes']
        self.urban_cache = {}

    async def add_meme(self, name, value, author_id, uses=0, nsfw=None):
        """Add a meme to the database."""

        meme = {
            'name': name,
            'value': value,
            'uses': uses,
            'author_id': author_id,
            'nsfw': nsfw,
        }

        await self.memes_coll.insert_one(meme)

    async def get_meme(self, name):
        """Get a meme from the database."""
        return await self.memes_coll.find_one({'name': name})

    async def delete_meme(self, name):
        """Remove a meme from the database."""
        return await self.memes_coll.delete_many({'name': name})

    async def update_meme(self, new_meme):
        """Update a new meme into the database."""

        await self.memes_coll.update_one({'name': new_meme['name']},
                                         {'$set': new_meme})

    async def search_memes(self, name):
        """Get a list of memes that have names that are close to
        the name given."""

        cur = self.memes_coll.find({
            'name': {
                '$regex': name,
            }
        })

        return [m['name'] for m in await cur.to_list(length=None)]

    @commands.group(aliases=['m'], invoke_without_command=True)
    async def meme(self, ctx):
        """The meme database."""
        await ctx.send("`j!help m`")

    @meme.command(name='add')
    async def add(self, ctx, name: str, *, value: str):
        """Add a meme."""
        name, value = name.strip(), value.strip()

        if len(name) > MAX_MEME_NAME:
            await ctx.send('2 long 5 me')
            return

        meme = await self.get_meme(name)
        if meme is not None:
            await ctx.send('meme already exists')
            return

        await self.add_meme(name, value, ctx.author.id, 0,
                            ctx.channel.is_nsfw())
        await ctx.ok()

    @meme.command(name='get')
    async def get(self, ctx, *, name: str):
        """Retrieve a meme.

        Doesn't allow NSFW memes(memes created in NSFW channels) to be shown
        in non-NSFW channels
        """
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            probables = await self.search_memes(name)
            if len(probables) > 0:
                await ctx.send(f'Didn\'t you mean `{",".join(probables)}`')
                return
            await ctx.send('Meme not found.')
            return

        nsfw_meme = meme.get('nsfw', False)
        if nsfw_meme and not ctx.channel.is_nsfw():
            await ctx.send('Can\'t show NSFW memes in a non-NSFW channel')
            return

        meme['uses'] += 1
        await self.update_meme(meme)
        await ctx.send(meme['value'])

    @meme.command()
    async def rm(self, ctx, name: str):
        """Remove a meme."""
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        owner = (await self.bot.application_info()).owner
        authorized = (meme.get('author_id') == ctx.author.id) or \
                     (ctx.author == owner)
        if not authorized:
            await ctx.send('Unauthorized')
            return

        await self.delete_meme(name)
        await ctx.ok()

    @meme.command()
    async def rename(self, ctx, name: str, new_name: str):
        """Rename a meme's name."""
        name, new_name = name.strip(), new_name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        new_name_meme = await self.get_meme(new_name)
        if new_name_meme is not None:
            await ctx.send('New name already used.')
            return

        await self.delete_meme(meme['name'])
        await self.add_meme(new_name, meme['value'], meme.get('author_id'),
                            meme['uses'], meme['nsfw'])
        await ctx.send(f'Renamed {meme["name"]!r} to {new_name!r}')

    @meme.command()
    async def owner(self, ctx, name: str):
        """Get a meme's owner (who created it)."""
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        await ctx.send(f'`{name}` was made by '
                       f'{self.get_name(meme.get("author_id"))}')

    @meme.command()
    async def count(self, ctx):
        """Amount of memes in the database."""
        amnt = await self.memes_coll.count()
        await ctx.send(f'amount: {amnt}')

    @meme.command()
    async def top(self, ctx):
        """Shows the top 15 most used memes."""
        res = []
        cur = self.memes_coll.find({}).sort('uses', pymongo.DESCENDING)

        for (idx, meme) in enumerate(await cur.to_list(length=15)):
            res.append(f'[{idx}] {meme["name"]} used {meme["uses"]} times')

        _joined = "\n".join(res)
        await ctx.send(f'```\n{_joined}\n```')

    @meme.command()
    async def used(self, ctx, name: str):
        """Shows the amount of times this meme was used."""
        name = name.strip()

        meme = await self.get_meme(name)
        if meme is None:
            await ctx.send('Meme not found')
            return

        await ctx.send(f'`{meme["name"]}` => {meme["uses"]} times')

    @meme.command()
    async def see(self, ctx, owner: discord.User, page: int = 0):
        """See memes made by someone."""
        cur = self.memes_coll.find({'author_id': owner.id})

        from_owner = [m['name'] for m in await cur.to_list(length=None)]

        if len(from_owner) < 1:
            await ctx.send("No memes found.")
            return

        page_slice = from_owner[page * 50:(page + 1) * 50]

        res = len(from_owner), len(page_slice), ', '.join(page_slice)
        report = f'Showing {res[0]}[{res[1]} in page],\nMemes: {res[2]}'
        await ctx.send(report)

    @meme.command()
    async def rand(self, ctx):
        """Get a random meme."""
        total = await self.memes_coll.count()
        rand_idx = random.randint(0, total - 1)

        cur = self.memes_coll.find()
        async for meme in cur.limit(-1).skip(rand_idx):
            await ctx.send(f'{meme["name"]}: {meme["value"]}')

    @meme.command()
    async def search(self, ctx, term: str):
        """Search for memes"""
        term = term.strip()
        if len(term) < 1:
            await ctx.send('lul')
            return

        memes = await self.search_memes(term)
        if len(memes) < 1:
            await ctx.send('No memes found')
            return

        await ctx.send(f'```\n{",".join(memes)}\n```')

    @commands.command(aliases=['fw'])
    async def fullwidth(self, ctx, *, text: str):
        """Convert text to full width."""
        if len(text.strip()) <= 0:
            return

        await ctx.send(text.translate(WIDE_MAP))

    @commands.command()
    async def ri(self, ctx, *, inputstr: str):
        """Convert text to Regional Indicators."""
        inputstr = inputstr.strip().lower()
        l_inputstr = len(inputstr)
        if l_inputstr < 1 or l_inputstr > 1995:
            await ctx.send('lul')
            return

        inputstr = list(inputstr)

        for (index, char) in enumerate(inputstr):
            if char in RI_TABLE:
                inputstr[index] = f'{RI_TABLE[char]}\u200b'

        res = ''.join(inputstr)
        await ctx.send(res)

    @commands.command(hidden=True)
    async def pupper(self, ctx):
        await ctx.send('http://i.imgur.com/9Le8rW7.jpg :sob:')

    @commands.command(name='8ball')
    async def _8ball(self, ctx):
        """8ball lul"""
        answer = random.choice([
            'Yes',
            'No',
            'Maybe',
            'Potentially',
            'Answer hazy',
            'Only in your dreams',
            'Ask later',
        ])

        await ctx.send(f'**{ctx.author.name}**, :8ball: said {answer}.')

    @commands.command(hidden=True)
    async def br(self, ctx):
        """br?"""
        await ctx.send('br?')

    @commands.command()
    async def urban(self, ctx, *, term: str):
        """Search for a word in the Urban Dictionary."""

        # cache :DDDDDDDDD
        if term in self.urban_cache:
            definition = self.urban_cache[term][0]
            example = self.urban_cache[term][1]
            return await ctx.send(f'```\n{term!r}:\n'
                                  f'{definition}\n{example}```')

        await self.jcoin.pricing(ctx, self.prices['API'])

        urban_url = f'https://api.urbandictionary.com/v0/define?term={urllib.parse.quote(term)}'
        content = await self.get_json(urban_url)
        c_list = content['list']

        if len(c_list) < 1:
            raise self.SayException('No results found')

        definition = c_list[0]['definition']
        example = ''
        if len(definition) < 1940:
            example = c_list[0]['example']

        self.urban_cache[term] = [definition, example]
        await ctx.send(f'```\n{term!r}:\n{definition}\n{example}```')

    @commands.command(hidden=True)
    async def ejaculate(self, ctx):
        """PUMPE THE MOOSCLES YIIIS"""
        await ctx.send('https://www.youtube.com/watch?v=S6UqgjaBt4w')

    @commands.command()
    @commands.is_owner()
    async def blink(self, ctx, *, text: str):
        m = await ctx.send(text)
        for i in range(10):
            if i % 2 == 0:
                await m.edit(content=f'**{text}**')
            else:
                await m.edit(content=f'{text}')
            await asyncio.sleep(2)

    @commands.command()
    @commands.is_nsfw()
    async def neko(self, ctx):
        """Posts a random neko picture."""
        api_url = 'http://nekos.life/api/neko'

        response = await self.get_json(api_url)
        image_url = response['neko']

        embed = discord.Embed(color=discord.Color(0xf84a6e))
        embed.set_image(url=image_url)

        await ctx.send(embed=embed)

def setup(bot):
    bot.add_cog(Memes(bot))
    bot.remove_command('meme')
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q

from celery import chord, group

from olympia import amo
from olympia.addons.models import Addon
from olympia.addons.tasks import (
    add_firefox57_tag, bump_appver_for_legacy_addons,
    find_inconsistencies_between_es_and_db)
from olympia.amo.utils import chunked
from olympia.devhub.tasks import convert_purified, get_preview_sizes
from olympia.editors.tasks import recalculate_post_review_weight
from olympia.lib.crypto.tasks import sign_addons

tasks = {
    'find_inconsistencies_between_es_and_db': {
        'method': find_inconsistencies_between_es_and_db, 'qs': []},
    'get_preview_sizes': {'method': get_preview_sizes, 'qs': []},
    'convert_purified': {'method': convert_purified, 'qs': []},
    'recalculate_post_review_weight': {
        'method': recalculate_post_review_weight,
        'qs': [
            Q(_current_version__autoapprovalsummary__verdict=amo.AUTO_APPROVED)
        ]},
    'sign_addons': {'method': sign_addons, 'qs': []},
    'add_firefox57_tag_to_webextensions': {
        'method': add_firefox57_tag,
        'qs': [Q(status=amo.STATUS_PUBLIC,
                 _current_version__files__is_webextension=True)]},
    'bump_appver_for_legacy_addons': {
        'method': bump_appver_for_legacy_addons,
        'qs': [
            Q(
                type__in=(amo.ADDON_EXTENSION, amo.ADDON_THEME),
                _current_version__files__is_webextension=False,
                _current_version__apps__max__version_int__lt=56990000200100,
                _current_version__apps__application__in=(
                    amo.FIREFOX.id, amo.ANDROID.id))
        ],
        'pre': lambda values_qs: values_qs.distinct(),
    }
}

class Command(BaseCommand):
    """
    A generic command to run a task on addons.
    Add tasks to the tasks dictionary, providing a list of Q objects if you'd
    like to filter the list down.

    method: the method to delay
    pre: a method to further pre process the pks, must return the pks (opt.)
    qs: a list of Q objects to apply to the method
    kwargs: any extra kwargs you want to apply to the delay method (optional)
    """
    def add_arguments(self, parser):
        """Handle command arguments."""
        parser.add_argument(
            '--task',
            action='store',
            dest='task',
            type=str,
            help='Run task on the addons.')

        parser.add_argument(
            '--with-deleted',
            action='store_true',
            dest='with_deleted',
            help='Include deleted add-ons when determining which '
                 'add-ons to process.')

    def handle(self, *args, **options):
        task = tasks.get(options.get('task'))
        if not task:
            raise CommandError('Unknown task provided. Options are: %s'
                               % ', '.join(tasks.keys()))
        if options.get('with_deleted'):
            addon_manager = Addon.unfiltered
        else:
            addon_manager = Addon.objects
        pks = (addon_manager.filter(*task['qs'])
                            .values_list('pk', flat=True)
                            .order_by('id'))
        if 'pre' in task:
            # This is run in process to ensure its run before the tasks.
            pks = task['pre'](pks)
        if pks:
            kw = task.get('kwargs', {})
            # All the remaining tasks go in one group.
            grouping = []
            for chunk in chunked(pks, 100):
                grouping.append(
                    task['method'].subtask(args=[chunk], kwargs=kw))

            # Add the post task on to the end.
            post = None
            if 'post' in task:
                post = task['post'].subtask(args=[], kwargs=kw, immutable=True)
                ts = chord(grouping, post)
            else:
                ts = group(grouping)
            ts.apply_async()
<commit_msg>Avoid harcoding the value of 56.* for readability<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q

from celery import chord, group

from olympia import amo
from olympia.addons.models import Addon
from olympia.addons.tasks import (
    add_firefox57_tag, bump_appver_for_legacy_addons,
    find_inconsistencies_between_es_and_db)
from olympia.amo.utils import chunked
from olympia.devhub.tasks import convert_purified, get_preview_sizes
from olympia.editors.tasks import recalculate_post_review_weight
from olympia.lib.crypto.tasks import sign_addons
from olympia.versions.compare import version_int

firefox_56_star = version_int('56.*')

tasks = {
    'find_inconsistencies_between_es_and_db': {
        'method': find_inconsistencies_between_es_and_db, 'qs': []},
    'get_preview_sizes': {'method': get_preview_sizes, 'qs': []},
    'convert_purified': {'method': convert_purified, 'qs': []},
    'recalculate_post_review_weight': {
        'method': recalculate_post_review_weight,
        'qs': [
            Q(_current_version__autoapprovalsummary__verdict=amo.AUTO_APPROVED)
        ]},
    'sign_addons': {'method': sign_addons, 'qs': []},
    'add_firefox57_tag_to_webextensions': {
        'method': add_firefox57_tag,
        'qs': [Q(status=amo.STATUS_PUBLIC,
                 _current_version__files__is_webextension=True)]},
    'bump_appver_for_legacy_addons': {
        'method': bump_appver_for_legacy_addons,
        'qs': [
            Q(
                type__in=(amo.ADDON_EXTENSION, amo.ADDON_THEME),
                _current_version__files__is_webextension=False,
                _current_version__apps__max__version_int__lt=firefox_56_star,
                _current_version__apps__application__in=(
                    amo.FIREFOX.id, amo.ANDROID.id))
        ],
        'pre': lambda values_qs: values_qs.distinct(),
    },
}

class Command(BaseCommand):
    """
    A generic command to run a task on addons.
    Add tasks to the tasks dictionary, providing a list of Q objects if you'd
    like to filter the list down.

    method: the method to delay
    pre: a method to further pre process the pks, must return the pks (opt.)
    qs: a list of Q objects to apply to the method
    kwargs: any extra kwargs you want to apply to the delay method (optional)
    """
    def add_arguments(self, parser):
        """Handle command arguments."""
        parser.add_argument(
            '--task',
            action='store',
            dest='task',
            type=str,
            help='Run task on the addons.')

        parser.add_argument(
            '--with-deleted',
            action='store_true',
            dest='with_deleted',
            help='Include deleted add-ons when determining which '
                 'add-ons to process.')

    def handle(self, *args, **options):
        task = tasks.get(options.get('task'))
        if not task:
            raise CommandError('Unknown task provided. Options are: %s'
                               % ', '.join(tasks.keys()))
        if options.get('with_deleted'):
            addon_manager = Addon.unfiltered
        else:
            addon_manager = Addon.objects
        pks = (addon_manager.filter(*task['qs'])
                            .values_list('pk', flat=True)
                            .order_by('id'))
        if 'pre' in task:
            # This is run in process to ensure its run before the tasks.
            pks = task['pre'](pks)
        if pks:
            kw = task.get('kwargs', {})
            # All the remaining tasks go in one group.
            grouping = []
            for chunk in chunked(pks, 100):
                grouping.append(
                    task['method'].subtask(args=[chunk], kwargs=kw))

            # Add the post task on to the end.
            post = None
            if 'post' in task:
                post = task['post'].subtask(args=[], kwargs=kw, immutable=True)
                ts = chord(grouping, post)
            else:
                ts = group(grouping)
            ts.apply_async()
<|endoftext|>


<commit_before>import logging
import os.path

from distutils.version import LooseVersion

from teuthology.config import config as teuth_config
from teuthology.orchestra import run
from teuthology import packaging

from teuthology.task.install.util import _get_builder_project, _get_local_dir

log = logging.getLogger(__name__)

def _remove(ctx, config, remote, rpm):
    """
    Removes RPM packages from remote

    :param ctx: the argparse.Namespace object
    :param config: the config dict
    :param remote: the teuthology.orchestra.remote.Remote object
    :param rpm: list of packages names to remove
    """
    remote_os = remote.os
    dist_release = remote_os.name

    install_ceph_packages = config.get('install_ceph_packages')
    if install_ceph_packages:
        log.info("Removing packages: {pkglist} on rpm system.".format(
            pkglist=", ".join(rpm)))
        if dist_release in ['opensuse', 'sle']:
            remote.run(args='''
                for d in {rpms} ; do
                    sudo zypper -n --no-gpg-checks remove --capability $d || true
                done'''.format(rpms=' '.join(rpm)))
            remote.run(args='sudo zypper clean -a')
        else:
            remote.run(args='''
                for d in {rpms} ; do
                    sudo yum -y remove $d || true
                done'''.format(rpms=' '.join(rpm)))
            remote.run(args='sudo yum clean all')
    else:
        log.info("install task did not install any packages, "
                 "so not removing any, either")

    repos = config.get('repos')
    if repos:
        if dist_release in ['opensuse', 'sle']:
            _zypper_removerepo(remote, repos)
        else:
            raise Exception('Custom repos were specified for %s ' % remote_os +
                            'but these are currently not supported')
    else:
        builder = _get_builder_project(ctx, remote, config)
        builder.remove_repo()

    if dist_release in ['opensuse', 'sle']:
        #remote.run(args='sudo zypper clean -a')
        log.info("Not cleaning zypper cache: this might fail, and is not needed "
                 "because the test machine will be destroyed or reimaged anyway")
    else:
        remote.run(args='sudo yum clean expire-cache')

def _zypper_addrepo(remote, repo_list):
    """
    Add zypper repos to the remote system.

    :param remote: remote node where to add packages
    :param repo_list: list of dictionaries with keys 'name', 'url'
    :return:
    """
    for repo in repo_list:
        if 'priority' in repo:
            remote.run(args=[
                'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck',
                '-p', str(repo['priority']), repo['url'], repo['name'],
            ])
        else:
            remote.run(args=[
                'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck',
                repo['url'], repo['name'],
            ])

def _zypper_removerepo(remote, repo_list):
    """
    Remove zypper repos on the remote system.

    :param remote: remote node where to remove packages from
    :param repo_list: list of dictionaries with keys 'name', 'url'
    :return:
    """
    for repo in repo_list:
        remote.run(args=[
            'sudo', 'zypper', '-n', 'removerepo', repo['name'],
        ])

def _zypper_wipe_all_repos(remote):
    """
    Completely "wipe" (remove) all zypper repos

    :param remote: remote node where to wipe zypper repos
    :return:
    """
    log.info("Wiping zypper repos (if any)")
    remote.sh('sudo zypper repos -upEP && sudo rm -f /etc/zypp/repos.d/*')

def _downgrade_packages(ctx, remote, pkgs, pkg_version, config):
    """
    Downgrade packages listed by 'downgrade_packages'

    Downgrade specified packages to given version. The list of packages
    downgrade is provided by 'downgrade_packages' as a property of "install"
    task.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    :param pkgs: list of package names to install
    :param pkg_version: the version to which all packages will be downgraded
    :param config: the config dict
    :return: list of package names from 'pkgs' which are not yet
             installed/downgraded
    """
    downgrade_pkgs = config.get('downgrade_packages', [])
    if not downgrade_pkgs:
        return pkgs
    log.info('Downgrading packages: {pkglist}'.format(
        pkglist=', '.join(downgrade_pkgs)))
    # assuming we are going to downgrade packages with the same version
    first_pkg = downgrade_pkgs[0]
    installed_version = packaging.get_package_version(remote, first_pkg)
    assert installed_version, "failed to get version of {}".format(first_pkg)
    assert LooseVersion(installed_version) > LooseVersion(pkg_version)
    # to compose package name like "librados2-0.94.10-87.g116a558.el7"
    pkgs_opt = ['-'.join([pkg, pkg_version]) for pkg in downgrade_pkgs]
    remote.run(args='sudo yum -y downgrade {}'.format(' '.join(pkgs_opt)))
    return [pkg for pkg in pkgs if pkg not in downgrade_pkgs]

def _update_package_list_and_install(ctx, remote, rpm, config):
    """
    Installs the repository for the relevant branch, then installs
    the requested packages on the remote system.

    TODO: split this into at least two functions.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    :param rpm: list of packages names to install
    :param config: the config dict
    """
    # rpm does not force installation of a particular version of the project
    # packages, so we can put extra_system_packages together with the rest
    system_pkglist = config.get('extra_system_packages')
    if system_pkglist:
        if isinstance(system_pkglist, dict):
            rpm += system_pkglist.get('rpm')
        else:
            rpm += system_pkglist
    remote_os = remote.os

    dist_release = remote_os.name
    log.debug("_update_package_list_and_install: config is {}".format(config))
    repos = config.get('repos')
    install_ceph_packages = config.get('install_ceph_packages')
    repos_only = config.get('repos_only')

    if repos:
        log.debug("Adding repos: %s" % repos)
        if dist_release in ['opensuse', 'sle']:
            _zypper_wipe_all_repos(remote)
            _zypper_addrepo(remote, repos)
        else:
            raise Exception('Custom repos were specified for %s ' % remote_os +
                            'but these are currently not supported')
    else:
        builder = _get_builder_project(ctx, remote, config)
        log.info('Pulling from %s', builder.base_url)
        log.info('Package version is %s', builder.version)
        builder.install_repo()

    if repos_only:
        log.info("repos_only was specified: not installing any packages")
        return None

    if not install_ceph_packages:
        log.info("install_ceph_packages set to False: not installing Ceph packages")
        # Although "librados2" is an indirect dependency of ceph-test, we
        # install it separately because, otherwise, ceph-test cannot be
        # installed (even with --force) when there are several conflicting
        # repos from different vendors.
        rpm = ["librados2", "ceph-test"]

    # rpm does not force installation of a particular version of the project
    # packages, so we can put extra_system_packages together with the rest
    system_pkglist = config.get('extra_system_packages', [])
    if system_pkglist:
        if isinstance(system_pkglist, dict):
            rpm += system_pkglist.get('rpm')
        else:
            rpm += system_pkglist

    log.info("Installing packages: {pkglist} on remote rpm {arch}".format(
        pkglist=", ".join(rpm), arch=remote.arch))

    if dist_release not in ['opensuse', 'sle']:
        project = builder.project
        uri = builder.uri_reference
        _yum_fix_repo_priority(remote, project, uri)
        _yum_fix_repo_host(remote, project)
        _yum_set_check_obsoletes(remote)

    if dist_release in ['opensuse', 'sle']:
        remote.run(args='sudo zypper clean -a')
    else:
        remote.run(args='sudo yum clean all')

    ldir = _get_local_dir(config, remote)

    if dist_release in ['opensuse', 'sle']:
        remove_cmd = 'sudo zypper -n remove --capability'
        # NOTE: --capability contradicts --force
        install_cmd = 'sudo zypper -n --no-gpg-checks install --force --no-recommends'
    else:
        remove_cmd = 'sudo yum -y remove'
        install_cmd = 'sudo yum -y install'
        # to compose version string like "0.94.10-87.g116a558.el7"
        pkg_version = '.'.join([builder.version, builder.dist_release])
        rpm = _downgrade_packages(ctx, remote, rpm, pkg_version, config)

    if system_pkglist:
        remote.run(
            args='{install_cmd} {rpms}'
                 .format(install_cmd=install_cmd, rpms=' '.join(rpm))
            )
    else:
        for cpack in rpm:
            if ldir:
                remote.run(args='''
                  if test -e {pkg} ; then
                    {remove_cmd} {pkg} ;
                    {install_cmd} {pkg} ;
                  else
                    {install_cmd} {cpack} ;
                  fi
                '''.format(remove_cmd=remove_cmd,
                           install_cmd=install_cmd,
                           pkg=os.path.join(ldir, cpack),
                           cpack=cpack))
            else:
                remote.run(
                    args='{install_cmd} {cpack}'
                         .format(install_cmd=install_cmd, cpack=cpack)
                    )

def _yum_fix_repo_priority(remote, project, uri):
    """
    On the remote, 'priority=1' lines to each enabled repo in:

        /etc/yum.repos.d/{project}.repo

    :param remote: the teuthology.orchestra.remote.Remote object
    :param project: the project whose repos need modification
    """
    repo_path = '/etc/yum.repos.d/%s.repo' % project
    remote.run(
        args=[
            'if', 'test', '-f', repo_path, run.Raw(';'), 'then',
            'sudo', 'sed', '-i', '-e',
            run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''),
            '-e',
            run.Raw("'s;ref/[a-zA-Z0-9_-]*/;{uri}/;g'".format(uri=uri)),
            repo_path, run.Raw(';'), 'fi'
        ]
    )

def _yum_fix_repo_host(remote, project):
    """
    Update the hostname to reflect the gitbuilder_host setting.
    """
    # Skip this bit if we're not using gitbuilder
    if not isinstance(packaging.get_builder_project(),
                      packaging.GitbuilderProject):
        return
    old_host = teuth_config._defaults['gitbuilder_host']
    new_host = teuth_config.gitbuilder_host
    if new_host == old_host:
        return
    repo_path = '/etc/yum.repos.d/%s.repo' % project
    host_sed_expr = "'s/{0}/{1}/'".format(old_host, new_host)
    remote.run(
        args=[
            'if', 'test', '-f', repo_path, run.Raw(';'), 'then',
            'sudo', 'sed', '-i', '-e', run.Raw(host_sed_expr),
            repo_path, run.Raw(';'), 'fi']
    )

def _yum_set_check_obsoletes(remote):
    """
    Set check_obsoletes = 1 in /etc/yum/pluginconf.d/priorities.conf

    Creates a backup at /etc/yum/pluginconf.d/priorities.conf.orig so we can
    restore later.
    """
    conf_path = '/etc/yum/pluginconf.d/priorities.conf'
    conf_path_orig = conf_path + '.orig'
    cmd = [
        'test', '-e', conf_path_orig, run.Raw('||'), 'sudo', 'cp', '-af',
        conf_path, conf_path_orig,
    ]
    remote.run(args=cmd)
    cmd = [
        'grep', 'check_obsoletes', conf_path, run.Raw('&&'), 'sudo', 'sed',
        '-i', 's/check_obsoletes.*0/check_obsoletes = 1/g', conf_path,
        run.Raw('||'), 'echo', 'check_obsoletes = 1', run.Raw('|'), 'sudo',
        'tee', '-a', conf_path,
    ]
    remote.run(args=cmd)

def _yum_unset_check_obsoletes(remote):
    """
    Restore the /etc/yum/pluginconf.d/priorities.conf backup
    """
    conf_path = '/etc/yum/pluginconf.d/priorities.conf'
    conf_path_orig = conf_path + '.orig'
    remote.run(args=['sudo', 'mv', '-f', conf_path_orig, conf_path],
               check_status=False)

def _remove_sources_list(ctx, config, remote):
    """
    Removes /etc/yum.repos.d/{proj}.repo

    :param remote: the teuthology.orchestra.remote.Remote object
    :param proj: the project whose .repo needs removing
    """
    builder = _get_builder_project(ctx, remote, config)
    builder.remove_repo()
    if remote.os.name not in ['opensuse', 'sle']:
        _yum_unset_check_obsoletes(remote)

def _upgrade_packages(ctx, config, remote, pkgs):
    """
    Upgrade project's packages on remote RPM-based host
    Before doing so, it makes sure the project's repository is installed -
    removing any previous version first.

    :param ctx: the argparse.Namespace object
    :param config: the config dict
    :param remote: the teuthology.orchestra.remote.Remote object
    :param pkgs: the RPM packages to be installed
    :param branch: the branch of the project to be used
    """
    builder = _get_builder_project(ctx, remote, config)
    log.info(
        "Host {host} is: {distro} {ver} {arch}".format(
            host=remote.shortname,
            distro=builder.os_type,
            ver=builder.os_version,
            arch=builder.arch,)
    )

    base_url = builder.base_url
    log.info('Repo base URL: %s', base_url)
    project = builder.project

    # Remove the repository before re-adding it
    builder.remove_repo()
    builder.install_repo()

    if builder.dist_release not in ['opensuse', 'sle']:
        uri = builder.uri_reference
        _yum_fix_repo_priority(remote, project, uri)
        _yum_fix_repo_host(remote, project)
        _yum_set_check_obsoletes(remote)

    if builder.dist_release in ['opensuse', 'sle']:
        pkg_mng_cmd = 'zypper'
        pkg_mng_opts = '-a'
    else:
        pkg_mng_cmd = 'yum'
        pkg_mng_opts = 'all'

    remote.run(
        args=[
            'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts,
        ])

    # Actually upgrade the project packages
    if builder.dist_release in ['opensuse', 'sle']:
        pkg_mng_opts = '-n'
        pkg_mng_subcommand_opts = '--capability'
        pkg_mng_install_opts = '--no-recommends'
    else:
        pkg_mng_opts = '-y'
        pkg_mng_subcommand_opts = ''
        pkg_mng_install_opts = ''
    args = ['sudo', pkg_mng_cmd, pkg_mng_opts,
            'install', pkg_mng_subcommand_opts,
            pkg_mng_install_opts]
    args += pkgs
    remote.run(args=args)
<commit_msg>install/rpm: fix use the 'zypper addrepo' command<commit_after>import logging
import os.path

from distutils.version import LooseVersion

from teuthology.config import config as teuth_config
from teuthology.orchestra import run
from teuthology import packaging

from teuthology.task.install.util import _get_builder_project, _get_local_dir

log = logging.getLogger(__name__)

def _remove(ctx, config, remote, rpm):
    """
    Removes RPM packages from remote

    :param ctx: the argparse.Namespace object
    :param config: the config dict
    :param remote: the teuthology.orchestra.remote.Remote object
    :param rpm: list of packages names to remove
    """
    remote_os = remote.os
    dist_release = remote_os.name

    install_ceph_packages = config.get('install_ceph_packages')
    if install_ceph_packages:
        log.info("Removing packages: {pkglist} on rpm system.".format(
            pkglist=", ".join(rpm)))
        if dist_release in ['opensuse', 'sle']:
            remote.run(args='''
                for d in {rpms} ; do
                    sudo zypper -n --no-gpg-checks remove --capability $d || true
                done'''.format(rpms=' '.join(rpm)))
            remote.run(args='sudo zypper clean -a')
        else:
            remote.run(args='''
                for d in {rpms} ; do
                    sudo yum -y remove $d || true
                done'''.format(rpms=' '.join(rpm)))
            remote.run(args='sudo yum clean all')
    else:
        log.info("install task did not install any packages, "
                 "so not removing any, either")

    repos = config.get('repos')
    if repos:
        if dist_release in ['opensuse', 'sle']:
            _zypper_removerepo(remote, repos)
        else:
            raise Exception('Custom repos were specified for %s ' % remote_os +
                            'but these are currently not supported')
    else:
        builder = _get_builder_project(ctx, remote, config)
        builder.remove_repo()

    if dist_release in ['opensuse', 'sle']:
        #remote.run(args='sudo zypper clean -a')
        log.info("Not cleaning zypper cache: this might fail, and is not needed "
                 "because the test machine will be destroyed or reimaged anyway")
    else:
        remote.run(args='sudo yum clean expire-cache')

def _zypper_addrepo(remote, repo_list):
    """
    Add zypper repos to the remote system.

    :param remote: remote node where to add packages
    :param repo_list: list of dictionaries with keys 'name', 'url'
    :return:
    """
    for repo in repo_list:
        if 'priority' in repo:
            remote.run(args=[
                'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck',
                '-p', str(repo['priority']), repo['url'], repo['name'],
            ])
        else:
            remote.run(args=[
                'sudo', 'zypper', '-n', 'addrepo', '--refresh', '--no-gpgcheck',
                repo['url'], repo['name'],
            ])

def _zypper_removerepo(remote, repo_list):
    """
    Remove zypper repos on the remote system.

    :param remote: remote node where to remove packages from
    :param repo_list: list of dictionaries with keys 'name', 'url'
    :return:
    """
    for repo in repo_list:
        remote.run(args=[
            'sudo', 'zypper', '-n', 'removerepo', repo['name'],
        ])

def _zypper_wipe_all_repos(remote):
    """
    Completely "wipe" (remove) all zypper repos

    :param remote: remote node where to wipe zypper repos
    :return:
    """
    log.info("Wiping zypper repos (if any)")
    remote.sh('sudo zypper repos -upEP && '
              'sudo rm -f /etc/zypp/repos.d/* || '
              'true')

def _downgrade_packages(ctx, remote, pkgs, pkg_version, config):
    """
    Downgrade packages listed by 'downgrade_packages'

    Downgrade specified packages to given version. The list of packages
    downgrade is provided by 'downgrade_packages' as a property of "install"
    task.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    :param pkgs: list of package names to install
    :param pkg_version: the version to which all packages will be downgraded
    :param config: the config dict
    :return: list of package names from 'pkgs' which are not yet
             installed/downgraded
    """
    downgrade_pkgs = config.get('downgrade_packages', [])
    if not downgrade_pkgs:
        return pkgs
    log.info('Downgrading packages: {pkglist}'.format(
        pkglist=', '.join(downgrade_pkgs)))
    # assuming we are going to downgrade packages with the same version
    first_pkg = downgrade_pkgs[0]
    installed_version = packaging.get_package_version(remote, first_pkg)
    assert installed_version, "failed to get version of {}".format(first_pkg)
    assert LooseVersion(installed_version) > LooseVersion(pkg_version)
    # to compose package name like "librados2-0.94.10-87.g116a558.el7"
    pkgs_opt = ['-'.join([pkg, pkg_version]) for pkg in downgrade_pkgs]
    remote.run(args='sudo yum -y downgrade {}'.format(' '.join(pkgs_opt)))
    return [pkg for pkg in pkgs if pkg not in downgrade_pkgs]

def _update_package_list_and_install(ctx, remote, rpm, config):
    """
    Installs the repository for the relevant branch, then installs
    the requested packages on the remote system.

    TODO: split this into at least two functions.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    :param rpm: list of packages names to install
    :param config: the config dict
    """
    # rpm does not force installation of a particular version of the project
    # packages, so we can put extra_system_packages together with the rest
    system_pkglist = config.get('extra_system_packages')
    if system_pkglist:
        if isinstance(system_pkglist, dict):
            rpm += system_pkglist.get('rpm')
        else:
            rpm += system_pkglist
    remote_os = remote.os

    dist_release = remote_os.name
    log.debug("_update_package_list_and_install: config is {}".format(config))
    repos = config.get('repos')
    install_ceph_packages = config.get('install_ceph_packages')
    repos_only = config.get('repos_only')

    if repos:
        log.debug("Adding repos: %s" % repos)
        if dist_release in ['opensuse', 'sle']:
            _zypper_wipe_all_repos(remote)
            _zypper_addrepo(remote, repos)
        else:
            raise Exception('Custom repos were specified for %s ' % remote_os +
                            'but these are currently not supported')
    else:
        builder = _get_builder_project(ctx, remote, config)
        log.info('Pulling from %s', builder.base_url)
        log.info('Package version is %s', builder.version)
        builder.install_repo()

    if repos_only:
        log.info("repos_only was specified: not installing any packages")
        return None

    if not install_ceph_packages:
        log.info("install_ceph_packages set to False: not installing Ceph packages")
        # Although "librados2" is an indirect dependency of ceph-test, we
        # install it separately because, otherwise, ceph-test cannot be
        # installed (even with --force) when there are several conflicting
        # repos from different vendors.
        rpm = ["librados2", "ceph-test"]

    # rpm does not force installation of a particular version of the project
    # packages, so we can put extra_system_packages together with the rest
    system_pkglist = config.get('extra_system_packages', [])
    if system_pkglist:
        if isinstance(system_pkglist, dict):
            rpm += system_pkglist.get('rpm')
        else:
            rpm += system_pkglist

    log.info("Installing packages: {pkglist} on remote rpm {arch}".format(
        pkglist=", ".join(rpm), arch=remote.arch))

    if dist_release not in ['opensuse', 'sle']:
        project = builder.project
        uri = builder.uri_reference
        _yum_fix_repo_priority(remote, project, uri)
        _yum_fix_repo_host(remote, project)
        _yum_set_check_obsoletes(remote)

    if dist_release in ['opensuse', 'sle']:
        remote.run(args='sudo zypper clean -a')
    else:
        remote.run(args='sudo yum clean all')

    ldir = _get_local_dir(config, remote)

    if dist_release in ['opensuse', 'sle']:
        remove_cmd = 'sudo zypper -n remove --capability'
        # NOTE: --capability contradicts --force
        install_cmd = 'sudo zypper -n --no-gpg-checks install --force --no-recommends'
    else:
        remove_cmd = 'sudo yum -y remove'
        install_cmd = 'sudo yum -y install'
        # to compose version string like "0.94.10-87.g116a558.el7"
        pkg_version = '.'.join([builder.version, builder.dist_release])
        rpm = _downgrade_packages(ctx, remote, rpm, pkg_version, config)

    if system_pkglist:
        remote.run(
            args='{install_cmd} {rpms}'
                 .format(install_cmd=install_cmd, rpms=' '.join(rpm))
            )
    else:
        for cpack in rpm:
            if ldir:
                remote.run(args='''
                  if test -e {pkg} ; then
                    {remove_cmd} {pkg} ;
                    {install_cmd} {pkg} ;
                  else
                    {install_cmd} {cpack} ;
                  fi
                '''.format(remove_cmd=remove_cmd,
                           install_cmd=install_cmd,
                           pkg=os.path.join(ldir, cpack),
                           cpack=cpack))
            else:
                remote.run(
                    args='{install_cmd} {cpack}'
                         .format(install_cmd=install_cmd, cpack=cpack)
                    )

def _yum_fix_repo_priority(remote, project, uri):
    """
    On the remote, 'priority=1' lines to each enabled repo in:

        /etc/yum.repos.d/{project}.repo

    :param remote: the teuthology.orchestra.remote.Remote object
    :param project: the project whose repos need modification
    """
    repo_path = '/etc/yum.repos.d/%s.repo' % project
    remote.run(
        args=[
            'if', 'test', '-f', repo_path, run.Raw(';'), 'then',
            'sudo', 'sed', '-i', '-e',
            run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''),
            '-e',
            run.Raw("'s;ref/[a-zA-Z0-9_-]*/;{uri}/;g'".format(uri=uri)),
            repo_path, run.Raw(';'), 'fi'
        ]
    )

def _yum_fix_repo_host(remote, project):
    """
    Update the hostname to reflect the gitbuilder_host setting.
    """
    # Skip this bit if we're not using gitbuilder
    if not isinstance(packaging.get_builder_project(),
                      packaging.GitbuilderProject):
        return
    old_host = teuth_config._defaults['gitbuilder_host']
    new_host = teuth_config.gitbuilder_host
    if new_host == old_host:
        return
    repo_path = '/etc/yum.repos.d/%s.repo' % project
    host_sed_expr = "'s/{0}/{1}/'".format(old_host, new_host)
    remote.run(
        args=[
            'if', 'test', '-f', repo_path, run.Raw(';'), 'then',
            'sudo', 'sed', '-i', '-e', run.Raw(host_sed_expr),
            repo_path, run.Raw(';'), 'fi']
    )

def _yum_set_check_obsoletes(remote):
    """
    Set check_obsoletes = 1 in /etc/yum/pluginconf.d/priorities.conf

    Creates a backup at /etc/yum/pluginconf.d/priorities.conf.orig so we can
    restore later.
    """
    conf_path = '/etc/yum/pluginconf.d/priorities.conf'
    conf_path_orig = conf_path + '.orig'
    cmd = [
        'test', '-e', conf_path_orig, run.Raw('||'), 'sudo', 'cp', '-af',
        conf_path, conf_path_orig,
    ]
    remote.run(args=cmd)
    cmd = [
        'grep', 'check_obsoletes', conf_path, run.Raw('&&'), 'sudo', 'sed',
        '-i', 's/check_obsoletes.*0/check_obsoletes = 1/g', conf_path,
        run.Raw('||'), 'echo', 'check_obsoletes = 1', run.Raw('|'), 'sudo',
        'tee', '-a', conf_path,
    ]
    remote.run(args=cmd)

def _yum_unset_check_obsoletes(remote):
    """
    Restore the /etc/yum/pluginconf.d/priorities.conf backup
    """
    conf_path = '/etc/yum/pluginconf.d/priorities.conf'
    conf_path_orig = conf_path + '.orig'
    remote.run(args=['sudo', 'mv', '-f', conf_path_orig, conf_path],
               check_status=False)

def _remove_sources_list(ctx, config, remote):
    """
    Removes /etc/yum.repos.d/{proj}.repo

    :param remote: the teuthology.orchestra.remote.Remote object
    :param proj: the project whose .repo needs removing
    """
    builder = _get_builder_project(ctx, remote, config)
    builder.remove_repo()
    if remote.os.name not in ['opensuse', 'sle']:
        _yum_unset_check_obsoletes(remote)

def _upgrade_packages(ctx, config, remote, pkgs):
    """
    Upgrade project's packages on remote RPM-based host
    Before doing so, it makes sure the project's repository is installed -
    removing any previous version first.

    :param ctx: the argparse.Namespace object
    :param config: the config dict
    :param remote: the teuthology.orchestra.remote.Remote object
    :param pkgs: the RPM packages to be installed
    :param branch: the branch of the project to be used
    """
    builder = _get_builder_project(ctx, remote, config)
    log.info(
        "Host {host} is: {distro} {ver} {arch}".format(
            host=remote.shortname,
            distro=builder.os_type,
            ver=builder.os_version,
            arch=builder.arch,)
    )

    base_url = builder.base_url
    log.info('Repo base URL: %s', base_url)
    project = builder.project

    # Remove the repository before re-adding it
    builder.remove_repo()
    builder.install_repo()

    if builder.dist_release not in ['opensuse', 'sle']:
        uri = builder.uri_reference
        _yum_fix_repo_priority(remote, project, uri)
        _yum_fix_repo_host(remote, project)
        _yum_set_check_obsoletes(remote)

    if builder.dist_release in ['opensuse', 'sle']:
        pkg_mng_cmd = 'zypper'
        pkg_mng_opts = '-a'
    else:
        pkg_mng_cmd = 'yum'
        pkg_mng_opts = 'all'

    remote.run(
        args=[
            'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts,
        ])

    # Actually upgrade the project packages
    if builder.dist_release in ['opensuse', 'sle']:
        pkg_mng_opts = '-n'
        pkg_mng_subcommand_opts = '--capability'
        pkg_mng_install_opts = '--no-recommends'
    else:
        pkg_mng_opts = '-y'
        pkg_mng_subcommand_opts = ''
        pkg_mng_install_opts = ''
    args = ['sudo', pkg_mng_cmd, pkg_mng_opts,
            'install', pkg_mng_subcommand_opts,
            pkg_mng_install_opts]
    args += pkgs
    remote.run(args=args)
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
@file
@brief Run some checkings about the distribution

.. versionadded:: 1.3
"""
from __future__ import print_function

import os
import sys

from .win_exception import WinInstallDistributionError

if sys.version_info[0] == 2:
    FileNotFoundError = Exception

def distribution_checkings(python_path, tools_path, fLOG=print):
    """
    checks a distribution was properly executed

    @param      python_path     path for python
    @param      tools_path      path for tools
    @param      fLOG            logging function

    The function raises @see cl WinInstallDistributionError if an issue is detected.

    If *python_path* is None, it is replace by ``os.path.dirname(sys.executable)``.

    .. versionadded:: 1.3
    """
    exceptions = []

    if python_path is None:
        python_path = os.path.dirname(sys.executable)
    scripts = os.path.join(python_path, "Scripts")

    files_to_check = ["rodeo.exe", "jupyter-console.exe", "jupyter-qtconsole.exe",
                      "jupyter-notebook.exe", "jupyter.exe", "spyder.exe", "pip.exe",
                      "autopep8.exe"]

    #############################################
    # check Jupyter, Rodeo, numpy works properly
    #############################################
    if sys.platform.startswith("win"):
        for file in files_to_check:
            f = os.path.join(scripts, file)
            if not os.path.exists(f):
                try:
                    raise FileNotFoundError(f)
                except Exception as e:
                    exceptions.append(e)

    ########
    # final
    ########
    if len(exceptions) > 0:
        typstr = str  # unicode#
        errors = "\n----\n".join("{0}\n{1}".format(type(_), typstr(_))
                                 for _ in exceptions)
        raise WinInstallDistributionError("\n" + errors)
<commit_msg>fix unit test in a virtual environment<commit_after>#-*- coding: utf-8 -*-
"""
@file
@brief Run some checkings about the distribution

.. versionadded:: 1.3
"""
from __future__ import print_function

import os
import sys

from .win_exception import WinInstallDistributionError

if sys.version_info[0] == 2:
    FileNotFoundError = Exception

def distribution_checkings(python_path, tools_path, fLOG=print):
    """
    checks a distribution was properly executed

    @param      python_path     path for python
    @param      tools_path      path for tools
    @param      fLOG            logging function

    The function raises @see cl WinInstallDistributionError if an issue is detected.

    If *python_path* is None, it is replace by ``os.path.dirname(sys.executable)``.

    .. versionadded:: 1.3
    """
    exceptions = []
    files_to_check = ["jupyter-console.exe", "jupyter-qtconsole.exe",
                      "jupyter-notebook.exe", "jupyter.exe"]

    if python_path is None:
        python_path = os.path.dirname(sys.executable)
    pip = os.path.join(python_path, "pip.exe")
    if not os.path.exists(pip):
        scripts = os.path.join(python_path, "Scripts")
        files_to_check2.extend(
            ["rodeo.exe", "spyder.exe", "pip.exe", "autopep8.exe"])
    else:
        scripts = python_path

    #############################################
    # check Jupyter, Rodeo, numpy works properly
    #############################################
    if sys.platform.startswith("win"):
        for file in files_to_check:
            f = os.path.join(scripts, file)
            if not os.path.exists(f):
                try:
                    raise FileNotFoundError(f)
                except Exception as e:
                    exceptions.append(e)

    ########
    # final
    ########
    if len(exceptions) > 0:
        typstr = str  # unicode#
        errors = "\n----\n".join("{0}\n{1}".format(type(_), typstr(_))
                                 for _ in exceptions)
        raise WinInstallDistributionError("\n" + errors)
<|endoftext|>


<commit_before> # -*- coding: utf-8 -*-
from __future__ import print_function

import sys
import re
import csv
from itertools import izip
from collections import OrderedDict, defaultdict

timestamp_regex = re.compile('''(?x)
    ^
        (?P<date>[0-9]{4}-[0-9]{2}-[0-9]{2})    # match date like 2012-07-31
        \s                                      # whitespace
        (?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2})    # match time like 22:00:00
        \s                                      # whitespace
        Timestamp                               # word 'Timestamp'
        r\r?\n                                 # line ending
''')

class BadgeData(object):
    """
    Represents the whole data file for a given badge. Contains <sensors>,
    a dictionary containing mapping from sensor type (int) to SensorData
    objects.

    """

    name_pattern = re.compile('''(?x)
        (?P<pin>[0-9]{1,3})
        b
        (?P<b>[0-9]{1,2})
        a
        (?P<a>[0-9]{1,2})
        \.
        (?P<num>[0-9]{1})
    ''')

    def __init__(self, data_file, header=''):
        self.header = header
        self.sensors = {}

        # extract data from the name
        self.name = data_file.split('/')[-1]
        groups = self.name_pattern.match(self.name).groupdict()
        self.pin = groups['pin']
        self.a = groups['a']
        self.b = groups['b']

    def add_reading(self, sensor_reading):
        """Add reading to the appropriate SensorData instance."""
        sensor_type = sensor_reading.sensor_type
        if sensor_type not in self.sensors.keys():
            self.sensors[sensor_type] = SensorData(sensor_type)

        self.sensors[sensor_type].add_reading(sensor_reading)

class SensorData(object):
    """Represents all sensor readings from data file."""

    def __init__(self, sensor_type):
        self.sensor_type = sensor_type

        self.readings = []
        self.values = defaultdict(lambda: None)

        self._times = set()
        # we want dates to be unique too, but also to preserve ordering, so we
        # manually keep track of what goes on there
        self.dates = []

    def add_reading(self, reading):
        """
        Add reading to the list. Also add the reading value to the dict for
        faster lookup.

        """
        self._times.add(reading.time)
        if reading.date not in self.dates:
            self.dates.append(reading.date)

        self.values['{0} {1}'.format(reading.date, reading.time)] = reading.value
        self.readings.append(reading)

    @property
    def times(self):
        return sorted(list(self._times))

    def get_value(self, date, time):
        return self.values['{0} {1}'.format(date, time)]

    def get_n_days(self, min_val=None):
        """
        Lazily compute and return the number of unique days in the data. If
        <min_val> is specified, only count days with sum of values at least
        <min_val>.

        """
        if min_val is None:
            if hasattr(self, 'n_days'):
                return self.n_days
            else:
                return len(set([reading.date for reading in self.readings]))
        else:
            return len(set([
                reading.date
                for reading in self.readings
                if self.get_day_sums()[reading.date] > min_val
            ]))

    def get_day_sums(self):
        if hasattr(self, 'day_sums'):
            return self.day_sums
        else:
            self.day_sums = defaultdict(lambda: 0.0)
            for reading in self.readings:
                self.day_sums[reading.date] += reading.value
            return self.day_sums

    def get_sum(self):
        day_sums = self.get_day_sums()
        sensor_sum = 0.0
        return sum(day_sums[day] for day in day_sums.keys())

class SensorReading(object):
    """Single reading from a sensor."""

    # sensor reading regex
    # float regex from http://stackoverflow.com/questions/385558/python-and-regex-question-extract-float-double-value
    pattern = re.compile('''(?x)
        ^
            (?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2})    # time in HH:MM:SS format
            \s                                      # whitespace
            (?P<val>
                [+-]?\ *            # first, match an optional sign *and space*
                (                   # then match integers or f.p. mantissas:
                    \d+             # start out with a ...
                    (
                        \.\d*       # mantissa of the form a.b or a.
                    )?              # ? takes care of integers of the form a
                    |\.\d+          # mantissa of the form .b
                )
                ([eE][+-]?\d+)?     # finally, optionally match an exponent
            )
            \s                      # whitespace
            (?P<sensor_type>[1-3]*) # second value
            r\r?\n                 # line ending
    ''')

    def __init__(self, date='', time='', val=0.0, sensor_type=0):
        self.date = date
        self.time = time
        self.value = float(val)
        self.sensor_type = int(sensor_type)

    def __str__(self):
        return str(self.value)

    def __repr__(self):
        return 'SensorReading(\'{0}\',\'{1}\', {2}, {3})'.format(
            self.date, self.time, self.sensor_type, self.value
        )

    @classmethod
    def from_line(cls, line, date):
        re_obj = cls.pattern.match(line)
        if re_obj is None:
            raise RuntimeError('The following line does not match a '
                               'SensorReading pattern: {0}'.format(line))
        else:
            return SensorReading(
                date=date,
                time=re_obj.group('time'),
                val=re_obj.group('val'),
                sensor_type=re_obj.group('sensor_type')
            )

    @classmethod
    def legit(cls, line):
        """Return True if the given line represents a valid SensorReading."""
        return cls.pattern.match(line) is not None

def is_timestamp(line):
    global timestamp_regex
    return timestamp_regex.match(line) is not None

def get_date_from_timestamp(timestamp):
    global timestamp_regex
    re_obj = timestamp_regex.match(timestamp)
    if re_obj:
        return re_obj.group('date')
    else:
        return None

if __name__ == '__main__':
    n_args = len(sys.argv)
    if not (n_args == 2 or n_args == 3):
        print('Need arguments: data_file [sensor_type]')
        sys.exit()

    data_file = sys.argv[1]
    out_file = data_file + '.csv'
    sensor_type = 1
    if len(sys.argv) is 3:
        sensor_type = int(sys.argv[2])

    badge_data = BadgeData(data_file)

    current_date = ''
    # read the data
    with open(data_file, 'r') as f:
        for line in f:
            if is_timestamp(line):
                # start recording a given date
                current_date = get_date_from_timestamp(line)
            elif SensorReading.legit(line):
                # sensor reading line, add it to the dict
                reading = SensorReading.from_line(line, current_date)
                badge_data.add_reading(reading)
            else:
                badge_data.header += line
    print('done reading data')

    sensor_data = badge_data.sensors[sensor_type]

    for date in sensor_data.dates:
        print(date, sensor_data.sensor_type, sensor_data.get_day_sums()[date])

    # write the CSV file
    print('writing to ',  out_file)
    with open(out_file, 'w') as f:
        # write row of times
        # with one empty cell first
        f.write(' ,')
        for time in sensor_data.times:
            f.write(time + ',')

        # start new line
        f.write('\n')

        # for each date write values at all times
        for date in sensor_data.dates:
            f.write(date + ',')
            for time in sensor_data.times:
                value = sensor_data.get_value(date, time)
                if value is None:
                    value = '-'
                f.write(str(value) + ',')
            f.write('\n')

    days = sensor_data.get_n_days()
    valid_days = sensor_data.get_n_days(50.0)

    # write the report file
    report_path = data_file + '_report.csv'
    with open(report_path, 'w') as f:
        f.write('pin,a,b,days,valid_days,sum\n')
        f.write('{0},{1},{2},{3},{4},{5}\n'.format(
            badge_data.pin,
            badge_data.a,
            badge_data.b,
            days,
            valid_days,
            sensor_data.get_sum()
        ))

    print(open(report_path, 'r').read())
<commit_msg>added threshold to get_sum<commit_after> # -*- coding: utf-8 -*-
from __future__ import print_function

import sys
import re
import csv
from itertools import izip
from collections import OrderedDict, defaultdict

THRESHOLD = 50.0

timestamp_regex = re.compile('''(?x)
    ^
        (?P<date>[0-9]{4}-[0-9]{2}-[0-9]{2})    # match date like 2012-07-31
        \s                                      # whitespace
        (?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2})    # match time like 22:00:00
        \s                                      # whitespace
        Timestamp                               # word 'Timestamp'
        r\r?\n                                 # line ending
''')

class BadgeData(object):
    """
    Represents the whole data file for a given badge. Contains <sensors>,
    a dictionary containing mapping from sensor type (int) to SensorData
    objects.

    """

    name_pattern = re.compile('''(?x)
        (?P<pin>[0-9]{1,3})
        b
        (?P<b>[0-9]{1,2})
        a
        (?P<a>[0-9]{1,2})
        \.
        (?P<num>[0-9]{1})
    ''')

    def __init__(self, data_file, header=''):
        self.header = header
        self.sensors = {}

        # extract data from the name
        self.name = data_file.split('/')[-1]
        groups = self.name_pattern.match(self.name).groupdict()
        self.pin = groups['pin']
        self.a = groups['a']
        self.b = groups['b']

    def add_reading(self, sensor_reading):
        """Add reading to the appropriate SensorData instance."""
        sensor_type = sensor_reading.sensor_type
        if sensor_type not in self.sensors.keys():
            self.sensors[sensor_type] = SensorData(sensor_type)

        self.sensors[sensor_type].add_reading(sensor_reading)

class SensorData(object):
    """Represents all sensor readings from data file."""

    def __init__(self, sensor_type):
        self.sensor_type = sensor_type

        self.readings = []
        self.values = defaultdict(lambda: None)

        self._times = set()
        # we want dates to be unique too, but also to preserve ordering, so we
        # manually keep track of what goes on there
        self.dates = []

    def add_reading(self, reading):
        """
        Add reading to the list. Also add the reading value to the dict for
        faster lookup.

        """
        self._times.add(reading.time)
        if reading.date not in self.dates:
            self.dates.append(reading.date)

        self.values['{0} {1}'.format(reading.date, reading.time)] = reading.value
        self.readings.append(reading)

    @property
    def times(self):
        return sorted(list(self._times))

    def get_value(self, date, time):
        return self.values['{0} {1}'.format(date, time)]

    def get_n_days(self, min_val=None):
        """
        Lazily compute and return the number of unique days in the data. If
        <min_val> is specified, only count days with sum of values at least
        <min_val>.

        """
        if min_val is None:
            if hasattr(self, 'n_days'):
                return self.n_days
            else:
                return len(set([reading.date for reading in self.readings]))
        else:
            return len(set([
                reading.date
                for reading in self.readings
                if self.get_day_sums()[reading.date] > min_val
            ]))

    def get_day_sums(self):
        if hasattr(self, 'day_sums'):
            return self.day_sums
        else:
            self.day_sums = defaultdict(lambda: 0.0)
            for reading in self.readings:
                self.day_sums[reading.date] += reading.value
            return self.day_sums

    def get_sum(self, min_val=None):
        day_sums = self.get_day_sums()
        sensor_sum = 0.0
        if min_val is None:
            return sum(day_sums[day] for day in day_sums.keys())
        else:
            return sum(
                day_sums[day]
                for day in day_sums.keys()
                if day_sums[day] > min_val
            )

class SensorReading(object):
    """Single reading from a sensor."""

    # sensor reading regex
    # float regex from http://stackoverflow.com/questions/385558/python-and-regex-question-extract-float-double-value
    pattern = re.compile('''(?x)
        ^
            (?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2})    # time in HH:MM:SS format
            \s                                      # whitespace
            (?P<val>
                [+-]?\ *            # first, match an optional sign *and space*
                (                   # then match integers or f.p. mantissas:
                    \d+             # start out with a ...
                    (
                        \.\d*       # mantissa of the form a.b or a.
                    )?              # ? takes care of integers of the form a
                    |\.\d+          # mantissa of the form .b
                )
                ([eE][+-]?\d+)?     # finally, optionally match an exponent
            )
            \s                      # whitespace
            (?P<sensor_type>[1-3]*) # second value
            r\r?\n                 # line ending
    ''')

    def __init__(self, date='', time='', val=0.0, sensor_type=0):
        self.date = date
        self.time = time
        self.value = float(val)
        self.sensor_type = int(sensor_type)

    def __str__(self):
        return str(self.value)

    def __repr__(self):
        return 'SensorReading(\'{0}\',\'{1}\', {2}, {3})'.format(
            self.date, self.time, self.sensor_type, self.value
        )

    @classmethod
    def from_line(cls, line, date):
        re_obj = cls.pattern.match(line)
        if re_obj is None:
            raise RuntimeError('The following line does not match a '
                               'SensorReading pattern: {0}'.format(line))
        else:
            return SensorReading(
                date=date,
                time=re_obj.group('time'),
                val=re_obj.group('val'),
                sensor_type=re_obj.group('sensor_type')
            )

    @classmethod
    def legit(cls, line):
        """Return True if the given line represents a valid SensorReading."""
        return cls.pattern.match(line) is not None

def is_timestamp(line):
    global timestamp_regex
    return timestamp_regex.match(line) is not None

def get_date_from_timestamp(timestamp):
    global timestamp_regex
    re_obj = timestamp_regex.match(timestamp)
    if re_obj:
        return re_obj.group('date')
    else:
        return None

if __name__ == '__main__':
    n_args = len(sys.argv)
    if not (n_args == 2 or n_args == 3):
        print('Need arguments: data_file [sensor_type]')
        sys.exit()

    data_file = sys.argv[1]
    out_file = data_file + '.csv'
    sensor_type = 1
    if len(sys.argv) is 3:
        sensor_type = int(sys.argv[2])

    badge_data = BadgeData(data_file)

    current_date = ''
    # read the data
    with open(data_file, 'r') as f:
        for line in f:
            if is_timestamp(line):
                # start recording a given date
                current_date = get_date_from_timestamp(line)
            elif SensorReading.legit(line):
                # sensor reading line, add it to the dict
                reading = SensorReading.from_line(line, current_date)
                badge_data.add_reading(reading)
            else:
                badge_data.header += line
    print('done reading data')

    sensor_data = badge_data.sensors[sensor_type]

    for date in sensor_data.dates:
        print(date, sensor_data.sensor_type, sensor_data.get_day_sums()[date])

    # write the CSV file
    print('writing to ',  out_file)
    with open(out_file, 'w') as f:
        # write row of times
        # with one empty cell first
        f.write(' ,')
        for time in sensor_data.times:
            f.write(time + ',')

        # start new line
        f.write('\n')

        # for each date write values at all times
        for date in sensor_data.dates:
            f.write(date + ',')
            for time in sensor_data.times:
                value = sensor_data.get_value(date, time)
                if value is None:
                    value = '-'
                f.write(str(value) + ',')
            f.write('\n')

    days = sensor_data.get_n_days()
    valid_days = sensor_data.get_n_days(THRESHOLD)

    # write the report file
    report_path = data_file + '_report.csv'
    with open(report_path, 'w') as f:
        f.write('pin,a,b,days,valid_days,sum\n')
        f.write('{0},{1},{2},{3},{4},{5}\n'.format(
            badge_data.pin,
            badge_data.a,
            badge_data.b,
            days,
            valid_days,
            sensor_data.get_sum(THRESHOLD)
        ))

    print(open(report_path, 'r').read())
<|endoftext|>