<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2014 by Filip H.F. "FiXato" Slagter <fixato+weechat@gmail.com>
#
# Shutup: a quick WeeChat script to replace text from specified users with
#         random or preset text as a way to hide their actual text.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# 2014-01-31: FiXato, (freenode.#weechat)
#       0.1 : initial release
#
# requires: WeeChat version 0.3.6 or higher
#
# Thanks go out to nils_2 for providing his skeleton.py template.
#
# Development is currently hosted at
# https://github.com/FiXato/weechat_scripts

try:
  import weechat,re
  from random import choice

except Exception:
  print "This script must be run under WeeChat."
  print "Get WeeChat now at: http://www.weechat.org/"
  quit()

SCRIPT_NAME     = "shutup"
SCRIPT_AUTHOR   = 'Filip H.F. "FiXato" Slagter <FiXato+weechat@gmail.com>'
SCRIPT_VERSION  = "0.1"
SCRIPT_LICENSE  = "GPL"
SCRIPT_DESC     = "Replace text from specified IRC users with random or preset text as a way to hide their actual text. Unlike /filter it won't hide the line (and thus can't be toggled either), and has access to the entire hostmask for comparison. Can be useful to mute users while still seeing that they are active."

OPTIONS         = {
                    'replacement_text'        : ('','Replacement text for everything the muted user says. Leave empty to use random lines from the Jabberwocky poem.'),
                    'muted_masks'             : ('','Space-separated regular expressions that will be matched against the nick!ident@host.mask. Any user matching will get their message muted. Can also include a comma-separated list of channels for every regular expression separated from the regexp by a colon. Prefix regexp with (?i) if you want it to be case insensitive. Example: "@\S+\.aol\.com$:#comcast,#AT&T (?i)!root@\S+" would mute messages in channels #comcast and #AT&T from users whose hosts end in *.aol.com, as well as all users who have any case variation of root as ident regardless of channel.'),
                  }

jabberwocky = """
'Twas brillig, and the slithy toves
Did gyre and gimble in the wabe;
All mimsy were the borogoves,
And the mome raths outgrabe.

"Beware the Jabberwock, my son!
The jaws that bite, the claws that catch!
Beware the Jubjub bird, and shun
The frumious Bandersnatch!"

He took his vorpal sword in hand:
Long time the manxome foe he sought—
So rested he by the Tumtum tree,
And stood awhile in thought.

And as in uffish thought he stood,
The Jabberwock, with eyes of flame,
Came whiffling through the tulgey wood,
And burbled as it came!

One, two! One, two! and through and through
The vorpal blade went snicker-snack!
He left it dead, and with its head
He went galumphing back.

"And hast thou slain the Jabberwock?
Come to my arms, my beamish boy!
O frabjous day! Callooh! Callay!"
He chortled in his joy.

'Twas brillig, and the slithy toves
Did gyre and gimble in the wabe;
All mimsy were the borogoves,
And the mome raths outgrabe.
"""
replacement_lines = filter(None, jabberwocky.splitlines())

def random_replacement_line(lines = replacement_lines):
  return choice(lines)

def replacement_line():
  global OPTIONS
  if OPTIONS['replacement_text'] == '':
    return random_replacement_line()
  return OPTIONS['replacement_text']

# Easily use weechat colors in the script
#   text = substitute_colors('my text ${color:yellow}yellow${color:default} colored.')
# eval_expression():  to match ${color:nn} tags
regex_color=re.compile('\$\{color:([^\{\}]+)\}')
def substitute_colors(text):
  if int(version) >= 0x00040200:
    return weechat.string_eval_expression(text,{},{},{})
  # substitute colors in output
  return re.sub(regex_color, lambda match: weechat.color(match.group(1)), text)

# ===================[ weechat options & description ]===================
def init_options():
  for option,value in OPTIONS.items():
    if not weechat.config_is_set_plugin(option):
      weechat.config_set_plugin(option, value[0])
      toggle_refresh(None, 'plugins.var.python.' + SCRIPT_NAME + '.' + option, value[0])
    else:
      toggle_refresh(None, 'plugins.var.python.' + SCRIPT_NAME + '.' + option, weechat.config_get_plugin(option))
    weechat.config_set_desc_plugin(option, '%s (default: "%s")' % (value[1], value[0]))

def update_muted_masks(masks):
  global muted_masks
  muted_masks = {}
  for mask in masks.split():
    if '#' in mask:
      mask, chan = mask.split(':',1)
      channels = [channel.lower() for channel in chan.split(',')]
    else:
      channels = []
    muted_masks[mask] = [re.compile(mask), channels]
  weechat.prnt("", 'muted masks: %s' % muted_masks)

def toggle_refresh(pointer, name, value):
  global OPTIONS
  option = name[len('plugins.var.python.' + SCRIPT_NAME + '.'):]        # get optionname
  OPTIONS[option] = value                                               # save new value
  if option == 'muted_masks':
    update_muted_masks(value)
  return weechat.WEECHAT_RC_OK

def shutup_cb(data, modifier, modifier_data, string):
  dict_in = { "message": string }
  message_ht = weechat.info_get_hashtable("irc_message_parse", dict_in)
  # weechat.prnt("", "message parsed: %s" % message_ht)

  hostmask = message_ht['host']
  arguments = message_ht['arguments']
  channel = message_ht['channel']

  new_arguments = re.sub(r'^%s :.+' % channel, lambda x: '%s :%s' % (channel, replacement_line()), arguments)
  new_string = re.sub(r'%s$' % re.escape(arguments), lambda x: new_arguments, string)

  # weechat.prnt("", "string: %s" % string)
  #   weechat.prnt("", "hostmask: %s" % hostmask)
  #   weechat.prnt("", "arguments: %s" % arguments)
  #   weechat.prnt("", "new arguments: %s" % new_arguments)
  #   weechat.prnt("", "new string: %s" % new_string)

  for key, [mask_regexp, channels] in muted_masks.iteritems():
    # If there is one or more channels listed for this mask regexp, and none of them match the current channel, continue to the next mute mask
    if len(channels) > 0 and channel.lower() not in channels:
      # weechat.prnt("", "%s doesn't match any of the listed channels: %s" % (channel, channels))
      continue

    # If the hostmask matches the mask regular expression, return the new, manipulated, string.
    weechat.prnt("", "comparing %s to %s" % (mask_regexp.pattern, hostmask))
    if mask_regexp.search(hostmask):
      weechat.prnt("", "  %s matches %s" % (mask_regexp.pattern, hostmask))
      return new_string
  # Nothing matches, so return the original, unmodified, string
  return string

# ================================[ main ]===============================
if __name__ == "__main__":
  if weechat.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION, SCRIPT_LICENSE, SCRIPT_DESC, '', ''):
    version = weechat.info_get("version_number", "") or 0

    if int(version) >= 0x00030600:
      # init options from your script
      init_options()
      # create a hook for your options
      weechat.hook_config( 'plugins.var.python.' + SCRIPT_NAME + '.*', 'toggle_refresh', '' )
    else:
      weechat.prnt("","%s%s %s" % (weechat.prefix("error"),SCRIPT_NAME,": needs version 0.3.6 or higher"))
      weechat.command("","/wait 1ms /python unload %s" % SCRIPT_NAME)

    hook = weechat.hook_modifier("irc_in_privmsg", "shutup_cb", "")

<commit_msg>[shutup.py] Added url to nils_2's skeleton.py<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2014 by Filip H.F. "FiXato" Slagter <fixato+weechat@gmail.com>
#
# Shutup: a quick WeeChat script to replace text from specified users with
#         random or preset text as a way to hide their actual text.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# 2014-01-31: FiXato, (freenode.#weechat)
#       0.1 : initial release
#
# requires: WeeChat version 0.3.6 or higher
#
# Thanks go out to nils_2 for providing his skeleton.py template, available at https://github.com/weechatter/weechat-scripts/blob/master/python/skeleton.py
#
# Development is currently hosted at
# https://github.com/FiXato/weechat_scripts

try:
  import weechat,re
  from random import choice

except Exception:
  print "This script must be run under WeeChat."
  print "Get WeeChat now at: http://www.weechat.org/"
  quit()

SCRIPT_NAME     = "shutup"
SCRIPT_AUTHOR   = 'Filip H.F. "FiXato" Slagter <FiXato+weechat@gmail.com>'
SCRIPT_VERSION  = "0.1"
SCRIPT_LICENSE  = "GPL"
SCRIPT_DESC     = "Replace text from specified IRC users with random or preset text as a way to hide their actual text. Unlike /filter it won't hide the line (and thus can't be toggled either), and has access to the entire hostmask for comparison. Can be useful to mute users while still seeing that they are active."

OPTIONS         = {
                    'replacement_text'        : ('','Replacement text for everything the muted user says. Leave empty to use random lines from the Jabberwocky poem.'),
                    'muted_masks'             : ('','Space-separated regular expressions that will be matched against the nick!ident@host.mask. Any user matching will get their message muted. Can also include a comma-separated list of channels for every regular expression separated from the regexp by a colon. Prefix regexp with (?i) if you want it to be case insensitive. Example: "@\S+\.aol\.com$:#comcast,#AT&T (?i)!root@\S+" would mute messages in channels #comcast and #AT&T from users whose hosts end in *.aol.com, as well as all users who have any case variation of root as ident regardless of channel.'),
                  }

jabberwocky = """
'Twas brillig, and the slithy toves
Did gyre and gimble in the wabe;
All mimsy were the borogoves,
And the mome raths outgrabe.

"Beware the Jabberwock, my son!
The jaws that bite, the claws that catch!
Beware the Jubjub bird, and shun
The frumious Bandersnatch!"

He took his vorpal sword in hand:
Long time the manxome foe he sought—
So rested he by the Tumtum tree,
And stood awhile in thought.

And as in uffish thought he stood,
The Jabberwock, with eyes of flame,
Came whiffling through the tulgey wood,
And burbled as it came!

One, two! One, two! and through and through
The vorpal blade went snicker-snack!
He left it dead, and with its head
He went galumphing back.

"And hast thou slain the Jabberwock?
Come to my arms, my beamish boy!
O frabjous day! Callooh! Callay!"
He chortled in his joy.

'Twas brillig, and the slithy toves
Did gyre and gimble in the wabe;
All mimsy were the borogoves,
And the mome raths outgrabe.
"""
replacement_lines = filter(None, jabberwocky.splitlines())

def random_replacement_line(lines = replacement_lines):
  return choice(lines)

def replacement_line():
  global OPTIONS
  if OPTIONS['replacement_text'] == '':
    return random_replacement_line()
  return OPTIONS['replacement_text']

# Easily use weechat colors in the script
#   text = substitute_colors('my text ${color:yellow}yellow${color:default} colored.')
# eval_expression():  to match ${color:nn} tags
regex_color=re.compile('\$\{color:([^\{\}]+)\}')
def substitute_colors(text):
  if int(version) >= 0x00040200:
    return weechat.string_eval_expression(text,{},{},{})
  # substitute colors in output
  return re.sub(regex_color, lambda match: weechat.color(match.group(1)), text)

# ===================[ weechat options & description ]===================
def init_options():
  for option,value in OPTIONS.items():
    if not weechat.config_is_set_plugin(option):
      weechat.config_set_plugin(option, value[0])
      toggle_refresh(None, 'plugins.var.python.' + SCRIPT_NAME + '.' + option, value[0])
    else:
      toggle_refresh(None, 'plugins.var.python.' + SCRIPT_NAME + '.' + option, weechat.config_get_plugin(option))
    weechat.config_set_desc_plugin(option, '%s (default: "%s")' % (value[1], value[0]))

def update_muted_masks(masks):
  global muted_masks
  muted_masks = {}
  for mask in masks.split():
    if '#' in mask:
      mask, chan = mask.split(':',1)
      channels = [channel.lower() for channel in chan.split(',')]
    else:
      channels = []
    muted_masks[mask] = [re.compile(mask), channels]
  weechat.prnt("", 'muted masks: %s' % muted_masks)

def toggle_refresh(pointer, name, value):
  global OPTIONS
  option = name[len('plugins.var.python.' + SCRIPT_NAME + '.'):]        # get optionname
  OPTIONS[option] = value                                               # save new value
  if option == 'muted_masks':
    update_muted_masks(value)
  return weechat.WEECHAT_RC_OK

def shutup_cb(data, modifier, modifier_data, string):
  dict_in = { "message": string }
  message_ht = weechat.info_get_hashtable("irc_message_parse", dict_in)
  # weechat.prnt("", "message parsed: %s" % message_ht)

  hostmask = message_ht['host']
  arguments = message_ht['arguments']
  channel = message_ht['channel']

  new_arguments = re.sub(r'^%s :.+' % channel, lambda x: '%s :%s' % (channel, replacement_line()), arguments)
  new_string = re.sub(r'%s$' % re.escape(arguments), lambda x: new_arguments, string)

  # weechat.prnt("", "string: %s" % string)
  #   weechat.prnt("", "hostmask: %s" % hostmask)
  #   weechat.prnt("", "arguments: %s" % arguments)
  #   weechat.prnt("", "new arguments: %s" % new_arguments)
  #   weechat.prnt("", "new string: %s" % new_string)

  for key, [mask_regexp, channels] in muted_masks.iteritems():
    # If there is one or more channels listed for this mask regexp, and none of them match the current channel, continue to the next mute mask
    if len(channels) > 0 and channel.lower() not in channels:
      # weechat.prnt("", "%s doesn't match any of the listed channels: %s" % (channel, channels))
      continue

    # If the hostmask matches the mask regular expression, return the new, manipulated, string.
    weechat.prnt("", "comparing %s to %s" % (mask_regexp.pattern, hostmask))
    if mask_regexp.search(hostmask):
      weechat.prnt("", "  %s matches %s" % (mask_regexp.pattern, hostmask))
      return new_string
  # Nothing matches, so return the original, unmodified, string
  return string

# ================================[ main ]===============================
if __name__ == "__main__":
  if weechat.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION, SCRIPT_LICENSE, SCRIPT_DESC, '', ''):
    version = weechat.info_get("version_number", "") or 0

    if int(version) >= 0x00030600:
      # init options from your script
      init_options()
      # create a hook for your options
      weechat.hook_config( 'plugins.var.python.' + SCRIPT_NAME + '.*', 'toggle_refresh', '' )
    else:
      weechat.prnt("","%s%s %s" % (weechat.prefix("error"),SCRIPT_NAME,": needs version 0.3.6 or higher"))
      weechat.command("","/wait 1ms /python unload %s" % SCRIPT_NAME)

    hook = weechat.hook_modifier("irc_in_privmsg", "shutup_cb", "")

<|endoftext|>


<commit_before>import asyncio
import logging
import math
import os
import re
import shutil
import subprocess
from functools import partial
from typing import Optional, Tuple

import aiohttp
import discord
import httpx
from discord.errors import DiscordException
from discord.ext import commands
from slugify import slugify
from youtube_dl import DownloadError, YoutubeDL

from ..tools import Cog, is_in_guild, timing

log = logging.getLogger(__name__)

FIREFOX_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0'

REDDIT_REGEX = re.compile(r'https?://(?:\w{2,32}\.)?reddit\.com/(?:r\/\w+\/)?(?:comments|gallery)\/[\w]+\/?\w*')
REDDIT_APP_REGEX = re.compile(r'https?:\/\/reddit\.app\.link\/[a-zA-Z0-9]{0,20}')
TIKTOK_SHORT_REGEX = re.compile(r'https?:\/\/(vm|www)\.tiktok\.com\/?t?\/[a-zA-Z0-9]{6,}/')
TIKTOK_REGEX = re.compile(r'https?:\/\/(?:\w{0,32}\.)?tiktok\.com\/@.+\b\/video\/[\d]+\b')
TWITTER_REGEX = re.compile(
    r'https?:\/\/twitter\.com\/([a-zA-Z0-9#-_!*\(\),]{0,20})\/status\/(\d{0,25})\??[a-zA-Z0-9#-_!*\(\),]*'
)

REGEXES = [
    re.compile(r'https?:\/\/(?:.{3,42}\.)?tiktokcdn-\w{1,8}.com\/\w+/\w+/video/tos/\w+/[^/]+/[^/]+/\?.+vr='),
    re.compile(r'https?://(?:v\.)?redd\.it/[a-zA-Z0-9#-_!*\(\),]{6,}'),
    re.compile(r'https?://video.twimg.com/ext_tw_video/\S*'),
    re.compile(r'https?://t\.co\/[a-zA-Z0-9#-_!*\(\),]{0,10}'),
    re.compile(r'https?://(?:www\.)instagram\.com/(?:p|reel)/[a-zA-Z0-9-_]{11}/'),
    re.compile(r'https?:\/\/clips.twitch.tv\/[a-zA-Z0-9-]{0,64}'),
]

MAX_VIDEO_LENGTH = 5 * 60  # 5 Minutes
AUDIO_BITRATE = 64 * 1000  # 64 Kbits
BUFFER_CONSTANT = 20  # Magic number, see https://unix.stackexchange.com/a/598360

FFPROBE_CMD = 'ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 in.mp4'
FFMPEG_CMD = 'ffmpeg -i in.mp4 -y -b:v {0} -maxrate:v {0} -b:a {1} -maxrate:a {1} -bufsize:v {2} {3}.mp4'

class NotAVideo(Exception):
    pass

class Video_DL(Cog):
    encode_lock = asyncio.Lock()
    mirror_upload_lock = asyncio.Lock()

    @staticmethod
    def download_video(url, id):
        ytdl = YoutubeDL({'outtmpl': f'{id}.mp4'})
        try:
            data = ytdl.extract_info(url, download=True)
        except DownloadError:
            raise NotAVideo(False)
        try:
            if data['format_id'] not in ['mp4', 'gif', 'm4a', 'mov']:
                raise NotAVideo(data['url'])
        except KeyError:
            pass
        return REGEXES[4].sub(
            '', f"{data['title']} - {data['description']}" if data.get('description') else data['title']
        )

    @staticmethod
    def download_audio(url, id):
        ydl_opts = {
            'outtmpl': f'{id}.%(ext)s',
            'format': 'bestaudio/best',
            'postprocessors': [
                {
                    'key': 'FFmpegExtractAudio',
                    'preferredcodec': 'm4a',
                    'preferredquality': '64',
                }
            ],
        }
        ytdl = YoutubeDL(ydl_opts)
        data = ytdl.extract_info(url, download=True)
        try:
            return data['title']
        except KeyError:
            return "audio"

    async def convert_twitter(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        matches = TWITTER_REGEX.match(message.content)
        if matches:
            async with aiohttp.ClientSession() as client:
                resp = await client.get(f'https://api.fxtwitter.com/{matches.group(1)}/status/{matches.group(2)}')
                data = await resp.json()
                if data.get('tweet') and data['tweet'].get('media') and data['tweet']['media'].get('video'):
                    # we have a video! let's download it
                    return (
                        data['tweet']['media']['video']['url'],
                        f"{data['tweet']['author']['name']} - {data['tweet']['text']}",
                    )
                elif data.get('tweet') and data['tweet'].get('quote'):
                    await self.on_message(
                        await message.channel.send(data['tweet']['quote']['url']),
                        override=True,
                        new_deleter=message.author.id,
                    )

    async def convert_reddit_app(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        matches = REDDIT_APP_REGEX.match(message.content)
        if matches:
            # get the forwarded url
            async with httpx.AsyncClient() as session:
                resp = await session.get(url=matches.group(0), headers={'User-Agent': FIREFOX_UA})
                message.content = str(resp.next_request.url)

    async def convert_reddit(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        matches = REDDIT_REGEX.match(message.content)
        if matches:
            async with httpx.AsyncClient() as session:
                resp = await session.get(url=matches.group(0) + '.json', headers={'User-Agent': 'AlexBot:v1.0.0'})
                data = resp.json()[0]['data']['children'][0]['data']
                # handle gallery
                if 'gallery_data' in data:
                    images = [item for item in data['gallery_data']['items']]
                    counter = 0
                    resp_text = ''
                    for image in images:
                        image_type = data['media_metadata'][image['media_id']]['m'].split('/')[1]
                        if counter == 5:
                            counter = 0
                            await message.reply(resp_text)
                            resp_text = ''
                        resp_text += f'https://i.redd.it/{image["media_id"]}.{image_type}'
                        if caption := image.get('caption'):
                            resp_text += f' ; {caption}'
                        if link := image.get('outbound_url'):
                            resp_text += f' ; <{link}>'

                        resp_text += '\n'
                        counter += 1
                    await message.reply(resp_text)
                # handle videos
                elif data['domain'] == 'v.redd.it':
                    return (
                        data['url_overridden_by_dest'],
                        f"{data['title']} \N{BULLET} {data['subreddit_name_prefixed']}",
                    )
                # everything else
                else:
                    await message.reply(data['url_overridden_by_dest'])
        return None

    async def convert_tiktok(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        text = message.content
        # convert short links to full links
        matches = TIKTOK_SHORT_REGEX.match(text)
        if matches:
            log.debug("Converting short TikTok link to full link")
            async with httpx.AsyncClient() as session:
                resp = await session.get(url=matches.group(0), headers={'User-Agent': FIREFOX_UA})
                text = str(resp.next_request.url)
        # match full tiktok link and get video download link
        matches = TIKTOK_REGEX.match(text)
        if matches:
            log.debug("Converting TikTok link to video link")
            video_params = '/'.join(matches.group(0).split('/')[-3::2])
            async with httpx.AsyncClient() as session:
                # this User-Agent is required because the API will not respond to other headers
                resp = await session.get(
                    url='https://www.tiktok.com/node/share/video/' + video_params,
                    headers={
                        'User-Agent': FIREFOX_UA,
                    },
                )
                data = resp.json()
                video_url = data["itemInfo"]["itemStruct"]["video"]["downloadAddr"]
                title = data['seoProps']['metaParams']['title']
                return video_url, title
        return None

    @Cog.listener()
    async def on_message(self, message: discord.Message, override=False, new_deleter=None):
        loop = asyncio.get_running_loop()
        if message.guild is None or (message.author == self.bot.user and not override):
            return
        if not (await self.bot.db.get_guild_data(message.guild.id)).config.tikTok:
            return

        await self.convert_reddit_app(message) # convert reddit app links to full links

        pack = (
            await self.convert_tiktok(message)
            or await self.convert_reddit(message)
            or await self.convert_twitter(message)
        )
        override_title = None
        ttt = None
        if pack:
            ttt = pack[0]
            override_title = pack[1]

        content = ttt or message.content
        matches = None

        for regex in REGEXES:
            matches = regex.match(content)
            if matches:
                break

        if matches is None and not ttt:
            return

        if ttt:
            match = ttt
        else:
            match = matches.group(0)
        log.info(f'collecting {match} for {message.author}')
        uploaded = None
        async with message.channel.typing():
            try:

                if match:
                    await message.channel.typing()
                    try:
                        await message.add_reaction('📥')
                    except discord.Forbidden:
                        pass

                    task = partial(self.download_video, match, message.id)
                    try:
                        title = await self.bot.loop.run_in_executor(None, task)
                        title = override_title if override_title else title
                    except NotAVideo as e:
                        if os.path.exists(f'{message.id}.mp4'):
                            os.remove(f'{message.id}.mp4')
                        if e.args[0]:
                            await message.reply(e, mention_author=False)
                            try:
                                await message.add_reaction('✅')
                            except DiscordException:
                                pass
                        return
                    loop.create_task(message.remove_reaction('📥', self.bot.user))

                    if os.path.getsize(f'{message.id}.mp4') > message.guild.filesize_limit:
                        loop.create_task(message.add_reaction('🪄'))  # magic wand

                        async with self.encode_lock:
                            task = partial(self.transcode_shrink, message.id, message.guild.filesize_limit * 0.95)
                            await self.bot.loop.run_in_executor(None, task)

                    # file is MESSAGE.ID.mp4, need to create discord.File and upload it to channel then delete out.mp4
                    file = discord.File(f'{message.id}.mp4', 'vid.mp4')
                    loop.create_task(message.add_reaction('📤'))
                    uploaded = await message.reply(title, file=file, mention_author=False)
                    loop.create_task(message.remove_reaction('📤', self.bot.user))
                    try:
                        await message.add_reaction('✅')
                    except DiscordException:
                        pass

            except Exception as e:
                log.warn(f'Exception occurred processing video {e} -- {os.path.getsize(f"{message.id}.mp4")}')

                try:
                    await message.add_reaction('❌')
                except discord.Forbidden:
                    await message.channel.send('Something broke')

            finally:
                await message.remove_reaction('📥', self.bot.user)

                if os.path.exists(f'{message.id}.mp4'):
                    os.remove(f'{message.id}.mp4')

        if uploaded:
            try:
                await uploaded.add_reaction("🗑")
            except DiscordException:
                return
            try:
                await message.edit(suppress=True)
            except DiscordException:
                pass

            def check(reaction: discord.Reaction, user: discord.User):
                return (
                    reaction.emoji == "🗑"
                    and user.id in [message.author.id, new_deleter]
                    and reaction.message.id == uploaded.id
                )

            try:
                await self.bot.wait_for('reaction_add', timeout=60 * 5, check=check)
            except asyncio.TimeoutError:
                await uploaded.remove_reaction("🗑", self.bot.user)
            else:
                # if we are here, someone with the power to do so want's to delete the upload
                await uploaded.delete()

    @staticmethod
    @timing(log=log)
    def transcode_shrink(id, limit: float):
        shutil.copyfile(f'{id}.mp4', 'in.mp4')
        os.remove(f'{id}.mp4')
        limit = limit * 8
        try:
            video_length = math.ceil(float(subprocess.check_output(FFPROBE_CMD.split(' ')).decode("utf-8")))

            if video_length > MAX_VIDEO_LENGTH:
                raise commands.CommandInvokeError('Video is too large.')

            target_total_bitrate = limit / video_length
            buffer_size = math.floor(limit / BUFFER_CONSTANT)
            target_video_bitrate = target_total_bitrate - AUDIO_BITRATE

            command_formatted = FFMPEG_CMD.format(
                str(target_video_bitrate), str(AUDIO_BITRATE), str(buffer_size), str(id)
            )

            subprocess.check_call(command_formatted.split(' '))

        except Exception as e:
            log.warn(f'Exception occurred transcoding video {e}')

        finally:
            if os.path.exists('in.mp4'):
                os.remove('in.mp4')

    @commands.command()
    @is_in_guild(791528974442299412)
    async def mirror(self, ctx: commands.Context, url: str):
        """Mirrors a youtube-dl compatible URL to a discord file upload.
        also connects to your voice channel, requests the bot play the song, and leaves."""
        async with self.mirror_upload_lock:
            async with ctx.typing():
                try:
                    task = partial(self.download_audio, url, ctx.message.id)
                    title = await self.bot.loop.run_in_executor(None, task)

                    try:
                        msg = await ctx.send(
                            file=discord.File(f"{ctx.message.id}.m4a", filename=f'{slugify(title)}.m4a'),
                            reference=ctx.message,
                        )
                    except discord.errors.HTTPException:
                        return await ctx.send("file too large :(", reference=ctx.message)
                    await ctx.send(
                        f"!play {msg.attachments[0].url}",
                    )
                finally:
                    if os.path.exists(f"{ctx.message.id}.m4a"):
                        os.remove(f"{ctx.message.id}.m4a")

async def setup(bot):
    await bot.add_cog(Video_DL(bot))
<commit_msg>auto: format python code<commit_after>import asyncio
import logging
import math
import os
import re
import shutil
import subprocess
from functools import partial
from typing import Optional, Tuple

import aiohttp
import discord
import httpx
from discord.errors import DiscordException
from discord.ext import commands
from slugify import slugify
from youtube_dl import DownloadError, YoutubeDL

from ..tools import Cog, is_in_guild, timing

log = logging.getLogger(__name__)

FIREFOX_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0'

REDDIT_REGEX = re.compile(r'https?://(?:\w{2,32}\.)?reddit\.com/(?:r\/\w+\/)?(?:comments|gallery)\/[\w]+\/?\w*')
REDDIT_APP_REGEX = re.compile(r'https?:\/\/reddit\.app\.link\/[a-zA-Z0-9]{0,20}')
TIKTOK_SHORT_REGEX = re.compile(r'https?:\/\/(vm|www)\.tiktok\.com\/?t?\/[a-zA-Z0-9]{6,}/')
TIKTOK_REGEX = re.compile(r'https?:\/\/(?:\w{0,32}\.)?tiktok\.com\/@.+\b\/video\/[\d]+\b')
TWITTER_REGEX = re.compile(
    r'https?:\/\/twitter\.com\/([a-zA-Z0-9#-_!*\(\),]{0,20})\/status\/(\d{0,25})\??[a-zA-Z0-9#-_!*\(\),]*'
)

REGEXES = [
    re.compile(r'https?:\/\/(?:.{3,42}\.)?tiktokcdn-\w{1,8}.com\/\w+/\w+/video/tos/\w+/[^/]+/[^/]+/\?.+vr='),
    re.compile(r'https?://(?:v\.)?redd\.it/[a-zA-Z0-9#-_!*\(\),]{6,}'),
    re.compile(r'https?://video.twimg.com/ext_tw_video/\S*'),
    re.compile(r'https?://t\.co\/[a-zA-Z0-9#-_!*\(\),]{0,10}'),
    re.compile(r'https?://(?:www\.)instagram\.com/(?:p|reel)/[a-zA-Z0-9-_]{11}/'),
    re.compile(r'https?:\/\/clips.twitch.tv\/[a-zA-Z0-9-]{0,64}'),
]

MAX_VIDEO_LENGTH = 5 * 60  # 5 Minutes
AUDIO_BITRATE = 64 * 1000  # 64 Kbits
BUFFER_CONSTANT = 20  # Magic number, see https://unix.stackexchange.com/a/598360

FFPROBE_CMD = 'ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 in.mp4'
FFMPEG_CMD = 'ffmpeg -i in.mp4 -y -b:v {0} -maxrate:v {0} -b:a {1} -maxrate:a {1} -bufsize:v {2} {3}.mp4'

class NotAVideo(Exception):
    pass

class Video_DL(Cog):
    encode_lock = asyncio.Lock()
    mirror_upload_lock = asyncio.Lock()

    @staticmethod
    def download_video(url, id):
        ytdl = YoutubeDL({'outtmpl': f'{id}.mp4'})
        try:
            data = ytdl.extract_info(url, download=True)
        except DownloadError:
            raise NotAVideo(False)
        try:
            if data['format_id'] not in ['mp4', 'gif', 'm4a', 'mov']:
                raise NotAVideo(data['url'])
        except KeyError:
            pass
        return REGEXES[4].sub(
            '', f"{data['title']} - {data['description']}" if data.get('description') else data['title']
        )

    @staticmethod
    def download_audio(url, id):
        ydl_opts = {
            'outtmpl': f'{id}.%(ext)s',
            'format': 'bestaudio/best',
            'postprocessors': [
                {
                    'key': 'FFmpegExtractAudio',
                    'preferredcodec': 'm4a',
                    'preferredquality': '64',
                }
            ],
        }
        ytdl = YoutubeDL(ydl_opts)
        data = ytdl.extract_info(url, download=True)
        try:
            return data['title']
        except KeyError:
            return "audio"

    async def convert_twitter(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        matches = TWITTER_REGEX.match(message.content)
        if matches:
            async with aiohttp.ClientSession() as client:
                resp = await client.get(f'https://api.fxtwitter.com/{matches.group(1)}/status/{matches.group(2)}')
                data = await resp.json()
                if data.get('tweet') and data['tweet'].get('media') and data['tweet']['media'].get('video'):
                    # we have a video! let's download it
                    return (
                        data['tweet']['media']['video']['url'],
                        f"{data['tweet']['author']['name']} - {data['tweet']['text']}",
                    )
                elif data.get('tweet') and data['tweet'].get('quote'):
                    await self.on_message(
                        await message.channel.send(data['tweet']['quote']['url']),
                        override=True,
                        new_deleter=message.author.id,
                    )

    async def convert_reddit_app(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        matches = REDDIT_APP_REGEX.match(message.content)
        if matches:
            # get the forwarded url
            async with httpx.AsyncClient() as session:
                resp = await session.get(url=matches.group(0), headers={'User-Agent': FIREFOX_UA})
                message.content = str(resp.next_request.url)

    async def convert_reddit(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        matches = REDDIT_REGEX.match(message.content)
        if matches:
            async with httpx.AsyncClient() as session:
                resp = await session.get(url=matches.group(0) + '.json', headers={'User-Agent': 'AlexBot:v1.0.0'})
                data = resp.json()[0]['data']['children'][0]['data']
                # handle gallery
                if 'gallery_data' in data:
                    images = [item for item in data['gallery_data']['items']]
                    counter = 0
                    resp_text = ''
                    for image in images:
                        image_type = data['media_metadata'][image['media_id']]['m'].split('/')[1]
                        if counter == 5:
                            counter = 0
                            await message.reply(resp_text)
                            resp_text = ''
                        resp_text += f'https://i.redd.it/{image["media_id"]}.{image_type}'
                        if caption := image.get('caption'):
                            resp_text += f' ; {caption}'
                        if link := image.get('outbound_url'):
                            resp_text += f' ; <{link}>'

                        resp_text += '\n'
                        counter += 1
                    await message.reply(resp_text)
                # handle videos
                elif data['domain'] == 'v.redd.it':
                    return (
                        data['url_overridden_by_dest'],
                        f"{data['title']} \N{BULLET} {data['subreddit_name_prefixed']}",
                    )
                # everything else
                else:
                    await message.reply(data['url_overridden_by_dest'])
        return None

    async def convert_tiktok(self, message: discord.Message) -> Optional[Tuple[str, str]]:
        text = message.content
        # convert short links to full links
        matches = TIKTOK_SHORT_REGEX.match(text)
        if matches:
            log.debug("Converting short TikTok link to full link")
            async with httpx.AsyncClient() as session:
                resp = await session.get(url=matches.group(0), headers={'User-Agent': FIREFOX_UA})
                text = str(resp.next_request.url)
        # match full tiktok link and get video download link
        matches = TIKTOK_REGEX.match(text)
        if matches:
            log.debug("Converting TikTok link to video link")
            video_params = '/'.join(matches.group(0).split('/')[-3::2])
            async with httpx.AsyncClient() as session:
                # this User-Agent is required because the API will not respond to other headers
                resp = await session.get(
                    url='https://www.tiktok.com/node/share/video/' + video_params,
                    headers={
                        'User-Agent': FIREFOX_UA,
                    },
                )
                data = resp.json()
                video_url = data["itemInfo"]["itemStruct"]["video"]["downloadAddr"]
                title = data['seoProps']['metaParams']['title']
                return video_url, title
        return None

    @Cog.listener()
    async def on_message(self, message: discord.Message, override=False, new_deleter=None):
        loop = asyncio.get_running_loop()
        if message.guild is None or (message.author == self.bot.user and not override):
            return
        if not (await self.bot.db.get_guild_data(message.guild.id)).config.tikTok:
            return

        await self.convert_reddit_app(message)  # convert reddit app links to full links

        pack = (
            await self.convert_tiktok(message)
            or await self.convert_reddit(message)
            or await self.convert_twitter(message)
        )
        override_title = None
        ttt = None
        if pack:
            ttt = pack[0]
            override_title = pack[1]

        content = ttt or message.content
        matches = None

        for regex in REGEXES:
            matches = regex.match(content)
            if matches:
                break

        if matches is None and not ttt:
            return

        if ttt:
            match = ttt
        else:
            match = matches.group(0)
        log.info(f'collecting {match} for {message.author}')
        uploaded = None
        async with message.channel.typing():
            try:

                if match:
                    await message.channel.typing()
                    try:
                        await message.add_reaction('📥')
                    except discord.Forbidden:
                        pass

                    task = partial(self.download_video, match, message.id)
                    try:
                        title = await self.bot.loop.run_in_executor(None, task)
                        title = override_title if override_title else title
                    except NotAVideo as e:
                        if os.path.exists(f'{message.id}.mp4'):
                            os.remove(f'{message.id}.mp4')
                        if e.args[0]:
                            await message.reply(e, mention_author=False)
                            try:
                                await message.add_reaction('✅')
                            except DiscordException:
                                pass
                        return
                    loop.create_task(message.remove_reaction('📥', self.bot.user))

                    if os.path.getsize(f'{message.id}.mp4') > message.guild.filesize_limit:
                        loop.create_task(message.add_reaction('🪄'))  # magic wand

                        async with self.encode_lock:
                            task = partial(self.transcode_shrink, message.id, message.guild.filesize_limit * 0.95)
                            await self.bot.loop.run_in_executor(None, task)

                    # file is MESSAGE.ID.mp4, need to create discord.File and upload it to channel then delete out.mp4
                    file = discord.File(f'{message.id}.mp4', 'vid.mp4')
                    loop.create_task(message.add_reaction('📤'))
                    uploaded = await message.reply(title, file=file, mention_author=False)
                    loop.create_task(message.remove_reaction('📤', self.bot.user))
                    try:
                        await message.add_reaction('✅')
                    except DiscordException:
                        pass

            except Exception as e:
                log.warn(f'Exception occurred processing video {e} -- {os.path.getsize(f"{message.id}.mp4")}')

                try:
                    await message.add_reaction('❌')
                except discord.Forbidden:
                    await message.channel.send('Something broke')

            finally:
                await message.remove_reaction('📥', self.bot.user)

                if os.path.exists(f'{message.id}.mp4'):
                    os.remove(f'{message.id}.mp4')

        if uploaded:
            try:
                await uploaded.add_reaction("🗑")
            except DiscordException:
                return
            try:
                await message.edit(suppress=True)
            except DiscordException:
                pass

            def check(reaction: discord.Reaction, user: discord.User):
                return (
                    reaction.emoji == "🗑"
                    and user.id in [message.author.id, new_deleter]
                    and reaction.message.id == uploaded.id
                )

            try:
                await self.bot.wait_for('reaction_add', timeout=60 * 5, check=check)
            except asyncio.TimeoutError:
                await uploaded.remove_reaction("🗑", self.bot.user)
            else:
                # if we are here, someone with the power to do so want's to delete the upload
                await uploaded.delete()

    @staticmethod
    @timing(log=log)
    def transcode_shrink(id, limit: float):
        shutil.copyfile(f'{id}.mp4', 'in.mp4')
        os.remove(f'{id}.mp4')
        limit = limit * 8
        try:
            video_length = math.ceil(float(subprocess.check_output(FFPROBE_CMD.split(' ')).decode("utf-8")))

            if video_length > MAX_VIDEO_LENGTH:
                raise commands.CommandInvokeError('Video is too large.')

            target_total_bitrate = limit / video_length
            buffer_size = math.floor(limit / BUFFER_CONSTANT)
            target_video_bitrate = target_total_bitrate - AUDIO_BITRATE

            command_formatted = FFMPEG_CMD.format(
                str(target_video_bitrate), str(AUDIO_BITRATE), str(buffer_size), str(id)
            )

            subprocess.check_call(command_formatted.split(' '))

        except Exception as e:
            log.warn(f'Exception occurred transcoding video {e}')

        finally:
            if os.path.exists('in.mp4'):
                os.remove('in.mp4')

    @commands.command()
    @is_in_guild(791528974442299412)
    async def mirror(self, ctx: commands.Context, url: str):
        """Mirrors a youtube-dl compatible URL to a discord file upload.
        also connects to your voice channel, requests the bot play the song, and leaves."""
        async with self.mirror_upload_lock:
            async with ctx.typing():
                try:
                    task = partial(self.download_audio, url, ctx.message.id)
                    title = await self.bot.loop.run_in_executor(None, task)

                    try:
                        msg = await ctx.send(
                            file=discord.File(f"{ctx.message.id}.m4a", filename=f'{slugify(title)}.m4a'),
                            reference=ctx.message,
                        )
                    except discord.errors.HTTPException:
                        return await ctx.send("file too large :(", reference=ctx.message)
                    await ctx.send(
                        f"!play {msg.attachments[0].url}",
                    )
                finally:
                    if os.path.exists(f"{ctx.message.id}.m4a"):
                        os.remove(f"{ctx.message.id}.m4a")

async def setup(bot):
    await bot.add_cog(Video_DL(bot))
<|endoftext|>


<commit_before><commit_msg>Retrying seeding local users on AutoReconnect errors<commit_after><|endoftext|>


<commit_before>from .base import *
import dj_database_url

DEBUG = False
ALLOWED_HOSTS = ['*']

DATABASES = {
    'default': dj_database_url.config()
}

# Enable Persistent Connections
DATABASES['default']['CONN_MAX_AGE'] = 500

# Enable Connection Pooling
DATABASES['default']['ENGINE'] = 'django_postgrespool'

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
<commit_msg>Fixing deploy<commit_after>from .base import *
import dj_database_url

DEBUG = False
ALLOWED_HOSTS = ['*']

DATABASES = {
    'default': dj_database_url.config()
}

# Enable Persistent Connections
DATABASES['default']['CONN_MAX_AGE'] = 500

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
import platform

import pep8

from flake8 import __version__
from flake8.reporter import multiprocessing, BaseQReport, QueueReport
from flake8.util import OrderedSet, is_windows, is_using_stdin

_flake8_noqa = re.compile(r'\s*# flake8[:=]\s*noqa', re.I).search

EXTRA_EXCLUDE = '.tox'

def _register_extensions():
    """Register all the extensions."""
    extensions = OrderedSet()
    extensions.add(('pep8', pep8.__version__))
    parser_hooks = []
    options_hooks = []
    try:
        from pkg_resources import iter_entry_points
    except ImportError:
        pass
    else:
        for entry in iter_entry_points('flake8.extension'):
            checker = entry.load()
            pep8.register_check(checker, codes=[entry.name])
            extensions.add((checker.name, checker.version))
            if hasattr(checker, 'add_options'):
                parser_hooks.append(checker.add_options)
            if hasattr(checker, 'parse_options'):
                options_hooks.append(checker.parse_options)
    return extensions, parser_hooks, options_hooks

def _install_hook_cb(option, option_str, value, parser):
    # For now, there's no way to affect a change in how pep8 processes
    # options. If no args are provided and there's no config file present,
    # it will error out because no input was provided. To get around this,
    # when we're using --install-hook, we'll say that there were arguments so
    # we can actually attempt to install the hook.
    # See: https://gitlab.com/pycqa/flake8/issues/2 and
    # https://github.com/jcrocholl/pep8/blob/4c5bf00cb613be617c7f48d3b2b82a1c7b895ac1/pep8.py#L1912
    # for more context.
    parser.values.install_hook = True
    parser.rargs.append('.')

def get_parser():
    """This returns an instance of optparse.OptionParser with all the
    extensions registered and options set. This wraps ``pep8.get_parser``.
    """
    (extensions, parser_hooks, options_hooks) = _register_extensions()
    details = ', '.join(['%s: %s' % ext for ext in extensions])
    python_version = get_python_version()
    parser = pep8.get_parser('flake8', '%s (%s) %s' % (
        __version__, details, python_version
    ))
    for opt in ('--repeat', '--testsuite', '--doctest'):
        try:
            parser.remove_option(opt)
        except ValueError:
            pass

    if multiprocessing:
        parser.config_options.append('jobs')
        parser.add_option('-j', '--jobs', type='string', default='auto',
                          help="number of jobs to run simultaneously, "
                          "or 'auto'. This is ignored on Windows.")

    parser.add_option('--exit-zero', action='store_true',
                      help="exit with code 0 even if there are errors")
    for parser_hook in parser_hooks:
        parser_hook(parser)
    # See comment above regarding why this has to be a callback.
    parser.add_option('--install-hook', default=False, dest='install_hook',
                      help='Install the appropriate hook for this '
                      'repository.', action='callback',
                      callback=_install_hook_cb)
    return parser, options_hooks

class StyleGuide(pep8.StyleGuide):

    def input_file(self, filename, lines=None, expected=None, line_offset=0):
        """Run all checks on a Python source file."""
        if self.options.verbose:
            print('checking %s' % filename)
        fchecker = self.checker_class(
            filename, lines=lines, options=self.options)
        # Any "flake8: noqa" comments to ignore the entire file?
        if any(_flake8_noqa(line) for line in fchecker.lines):
            return 0
        return fchecker.check_all(expected=expected, line_offset=line_offset)

def get_style_guide(**kwargs):
    """Parse the options and configure the checker. This returns a sub-class
    of ``pep8.StyleGuide``."""
    kwargs['parser'], options_hooks = get_parser()
    styleguide = StyleGuide(**kwargs)
    options = styleguide.options

    if options.exclude and not isinstance(options.exclude, list):
        options.exclude = pep8.normalize_paths(options.exclude)
    elif not options.exclude:
        options.exclude = []

    # Add pattersn in EXTRA_EXCLUDE to the list of excluded patterns
    options.exclude.extend(pep8.normalize_paths(EXTRA_EXCLUDE))

    for options_hook in options_hooks:
        options_hook(options)

    if options.diff:
        options.jobs = None

    force_disable_jobs = is_windows() or is_using_stdin(styleguide.paths)

    if multiprocessing and options.jobs and not force_disable_jobs:
        if options.jobs.isdigit():
            n_jobs = int(options.jobs)
        else:
            try:
                n_jobs = multiprocessing.cpu_count()
            except NotImplementedError:
                n_jobs = 1
        if n_jobs > 1:
            options.jobs = n_jobs
            reporter = BaseQReport if options.quiet else QueueReport
            report = styleguide.init_report(reporter)
            report.input_file = styleguide.input_file
            styleguide.runner = report.task_queue.put

    return styleguide

def get_python_version():
    # The implementation isn't all that important.
    try:
        impl = platform.python_implementation() + " "
    except AttributeError:  # Python 2.5
        impl = ''
    return '%s%s on %s' % (impl, platform.python_version(), platform.system())
<commit_msg>Add .eggs and *.egg to EXTRA_EXCLUDE<commit_after># -*- coding: utf-8 -*-
import re
import platform

import pep8

from flake8 import __version__
from flake8.reporter import multiprocessing, BaseQReport, QueueReport
from flake8.util import OrderedSet, is_windows, is_using_stdin

_flake8_noqa = re.compile(r'\s*# flake8[:=]\s*noqa', re.I).search

EXTRA_EXCLUDE = ['.tox', '.eggs', '*.egg']

def _register_extensions():
    """Register all the extensions."""
    extensions = OrderedSet()
    extensions.add(('pep8', pep8.__version__))
    parser_hooks = []
    options_hooks = []
    try:
        from pkg_resources import iter_entry_points
    except ImportError:
        pass
    else:
        for entry in iter_entry_points('flake8.extension'):
            checker = entry.load()
            pep8.register_check(checker, codes=[entry.name])
            extensions.add((checker.name, checker.version))
            if hasattr(checker, 'add_options'):
                parser_hooks.append(checker.add_options)
            if hasattr(checker, 'parse_options'):
                options_hooks.append(checker.parse_options)
    return extensions, parser_hooks, options_hooks

def _install_hook_cb(option, option_str, value, parser):
    # For now, there's no way to affect a change in how pep8 processes
    # options. If no args are provided and there's no config file present,
    # it will error out because no input was provided. To get around this,
    # when we're using --install-hook, we'll say that there were arguments so
    # we can actually attempt to install the hook.
    # See: https://gitlab.com/pycqa/flake8/issues/2 and
    # https://github.com/jcrocholl/pep8/blob/4c5bf00cb613be617c7f48d3b2b82a1c7b895ac1/pep8.py#L1912
    # for more context.
    parser.values.install_hook = True
    parser.rargs.append('.')

def get_parser():
    """This returns an instance of optparse.OptionParser with all the
    extensions registered and options set. This wraps ``pep8.get_parser``.
    """
    (extensions, parser_hooks, options_hooks) = _register_extensions()
    details = ', '.join(['%s: %s' % ext for ext in extensions])
    python_version = get_python_version()
    parser = pep8.get_parser('flake8', '%s (%s) %s' % (
        __version__, details, python_version
    ))
    for opt in ('--repeat', '--testsuite', '--doctest'):
        try:
            parser.remove_option(opt)
        except ValueError:
            pass

    if multiprocessing:
        parser.config_options.append('jobs')
        parser.add_option('-j', '--jobs', type='string', default='auto',
                          help="number of jobs to run simultaneously, "
                          "or 'auto'. This is ignored on Windows.")

    parser.add_option('--exit-zero', action='store_true',
                      help="exit with code 0 even if there are errors")
    for parser_hook in parser_hooks:
        parser_hook(parser)
    # See comment above regarding why this has to be a callback.
    parser.add_option('--install-hook', default=False, dest='install_hook',
                      help='Install the appropriate hook for this '
                      'repository.', action='callback',
                      callback=_install_hook_cb)
    return parser, options_hooks

class StyleGuide(pep8.StyleGuide):

    def input_file(self, filename, lines=None, expected=None, line_offset=0):
        """Run all checks on a Python source file."""
        if self.options.verbose:
            print('checking %s' % filename)
        fchecker = self.checker_class(
            filename, lines=lines, options=self.options)
        # Any "flake8: noqa" comments to ignore the entire file?
        if any(_flake8_noqa(line) for line in fchecker.lines):
            return 0
        return fchecker.check_all(expected=expected, line_offset=line_offset)

def get_style_guide(**kwargs):
    """Parse the options and configure the checker. This returns a sub-class
    of ``pep8.StyleGuide``."""
    kwargs['parser'], options_hooks = get_parser()
    styleguide = StyleGuide(**kwargs)
    options = styleguide.options

    if options.exclude and not isinstance(options.exclude, list):
        options.exclude = pep8.normalize_paths(options.exclude)
    elif not options.exclude:
        options.exclude = []

    # Add pattersn in EXTRA_EXCLUDE to the list of excluded patterns
    options.exclude.extend(pep8.normalize_paths(EXTRA_EXCLUDE))

    for options_hook in options_hooks:
        options_hook(options)

    if options.diff:
        options.jobs = None

    force_disable_jobs = is_windows() or is_using_stdin(styleguide.paths)

    if multiprocessing and options.jobs and not force_disable_jobs:
        if options.jobs.isdigit():
            n_jobs = int(options.jobs)
        else:
            try:
                n_jobs = multiprocessing.cpu_count()
            except NotImplementedError:
                n_jobs = 1
        if n_jobs > 1:
            options.jobs = n_jobs
            reporter = BaseQReport if options.quiet else QueueReport
            report = styleguide.init_report(reporter)
            report.input_file = styleguide.input_file
            styleguide.runner = report.task_queue.put

    return styleguide

def get_python_version():
    # The implementation isn't all that important.
    try:
        impl = platform.python_implementation() + " "
    except AttributeError:  # Python 2.5
        impl = ''
    return '%s%s on %s' % (impl, platform.python_version(), platform.system())
<|endoftext|>


<commit_before>#!/bin/env python

# Put your files on ice. Compress, encrypt, obfuscate and archive them on Amazon Glacier.
#
# Inspired by duply/duplicity and bakthat.

import aaargh
import boto.glacier
import boto.s3
from boto.s3.connection import Location
import ConfigParser
import getpass
import logging
import os
from Crypto.PublicKey import RSA
from Crypto import Random
import re
import sqlite3

log = logging.getLogger(__name__)

if not log.handlers:
    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s')

class Catalogue(object):
    """
    Encapsulates the catalogue - the database of files stored, their modification times and hashes.
    """
    def __init__(self, dbpath):
        self.conn = sqlite3.connect(dbpath)

class Config(object):
    """
    Manages the application configuration. Different config profiles correspond to subdirectories of the
    main config directory and allow different sets of configs to be used.
    """
    def __init__(self, config_profile, config_dir=os.path.expanduser("~/.iceit")):
        self.config_profile = config_profile
        self.config_dir = os.path.join(config_dir, self.config_profile)
        self.config = ConfigParser.SafeConfigParser()
        self.config.read(self.get_config_file_path())

    def is_valid(self):
        "Return a boolean indicating whether the current config profile exists and is valid"
        return os.path.exists(self.get_config_file_path())

    def get_config_file_path(self):
        "Return the path to the config file"
        return os.path.join(self.config_dir, 'iceit.conf')

    def get_public_key_path(self):
        "Return the path to the public key file"
        return os.path.join(self.config_dir, 'public.key')

    def get_private_key_path(self):
        "Return the path to the private key file"
        return os.path.join(self.config_dir, 'private.key')

    def get_catalogue_path(self):
        "Return the full path to the catalogue file"
        return os.path.join(self.config_dir, self.config.get('catalogue', 'name'))

    def get(self, *args, **kwargs):
        "Get a config value from the underlying config system"
        return self.config.get(*args, **kwargs)

    def write_config_file(self, settings):
        """
        Create default config.

        @param settings - dict containing AWS credentials
        """
        if not os.path.isdir(self.config_dir):
            os.makedirs(self.config_dir)

        if not self.config.has_section('aws'):
            self.config.add_section("aws")

        self.config.set("aws", "access_key", settings['aws']['access_key'])
        self.config.set("aws", "secret_key", settings['aws']['secret_key'])
        self.config.set("aws", "s3_location", settings['aws']['s3_location'])
        self.config.set("aws", "s3_bucket", settings['aws']['s3_bucket'])
        self.config.set("aws", "glacier_region", settings['aws']['glacier_region'])
        self.config.set("aws", "glacier_vault", settings['aws']['glacier_vault'])

        # default config values
        if not self.config.has_section('catalogue'):
            self.config.add_section('catalogue')
            # name of the sqlite3 database file (under the config directory) to use as the catalogue
            self.config.set('catalogue', 'name', 'catalogue.db')

        if not self.config.has_section('encryption'):
            self.config.add_section('encryption')
            # Whether to encrypt files or not
            self.config.set('encryption', 'encrypt_files', 'True')
            # path to public key
            self.config.set('encryption', 'public_key_path', self.get_public_key_path())
            # path to private key
            self.config.set('encryption', 'private_key_path', self.get_private_key_path())

        if not self.config.has_section('processing'):
            self.config.add_section('processing')
            # files whose extensions match the following regexes will not be compressed
            self.config.set('processing', 'disable_compression_extensions', 'avi,mp4,mpe?g,jpe?g,mp3,bz2,gz(ip)?')
            # Whether to create a single archive per directory (True), or to process files individually
            self.config.set('processing', 'create_one_archive_per_directory', 'true')
            # The minimum number of files to add to an archive when creating one per directory automatically
            self.config.set('processing', 'min_files_per_directory_archive', '2')
            # Whether to obfuscate file names before uploading them
            self.config.set('processing', 'obfuscate_file_names', 'true')
            # File patterns (as reg exes) to exclude from backing up. Separate multiple with commas.
            self.config.set('processing', 'exclude_patterns', '^.*/desktop\.ini$')

        self.config.write(open(self.get_config_file_path(), "w"))

        log.info("Config written to %s" % self.get_config_file_path())

class Encryptor(object):
    """
    All crypto-related methods.

    @todo - replace with gnupg
    """
    def generate_key_pair(self, length=2048, passphrase=None):
        """
        Generates an RSA key pair.
        """
        log.info("Generating RSA key pair")
        self.key = RSA.generate(length, Random.new().read)
        log.info("Key generated")

    def export_keys(self, public_key_path, private_key_path, passphrase):
        """
        Export the public and private keys, with the private key protected by the given passphrase.
        """
        log.info("Writing public key to %s" % public_key_path)
        with open(public_key_path, 'w') as pub_key_file:
            pub_key_file.write(self.key.publickey().exportKey())
        log.info("Public key written.")

        log.info("Writing private key to %s" % private_key_path)
        with open(private_key_path, 'w') as private_key_file:
            private_key_file.write(self.key.exportKey('PEM', passphrase))
        log.info("Private key written.")

class FileFinder(object):
    "Finds files using different methods"

    def __init__(self, path, recursive=False):
        """
        @param string start - Path to scan
        @param bool recursive - Whether to scan recursively or just return
            files in the input directory.
        """
        self.path = path
        self.recursive = recursive
        self.files = None

    def get_files(self):
        "Return a set of files for the given operating mode."
        if self.recursive:
            self.files = self.__get_files_recursive()
        else:
            self.files = self.__get_files_non_recursive()

        return self.files

    def __get_files_non_recursive(self):
        "Only return files from the input directory."
        for (path, dirs, files) in os.walk(self.path):
            return set([os.path.join(path, f) for f in files])

    def __get_files_recursive(self):
        "Return all matching files"
        output = set()

        for (path, dirs, files) in os.walk(self.path):
            full_files = [os.path.join(path, f) for f in files]
            if full_files:
                output.update(full_files)

        return output

class SetUtils(object):
    "Provides utility methods on sets"

    @staticmethod
    def match_patterns(search_set, pattern):
        "Return a new set containing elements from search_set that match the given regex pattern"
        log.info("Excluding files matching pattern '%s'" % pattern)
        matching_set = set()
        regex = re.compile(pattern)
        for item in search_set:
            if regex.match(item) != None:
                matching_set.add(item)

        return matching_set

class IceItException(Exception):
    "Base exception class"
    pass

class IceIt(object):
    def __init__(self, config_profile):
        self.config = Config(config_profile)
        self.encryptor = Encryptor()
        self.catalogue = None           # Need to open it when need it because if there's no config we'll be in trouble

    def __open_catalogue(self):
        "Open the catalogue"
        if not self.catalogue:
            self.catalogue = Catalogue(self.config.get_catalogue_path())

    def write_config_file(self, settings):
        return self.config.write_config_file(settings)

    def key_pair_exists(self):
        "Returns a boolean indicating whether a key pair already exists"
        return os.path.exists(self.config.get_public_key_path()) or os.path.exists(self.config.get_private_key_path())

    def generate_key_pair(self):
        "Generate a new key pair"
        return self.encryptor.generate_key_pair()

    def export_keys(self, passphrase):
        "Export the key pair using the given passphrase to secure the private key"
        return self.encryptor.export_keys(self.config.get_public_key_path(), self.config.get_private_key_path(), passphrase)

    def is_configured(self):
        "Return a boolean indicating whether the current config profile is valid and complete"
        return self.config.is_valid()

    def __trim_ineligible_files(self, potential_files):
        """
        Return the supplied set with all files that shouldn't be backed up removed.
        """
        # apply configured exclude patterns
        total_excluded = 0
        exclude_patterns = self.config.get('processing', 'exclude_patterns').split(',')
        for pattern in exclude_patterns:
            remove_set = SetUtils.match_patterns(potential_files, pattern)
            total_excluded += len(remove_set)
            potential_files -= remove_set

        log.info("%d files excluded by %d exclude patterns." % (total_excluded, len(exclude_patterns)))

        if len(potential_files) == 0:
            return

        self.__open_catalogue()

    def backup(self, paths, recursive):
        """
        Backup the given paths under the given config profile, optionally recursively.
        """
        potential_files = set()
        # find all files in the given paths and add to a set
        for path in  paths:
            log.info("Finding files in path %s (recursive=%s)" % (path, recursive))
            if os.path.isdir(path):
                file_finder = FileFinder(path, recursive)
                potential_files.update(file_finder.get_files())
            else:
                potential_files.update([path])
        log.info("%d files found in %d paths" % (len(potential_files), len(paths)))

        # remove ineligible files from the backup list, e.g. files that match exclusion patterns, files that have
        # been backed up previously and haven't since been modified, etc.
        eligible_files = self.__trim_ineligible_files(potential_files)
        # Perform all necessary processing prior to initiating an upload to the file store, e.g. combine files that
        # need archiving into archives, compress files that should be compressed, encrypt files
        # as necessary and obfuscate file names.
#            self.process_files(eligible_files)
        # upload to storage backend
        # if all went well, save new catalogue to highly available storage backend

# CLI application

app = aaargh.App(description="Compress, encrypt, obfuscate and archive files to Amazon S3/Glacier.")

@app.cmd(help="Set AWS S3/Glacier credentials.")
@app.cmd_arg('profile', type=str, help="Configuration profile name. Configuration "
                                        "profiles allow you to back up different parts of "
                                        "your system using different settings.")
def configure(profile):
    "Prompt for AWS credentials and write to config file"
    settings = {"aws": {}}

    settings['aws']['access_key'] = raw_input("AWS Access Key: ")
    settings['aws']["secret_key"] = raw_input("AWS Secret Key: ")

    s3_locations = [l for l in dir(Location) if not '_' in l]
    print "S3 settings: The list of your files along with any encryption keys will be stored in S3."
    settings['aws']["s3_location"] = raw_input("S3 location (possible values are %s): " % ', '.join(s3_locations))
    settings['aws']["s3_bucket"] = raw_input("S3 Bucket Name: ")

    glacier_regions = boto.glacier.regions()
    print "Your files will be backed up to Glacier."
    settings['aws']["glacier_region"] = raw_input("Glacier region (possible values are %s): " % ', '.join([r.name for r in glacier_regions]))
    settings['aws']["glacier_vault"] = raw_input("Glacier Vault Name: ")

    iceit = IceIt(profile)
    iceit.write_config_file(settings)

    print "Config file written. Please edit it to change further options."

    if not iceit.key_pair_exists():
        iceit.generate_key_pair()
        print "\nSecure your private key with a passphrase."
        print "IMPORTANT: If you forget this passphrase there will be no way to recover your files! "
        passphrase_1 = getpass.getpass("Enter a strong passphrase to secure your private key: ")
        passphrase_2 = getpass.getpass("Enter again to confirm: ")
        if passphrase_1 != passphrase_2:
            raise IceItException("Passphrases don't match.")
        iceit.export_keys(passphrase_1)

@app.cmd(help="Backup the given path(s) using the specified backup profile.")
@app.cmd_arg('profile', type=str, help="Configuration profile name. Configuration "
                                        "profiles allow you to back up different parts of "
                                        "your system using different settings.")
@app.cmd_arg('-r', '--recursive', action="store_true", help="Backup directories recursively")
@app.cmd_arg('paths', type=str, nargs="+", help="Directories/files to backup")
def backup(profile, recursive, paths):
    iceit = IceIt(profile)
    if not iceit.is_configured():
        raise IceItException("Configuration profile '%s' doesn't exist or is corrupt." % profile)

    iceit.backup(paths, recursive)

if __name__ == '__main__':
    app.run()<commit_msg>Working on processing files<commit_after>#!/bin/env python

# Put your files on ice. Compress, encrypt, obfuscate and archive them on Amazon Glacier.
#
# Inspired by duply/duplicity and bakthat.

import aaargh
import boto.glacier
import boto.s3
from boto.s3.connection import Location
import ConfigParser
import getpass
import logging
import os
from Crypto.PublicKey import RSA
from Crypto import Random
import random
import re
from sqlalchemy import create_engine, Table, Column, Integer, String, MetaData, DateTime, select
import string

log = logging.getLogger(__name__)

if not log.handlers:
    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s')

class Catalogue(object):
    """
    Encapsulates the catalogue - the database of files stored, their modification times and hashes.
    """
    def __init__(self, dbpath):
        self.tables = {}
        self.engine = create_engine('sqlite:///%s' % dbpath)
        self.__create_tables()

        self.conn = self.engine.connect()

    def __create_tables(self):
        "Create necessary tables"
        log.info("Creating DB tables...")
        metadata = MetaData()
        self.tables['files'] = Table('files', metadata,
            Column('id', Integer, primary_key=True),
            Column('path', String),
            Column('obfuscated_name', String),
            Column('file_mtime', DateTime),
            Column('hash', String),
            Column('last_backed_up', DateTime)
        )

        metadata.create_all(self.engine)
        log.info("DB tables created...")

    def get(self, file_path):
        "Get a file entry or return an empty list if not found"
        file_table = self.tables['files']
        query = select([file_table], file_table.c.path==file_path)
        result = self.conn.execute(query)

        rows = result.fetchall()

        print rows

class Config(object):
    """
    Manages the application configuration. Different config profiles correspond to subdirectories of the
    main config directory and allow different sets of configs to be used.
    """
    def __init__(self, config_profile, config_dir=os.path.expanduser("~/.iceit")):
        self.config_profile = config_profile
        self.config_dir = os.path.join(config_dir, self.config_profile)
        self.config = ConfigParser.SafeConfigParser()
        self.config.read(self.get_config_file_path())

    def is_valid(self):
        "Return a boolean indicating whether the current config profile exists and is valid"
        return os.path.exists(self.get_config_file_path())

    def get_config_file_path(self):
        "Return the path to the config file"
        return os.path.join(self.config_dir, 'iceit.conf')

    def get_public_key_path(self):
        "Return the path to the public key file"
        return os.path.join(self.config_dir, 'public.key')

    def get_private_key_path(self):
        "Return the path to the private key file"
        return os.path.join(self.config_dir, 'private.key')

    def get_catalogue_path(self):
        "Return the full path to the catalogue file"
        return os.path.join(self.config_dir, self.config.get('catalogue', 'name'))

    def get(self, *args, **kwargs):
        "Get a config value from the underlying config system"
        return self.config.get(*args, **kwargs)

    def write_config_file(self, settings):
        """
        Create default config.

        @param settings - dict containing AWS credentials
        """
        if not os.path.isdir(self.config_dir):
            os.makedirs(self.config_dir)

        if not self.config.has_section('aws'):
            self.config.add_section("aws")

        self.config.set("aws", "access_key", settings['aws']['access_key'])
        self.config.set("aws", "secret_key", settings['aws']['secret_key'])
        self.config.set("aws", "s3_location", settings['aws']['s3_location'])
        self.config.set("aws", "s3_bucket", settings['aws']['s3_bucket'])
        self.config.set("aws", "glacier_region", settings['aws']['glacier_region'])
        self.config.set("aws", "glacier_vault", settings['aws']['glacier_vault'])

        # default config values
        if not self.config.has_section('catalogue'):
            self.config.add_section('catalogue')
            # name of the sqlite3 database file (under the config directory) to use as the catalogue
            self.config.set('catalogue', 'name', 'catalogue.db')

        if not self.config.has_section('encryption'):
            self.config.add_section('encryption')
            # Whether to encrypt files or not
            self.config.set('encryption', 'encrypt_files', 'True')
            # path to public key
            self.config.set('encryption', 'public_key_path', self.get_public_key_path())
            # path to private key
            self.config.set('encryption', 'private_key_path', self.get_private_key_path())

        if not self.config.has_section('processing'):
            self.config.add_section('processing')
            # files whose extensions match the following regexes will not be compressed
            self.config.set('processing', 'disable_compression_extensions', 'avi,mp4,mpe?g,jpe?g,mp3,bz2,gz(ip)?')
            # Whether to create a single archive per directory (True), or to process files individually
            self.config.set('processing', 'create_one_archive_per_directory', 'true')
            # The minimum number of files to add to an archive when creating one per directory automatically
            self.config.set('processing', 'min_files_per_directory_archive', '2')
            # Whether to obfuscate file names before uploading them
            self.config.set('processing', 'obfuscate_file_names', 'true')
            # File patterns (as reg exes) to exclude from backing up. Separate multiple with commas.
            self.config.set('processing', 'exclude_patterns', '^.*/desktop\.ini$')

        self.config.write(open(self.get_config_file_path(), "w"))

        log.info("Config written to %s" % self.get_config_file_path())

class Encryptor(object):
    """
    All crypto-related methods.

    @todo - replace with gnupg
    """
    def generate_key_pair(self, length=2048, passphrase=None):
        """
        Generates an RSA key pair.
        """
        log.info("Generating RSA key pair")
        self.key = RSA.generate(length, Random.new().read)
        log.info("Key generated")

    def export_keys(self, public_key_path, private_key_path, passphrase):
        """
        Export the public and private keys, with the private key protected by the given passphrase.
        """
        log.info("Writing public key to %s" % public_key_path)
        with open(public_key_path, 'w') as pub_key_file:
            pub_key_file.write(self.key.publickey().exportKey())
        log.info("Public key written.")

        log.info("Writing private key to %s" % private_key_path)
        with open(private_key_path, 'w') as private_key_file:
            private_key_file.write(self.key.exportKey('PEM', passphrase))
        log.info("Private key written.")

class FileFinder(object):
    "Finds files using different methods"

    def __init__(self, path, recursive=False):
        """
        @param string start - Path to scan
        @param bool recursive - Whether to scan recursively or just return
            files in the input directory.
        """
        self.path = path
        self.recursive = recursive
        self.files = None

    def get_files(self):
        "Return a set of files for the given operating mode."
        if self.recursive:
            self.files = self.__get_files_recursive()
        else:
            self.files = self.__get_files_non_recursive()

        return self.files

    def __get_files_non_recursive(self):
        "Only return files from the input directory."
        for (path, dirs, files) in os.walk(self.path):
            return set([os.path.join(path, f) for f in files])

    def __get_files_recursive(self):
        "Return all matching files"
        output = set()

        for (path, dirs, files) in os.walk(self.path):
            full_files = [os.path.join(path, f) for f in files]
            if full_files:
                output.update(full_files)

        return output

class SetUtils(object):
    "Provides utility methods on sets"

    @staticmethod
    def match_patterns(search_set, pattern):
        "Return a new set containing elements from search_set that match the given regex pattern"
        log.info("Excluding files matching pattern '%s'" % pattern)
        matching_set = set()
        regex = re.compile(pattern)
        for item in search_set:
            if regex.match(item) != None:
                matching_set.add(item)

        return matching_set

class StringUtils(object):
    "Utilities on strings"

    @staticmethod
    def get_random_string(length=32):
        return ''.join(random.choice(
            string.ascii_letters + string.digits) for x in range(length))

class IceItException(Exception):
    "Base exception class"
    pass

class IceIt(object):
    def __init__(self, config_profile):
        self.config = Config(config_profile)
        self.encryptor = Encryptor()
        self.catalogue = None           # Need to open it when need it because if there's no config we'll be in trouble

    def __open_catalogue(self):
        "Open the catalogue"
        if not self.catalogue:
            self.catalogue = Catalogue(self.config.get_catalogue_path())

    def write_config_file(self, settings):
        return self.config.write_config_file(settings)

    def key_pair_exists(self):
        "Returns a boolean indicating whether a key pair already exists"
        return os.path.exists(self.config.get_public_key_path()) or os.path.exists(self.config.get_private_key_path())

    def generate_key_pair(self):
        "Generate a new key pair"
        return self.encryptor.generate_key_pair()

    def export_keys(self, passphrase):
        "Export the key pair using the given passphrase to secure the private key"
        return self.encryptor.export_keys(self.config.get_public_key_path(), self.config.get_private_key_path(), passphrase)

    def is_configured(self):
        "Return a boolean indicating whether the current config profile is valid and complete"
        return self.config.is_valid()

    def __trim_ineligible_files(self, potential_files):
        """
        Return the supplied set with all files that shouldn't be backed up removed.
        """
        # apply configured exclude patterns
        total_excluded = 0
        exclude_patterns = self.config.get('processing', 'exclude_patterns').split(',')
        for pattern in exclude_patterns:
            remove_set = SetUtils.match_patterns(potential_files, pattern)
            total_excluded += len(remove_set)
            potential_files -= remove_set

        log.info("%d files excluded by %d exclude patterns." % (total_excluded, len(exclude_patterns)))

        if len(potential_files) == 0:
            return

        self.__open_catalogue()

        for file in potential_files:
            existing_backups = self.catalogue.get(file)
            if len(existing_backups):
                # if the mtime hasn't changed, remove from potential_files

                # if it has, hash the file and remove from potential_files if the old and current hashes are the same

# @todo - complete this once there is data in the catalogue
                pass

    def __process_files(self, eligible_files):
        """
        Perform all necessary processing prior to initiating an upload to the file store, e.g. combine files that
        need archiving into archives, compress files that should be compressed, encrypt files as necessary and
        obfuscate file names.
        """
        # iterate over the file list building a dictionary of metadata about whether the file should be added to
        # an archive, compressed, encrypted, etc.
        for file in eligible_files:
# @todo - implement this and create a new data structure

        for file, metadata in new_data_structure_from_the_above_step:
# @todo - implement this
            # combine files into archives as necessary

            # compress files

            # encrypt files

            obfuscated_name = StringUtils.get_random_string()
            # update the catalogue, but don't commit until all files are processed (although uploading may
            # subsequently fail)

    def backup(self, paths, recursive):
        """
        Backup the given paths under the given config profile, optionally recursively.
        """
        potential_files = set()
        # find all files in the given paths and add to a set
        for path in  paths:
            log.info("Finding files in path %s (recursive=%s)" % (path, recursive))
            if os.path.isdir(path):
                file_finder = FileFinder(path, recursive)
                potential_files.update(file_finder.get_files())
            else:
                potential_files.update([path])
        log.info("%d files found in %d paths" % (len(potential_files), len(paths)))

        # remove ineligible files from the backup list, e.g. files that match exclusion patterns, files that have
        # been backed up previously and haven't since been modified, etc.
        eligible_files = self.__trim_ineligible_files(potential_files)
        # Perform all necessary processing prior to initiating an upload to the file store, e.g. combine files that
        # need archiving into archives, compress files that should be compressed, encrypt files
        # as necessary and obfuscate file names.
        self.__process_files(eligible_files)
        # upload to storage backend
        # if all went well, save new catalogue to highly available storage backend

# CLI application

app = aaargh.App(description="Compress, encrypt, obfuscate and archive files to Amazon S3/Glacier.")

@app.cmd(help="Set AWS S3/Glacier credentials.")
@app.cmd_arg('profile', type=str, help="Configuration profile name. Configuration "
                                        "profiles allow you to back up different parts of "
                                        "your system using different settings.")
def configure(profile):
    "Prompt for AWS credentials and write to config file"
    settings = {"aws": {}}

    settings['aws']['access_key'] = raw_input("AWS Access Key: ")
    settings['aws']["secret_key"] = raw_input("AWS Secret Key: ")

    s3_locations = [l for l in dir(Location) if not '_' in l]
    print "S3 settings: The list of your files along with any encryption keys will be stored in S3."
    settings['aws']["s3_location"] = raw_input("S3 location (possible values are %s): " % ', '.join(s3_locations))
    settings['aws']["s3_bucket"] = raw_input("S3 Bucket Name: ")

    glacier_regions = boto.glacier.regions()
    print "Your files will be backed up to Glacier."
    settings['aws']["glacier_region"] = raw_input("Glacier region (possible values are %s): " % ', '.join([r.name for r in glacier_regions]))
    settings['aws']["glacier_vault"] = raw_input("Glacier Vault Name: ")

    iceit = IceIt(profile)
    iceit.write_config_file(settings)

    print "Config file written. Please edit it to change further options."

    if not iceit.key_pair_exists():
        iceit.generate_key_pair()
        print "\nSecure your private key with a passphrase."
        print "IMPORTANT: If you forget this passphrase there will be no way to recover your files! "
        passphrase_1 = getpass.getpass("Enter a strong passphrase to secure your private key: ")
        passphrase_2 = getpass.getpass("Enter again to confirm: ")
        if passphrase_1 != passphrase_2:
            raise IceItException("Passphrases don't match.")
        iceit.export_keys(passphrase_1)

@app.cmd(help="Backup the given path(s) using the specified backup profile.")
@app.cmd_arg('profile', type=str, help="Configuration profile name. Configuration "
                                        "profiles allow you to back up different parts of "
                                        "your system using different settings.")
@app.cmd_arg('-r', '--recursive', action="store_true", help="Backup directories recursively")
@app.cmd_arg('paths', type=str, nargs="+", help="Directories/files to backup")
def backup(profile, recursive, paths):
    iceit = IceIt(profile)
    if not iceit.is_configured():
        raise IceItException("Configuration profile '%s' doesn't exist or is corrupt." % profile)

    iceit.backup(paths, recursive)

if __name__ == '__main__':
    app.run()<|endoftext|>