<commit_before>from __future__ import print_function
import code
import types
import inspect
import sys
import tempfile
import re
import shutil
import struct
import fcntl
from bdb import BdbQuit

try:
    import termios
except ImportError:
    termios = None

try:
    import pygments
    import pygments.lexers
    import pygments.formatters
    has_pygments = True
except ImportError:
    has_pygments = False
    pass

BdbQuit_excepthook = None
try:
    import bpython
    has_bpython = True
except ImportError:
    has_bpython = False
    try:
        import IPython
        from IPython.core.debugger import BdbQuit_excepthook
        from IPython.core import page
        from IPython.terminal.ipapp import load_default_config
        from IPython.core.magic import (magics_class, line_magic, Magics)

        ipython_config = load_default_config()
        ipython_config.TerminalInteractiveShell.confirm_exit = False

        old_init = IPython.terminal.embed.InteractiveShellEmbed.__init__

        def new_init(self, *k, **kw):
            frames = kw.pop("frames", None)
            old_init(self, *k, **kw)
            from pry import get_context, highlight, terminal_size

            @magics_class
            class MyMagics(Magics):
                def __init__(self, shell, frames):
                    # You must call the parent constructor
                    super(MyMagics, self).__init__(shell)

                    self.frame_offset = 0
                    self.frames = frames
                    self.calling_frame = frames[0]

                @property
                def active_frame(self):
                    return self.frames[self.frame_offset]

                def build_terminal_list(self, list, term_width=80):
                    line = " "
                    for name in sorted(list):
                        if len(line + name) > term_width:
                            yield line
                            line = " "
                        line += " %s" % name
                    if line != " ":
                        yield line

                @line_magic("ls")
                def ls(self, query):
                    """
                    Show local variables/methods/class properties
                    """
                    lines = []
                    width = terminal_size()[0]

                    methods = []
                    properties = []
                    has_query = True

                    that = self.shell.user_ns.get(query, None)

                    if that is None:
                        that = self.active_frame.locals.get("self", [])
                        has_query = False

                    # apparently there is no better way to check if the caller
                    # is a method
                    for attr in dir(that):
                        if isinstance(getattr(that, attr), types.MethodType):
                            methods.append(attr)
                        else:
                            properties.append(attr)

                    if len(methods) > 0:
                        lines.append("local methods:")
                        lines.extend(self.build_terminal_list(methods, width))

                    if len(properties) > 0:
                        lines.append("properties")
                        props = self.build_terminal_list(
                            properties, width)
                        lines.extend(props)

                    if not has_query:
                        lines.append("local variables:")
                        local_vars = self.build_terminal_list(
                            self.active_frame.locals.keys(), width)
                        lines.extend(local_vars)

                    page.page("\n".join(lines))

                @line_magic("editfile")
                def editfile(self, query):
                    """
                    open current breakpoint in editor.
                    """
                    IPython.get_ipython().hooks.editor(
                        self.active_frame.filename,
                        linenum=self.active_frame.lineno)

                @line_magic("where")
                def where(self, query):
                    """
                    Show backtrace
                    """
                    context = []
                    for f in reversed(self.frames[self.frame_offset:]):
                        context.append(get_context(f))
                    page.page("".join(context))

                @line_magic("showsource")
                def showsource(self, query):
                    """
                    Show source of object
                    """
                    obj = self.active_frame.locals.get(
                        query, self.active_frame.globals.get(query, None))
                    if obj is None:
                        return "Not found: %s" % query
                    try:
                        s = inspect.getsource(obj)
                    except TypeError as f:
                        print("%s" % f, file=sys.stderr)
                        return

                    if has_pygments:
                        s = "\n".join(highlight(s.split("\n")))
                    page.page(s)

                def update_context(self):
                    print(get_context(self.active_frame), file=sys.stderr)
                    # hacky
                    self.shell.user_ns.update(self.active_frame.locals)
                    self.shell.user_module.__dict__ = self.active_frame.globals

                @line_magic("up")
                def up(self, query):
                    """
                    Get from call frame up.
                    """
                    self.frame_offset += 1
                    self.frame_offset = min(self.frame_offset,
                                            len(self.frames) - 1)
                    self.update_context()

                @line_magic("down")
                def down(self, query):
                    """
                    Get from call frame down.
                    """
                    self.frame_offset -= 1
                    self.frame_offset = max(self.frame_offset, 0)
                    self.update_context()

                @line_magic("removepry")
                def removepry(self, query):
                    """
                    Remove pry call at current breakpoint.
                    """
                    f = self.calling_frame
                    with open(f.filename) as src, \
                            tempfile.NamedTemporaryFile(mode='w') as dst:
                        for i, line in enumerate(src):
                            if (i + 1) == f.lineno:
                                line = re.sub(r'(import pry;)?\s*pry\(\)', "",
                                              line)
                                if line.strip() == "":
                                    continue
                            dst.write(line)
                        dst.flush()
                        src.close()
                        shutil.copyfile(dst.name, f.filename)

            self.register_magics(MyMagics(self, frames))

        IPython.terminal.embed.InteractiveShellEmbed.__init__ = new_init

        has_ipython = True
    except ImportError:
        has_ipython = False
        pass

try:
    import readline
    has_readline = True
except ImportError:
    has_readline = False
    pass
else:
    import rlcompleter

class Frame():
    """
    Abstraction around old python traceback api
    """

    def __init__(self, *args):
        self.frame = args[0]
        self.filename = args[1]
        self.lineno = args[2]
        self.function = args[3]
        self.lines = args[4]
        self.index = args[5]
        self.locals = self.frame.f_locals
        self.globals = self.frame.f_globals

class Pry():
    def __init__(self, module):
        self.module = module
        self.lexer = None
        self.formatter = None

    def wrap_raw_frames(self, raw_frames):
        frames = []
        for raw_frame in raw_frames:
            frames.append(Frame(*raw_frame))
        return frames

    def highlight(self, lines):
        if not self.module.has_pygments:
            return lines
        p = self.module.pygments
        if self.lexer is None:
            self.lexer = p.lexers.PythonLexer()
        if self.formatter is None:
            self.formatter = p.formatters.Terminal256Formatter()
        tokens = self.lexer.get_tokens("\n".join(lines))
        source = p.format(tokens, self.formatter)
        return source.split("\n")

    def __enter__(self):
        pass

    def __exit__(self, type, value, tb):
        self.wrap_sys_excepthook()
        if tb is None:
            return
        frames = self.wrap_raw_frames(self.module.inspect.getinnerframes(tb))
        for frame in frames:
            print(self.get_context(frame), file=sys.stderr)
        print("%s: %s\n" % (type.__name__, str(value)), file=sys.stderr)
        self(frames)

    def wrap_sys_excepthook(self):
        m = self.module
        if not m.has_ipython:
            return
        # make sure we wrap it only once or we would end up with a cycle
        #  BdbQuit_excepthook.excepthook_ori == BdbQuit_excepthook
        if m.sys.excepthook != m.BdbQuit_excepthook:
            m.BdbQuit_excepthook.excepthook_ori = m.sys.excepthook
            m.sys.excepthook = m.BdbQuit_excepthook

    def get_context(self, frame, unwind=True):
        before = max(frame.lineno - 6, 0)
        after = frame.lineno + 4
        context = []
        try:
            f = open(frame.filename)

            for i, line in enumerate(f):
                if i >= before:
                    context.append(line.rstrip())
                if i > after:
                    break
            f.close()
        except IOError:
            context = frame.lines
        banner = "From: {} @ line {} :\n".format(frame.filename, frame.lineno)
        i = max(frame.lineno - 5, 0)

        if self.module.has_pygments and not self.module.has_bpython:
            context = self.highlight(context)

        for line in context:
            pointer = "-->" if i == frame.lineno else "   "
            banner += "{} {}: {}\n".format(pointer, i, line)
            i += 1
        return banner

    def fix_tty(self):
        m = self.module
        if m.termios is None:
            return
        # Sometimes when you do something funky, you may lose your terminal
        # echo. This should restore it when spawning new pdb.
        termios_fd = m.sys.stdin.fileno()
        termios_echo = m.termios.tcgetattr(termios_fd)
        termios_echo[3] = termios_echo[3] | m.termios.ECHO
        m.termios.tcsetattr(termios_fd, termios.TCSADRAIN, termios_echo)

    def terminal_size(self):
        m = self.module
        if m.termios is None:
            return 80, 24
        args = m.struct.pack('HHHH', 0, 0, 0, 0)
        res = m.fcntl.ioctl(0, m.termios.TIOCGWINSZ, args)
        h, w, hp, wp = m.struct.unpack('HHHH', res)
        return w, h

    def shell(self, context, frames):
        active_frame = frames[0]
        m = self.module
        if m.has_bpython:
            globals().update(active_frame.globals)
            m.bpython.embed(active_frame.locals.copy(), banner=context)
        if m.has_ipython:
            m.IPython.embed(
                user_ns=active_frame.locals.copy(),
                global_ns=active_frame.globals.copy(),
                banner1=context,
                config=m.ipython_config,
                frames=frames)
        else:
            if m.has_readline:
                m.readline.parse_and_bind("tab: complete")
            globals().update(active_frame.globals)
            m.code.interact(context, local=active_frame.locals.copy())

    def __call__(self, frames=None):
        if frames is None:
            frames = self.module.inspect.getouterframes(
                self.module.inspect.currentframe())
            if len(frames) > 1:
                frames = frames[1:]
            frames = self.wrap_raw_frames(frames)

        context = self.get_context(frames[0])
        self.fix_tty()
        self.shell(context, frames)

# hack for convenient access
sys.modules[__name__] = Pry(sys.modules[__name__])
<commit_msg>ls: catch potential exception in getattr<commit_after>from __future__ import print_function
import code
import types
import inspect
import sys
import tempfile
import re
import shutil
import struct
import fcntl
from bdb import BdbQuit

try:
    import termios
except ImportError:
    termios = None

try:
    import pygments
    import pygments.lexers
    import pygments.formatters
    has_pygments = True
except ImportError:
    has_pygments = False
    pass

BdbQuit_excepthook = None
try:
    import bpython
    has_bpython = True
except ImportError:
    has_bpython = False
    try:
        import IPython
        from IPython.core.debugger import BdbQuit_excepthook
        from IPython.core import page
        from IPython.terminal.ipapp import load_default_config
        from IPython.core.magic import (magics_class, line_magic, Magics)

        ipython_config = load_default_config()
        ipython_config.TerminalInteractiveShell.confirm_exit = False

        old_init = IPython.terminal.embed.InteractiveShellEmbed.__init__

        def new_init(self, *k, **kw):
            frames = kw.pop("frames", None)
            old_init(self, *k, **kw)
            from pry import get_context, highlight, terminal_size

            @magics_class
            class MyMagics(Magics):
                def __init__(self, shell, frames):
                    # You must call the parent constructor
                    super(MyMagics, self).__init__(shell)

                    self.frame_offset = 0
                    self.frames = frames
                    self.calling_frame = frames[0]

                @property
                def active_frame(self):
                    return self.frames[self.frame_offset]

                def build_terminal_list(self, list, term_width=80):
                    line = " "
                    for name in sorted(list):
                        if len(line + name) > term_width:
                            yield line
                            line = " "
                        line += " %s" % name
                    if line != " ":
                        yield line

                @line_magic("ls")
                def ls(self, query):
                    """
                    Show local variables/methods/class properties
                    """
                    lines = []
                    width = terminal_size()[0]

                    methods = []
                    properties = []
                    has_query = True

                    that = self.shell.user_ns.get(query, None)

                    if that is None:
                        that = self.active_frame.locals.get("self", [])
                        has_query = False

                    # apparently there is no better way to check if the caller
                    # is a method
                    for attr in dir(that):
                        try:
                            value = getattr(that, attr)
                        except Exception:
                            continue
                        if isinstance(value, types.MethodType):
                            methods.append(attr)
                        else:
                            properties.append(attr)

                    if len(methods) > 0:
                        lines.append("local methods:")
                        lines.extend(self.build_terminal_list(methods, width))

                    if len(properties) > 0:
                        lines.append("properties")
                        props = self.build_terminal_list(
                            properties, width)
                        lines.extend(props)

                    if not has_query:
                        lines.append("local variables:")
                        local_vars = self.build_terminal_list(
                            self.active_frame.locals.keys(), width)
                        lines.extend(local_vars)

                    page.page("\n".join(lines))

                @line_magic("editfile")
                def editfile(self, query):
                    """
                    open current breakpoint in editor.
                    """
                    IPython.get_ipython().hooks.editor(
                        self.active_frame.filename,
                        linenum=self.active_frame.lineno)

                @line_magic("where")
                def where(self, query):
                    """
                    Show backtrace
                    """
                    context = []
                    for f in reversed(self.frames[self.frame_offset:]):
                        context.append(get_context(f))
                    page.page("".join(context))

                @line_magic("showsource")
                def showsource(self, query):
                    """
                    Show source of object
                    """
                    obj = self.active_frame.locals.get(
                        query, self.active_frame.globals.get(query, None))
                    if obj is None:
                        return "Not found: %s" % query
                    try:
                        s = inspect.getsource(obj)
                    except TypeError as f:
                        print("%s" % f, file=sys.stderr)
                        return

                    if has_pygments:
                        s = "\n".join(highlight(s.split("\n")))
                    page.page(s)

                def update_context(self):
                    print(get_context(self.active_frame), file=sys.stderr)
                    # hacky
                    self.shell.user_ns.update(self.active_frame.locals)
                    self.shell.user_module.__dict__ = self.active_frame.globals

                @line_magic("up")
                def up(self, query):
                    """
                    Get from call frame up.
                    """
                    self.frame_offset += 1
                    self.frame_offset = min(self.frame_offset,
                                            len(self.frames) - 1)
                    self.update_context()

                @line_magic("down")
                def down(self, query):
                    """
                    Get from call frame down.
                    """
                    self.frame_offset -= 1
                    self.frame_offset = max(self.frame_offset, 0)
                    self.update_context()

                @line_magic("removepry")
                def removepry(self, query):
                    """
                    Remove pry call at current breakpoint.
                    """
                    f = self.calling_frame
                    with open(f.filename) as src, \
                            tempfile.NamedTemporaryFile(mode='w') as dst:
                        for i, line in enumerate(src):
                            if (i + 1) == f.lineno:
                                line = re.sub(r'(import pry;)?\s*pry\(\)', "",
                                              line)
                                if line.strip() == "":
                                    continue
                            dst.write(line)
                        dst.flush()
                        src.close()
                        shutil.copyfile(dst.name, f.filename)

            self.register_magics(MyMagics(self, frames))

        IPython.terminal.embed.InteractiveShellEmbed.__init__ = new_init

        has_ipython = True
    except ImportError:
        has_ipython = False
        pass

try:
    import readline
    has_readline = True
except ImportError:
    has_readline = False
    pass
else:
    import rlcompleter

class Frame():
    """
    Abstraction around old python traceback api
    """

    def __init__(self, *args):
        self.frame = args[0]
        self.filename = args[1]
        self.lineno = args[2]
        self.function = args[3]
        self.lines = args[4]
        self.index = args[5]
        self.locals = self.frame.f_locals
        self.globals = self.frame.f_globals

class Pry():
    def __init__(self, module):
        self.module = module
        self.lexer = None
        self.formatter = None

    def wrap_raw_frames(self, raw_frames):
        frames = []
        for raw_frame in raw_frames:
            frames.append(Frame(*raw_frame))
        return frames

    def highlight(self, lines):
        if not self.module.has_pygments:
            return lines
        p = self.module.pygments
        if self.lexer is None:
            self.lexer = p.lexers.PythonLexer()
        if self.formatter is None:
            self.formatter = p.formatters.Terminal256Formatter()
        tokens = self.lexer.get_tokens("\n".join(lines))
        source = p.format(tokens, self.formatter)
        return source.split("\n")

    def __enter__(self):
        pass

    def __exit__(self, type, value, tb):
        self.wrap_sys_excepthook()
        if tb is None:
            return
        frames = self.wrap_raw_frames(self.module.inspect.getinnerframes(tb))
        for frame in frames:
            print(self.get_context(frame), file=sys.stderr)
        print("%s: %s\n" % (type.__name__, str(value)), file=sys.stderr)
        self(frames)

    def wrap_sys_excepthook(self):
        m = self.module
        if not m.has_ipython:
            return
        # make sure we wrap it only once or we would end up with a cycle
        #  BdbQuit_excepthook.excepthook_ori == BdbQuit_excepthook
        if m.sys.excepthook != m.BdbQuit_excepthook:
            m.BdbQuit_excepthook.excepthook_ori = m.sys.excepthook
            m.sys.excepthook = m.BdbQuit_excepthook

    def get_context(self, frame, unwind=True):
        before = max(frame.lineno - 6, 0)
        after = frame.lineno + 4
        context = []
        try:
            f = open(frame.filename)

            for i, line in enumerate(f):
                if i >= before:
                    context.append(line.rstrip())
                if i > after:
                    break
            f.close()
        except IOError:
            context = frame.lines
        banner = "From: {} @ line {} :\n".format(frame.filename, frame.lineno)
        i = max(frame.lineno - 5, 0)

        if self.module.has_pygments and not self.module.has_bpython:
            context = self.highlight(context)

        for line in context:
            pointer = "-->" if i == frame.lineno else "   "
            banner += "{} {}: {}\n".format(pointer, i, line)
            i += 1
        return banner

    def fix_tty(self):
        m = self.module
        if m.termios is None:
            return
        # Sometimes when you do something funky, you may lose your terminal
        # echo. This should restore it when spawning new pdb.
        termios_fd = m.sys.stdin.fileno()
        termios_echo = m.termios.tcgetattr(termios_fd)
        termios_echo[3] = termios_echo[3] | m.termios.ECHO
        m.termios.tcsetattr(termios_fd, termios.TCSADRAIN, termios_echo)

    def terminal_size(self):
        m = self.module
        if m.termios is None:
            return 80, 24
        args = m.struct.pack('HHHH', 0, 0, 0, 0)
        res = m.fcntl.ioctl(0, m.termios.TIOCGWINSZ, args)
        h, w, hp, wp = m.struct.unpack('HHHH', res)
        return w, h

    def shell(self, context, frames):
        active_frame = frames[0]
        m = self.module
        if m.has_bpython:
            globals().update(active_frame.globals)
            m.bpython.embed(active_frame.locals.copy(), banner=context)
        if m.has_ipython:
            m.IPython.embed(
                user_ns=active_frame.locals.copy(),
                global_ns=active_frame.globals.copy(),
                banner1=context,
                config=m.ipython_config,
                frames=frames)
        else:
            if m.has_readline:
                m.readline.parse_and_bind("tab: complete")
            globals().update(active_frame.globals)
            m.code.interact(context, local=active_frame.locals.copy())

    def __call__(self, frames=None):
        if frames is None:
            frames = self.module.inspect.getouterframes(
                self.module.inspect.currentframe())
            if len(frames) > 1:
                frames = frames[1:]
            frames = self.wrap_raw_frames(frames)

        context = self.get_context(frames[0])
        self.fix_tty()
        self.shell(context, frames)

# hack for convenient access
sys.modules[__name__] = Pry(sys.modules[__name__])
<|endoftext|>


<commit_before>from mutagen.easyid3 import EasyID3
from mutagen.id3 import ID3, TORY, TYER, TPUB, APIC, USLT, COMM
from mutagen.mp4 import MP4, MP4Cover
from mutagen.flac import Picture, FLAC

import urllib.request
from logzero import logger as log

from spotdl.const import TAG_PRESET, M4A_TAG_PRESET

def compare(music_file, metadata):
    """Check if the input music file title matches the expected title."""
    already_tagged = False
    try:
        if music_file.endswith(".mp3"):
            audiofile = EasyID3(music_file)
            already_tagged = audiofile["title"][0] == metadata["name"]
        elif music_file.endswith(".m4a"):
            audiofile = MP4(music_file)
            already_tagged = audiofile["\xa9nam"][0] == metadata["name"]
    except (KeyError, TypeError):
        pass

    return already_tagged

def embed(music_file, meta_tags):
    """ Embed metadata. """
    embed = EmbedMetadata(music_file, meta_tags)
    if music_file.endswith(".m4a"):
        log.info("Applying metadata")
        return embed.as_m4a()
    elif music_file.endswith(".mp3"):
        log.info("Applying metadata")
        return embed.as_mp3()
    elif music_file.endswith(".flac"):
        log.info("Applying metadata")
        return embed.as_flac()
    else:
        log.warning("Cannot embed metadata into given output extension")
        return False

class EmbedMetadata:
    def __init__(self, music_file, meta_tags):
        self.music_file = music_file
        self.meta_tags = meta_tags
        self.spotify_metadata = meta_tags["spotify_metadata"]
        self.provider = "spotify" if meta_tags["spotify_metadata"] else "youtube"

    def as_mp3(self):
        """ Embed metadata to MP3 files. """
        music_file = self.music_file
        meta_tags = self.meta_tags
        # EasyID3 is fun to use ;)
        # For supported easyid3 tags:
        # https://github.com/quodlibet/mutagen/blob/master/mutagen/easyid3.py
        # Check out somewhere at end of above linked file
        audiofile = EasyID3(music_file)
        self._embed_basic_metadata(audiofile, preset=TAG_PRESET)
        audiofile["media"] = meta_tags["type"]
        audiofile["author"] = meta_tags["artists"][0]["name"]
        audiofile["lyricist"] = meta_tags["artists"][0]["name"]
        audiofile["arranger"] = meta_tags["artists"][0]["name"]
        audiofile["performer"] = meta_tags["artists"][0]["name"]
        audiofile["website"] = meta_tags["external_urls"][self.provider]
        audiofile["length"] = str(meta_tags["duration"])
        if meta_tags["publisher"]:
            audiofile["encodedby"] = meta_tags["publisher"]
        if meta_tags["external_ids"]["isrc"]:
            audiofile["isrc"] = meta_tags["external_ids"]["isrc"]
        audiofile.save(v2_version=3)

        # For supported id3 tags:
        # https://github.com/quodlibet/mutagen/blob/master/mutagen/id3/_frames.py
        # Each class represents an id3 tag
        audiofile = ID3(music_file)
        audiofile["TORY"] = TORY(encoding=3, text=meta_tags["year"])
        audiofile["TYER"] = TYER(encoding=3, text=meta_tags["year"])
        if meta_tags["publisher"]:
            audiofile["TPUB"] = TPUB(encoding=3, text=meta_tags["publisher"])
        audiofile["COMM"] = COMM(
            encoding=3, text=meta_tags["external_urls"][self.provider]
        )
        if meta_tags["lyrics"]:
            audiofile["USLT"] = USLT(
                encoding=3, desc=u"Lyrics", text=meta_tags["lyrics"]
            )
        try:
            albumart = urllib.request.urlopen(meta_tags["album"]["images"][0]["url"])
            audiofile["APIC"] = APIC(
                encoding=3,
                mime="image/jpeg",
                type=3,
                desc=u"Cover",
                data=albumart.read(),
            )
            albumart.close()
        except IndexError:
            pass

        audiofile.save(v2_version=3)
        return True

    def as_m4a(self):
        """ Embed metadata to M4A files. """
        music_file = self.music_file
        meta_tags = self.meta_tags
        audiofile = MP4(music_file)
        self._embed_basic_metadata(audiofile, preset=M4A_TAG_PRESET)
        audiofile[M4A_TAG_PRESET["year"]] = meta_tags["year"]
        audiofile[M4A_TAG_PRESET["comment"]] = meta_tags["external_urls"][self.provider]
        if meta_tags["lyrics"]:
            audiofile[M4A_TAG_PRESET["lyrics"]] = meta_tags["lyrics"]
        try:
            albumart = urllib.request.urlopen(meta_tags["album"]["images"][0]["url"])
            audiofile[M4A_TAG_PRESET["albumart"]] = [
                MP4Cover(albumart.read(), imageformat=MP4Cover.FORMAT_JPEG)
            ]
            albumart.close()
        except IndexError:
            pass

        audiofile.save()
        return True

    def as_flac(self):
        music_file = self.music_file
        meta_tags = self.meta_tags
        audiofile = FLAC(music_file)
        self._embed_basic_metadata(audiofile)
        audiofile["year"] = meta_tags["year"]
        audiofile["comment"] = meta_tags["external_urls"][self.provider]
        if meta_tags["lyrics"]:
            audiofile["lyrics"] = meta_tags["lyrics"]

        image = Picture()
        image.type = 3
        image.desc = "Cover"
        image.mime = "image/jpeg"
        albumart = urllib.request.urlopen(meta_tags["album"]["images"][0]["url"])
        image.data = albumart.read()
        albumart.close()
        audiofile.add_picture(image)

        audiofile.save()
        return True

    def _embed_basic_metadata(self, audiofile, preset=TAG_PRESET):
        meta_tags = self.meta_tags
        audiofile[preset["artist"]] = meta_tags["artists"][0]["name"]
        if meta_tags["album"]["artists"][0]["name"]:
            audiofile[preset["albumartist"]] = meta_tags["album"]["artists"][0]["name"]
        if meta_tags["album"]["name"]:
            audiofile[preset["album"]] = meta_tags["album"]["name"]
        audiofile[preset["title"]] = meta_tags["name"]
        audiofile[preset["date"]] = meta_tags["release_date"]
        audiofile[preset["originaldate"]] = meta_tags["release_date"]
        if meta_tags["genre"]:
            audiofile[preset["genre"]] = meta_tags["genre"]
        if meta_tags["copyright"]:
            audiofile[preset["copyright"]] = meta_tags["copyright"]
        if self.music_file.endswith(".flac"):
            audiofile[preset["discnumber"]] = str(meta_tags["disc_number"])
        else:
            audiofile[preset["discnumber"]] = [(meta_tags["disc_number"], 0)]
        if self.music_file.endswith(".flac"):
            audiofile[preset["tracknumber"]] = str(meta_tags["track_number"])
        else:
            if preset["tracknumber"] == TAG_PRESET["tracknumber"]:
                audiofile[preset["tracknumber"]] = "{}/{}".format(
                    meta_tags["track_number"], meta_tags["total_tracks"]
                )
            else:
                audiofile[preset["tracknumber"]] = [
                    (meta_tags["track_number"], meta_tags["total_tracks"])
                ]
<commit_msg>Embed release and year metadata only when available<commit_after>from mutagen.easyid3 import EasyID3
from mutagen.id3 import ID3, TORY, TYER, TPUB, APIC, USLT, COMM
from mutagen.mp4 import MP4, MP4Cover
from mutagen.flac import Picture, FLAC

import urllib.request
from logzero import logger as log

from spotdl.const import TAG_PRESET, M4A_TAG_PRESET

def compare(music_file, metadata):
    """Check if the input music file title matches the expected title."""
    already_tagged = False
    try:
        if music_file.endswith(".mp3"):
            audiofile = EasyID3(music_file)
            already_tagged = audiofile["title"][0] == metadata["name"]
        elif music_file.endswith(".m4a"):
            audiofile = MP4(music_file)
            already_tagged = audiofile["\xa9nam"][0] == metadata["name"]
    except (KeyError, TypeError):
        pass

    return already_tagged

def embed(music_file, meta_tags):
    """ Embed metadata. """
    embed = EmbedMetadata(music_file, meta_tags)
    if music_file.endswith(".m4a"):
        log.info("Applying metadata")
        return embed.as_m4a()
    elif music_file.endswith(".mp3"):
        log.info("Applying metadata")
        return embed.as_mp3()
    elif music_file.endswith(".flac"):
        log.info("Applying metadata")
        return embed.as_flac()
    else:
        log.warning("Cannot embed metadata into given output extension")
        return False

class EmbedMetadata:
    def __init__(self, music_file, meta_tags):
        self.music_file = music_file
        self.meta_tags = meta_tags
        self.spotify_metadata = meta_tags["spotify_metadata"]
        self.provider = "spotify" if meta_tags["spotify_metadata"] else "youtube"

    def as_mp3(self):
        """ Embed metadata to MP3 files. """
        music_file = self.music_file
        meta_tags = self.meta_tags
        # EasyID3 is fun to use ;)
        # For supported easyid3 tags:
        # https://github.com/quodlibet/mutagen/blob/master/mutagen/easyid3.py
        # Check out somewhere at end of above linked file
        audiofile = EasyID3(music_file)
        self._embed_basic_metadata(audiofile, preset=TAG_PRESET)
        audiofile["media"] = meta_tags["type"]
        audiofile["author"] = meta_tags["artists"][0]["name"]
        audiofile["lyricist"] = meta_tags["artists"][0]["name"]
        audiofile["arranger"] = meta_tags["artists"][0]["name"]
        audiofile["performer"] = meta_tags["artists"][0]["name"]
        audiofile["website"] = meta_tags["external_urls"][self.provider]
        audiofile["length"] = str(meta_tags["duration"])
        if meta_tags["publisher"]:
            audiofile["encodedby"] = meta_tags["publisher"]
        if meta_tags["external_ids"]["isrc"]:
            audiofile["isrc"] = meta_tags["external_ids"]["isrc"]
        audiofile.save(v2_version=3)

        # For supported id3 tags:
        # https://github.com/quodlibet/mutagen/blob/master/mutagen/id3/_frames.py
        # Each class represents an id3 tag
        audiofile = ID3(music_file)
        if meta_tags["year"]:
            audiofile["TORY"] = TORY(encoding=3, text=meta_tags["year"])
            audiofile["TYER"] = TYER(encoding=3, text=meta_tags["year"])
        if meta_tags["publisher"]:
            audiofile["TPUB"] = TPUB(encoding=3, text=meta_tags["publisher"])
        audiofile["COMM"] = COMM(
            encoding=3, text=meta_tags["external_urls"][self.provider]
        )
        if meta_tags["lyrics"]:
            audiofile["USLT"] = USLT(
                encoding=3, desc=u"Lyrics", text=meta_tags["lyrics"]
            )
        try:
            albumart = urllib.request.urlopen(meta_tags["album"]["images"][0]["url"])
            audiofile["APIC"] = APIC(
                encoding=3,
                mime="image/jpeg",
                type=3,
                desc=u"Cover",
                data=albumart.read(),
            )
            albumart.close()
        except IndexError:
            pass

        audiofile.save(v2_version=3)
        return True

    def as_m4a(self):
        """ Embed metadata to M4A files. """
        music_file = self.music_file
        meta_tags = self.meta_tags
        audiofile = MP4(music_file)
        self._embed_basic_metadata(audiofile, preset=M4A_TAG_PRESET)
        if meta_tags["year"]:
            audiofile[M4A_TAG_PRESET["year"]] = meta_tags["year"]
        audiofile[M4A_TAG_PRESET["comment"]] = meta_tags["external_urls"][self.provider]
        if meta_tags["lyrics"]:
            audiofile[M4A_TAG_PRESET["lyrics"]] = meta_tags["lyrics"]
        try:
            albumart = urllib.request.urlopen(meta_tags["album"]["images"][0]["url"])
            audiofile[M4A_TAG_PRESET["albumart"]] = [
                MP4Cover(albumart.read(), imageformat=MP4Cover.FORMAT_JPEG)
            ]
            albumart.close()
        except IndexError:
            pass

        audiofile.save()
        return True

    def as_flac(self):
        music_file = self.music_file
        meta_tags = self.meta_tags
        audiofile = FLAC(music_file)
        self._embed_basic_metadata(audiofile)
        if meta_tags["year"]:
            audiofile["year"] = meta_tags["year"]
        audiofile["comment"] = meta_tags["external_urls"][self.provider]
        if meta_tags["lyrics"]:
            audiofile["lyrics"] = meta_tags["lyrics"]

        image = Picture()
        image.type = 3
        image.desc = "Cover"
        image.mime = "image/jpeg"
        albumart = urllib.request.urlopen(meta_tags["album"]["images"][0]["url"])
        image.data = albumart.read()
        albumart.close()
        audiofile.add_picture(image)

        audiofile.save()
        return True

    def _embed_basic_metadata(self, audiofile, preset=TAG_PRESET):
        meta_tags = self.meta_tags
        audiofile[preset["artist"]] = meta_tags["artists"][0]["name"]
        if meta_tags["album"]["artists"][0]["name"]:
            audiofile[preset["albumartist"]] = meta_tags["album"]["artists"][0]["name"]
        if meta_tags["album"]["name"]:
            audiofile[preset["album"]] = meta_tags["album"]["name"]
        audiofile[preset["title"]] = meta_tags["name"]
        if meta_tags["release_date"]:
            audiofile[preset["date"]] = meta_tags["release_date"]
            audiofile[preset["originaldate"]] = meta_tags["release_date"]
        if meta_tags["genre"]:
            audiofile[preset["genre"]] = meta_tags["genre"]
        if meta_tags["copyright"]:
            audiofile[preset["copyright"]] = meta_tags["copyright"]
        if self.music_file.endswith(".flac"):
            audiofile[preset["discnumber"]] = str(meta_tags["disc_number"])
        else:
            audiofile[preset["discnumber"]] = [(meta_tags["disc_number"], 0)]
        if self.music_file.endswith(".flac"):
            audiofile[preset["tracknumber"]] = str(meta_tags["track_number"])
        else:
            if preset["tracknumber"] == TAG_PRESET["tracknumber"]:
                audiofile[preset["tracknumber"]] = "{}/{}".format(
                    meta_tags["track_number"], meta_tags["total_tracks"]
                )
            else:
                audiofile[preset["tracknumber"]] = [
                    (meta_tags["track_number"], meta_tags["total_tracks"])
                ]
<|endoftext|>


<commit_before>import code
import inspect
import sys
import traceback
from bdb import BdbQuit

try:
    import termios
except ImportError:
    termios = None

BdbQuit_excepthook = None
try:
    import bpython
    has_bpython = True
except ImportError:
    has_bpython = False
    try:
        import IPython
        from IPython.core.debugger import BdbQuit_excepthook
        from IPython.terminal.ipapp import load_default_config
        from IPython.core.magic import (magics_class, line_magic, Magics)

        ipython_config = load_default_config()
        ipython_config.TerminalInteractiveShell.confirm_exit = False

        old_init = IPython.terminal.embed.InteractiveShellEmbed.__init__

        def new_init(self, *k, **kw):
            old_init(self, *k, **kw)

            @magics_class
            class MyMagics(Magics):
                @line_magic("editfile")
                def editfile(self, query):
                    outer = inspect.getouterframes(inspect.currentframe())
                    found_pry = False
                    for frame in outer:
                        if frame.filename.endswith("pry.py"):
                            if frame.function in ("__call__", "__exit__"):
                                found_pry = True
                        elif found_pry:
                            break
                    IPython.get_ipython().hooks.editor(frame.filename, linenum=frame.lineno)

            self.register_magics(MyMagics)

        IPython.terminal.embed.InteractiveShellEmbed.__init__ = new_init

        has_ipython = True
    except ImportError:
        has_ipython = False
        pass

try:
    import pygments
    import pygments.lexers
    import pygments.formatters
    has_pygments = True
except ImportError:
    has_pygments = False
    pass

try:
    import readline
    has_readline = True
except ImportError:
    has_readline = False
    pass
else:
    import rlcompleter

class Pry():
    def __init__(self, module):
        self.module = module
        self.lexer = None
        self.formatter = None

    def highlight(self, lines):
        if not self.module.has_pygments:
            return lines
        p = self.module.pygments
        if self.lexer is None:
            self.lexer = p.lexers.PythonLexer()
        if self.formatter is None:
            self.formatter = p.formatters.Terminal256Formatter()
        tokens = self.lexer.get_tokens("\n".join(lines))
        source = p.format(tokens, self.formatter)
        return source.split("\n")

    def __enter__(self):
        pass

    def __exit__(self, type, value, tb):
        self.wrap_sys_excepthook()
        while tb.tb_next is not None:
            context = self.get_context(tb.tb_frame)[0]
            self.module.sys.stderr.write(context)
            tb = tb.tb_next
        self.module.sys.stderr.write("%s: %s\n" % (type.__name__, str(value)))
        self(tb.tb_frame)

    def wrap_sys_excepthook(self):
        m = self.module
        if not m.has_ipython:
            return
        # make sure we wrap it only once or we would end up with a cycle
        #  BdbQuit_excepthook.excepthook_ori == BdbQuit_excepthook
        if m.sys.excepthook != m.BdbQuit_excepthook:
            m.BdbQuit_excepthook.excepthook_ori = m.sys.excepthook
            m.sys.excepthook = m.BdbQuit_excepthook

    def get_context(self, currentframe):
        frames = self.module.inspect.getouterframes(currentframe)
        if len(frames) > 1:
            frames = frames[1:]
        frame, filename, line_number, function_name, lines, index = frames[0]
        before = max(line_number - 6, 0)
        after = line_number + 4
        context = []
        try:
            f = open(filename)

            for i, line in enumerate(f):
                if i >= before:
                    context.append(line.rstrip())
                if i > after:
                    break
            f.close()
        except IOError:
            context = lines
        banner = "From: {} @ line {} :\n".format(filename, line_number)
        i = max(line_number - 5, 0)

        if self.module.has_pygments and not self.module.has_bpython:
            context = self.highlight(context)

        for line in context:
            pointer = "-->" if i == line_number else "   "
            banner += "{} {}: {}\n".format(pointer, i, line)
            i += 1
        return banner, frame.f_locals, frame.f_globals

    def fix_tty(self):
        m = self.module
        if m.termios is None:
            return
        # Sometimes when you do something funky, you may lose your terminal
        # echo. This should restore it when spawning new pdb.
        termios_fd = m.sys.stdin.fileno()
        termios_echo = m.termios.tcgetattr(termios_fd)
        termios_echo[3] = termios_echo[3] | m.termios.ECHO
        m.termios.tcsetattr(termios_fd, termios.TCSADRAIN, termios_echo)

    def shell(self, context, local, global_):
        m = self.module
        if m.has_bpython:
            globals = global_
            m.bpython.embed(local, banner=context)
        if m.has_ipython:
            m.IPython.embed(
                user_ns=local,
                global_ns=global_,
                banner1=context,
                config=m.ipython_config)
        else:
            if m.has_readline:
                m.readline.parse_and_bind("tab: complete")
            globals = global_
            m.code.interact(context, local=local)

    def __call__(self, frame=None):
        if frame is None:
            frame = self.module.inspect.currentframe()
        context, local, global_ = self.get_context(frame)
        self.fix_tty()
        self.shell(context, local, global_)

# hack for convenient access
sys.modules[__name__] = Pry(sys.modules[__name__])
<commit_msg>add removepry magic<commit_after>import code
import inspect
import sys
import traceback
import tempfile
import re
import shutil
from bdb import BdbQuit

try:
    import termios
except ImportError:
    termios = None

BdbQuit_excepthook = None
try:
    import bpython
    has_bpython = True
except ImportError:
    has_bpython = False
    try:
        import IPython
        from IPython.core.debugger import BdbQuit_excepthook
        from IPython.terminal.ipapp import load_default_config
        from IPython.core.magic import (magics_class, line_magic, Magics)

        ipython_config = load_default_config()
        ipython_config.TerminalInteractiveShell.confirm_exit = False

        old_init = IPython.terminal.embed.InteractiveShellEmbed.__init__

        def new_init(self, *k, **kw):
            old_init(self, *k, **kw)

            @magics_class
            class MyMagics(Magics):
                def get_invocation_callsite(self):
                    outer = inspect.getouterframes(inspect.currentframe())
                    found_pry = False
                    for frame in outer:
                        if frame.filename.endswith("pry.py"):
                            if frame.function in ("__call__", "__exit__"):
                                found_pry = True
                        elif found_pry:
                            break
                    return frame

                @line_magic("editfile")
                def editfile(self, query):
                    frame = self.get_invocation_callsite()
                    IPython.get_ipython().hooks.editor(
                        frame.filename, linenum=frame.lineno)

                @line_magic("removepry")
                def editfile(self, query):
                    frame = self.get_invocation_callsite()
                    with open(frame.filename) as src, \
                            tempfile.NamedTemporaryFile(mode='w') as dst:
                        for i, line in enumerate(src):
                            if (i + 1) == frame.lineno:
                                line = re.sub(r'(import pry;)?\s*pry\(\)', "", line)
                                if line.strip() == "":
                                    continue
                            dst.write(line)
                        dst.flush()
                        src.close()
                        shutil.copyfile(dst.name, frame.filename)
            self.register_magics(MyMagics)

        IPython.terminal.embed.InteractiveShellEmbed.__init__ = new_init

        has_ipython = True
    except ImportError:
        has_ipython = False
        pass

try:
    import pygments
    import pygments.lexers
    import pygments.formatters
    has_pygments = True
except ImportError:
    has_pygments = False
    pass

try:
    import readline
    has_readline = True
except ImportError:
    has_readline = False
    pass
else:
    import rlcompleter

class Pry():
    def __init__(self, module):
        self.module = module
        self.lexer = None
        self.formatter = None

    def highlight(self, lines):
        if not self.module.has_pygments:
            return lines
        p = self.module.pygments
        if self.lexer is None:
            self.lexer = p.lexers.PythonLexer()
        if self.formatter is None:
            self.formatter = p.formatters.Terminal256Formatter()
        tokens = self.lexer.get_tokens("\n".join(lines))
        source = p.format(tokens, self.formatter)
        return source.split("\n")

    def __enter__(self):
        pass

    def __exit__(self, type, value, tb):
        self.wrap_sys_excepthook()
        while tb.tb_next is not None:
            context = self.get_context(tb.tb_frame)[0]
            self.module.sys.stderr.write(context)
            tb = tb.tb_next
        self.module.sys.stderr.write("%s: %s\n" % (type.__name__, str(value)))
        self(tb.tb_frame)

    def wrap_sys_excepthook(self):
        m = self.module
        if not m.has_ipython:
            return
        # make sure we wrap it only once or we would end up with a cycle
        #  BdbQuit_excepthook.excepthook_ori == BdbQuit_excepthook
        if m.sys.excepthook != m.BdbQuit_excepthook:
            m.BdbQuit_excepthook.excepthook_ori = m.sys.excepthook
            m.sys.excepthook = m.BdbQuit_excepthook

    def get_context(self, currentframe):
        frames = self.module.inspect.getouterframes(currentframe)
        if len(frames) > 1:
            frames = frames[1:]
        frame, filename, line_number, function_name, lines, index = frames[0]
        before = max(line_number - 6, 0)
        after = line_number + 4
        context = []
        try:
            f = open(filename)

            for i, line in enumerate(f):
                if i >= before:
                    context.append(line.rstrip())
                if i > after:
                    break
            f.close()
        except IOError:
            context = lines
        banner = "From: {} @ line {} :\n".format(filename, line_number)
        i = max(line_number - 5, 0)

        if self.module.has_pygments and not self.module.has_bpython:
            context = self.highlight(context)

        for line in context:
            pointer = "-->" if i == line_number else "   "
            banner += "{} {}: {}\n".format(pointer, i, line)
            i += 1
        return banner, frame.f_locals, frame.f_globals

    def fix_tty(self):
        m = self.module
        if m.termios is None:
            return
        # Sometimes when you do something funky, you may lose your terminal
        # echo. This should restore it when spawning new pdb.
        termios_fd = m.sys.stdin.fileno()
        termios_echo = m.termios.tcgetattr(termios_fd)
        termios_echo[3] = termios_echo[3] | m.termios.ECHO
        m.termios.tcsetattr(termios_fd, termios.TCSADRAIN, termios_echo)

    def shell(self, context, local, global_):
        m = self.module
        if m.has_bpython:
            globals = global_
            m.bpython.embed(local, banner=context)
        if m.has_ipython:
            m.IPython.embed(
                user_ns=local,
                global_ns=global_,
                banner1=context,
                config=m.ipython_config)
        else:
            if m.has_readline:
                m.readline.parse_and_bind("tab: complete")
            globals = global_
            m.code.interact(context, local=local)

    def __call__(self, frame=None):
        if frame is None:
            frame = self.module.inspect.currentframe()
        context, local, global_ = self.get_context(frame)
        self.fix_tty()
        self.shell(context, local, global_)

# hack for convenient access
sys.modules[__name__] = Pry(sys.modules[__name__])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Copyright (c) 2015 by Tobias Houska

This file is part of Statistical Parameter Optimization Tool (SPOTPY).

:author: Tobias Houska

This is the parent class of all algorithms, which can handle the database
structure during the sample.
'''
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import numpy as np
import io
from itertools import product

class database(object):
    """
    Parent class for database. It can handle the basic functionalities of all
    databases.
    """

    def __init__(self, dbname, parnames, like, randompar, simulations=None,
                 chains=1, save_sim=True, **kwargs):
        # Just needed for the first line in the database
        self.chains = chains
        self.dbname = dbname
        self.like = like
        self.randompar = randompar
        self.simulations = simulations
        self.save_sim = save_sim
        if not save_sim:
            simulations = None
        self.dim_dict = {}
        self.singular_data_lens = [self._check_dims(name, obj) for name, obj in [(
            'like', like), ('par', randompar), ('simulation', simulations)]]
        self._make_header(simulations,parnames)

    def _check_dims(self, name, obj):
        '''checks dimensionality of obj and stores function in dict'''
        if obj is None:
            # None object
            self.dim_dict[name] = self._empty_list
            return (0,)
        elif hasattr(obj, '__len__'):
            if hasattr(obj, 'shape'):
                # np.array style obj
                self.dim_dict[name] = self._array_to_list
                return obj.shape
            elif all([hasattr(x, '__len__') for x in obj]):
                # nested list, checked only for singular nesting
                # assumes all lists have same length
                self.dim_dict[name] = self._nestediterable_to_list
                return (len(obj), len(obj[0]))
            else:
                # simple iterable
                self.dim_dict[name] = self._iterable_to_list
                return (len(obj),)
        else:
            # scalar (int, float)
            self.dim_dict[name] = self._scalar_to_list
            return (1,)

    def _empty_list(self, obj):
        return []

    def _scalar_to_list(self, obj):
        #print('scalar')
        return [obj]

    def _iterable_to_list(self, obj):
        #print('iterable')
        return list(obj)

    def _array_to_list(self, obj):
        #print('array')
        values = []
        for val in obj:
            values.append(val)
        return values
        #return obj.flatten().tolist()

    def _nestediterable_to_list(self, obj):
        #print('nested')
        values = []
        for nestedlist in obj:
            #print(len(nestedlist))
            for val in nestedlist:
                values.append(val)
        #print(len(values))
        return values
        #return np.array(obj).flatten().tolist()

    def _make_header(self, simulations,parnames):
        self.header = []
        self.header.extend(['like' + '_'.join(map(str, x))
                            for x in product(*self._tuple_2_xrange(self.singular_data_lens[0]))])
        self.header.extend(['par{0}'.format(x.decode()) for x in parnames])
        #print(self.singular_data_lens[2])
        #print(type(self.singular_data_lens[2]))
        if self.save_sim:
            for i in range(len(simulations)):
                if type(simulations[0]) == type([]) or type(simulations[0]) == type(np.array([])):
                    for j in range(len(simulations[i])):
                        self.header.extend(['simulation' + str(i+1)+'_'+str(j+1)])
                else:
                    self.header.extend(['simulation' + '_'+str(i)])
                                    #for x in product(*self._tuple_2_xrange(self.singular_data_lens[2]))])

        self.header.append('chain')

    def _tuple_2_xrange(self, t):
        return (range(1, x + 1) for x in t)

class ram(database):
    """
    This class saves the process in the working storage. It can be used if
    time matters.
    """

    def __init__(self, *args, **kwargs):
        # init base class
        super(ram, self).__init__(*args, **kwargs)
        # init the status vars
        self.ram = []
        # store init item only if dbinit
        if kwargs.get('dbinit', True):
            self.save(self.like, self.randompar, self.simulations, self.chains)

    def save(self, objectivefunction, parameterlist, simulations=None,
             chains=1):
        self.ram.append(self.dim_dict['like'](objectivefunction) +
                        self.dim_dict['par'](parameterlist) +
                        self.dim_dict['simulation'](simulations) +
                        [chains])

    def finalize(self):
        #print(self.ram[0:])
        dt = {'names': self.header,
                       'formats': [np.float] * len(self.header)}

        #dt = np.dtype({'names': self.header, 'formats': [np.float] * len(self.header)})

        # ignore the first initialization run to reduce the risk of different
        # objectivefunction mixing
        i = 0
        Y = np.zeros(len(self.ram), dtype=dt)
        for name in dt["names"]:
            Y[name] =  np.transpose(self.ram)[i]
            i+=1

        self.data = Y

    def getdata(self):
        # Expects a finalized database
        return self.data

class csv(database):
    """
    This class saves the process in the working storage. It can be used if
    safety matters.
    """

    def __init__(self, *args, **kwargs):
        # init base class
        super(csv, self).__init__(*args, **kwargs)
        # store init item only if dbinit
        if kwargs.get('dbinit', True):
            # Create a open file, which needs to be closed after the sampling
            self.db = io.open(self.dbname + '.csv', 'w', buffering=1)
            # write header line
            self.db.write(','.join(self.header) + '\n')
            self.save(self.like, self.randompar, self.simulations, self.chains)
        else:
            # Continues writing file
            self.db = io.open(self.dbname + '.csv', 'a')
            self.save(self.like, self.randompar, self.simulations, self.chains)

    def save(self, objectivefunction, parameterlist, simulations=None, chains=1):
        coll = (self.dim_dict['like'](objectivefunction) +
                self.dim_dict['par'](parameterlist) +
                self.dim_dict['simulation'](simulations) +
                [chains])
        try:
            # maybe apply a rounding for the floats?!
            coll = map(np.float16, coll)
            self.db.write(
                ','.join(map(str, coll)) + '\n')
        except IOError:
            input("Please close the file " + self.dbname +
                  " When done press Enter to continue...")
            coll = map(np.float16, coll)
            self.db.write(
                ','.join(map(str, coll)) + '\n')

    def finalize(self):
        self.db.close()

    def getdata(self, dbname=None):
        data = np.genfromtxt(
            self.dbname + '.csv', delimiter=',', names=True)[0:]
        return data

class sql(database):

    """
    This class saves the process in the working storage. It can be used if
    safety matters.
    """

    def __init__(self, *args, **kwargs):
        import sqlite3
        import os
        # init base class
        super(sql, self).__init__(*args, **kwargs)
        # Create a open file, which needs to be closed after the sampling
        try:
            os.remove(self.dbname + '.db')
        except:
            pass
        self.db = sqlite3.connect(self.dbname + '.db')
        self.db_cursor = self.db.cursor()
        # Create Table
#        self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS  '''+self.dbname+'''
#                     (like1 real, parx real, pary real, simulation1 real, chain int)''')
        self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS  '''+self.dbname+'''
                     ('''+' real ,'.join(self.header)+''')''')
        # store init item only if dbinit
        if kwargs.get('dbinit', True):
            self.save(self.like, self.randompar, self.simulations, self.chains)

    def save(self, objectivefunction, parameterlist, simulations=None, chains=1):

        #maybe apply a rounding for the floats?!
        coll = (self.dim_dict['like'](objectivefunction) +
                self.dim_dict['par'](parameterlist) +
                self.dim_dict['simulation'](simulations) +
                [chains])
        try:
            self.db_cursor.execute("INSERT INTO "+self.dbname+" VALUES ("+str(','.join(map(str, coll)))+")")

        except Exception:
            input("Please close the file " + self.dbname +
                  " When done press Enter to continue...")
            self.db_cursor.execute("INSERT INTO "+self.dbname+" VALUES ("+str(','.join(map(str, coll)))+")")

        self.db.commit()

    def finalize(self):
        self.db.close()

    def getdata(self):
        import sqlite3
        self.db = sqlite3.connect(self.dbname + '.db')
        self.db_cursor = self.db.cursor()
        back = [row for row in self.db_cursor.execute('SELECT * FROM '+self.dbname)]
        self.db.close()
        return back

class noData(database):
    """
    This class saves the process in the working storage. It can be used if
    safety matters.
    """

    def __init__(self, *args, **kwargs):
        pass

    def save(self, objectivefunction, parameterlist, simulations=None, chains=1):
        pass

    def finalize(self):
        pass

    def getdata(self):
        pass
<commit_msg>Update database.py<commit_after># -*- coding: utf-8 -*-
'''
Copyright (c) 2015 by Tobias Houska

This file is part of Statistical Parameter Optimization Tool (SPOTPY).

:author: Tobias Houska

This is the parent class of all algorithms, which can handle the database
structure during the sample.
'''
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import numpy as np
import io
from itertools import product

class database(object):
    """
    Parent class for database. It can handle the basic functionalities of all
    databases.
    """

    def __init__(self, dbname, parnames, like, randompar, simulations=None,
                 chains=1, save_sim=True, **kwargs):
        # Just needed for the first line in the database
        self.chains = chains
        self.dbname = dbname
        self.like = like
        self.randompar = randompar
        self.simulations = simulations
        self.save_sim = save_sim
        if not save_sim:
            simulations = None
        self.dim_dict = {}
        self.singular_data_lens = [self._check_dims(name, obj) for name, obj in [(
            'like', like), ('par', randompar), ('simulation', simulations)]]
        self._make_header(simulations,parnames)

    def _check_dims(self, name, obj):
        '''checks dimensionality of obj and stores function in dict'''
        if obj is None:
            # None object
            self.dim_dict[name] = self._empty_list
            return (0,)
        elif hasattr(obj, '__len__'):
            if hasattr(obj, 'shape'):
                # np.array style obj
                self.dim_dict[name] = self._array_to_list
                return obj.shape
            elif all([hasattr(x, '__len__') for x in obj]):
                # nested list, checked only for singular nesting
                # assumes all lists have same length
                self.dim_dict[name] = self._nestediterable_to_list
                return (len(obj), len(obj[0]))
            else:
                # simple iterable
                self.dim_dict[name] = self._iterable_to_list
                return (len(obj),)
        else:
            # scalar (int, float)
            self.dim_dict[name] = self._scalar_to_list
            return (1,)

    def _empty_list(self, obj):
        return []

    def _scalar_to_list(self, obj):
        #print('scalar')
        return [obj]

    def _iterable_to_list(self, obj):
        #print('iterable')
        return list(obj)

    def _array_to_list(self, obj):
        #print('array')
        values = []
        for val in obj:
            values.append(val)
        return values
        #return obj.flatten().tolist()

    def _nestediterable_to_list(self, obj):
        #print('nested')
        values = []
        for nestedlist in obj:
            #print(len(nestedlist))
            for val in nestedlist:
                values.append(val)
        #print(len(values))
        return values
        #return np.array(obj).flatten().tolist()

    def _make_header(self, simulations,parnames):
        self.header = []
        self.header.extend(['like' + '_'.join(map(str, x))
                            for x in product(*self._tuple_2_xrange(self.singular_data_lens[0]))])
        self.header.extend(['par{0}'.format(x.decode()) for x in parnames])
        #print(self.singular_data_lens[2])
        #print(type(self.singular_data_lens[2]))
        if self.save_sim:
            for i in range(len(simulations)):
                if type(simulations[0]) == type([]) or type(simulations[0]) == type(np.array([])):
                    for j in range(len(simulations[i])):
                        self.header.extend(['simulation' + str(i+1)+'_'+str(j+1)])
                else:
                    self.header.extend(['simulation' + '_'+str(i)])
                                    #for x in product(*self._tuple_2_xrange(self.singular_data_lens[2]))])

        self.header.append('chain')

    def _tuple_2_xrange(self, t):
        return (range(1, x + 1) for x in t)

class ram(database):
    """
    This class saves the process in the working storage. It can be used if
    time matters.
    """

    def __init__(self, *args, **kwargs):
        # init base class
        super(ram, self).__init__(*args, **kwargs)
        # init the status vars
        self.ram = []
        # store init item only if dbinit
        if kwargs.get('dbinit', True):
            self.save(self.like, self.randompar, self.simulations, self.chains)

    def save(self, objectivefunction, parameterlist, simulations=None,
             chains=1):
        self.ram.append(self.dim_dict['like'](objectivefunction) +
                        self.dim_dict['par'](parameterlist) +
                        self.dim_dict['simulation'](simulations) +
                        [chains])

    def finalize(self):
        #print(self.ram[0:])
        dt = {'names': self.header,
                       'formats': [np.float] * len(self.header)}

        #dt = np.dtype({'names': self.header, 'formats': [np.float] * len(self.header)})

        # ignore the first initialization run to reduce the risk of different
        # objectivefunction mixing
        i = 0
        Y = np.zeros(len(self.ram), dtype=dt)
        for name in dt["names"]:
            Y[name] =  np.transpose(self.ram)[i]
            i+=1

        self.data = Y

    def getdata(self):
        # Expects a finalized database
        return self.data

class csv(database):
    """
    This class saves the process in the working storage. It can be used if
    safety matters.
    """

    def __init__(self, *args, **kwargs):
        # init base class
        super(csv, self).__init__(*args, **kwargs)
        # store init item only if dbinit
        if kwargs.get('dbinit', True):
            # Create a open file, which needs to be closed after the sampling
            self.db = io.open(self.dbname + '.csv', 'w')
            # write header line
            self.db.write(','.join(self.header) + '\n')
            self.save(self.like, self.randompar, self.simulations, self.chains)
        else:
            # Continues writing file
            self.db = io.open(self.dbname + '.csv', 'a')
            self.save(self.like, self.randompar, self.simulations, self.chains)

    def save(self, objectivefunction, parameterlist, simulations=None, chains=1):
        coll = (self.dim_dict['like'](objectivefunction) +
                self.dim_dict['par'](parameterlist) +
                self.dim_dict['simulation'](simulations) +
                [chains])
        try:
            # maybe apply a rounding for the floats?!
            coll = map(np.float16, coll)
            self.db.write(
                ','.join(map(str, coll)) + '\n')
        except IOError:
            input("Please close the file " + self.dbname +
                  " When done press Enter to continue...")
            coll = map(np.float16, coll)
            self.db.write(
                ','.join(map(str, coll)) + '\n')

    def finalize(self):
        self.db.close()

    def getdata(self, dbname=None):
        data = np.genfromtxt(
            self.dbname + '.csv', delimiter=',', names=True)[0:]
        return data

class sql(database):

    """
    This class saves the process in the working storage. It can be used if
    safety matters.
    """

    def __init__(self, *args, **kwargs):
        import sqlite3
        import os
        # init base class
        super(sql, self).__init__(*args, **kwargs)
        # Create a open file, which needs to be closed after the sampling
        try:
            os.remove(self.dbname + '.db')
        except:
            pass
        self.db = sqlite3.connect(self.dbname + '.db')
        self.db_cursor = self.db.cursor()
        # Create Table
#        self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS  '''+self.dbname+'''
#                     (like1 real, parx real, pary real, simulation1 real, chain int)''')
        self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS  '''+self.dbname+'''
                     ('''+' real ,'.join(self.header)+''')''')
        # store init item only if dbinit
        if kwargs.get('dbinit', True):
            self.save(self.like, self.randompar, self.simulations, self.chains)

    def save(self, objectivefunction, parameterlist, simulations=None, chains=1):

        #maybe apply a rounding for the floats?!
        coll = (self.dim_dict['like'](objectivefunction) +
                self.dim_dict['par'](parameterlist) +
                self.dim_dict['simulation'](simulations) +
                [chains])
        try:
            self.db_cursor.execute("INSERT INTO "+self.dbname+" VALUES ("+str(','.join(map(str, coll)))+")")

        except Exception:
            input("Please close the file " + self.dbname +
                  " When done press Enter to continue...")
            self.db_cursor.execute("INSERT INTO "+self.dbname+" VALUES ("+str(','.join(map(str, coll)))+")")

        self.db.commit()

    def finalize(self):
        self.db.close()

    def getdata(self):
        import sqlite3
        self.db = sqlite3.connect(self.dbname + '.db')
        self.db_cursor = self.db.cursor()
        back = [row for row in self.db_cursor.execute('SELECT * FROM '+self.dbname)]
        self.db.close()
        return back

class noData(database):
    """
    This class saves the process in the working storage. It can be used if
    safety matters.
    """

    def __init__(self, *args, **kwargs):
        pass

    def save(self, objectivefunction, parameterlist, simulations=None, chains=1):
        pass

    def finalize(self):
        pass

    def getdata(self):
        pass
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User
from .facebook_page_tools import page_graph
from django.conf import settings
from moderation.models import Moderator
from .wot_wrapper import is_safe
from .notifications import author_notification, target_notification
from project.loghandler import LogHandler
from facebook import GraphAPIError
# Create your models here.

logger = LogHandler(__name__).logger

# Initial spotted count
initial_count = int(settings.INITIAL_COUNT)

class Spotted(models.Model):
    """Spotted

    fields and methods
    """

    @staticmethod
    def total_spotteds():
        # Returns the latest spotted id + initial_count for the total number of spotteds ever sent
        return Spotted.objects.latest('id').spotted_count

    author = models.ForeignKey(  # User Obj from author
        User,
        related_name='authored',
        blank=True,
        null=True
    )
    target = models.ForeignKey(  # Targeted user
        User,
        related_name='targeted',
        blank=True,
        null=True
    )
    approver = models.ForeignKey(
        Moderator,
        related_name='approved_spotteds',
        blank=True,
        null=True
    )
    created = models.DateTimeField(auto_now_add=True, null=True)
    message = models.TextField()  # Spotted message
    attachment = models.URLField()    # Spotted attachment
    post_id = models.CharField(max_length=100, null=True)  # unique post_id to use with Graph
    api_id = models.IntegerField(default=0)  # unique API id
    dismissed = models.BooleanField(default=False)  # used to save the state of spotteds dismissed by the target
    public = models.BooleanField(default=False)  # Whether the spotted is public or not
    reported = models.CharField(default='', max_length=1000)  # When someone reports a spotted, this is the report text
    share_with_crush = models.BooleanField(default=True)   # Whether or not to let crush know who is the author

    @property
    def spotted_count(self):
        return self.id + initial_count

    @property
    def is_attachment_safe(self):
        return True

    def remove_spotted(self, db_remove=False):
        """Remove Spotted

        tries to delete the spotted from facebook and from the DB if db_remove is True
        """

        try:
            # Remove from facebook
            page_graph().delete_object(self.post_id)
        except Exception as e:
            logger.exception("Exception raised while trying to delete spotted(%s) from Facebook", self.post_id)
            pass
        if db_remove:
            # delete from db
            self.delete()
            return None
        # recreate pending spotted if db_remove is False
        ps = PendingSpotted(message=self.message, author=self.author, target=self.target, attachment=self.attachment)
        ps.save()
        self.delete()
        return ps

    def __str__(self):
        return "Spotted #" + str(self.id)

class PendingSpotted(models.Model):
    """Pending Spotted

    fields and methods
    """

    author = models.ForeignKey(  # User Obj from author
        User,
        on_delete=models.CASCADE,
        related_name='pending_authored',
        blank=True,
        null=True
    )
    target = models.ForeignKey(  # Targeted user
        User,
        on_delete=models.CASCADE,
        related_name='pending_targeted',
        blank=True,
        null=True
    )
    message = models.TextField()  # Spotted message
    attachment = models.URLField(null=True, blank=True)    # Spotted attachment
    public = models.BooleanField(default=False)  # Whether the spotted is public or not
    polemic = models.BooleanField(default=False)
    api_id = models.IntegerField(default=0)  # unique API id
    share_with_crush = models.BooleanField(default=True)   # Whether or not to let crush know who is the author

    suggestion = models.CharField(default='', max_length=100)

    attachment_safe = models.BooleanField(default=False)

    @property
    def is_attachment_safe(self):
        # If there is an attachment and it is not safe
        if not self.attachment_safe and self.attachment:
            # Check if really not safe
            res = is_safe(self.attachment)
            # Update spotted
            self.attachment_safe = res
            self.save()
            # Return its value
            return res

        # Else, is safe
        return True

    def post_spotted(self, mod=None):

        # Only post with attachment if safe
        attachment = self.attachment
        if not self.is_attachment_safe:
            attachment = ''
        if attachment is None:
            attachment = ''

        # Create new Spotted object from self
        s = Spotted(message=self.message, author=self.author, approver=mod, target=self.target, attachment=attachment, public=self.public, api_id=self.api_id, share_with_crush=self.share_with_crush)
        s.save()
        new_id = s.id + initial_count

        # format message
        f_message = "#" + str(new_id) + "\n\n" + self.message

        # post to facebook
        try:
            resp = page_graph().put_wall_post(f_message, {'link': self.attachment})

        # Try to catch invalid url exception
        except GraphAPIError as e:
            # If it was the case, resubmit without link
            if e.message == "The url you supplied is invalid":
                resp = page_graph().put_wall_post(f_message)
            # Else, raise the exception and log it
            else:
                logger.exception("Exception raised while trying to post spotted(%s) to Facebook", self.id)
                raise e
        except Exception as e:
            logger.exception("Exception raised while trying to post spotted(%s) to Facebook", self.id)
            raise e

        # save response post id
        s.post_id = resp['id']
        s.save()
        self.delete()

        # send notifications
        if s.author:
            author_notification(s)
        if s.target:
            target_notification(s)
        return s

    def __str__(self):
        return "PendingSpotted #" + str(self.id)
<commit_msg>set spotted model on delete to set_null<commit_after>from django.db import models
from django.contrib.auth.models import User
from .facebook_page_tools import page_graph
from django.conf import settings
from moderation.models import Moderator
from .wot_wrapper import is_safe
from .notifications import author_notification, target_notification
from project.loghandler import LogHandler
from facebook import GraphAPIError
# Create your models here.

logger = LogHandler(__name__).logger

# Initial spotted count
initial_count = int(settings.INITIAL_COUNT)

class Spotted(models.Model):
    """Spotted

    fields and methods
    """

    @staticmethod
    def total_spotteds():
        # Returns the latest spotted id + initial_count for the total number of spotteds ever sent
        return Spotted.objects.latest('id').spotted_count

    author = models.ForeignKey(  # User Obj from author
        User,
        related_name='authored',
        blank=True,
        null=True,
        on_delete=models.SET_NULL
    )
    target = models.ForeignKey(  # Targeted user
        User,
        related_name='targeted',
        blank=True,
        null=True,
        on_delete=models.SET_NULL
    )
    approver = models.ForeignKey(
        Moderator,
        related_name='approved_spotteds',
        blank=True,
        null=True,
        on_delete=models.SET_NULL
    )
    created = models.DateTimeField(auto_now_add=True, null=True)
    message = models.TextField()  # Spotted message
    attachment = models.URLField()    # Spotted attachment
    post_id = models.CharField(max_length=100, null=True)  # unique post_id to use with Graph
    api_id = models.IntegerField(default=0)  # unique API id
    dismissed = models.BooleanField(default=False)  # used to save the state of spotteds dismissed by the target
    public = models.BooleanField(default=False)  # Whether the spotted is public or not
    reported = models.CharField(default='', max_length=1000)  # When someone reports a spotted, this is the report text
    share_with_crush = models.BooleanField(default=True)   # Whether or not to let crush know who is the author

    @property
    def spotted_count(self):
        return self.id + initial_count

    @property
    def is_attachment_safe(self):
        return True

    def remove_spotted(self, db_remove=False):
        """Remove Spotted

        tries to delete the spotted from facebook and from the DB if db_remove is True
        """

        try:
            # Remove from facebook
            page_graph().delete_object(self.post_id)
        except Exception as e:
            logger.exception("Exception raised while trying to delete spotted(%s) from Facebook", self.post_id)
            pass
        if db_remove:
            # delete from db
            self.delete()
            return None
        # recreate pending spotted if db_remove is False
        ps = PendingSpotted(message=self.message, author=self.author, target=self.target, attachment=self.attachment)
        ps.save()
        self.delete()
        return ps

    def __str__(self):
        return "Spotted #" + str(self.id)

class PendingSpotted(models.Model):
    """Pending Spotted

    fields and methods
    """

    author = models.ForeignKey(  # User Obj from author
        User,
        on_delete=models.SET_NULL,
        related_name='pending_authored',
        blank=True,
        null=True
    )
    target = models.ForeignKey(  # Targeted user
        User,
        on_delete=models.SET_NULL,
        related_name='pending_targeted',
        blank=True,
        null=True
    )
    message = models.TextField()  # Spotted message
    attachment = models.URLField(null=True, blank=True)    # Spotted attachment
    public = models.BooleanField(default=False)  # Whether the spotted is public or not
    polemic = models.BooleanField(default=False)
    api_id = models.IntegerField(default=0)  # unique API id
    share_with_crush = models.BooleanField(default=True)   # Whether or not to let crush know who is the author

    suggestion = models.CharField(default='', max_length=100)

    attachment_safe = models.BooleanField(default=False)

    @property
    def is_attachment_safe(self):
        # If there is an attachment and it is not safe
        if not self.attachment_safe and self.attachment:
            # Check if really not safe
            res = is_safe(self.attachment)
            # Update spotted
            self.attachment_safe = res
            self.save()
            # Return its value
            return res

        # Else, is safe
        return True

    def post_spotted(self, mod=None):

        # Only post with attachment if safe
        attachment = self.attachment
        if not self.is_attachment_safe:
            attachment = ''
        if attachment is None:
            attachment = ''

        # Create new Spotted object from self
        s = Spotted(message=self.message, author=self.author, approver=mod, target=self.target, attachment=attachment, public=self.public, api_id=self.api_id, share_with_crush=self.share_with_crush)
        s.save()
        new_id = s.id + initial_count

        # format message
        f_message = "#" + str(new_id) + "\n\n" + self.message

        # post to facebook
        try:
            resp = page_graph().put_wall_post(f_message, {'link': self.attachment})

        # Try to catch invalid url exception
        except GraphAPIError as e:
            # If it was the case, resubmit without link
            if e.message == "The url you supplied is invalid":
                resp = page_graph().put_wall_post(f_message)
            # Else, raise the exception and log it
            else:
                logger.exception("Exception raised while trying to post spotted(%s) to Facebook", self.id)
                raise e
        except Exception as e:
            logger.exception("Exception raised while trying to post spotted(%s) to Facebook", self.id)
            raise e

        # save response post id
        s.post_id = resp['id']
        s.save()
        self.delete()

        # send notifications
        if s.author:
            author_notification(s)
        if s.target:
            target_notification(s)
        return s

    def __str__(self):
        return "PendingSpotted #" + str(self.id)
<|endoftext|>


<commit_before># Copyright 2020-2021 Foreseeti AB <https://foreseeti.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from securicad.enterprise.client import Client
from securicad.enterprise.models import ModelInfo
from securicad.enterprise.organizations import Organization
from securicad.enterprise.projects import AccessLevel, Project
from securicad.enterprise.scenarios import Scenario
from securicad.enterprise.simulations import Simulation
from securicad.enterprise.tunings import Tuning
from securicad.enterprise.users import Role, User

__version__ = "0.3.2"

def client(*args, **kwargs) -> Client:
    return Client(*args, **kwargs)
<commit_msg>Release 0.3.3 (#48)<commit_after># Copyright 2020-2021 Foreseeti AB <https://foreseeti.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from securicad.enterprise.client import Client
from securicad.enterprise.models import ModelInfo
from securicad.enterprise.organizations import Organization
from securicad.enterprise.projects import AccessLevel, Project
from securicad.enterprise.scenarios import Scenario
from securicad.enterprise.simulations import Simulation
from securicad.enterprise.tunings import Tuning
from securicad.enterprise.users import Role, User

__version__ = "0.3.3"

def client(*args, **kwargs) -> Client:
    return Client(*args, **kwargs)
<|endoftext|>


<commit_before>"""Watch and watch-file data structures"""

from __future__ import absolute_import, print_function, unicode_literals

import fnmatch

class Watch(object):
    """A filename pattern and a shell command"""

    __slots__ = ('pattern', 'command', 'final')

    def __init__(self, pattern, command, final=False):
        self.pattern = pattern
        self.command = command
        self.final = final

    def pattern_matches(self, path):
        """Return true if the given path matches the watch pattern"""
        return fnmatch.fnmatch(path, self.pattern) or path == self.pattern

    def __repr__(self):
        return "<Watch{}: {} -> {}>".format(
            " (final)" if self.final else "", self.pattern, self.command)

    def __str__(self):
        return "watch{}: {} -> {}".format(
            "-final" if self.final else "", self.pattern, self.command)

class WatchList(list):
    """A list of watches, and the associated definitions"""

    def __init__(self, iterable=None, definitions=None,
                 entry_commands=None, exit_commands=None):
        super(WatchList, self).__init__(iterable or list())
        self.definitions = definitions or dict()
        self.entry_commands = entry_commands or list()
        self.exit_commands = exit_commands or list()

    def __repr__(self):
        return "<WatchList: {}, definitions={}, entry={}, exit={}>".format(
            super(WatchList, self).__repr__(), self.definitions,
            self.entry_commands, self.exit_commands)

    def __str__(self):
        strings = []
        for key, value in self.definitions.items():
            strings.append("define: {} -> {}".format(key, value))
        for command in self.entry_commands:
            strings.append("start: {}".format(command))
        for watch in self:
            strings.append(str(watch))
        for command in self.exit_commands:
            strings.append("start: {}".format(command))
        return "\n".join(strings)

class WatchFile(WatchList):
    """A list of watches read directly from a watch-file"""

    def __init__(self, filename):
        super(WatchFile, self).__init__()
        self.read_file(filename)

    def read_file(self, filename):
        """Read each line of a watch-file"""
        with open(filename, 'r') as file:
            lines = [line.strip() for line in file]

        for line in lines:
            if line and not line.startswith('#'):
                self.read_line(line)

    def read_line(self, line):
        """Read in a single line and return a (directive, arguments) tuple"""
        directive, arguments = line.split(':')
        arguments = [a.strip() for a in arguments.split('->')]
        self.read_directive(directive.strip(), *arguments)

    def read_directive(self, directive, *arguments):
        """Read a parsed directive and add it to the current instance"""
        try:
            {
                'define':      self._add_definition,
                'start':       self._add_entry_command,
                'stop':        self._add_exit_command,
                'watch':       self._add_watch,
                'watch-final': self._add_final_watch,
            }[directive](*arguments)
        except KeyError:
            print("Unknown directive '{}'".format(directive))
            raise

    def _add_definition(self, key, value):
        self.definitions[key] = value

    def _add_entry_command(self, command):
        self.entry_commands.append(command)

    def _add_exit_command(self, command):
        self.exit_commands.append(command)

    def _add_watch(self, pattern, command, final=False):
        self.append(Watch(pattern, command, final))

    def _add_final_watch(self, pattern, command):
        self._add_watch(pattern, command, final=True)
<commit_msg>Fixes #5 Only perform one split on the argument.<commit_after>"""Watch and watch-file data structures"""

from __future__ import absolute_import, print_function, unicode_literals

import fnmatch

class Watch(object):
    """A filename pattern and a shell command"""

    __slots__ = ('pattern', 'command', 'final')

    def __init__(self, pattern, command, final=False):
        self.pattern = pattern
        self.command = command
        self.final = final

    def pattern_matches(self, path):
        """Return true if the given path matches the watch pattern"""
        return fnmatch.fnmatch(path, self.pattern) or path == self.pattern

    def __repr__(self):
        return "<Watch{}: {} -> {}>".format(
            " (final)" if self.final else "", self.pattern, self.command)

    def __str__(self):
        return "watch{}: {} -> {}".format(
            "-final" if self.final else "", self.pattern, self.command)

class WatchList(list):
    """A list of watches, and the associated definitions"""

    def __init__(self, iterable=None, definitions=None,
                 entry_commands=None, exit_commands=None):
        super(WatchList, self).__init__(iterable or list())
        self.definitions = definitions or dict()
        self.entry_commands = entry_commands or list()
        self.exit_commands = exit_commands or list()

    def __repr__(self):
        return "<WatchList: {}, definitions={}, entry={}, exit={}>".format(
            super(WatchList, self).__repr__(), self.definitions,
            self.entry_commands, self.exit_commands)

    def __str__(self):
        strings = []
        for key, value in self.definitions.items():
            strings.append("define: {} -> {}".format(key, value))
        for command in self.entry_commands:
            strings.append("start: {}".format(command))
        for watch in self:
            strings.append(str(watch))
        for command in self.exit_commands:
            strings.append("start: {}".format(command))
        return "\n".join(strings)

class WatchFile(WatchList):
    """A list of watches read directly from a watch-file"""

    def __init__(self, filename):
        super(WatchFile, self).__init__()
        self.read_file(filename)

    def read_file(self, filename):
        """Read each line of a watch-file"""
        with open(filename, 'r') as file:
            lines = [line.strip() for line in file]

        for line in lines:
            if line and not line.startswith('#'):
                self.read_line(line)

    def read_line(self, line):
        """Read in a single line and return a (directive, arguments) tuple"""
        directive, arguments = line.split(':', 1)
        arguments = [a.strip() for a in arguments.split('->')]
        self.read_directive(directive.strip(), *arguments)

    def read_directive(self, directive, *arguments):
        """Read a parsed directive and add it to the current instance"""
        try:
            {
                'define':      self._add_definition,
                'start':       self._add_entry_command,
                'stop':        self._add_exit_command,
                'watch':       self._add_watch,
                'watch-final': self._add_final_watch,
            }[directive](*arguments)
        except KeyError:
            print("Unknown directive '{}'".format(directive))
            raise

    def _add_definition(self, key, value):
        self.definitions[key] = value

    def _add_entry_command(self, command):
        self.entry_commands.append(command)

    def _add_exit_command(self, command):
        self.exit_commands.append(command)

    def _add_watch(self, pattern, command, final=False):
        self.append(Watch(pattern, command, final))

    def _add_final_watch(self, pattern, command):
        self._add_watch(pattern, command, final=True)
<|endoftext|>


<commit_before>"""

    PTP library.

"""

from libptp.tools.arachni.arachni import ArachniReport
from libptp.tools.skipfish.skipfish import SkipfishReport
from libptp.tools.w3af.w3af import W3AFReport
from libptp.tools.wapiti.wapiti import WapitiReport

class PTP(object):
    """PTP class definition.

    Usage:
        ptp = PTP()
        ptp.parse(path_to_report, filename)

    """

    # Reports for supported tools.
    supported = {
        'arachni': ArachniReport,
        'skipfish': SkipfishReport,
        'w3af': W3AFReport,
        'wapiti': WapitiReport
        }

    def __init__(self, tool_name=None):
        self.tool_name = tool_name
        self.report = None

    def parse(self, pathname=None, filename=None):
        if self.tool_name is None:
            for tool in self.supported.values():
                if tool.is_mine(pathname, filename=filename):
                    print('Report matched for: ' + tool.__name__)
                    self.report = tool()
                    break
        else:
            try:
                self.report = self.supported[self.tool_name]()
            except KeyError:
                pass
        if self.report is None:
            # TODO: Implement custom exception
            raise ValueError('Unsupported tool.')
        return self.report.parse(pathname, filename)

    def get_highest_ranking(self):
        return self.report.get_highest_ranking()
<commit_msg>Add __str__ for ptp.<commit_after>"""

    PTP library.

"""

from libptp.tools.arachni.arachni import ArachniReport
from libptp.tools.skipfish.skipfish import SkipfishReport
from libptp.tools.w3af.w3af import W3AFReport
from libptp.tools.wapiti.wapiti import WapitiReport

class PTP(object):
    """PTP class definition.

    Usage:
        ptp = PTP()
        ptp.parse(path_to_report, filename)

    """

    # Reports for supported tools.
    supported = {
        'arachni': ArachniReport,
        'skipfish': SkipfishReport,
        'w3af': W3AFReport,
        'wapiti': WapitiReport
        }

    def __init__(self, tool_name=None):
        self.tool_name = tool_name
        self.report = None

    def __str__(self):
        return self.report.__str__()

    def parse(self, pathname=None, filename=None):
        if self.tool_name is None:
            for tool in self.supported.values():
                if tool.is_mine(pathname, filename=filename):
                    print('Report matched for: ' + tool.__name__)
                    self.report = tool()
                    break
        else:
            try:
                self.report = self.supported[self.tool_name]()
            except KeyError:
                pass
        if self.report is None:
            # TODO: Implement custom exception
            raise ValueError('Unsupported tool.')
        return self.report.parse(pathname, filename)

    def get_highest_ranking(self):
        return self.report.get_highest_ranking()
<|endoftext|>