<commit_before>import os

class System(object):
    @staticmethod
    def is_unix():
        return os.name != 'nt'

    @staticmethod
    def hardlink(source, link_name):
        import ctypes
        from dvc.exceptions import DvcException

        if System.is_unix():
            try:
                os.link(source, link_name)
                return
            except Exception as exc:
                raise DvcException('link', cause=exc)

        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
        CreateHardLink.argtypes = [ctypes.c_wchar_p,
                                   ctypes.c_wchar_p,
                                   ctypes.c_void_p]
        CreateHardLink.restype = ctypes.wintypes.BOOL

        res = CreateHardLink(link_name, source, None)
        if res == 0:
            raise DvcException('CreateHardLinkW', cause=ctypes.WinError())

    @staticmethod
    def symlink(source, link_name):
        import ctypes
        from dvc.exceptions import DvcException

        if System.is_unix():
            try:
                os.symlink(source, link_name)
                return
            except Exception as exc:
                msg = "Failed to symlink '{}' -> '{}': {}"
                raise DvcException(msg.format(source, link_name, str(exc)))

        flags = 0
        if source is not None and os.path.isdir(source):
            flags = 1

        func = ctypes.windll.kernel32.CreateSymbolicLinkW
        func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
        func.restype = ctypes.c_ubyte

        if func(link_name, source, flags) == 0:
            raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError())

    @staticmethod
    def _reflink_darwin(src, dst):
        import ctypes

        clib = ctypes.CDLL('libc.dylib')
        if not hasattr(clib, 'clonefile'):
            return -1

        clonefile = clib.clonefile
        clonefile.argtypes = [ctypes.c_char_p,
                              ctypes.c_char_p,
                              ctypes.c_int]
        clonefile.restype = ctypes.c_int

        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
                         ctypes.c_char_p(dst.encode('utf-8')),
                         ctypes.c_int(0))

    @staticmethod
    def _reflink_windows(src, dst):
        return -1

    @staticmethod
    def _reflink_linux(src, dst):
        import os
        import fcntl

        FICLONE = 0x40049409

        s = open(src, 'r')
        d = open(dst, 'w+')

        try:
            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
        except IOError:
            s.close()
            d.close()
            os.unlink(dst)
            raise

        s.close()
        d.close()

        if ret != 0:
            os.unlink(dst)

        return ret

    @staticmethod
    def reflink(source, link_name):
        import platform
        from dvc.exceptions import DvcException

        system = platform.system()
        try:
            if system == 'Windows':
                ret = System._reflink_windows(source, link_name)
            elif system == 'Darwin':
                ret = System._reflink_darwin(source, link_name)
            elif system == 'Linux':
                ret = System._reflink_linux(source, link_name)
            else:
                ret = -1
        except IOError:
            ret = -1

        if ret != 0:
            raise DvcException('Reflink is not supported')

    @staticmethod
    def getdirinfo(path):
        import ctypes
        from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
        from ctypes.wintypes import DWORD, HANDLE, BOOL

        # NOTE: use this flag to open symlink itself and not the target
        # See https://docs.microsoft.com/en-us/windows/desktop/api/
        # fileapi/nf-fileapi-createfilew#symbolic-link-behavior
        FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000

        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
        FILE_SHARE_READ = 0x00000001
        OPEN_EXISTING = 3

        class FILETIME(Structure):
            _fields_ = [("dwLowDateTime", DWORD),
                        ("dwHighDateTime", DWORD)]

        class BY_HANDLE_FILE_INFORMATION(Structure):
            _fields_ = [("dwFileAttributes", DWORD),
                        ("ftCreationTime", FILETIME),
                        ("ftLastAccessTime", FILETIME),
                        ("ftLastWriteTime", FILETIME),
                        ("dwVolumeSerialNumber", DWORD),
                        ("nFileSizeHigh", DWORD),
                        ("nFileSizeLow", DWORD),
                        ("nNumberOfLinks", DWORD),
                        ("nFileIndexHigh", DWORD),
                        ("nFileIndexLow", DWORD)]

        flags = FILE_FLAG_BACKUP_SEMANTICS & FILE_FLAG_OPEN_REPARSE_POINT

        func = ctypes.windll.kernel32.CreateFileW
        func.argtypes = [c_wchar_p,
                         DWORD,
                         DWORD,
                         c_void_p,
                         DWORD,
                         DWORD,
                         HANDLE]
        func.restype = HANDLE

        hfile = func(path,
                     0,
                     FILE_SHARE_READ,
                     None,
                     OPEN_EXISTING,
                     flags,
                     None)
        if hfile is None:
            raise WinError()

        func = ctypes.windll.kernel32.GetFileInformationByHandle
        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
        func.restype = BOOL

        info = BY_HANDLE_FILE_INFORMATION()
        rv = func(hfile, info)

        func = ctypes.windll.kernel32.CloseHandle
        func.argtypes = [HANDLE]
        func.restype = BOOL

        func(hfile)

        if rv == 0:
            raise WinError()

        return info

    @staticmethod
    def inode(path):
        if System.is_unix():
            import ctypes
            inode = os.lstat(path).st_ino
            # NOTE: See https://bugs.python.org/issue29619 and
            # https://stackoverflow.com/questions/34643289/
            # pythons-os-stat-is-returning-wrong-inode-value
            inode = ctypes.c_ulong(inode).value
        else:
            # getdirinfo from ntfsutils works on both files and dirs
            info = System.getdirinfo(path)
            inode = abs(hash((info.dwVolumeSerialNumber,
                              info.nFileIndexHigh,
                              info.nFileIndexLow)))
        assert inode >= 0
        assert inode < 2**64
        return inode

    @staticmethod
    def _wait_for_input_windows(timeout):
        import sys
        import ctypes
        import msvcrt
        from ctypes.wintypes import DWORD, HANDLE

        # https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-waitforsingleobject
        WAIT_OBJECT_0 = 0
        WAIT_TIMEOUT = 0x00000102

        func = ctypes.windll.kernel32.WaitForSingleObject
        func.argtypes = [HANDLE, DWORD]
        func.restype = DWORD

        rc = func(msvcrt.get_osfhandle(sys.stdin.fileno()), timeout * 1000)
        if rc not in [WAIT_OBJECT_0, WAIT_TIMEOUT]:
            raise RuntimeError(rc)

    @staticmethod
    def _wait_for_input_posix(timeout):
        import sys
        import select
        try:
            select.select([sys.stdin], [], [], timeout)
        except select.error:
            pass

    @staticmethod
    def wait_for_input(timeout):
        if System.is_unix():
            return System._wait_for_input_posix(timeout)
        else:
            return System._wait_for_input_windows(timeout)

    @staticmethod
    def is_symlink(path):
        if System.is_unix():
            return os.path.islink(path)

        # https://docs.microsoft.com/en-us/windows/desktop/fileio/
        # file-attribute-constants
        FILE_ATTRIBUTE_REPARSE_POINT = 0x400
        info = System.getdirinfo(path)
        return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT

    @staticmethod
    def is_hardlink(path):
        if System.is_unix():
            return os.stat(path).st_nlink > 1

        info = System.getdirinfo(path)
        return info.nNumberOfLinks > 1
<commit_msg>system: fix typo<commit_after>import os

class System(object):
    @staticmethod
    def is_unix():
        return os.name != 'nt'

    @staticmethod
    def hardlink(source, link_name):
        import ctypes
        from dvc.exceptions import DvcException

        if System.is_unix():
            try:
                os.link(source, link_name)
                return
            except Exception as exc:
                raise DvcException('link', cause=exc)

        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
        CreateHardLink.argtypes = [ctypes.c_wchar_p,
                                   ctypes.c_wchar_p,
                                   ctypes.c_void_p]
        CreateHardLink.restype = ctypes.wintypes.BOOL

        res = CreateHardLink(link_name, source, None)
        if res == 0:
            raise DvcException('CreateHardLinkW', cause=ctypes.WinError())

    @staticmethod
    def symlink(source, link_name):
        import ctypes
        from dvc.exceptions import DvcException

        if System.is_unix():
            try:
                os.symlink(source, link_name)
                return
            except Exception as exc:
                msg = "Failed to symlink '{}' -> '{}': {}"
                raise DvcException(msg.format(source, link_name, str(exc)))

        flags = 0
        if source is not None and os.path.isdir(source):
            flags = 1

        func = ctypes.windll.kernel32.CreateSymbolicLinkW
        func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
        func.restype = ctypes.c_ubyte

        if func(link_name, source, flags) == 0:
            raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError())

    @staticmethod
    def _reflink_darwin(src, dst):
        import ctypes

        clib = ctypes.CDLL('libc.dylib')
        if not hasattr(clib, 'clonefile'):
            return -1

        clonefile = clib.clonefile
        clonefile.argtypes = [ctypes.c_char_p,
                              ctypes.c_char_p,
                              ctypes.c_int]
        clonefile.restype = ctypes.c_int

        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
                         ctypes.c_char_p(dst.encode('utf-8')),
                         ctypes.c_int(0))

    @staticmethod
    def _reflink_windows(src, dst):
        return -1

    @staticmethod
    def _reflink_linux(src, dst):
        import os
        import fcntl

        FICLONE = 0x40049409

        s = open(src, 'r')
        d = open(dst, 'w+')

        try:
            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
        except IOError:
            s.close()
            d.close()
            os.unlink(dst)
            raise

        s.close()
        d.close()

        if ret != 0:
            os.unlink(dst)

        return ret

    @staticmethod
    def reflink(source, link_name):
        import platform
        from dvc.exceptions import DvcException

        system = platform.system()
        try:
            if system == 'Windows':
                ret = System._reflink_windows(source, link_name)
            elif system == 'Darwin':
                ret = System._reflink_darwin(source, link_name)
            elif system == 'Linux':
                ret = System._reflink_linux(source, link_name)
            else:
                ret = -1
        except IOError:
            ret = -1

        if ret != 0:
            raise DvcException('Reflink is not supported')

    @staticmethod
    def getdirinfo(path):
        import ctypes
        from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
        from ctypes.wintypes import DWORD, HANDLE, BOOL

        # NOTE: use this flag to open symlink itself and not the target
        # See https://docs.microsoft.com/en-us/windows/desktop/api/
        # fileapi/nf-fileapi-createfilew#symbolic-link-behavior
        FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000

        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
        FILE_SHARE_READ = 0x00000001
        OPEN_EXISTING = 3

        class FILETIME(Structure):
            _fields_ = [("dwLowDateTime", DWORD),
                        ("dwHighDateTime", DWORD)]

        class BY_HANDLE_FILE_INFORMATION(Structure):
            _fields_ = [("dwFileAttributes", DWORD),
                        ("ftCreationTime", FILETIME),
                        ("ftLastAccessTime", FILETIME),
                        ("ftLastWriteTime", FILETIME),
                        ("dwVolumeSerialNumber", DWORD),
                        ("nFileSizeHigh", DWORD),
                        ("nFileSizeLow", DWORD),
                        ("nNumberOfLinks", DWORD),
                        ("nFileIndexHigh", DWORD),
                        ("nFileIndexLow", DWORD)]

        flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT

        func = ctypes.windll.kernel32.CreateFileW
        func.argtypes = [c_wchar_p,
                         DWORD,
                         DWORD,
                         c_void_p,
                         DWORD,
                         DWORD,
                         HANDLE]
        func.restype = HANDLE

        hfile = func(path,
                     0,
                     FILE_SHARE_READ,
                     None,
                     OPEN_EXISTING,
                     flags,
                     None)
        if hfile is None:
            raise WinError()

        func = ctypes.windll.kernel32.GetFileInformationByHandle
        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
        func.restype = BOOL

        info = BY_HANDLE_FILE_INFORMATION()
        rv = func(hfile, info)

        func = ctypes.windll.kernel32.CloseHandle
        func.argtypes = [HANDLE]
        func.restype = BOOL

        func(hfile)

        if rv == 0:
            raise WinError()

        return info

    @staticmethod
    def inode(path):
        if System.is_unix():
            import ctypes
            inode = os.lstat(path).st_ino
            # NOTE: See https://bugs.python.org/issue29619 and
            # https://stackoverflow.com/questions/34643289/
            # pythons-os-stat-is-returning-wrong-inode-value
            inode = ctypes.c_ulong(inode).value
        else:
            # getdirinfo from ntfsutils works on both files and dirs
            info = System.getdirinfo(path)
            inode = abs(hash((info.dwVolumeSerialNumber,
                              info.nFileIndexHigh,
                              info.nFileIndexLow)))
        assert inode >= 0
        assert inode < 2**64
        return inode

    @staticmethod
    def _wait_for_input_windows(timeout):
        import sys
        import ctypes
        import msvcrt
        from ctypes.wintypes import DWORD, HANDLE

        # https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-waitforsingleobject
        WAIT_OBJECT_0 = 0
        WAIT_TIMEOUT = 0x00000102

        func = ctypes.windll.kernel32.WaitForSingleObject
        func.argtypes = [HANDLE, DWORD]
        func.restype = DWORD

        rc = func(msvcrt.get_osfhandle(sys.stdin.fileno()), timeout * 1000)
        if rc not in [WAIT_OBJECT_0, WAIT_TIMEOUT]:
            raise RuntimeError(rc)

    @staticmethod
    def _wait_for_input_posix(timeout):
        import sys
        import select
        try:
            select.select([sys.stdin], [], [], timeout)
        except select.error:
            pass

    @staticmethod
    def wait_for_input(timeout):
        if System.is_unix():
            return System._wait_for_input_posix(timeout)
        else:
            return System._wait_for_input_windows(timeout)

    @staticmethod
    def is_symlink(path):
        if System.is_unix():
            return os.path.islink(path)

        # https://docs.microsoft.com/en-us/windows/desktop/fileio/
        # file-attribute-constants
        FILE_ATTRIBUTE_REPARSE_POINT = 0x400
        info = System.getdirinfo(path)
        return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT

    @staticmethod
    def is_hardlink(path):
        if System.is_unix():
            return os.stat(path).st_nlink > 1

        info = System.getdirinfo(path)
        return info.nNumberOfLinks > 1
<|endoftext|>


<commit_before>"""Page layout of mara app"""

import functools
import re
import subprocess

import flask
import mara_page.response
from mara_app import config
from mara_page import navigation, _, xml

def layout(response: mara_page.response.Response) -> str:
    """Renders a complete html page for the response"""
    return '<!DOCTYPE html>\n' + str(
        _.html(lang='en')[
            _.head[
                head_elements(response)
            ],
            _.body(_class='navigation-collapsed')[
                body_elements(response)
            ]
        ])

def head_elements(response: mara_page.response.Response) -> [xml.XMLElement]:
    """All elements in the head section of the html page"""
    return [
        _.meta(charset='utf-8'),
        _.meta(name='viewport', content='width=device-width, initial-scale=1, shrink-to-fit=no'),
        _.title[re.sub(r'<[^>]*?>', '', response.title)],
        [_.link(rel='stylesheet', href=url + '?' + _current_git_commit())[''] for url in css_files(response)],
        _.link(rel='icon', type='image/png', href=config.favicon_url() + '?' + _current_git_commit())
    ]

def body_elements(response: mara_page.response.Response) -> [xml.XMLElement]:
    """All elements inside the body section of the html page"""
    return [
        _.script['var isTouchDevice = ("ontouchstart" in document.documentElement); ',
                 'window.document.getElementsByTagName("body")[0].className += isTouchDevice ? " touch" : " no-touch";'],
        page_header(response),
        navigation_bar(),
        content_area(response),
        [_.script(src=url + '?' + _current_git_commit())[''] for url in js_files(response)],
        flash_messages(response)
    ]

def css_files(response: mara_page.response.Response) -> [xml.XMLElement]:
    """The list of all css files to include in the page"""
    return [flask.url_for('mara_app.static', filename='bootstrap-4.0.0-alpha.6/bootstrap.min.css'),
            flask.url_for('mara_app.static', filename='font-awesome-4.7.0/css/font-awesome.min.css'),
            flask.url_for('mara_app.static', filename='tether-1.3.3/tether.min.css'),
            flask.url_for('mara_app.static', filename='mara.css')
            ] + response.css_files

def js_files(response: mara_page.response.Response):
    """The list of all js files to include in the page"""
    return [flask.url_for('mara_app.static', filename='jquery-3.1.1.min.js'),
            flask.url_for('mara_app.static', filename='tether-1.3.3/tether.min.js'),
            flask.url_for('mara_app.static', filename='bootstrap-4.0.0-alpha.6/bootstrap.min.js'),
            flask.url_for('mara_app.static', filename='mara.js')
            ] + response.js_files

def page_header(response: mara_page.response.Response):
    """Renders the fixed top part of the page"""
    return _.nav(id='mara-page-header', class_='navbar fixed-top')[
        _.a(class_='navigation-toggle-button fa fa-lg fa-reorder', onclick='toggleNavigation()')[' '],
        _.h1()[response.title],
        _.img(src=config.logo_url() + '?' + _current_git_commit()),
        _.span(class_='action-buttons')[map(action_button, response.action_buttons)]]

def action_button(button: mara_page.response.ActionButton):
    """Renders an action button"""
    return [_.a(class_='btn', href=button.action, title=button.title)[
                _.span(class_='fa fa-' + button.icon)[''], ' ',
                button.label]]

@functools.lru_cache(maxsize=None)
def navigation_bar() -> str:
    """Renders the navigation sidebar"""

    def render_entries(entries: [navigation.NavigationEntry] = [], level: int = 1):
        def render_entry(entry: navigation.NavigationEntry, level: int = 1):
            attrs = {}
            if entry.children:
                attrs.update({'onClick': 'toggleNavigationEntry(this)'})
            else:
                attrs.update({'onClick': 'highlightNavigationEntry(\'' + entry.uri_fn() + '\');',
                              'href': entry.uri_fn()})

            if entry.description:
                attrs.update({'title': entry.description, 'data-toggle': 'tooltip',
                              'data-container': 'body', 'data-placement': 'right'})
            return _.div(class_='mara-nav-entry level-' + str(level),
                         style='display:none' if level > 1 else '')[
                _.a(**attrs)[
                    _.div(class_='mara-nav-entry-icon fa fa-fw fa-' + entry.icon + (' fa-lg' if level == 1 else ''))[
                        ''] if entry.icon else '',
                    _.div(class_='mara-nav-entry-text')[entry.label.replace('_', '_<wbr>')],
                    _.div(class_='mara-caret fa fa-caret-down')[''] if entry.children else ''],
                render_entries(entry.children, level + 1)
            ]

        return [functools.partial(render_entry, level=level)(entry)
                for entry in sorted([entry for entry in entries if entry.visible], key=lambda x: x.rank)]

    return str(_.nav(id='mara-navigation', class_='nav')[render_entries(flask.current_app.navigation_root.children)])

def content_area(response: mara_page.response.Response) -> xml.XMLElement:
    """Renders the main content area"""
    return _.div(id='mara-main', class_='container-fluid')[response.html]

def flash_messages(response: mara_page.response.Response) -> xml.XMLElement:
    """Displays flask flash messages"""
    return [_.div(id='alerts'),
            _.script(type='text/javascript')[
                map(lambda m: 'showAlert("' + m[1].replace('"', '"') + '","' + (
                    m[0] if m[0] != 'message' else 'info') + '");',
                    flask.get_flashed_messages(True))

            ]]

@functools.lru_cache(maxsize=None)
def _current_git_commit():
    """Returns the current git commit of the mara application"""
    command = f'git --git-dir {flask.current_app.root_path}/.git rev-parse HEAD'
    return subprocess.getoutput(command)
<commit_msg>remove redundant representation of response body<commit_after>"""Page layout of mara app"""

import functools
import re
import subprocess

import flask
import mara_page.response
from mara_app import config
from mara_page import navigation, _, xml

def layout(response: mara_page.response.Response) -> str:
    """Renders a complete html page for the response"""
    return '<!DOCTYPE html>\n' + str(
        _.html(lang='en')[
            _.head[
                head_elements(response)
            ],
            _.body(_class='navigation-collapsed')[
                body_elements(response)
            ]
        ])

def head_elements(response: mara_page.response.Response) -> [xml.XMLElement]:
    """All elements in the head section of the html page"""
    return [
        _.meta(charset='utf-8'),
        _.meta(name='viewport', content='width=device-width, initial-scale=1, shrink-to-fit=no'),
        _.title[re.sub(r'<[^>]*?>', '', response.title)],
        [_.link(rel='stylesheet', href=url + '?' + _current_git_commit())[''] for url in css_files(response)],
        _.link(rel='icon', type='image/png', href=config.favicon_url() + '?' + _current_git_commit())
    ]

def body_elements(response: mara_page.response.Response) -> [xml.XMLElement]:
    """All elements inside the body section of the html page"""
    return [
        _.script['var isTouchDevice = ("ontouchstart" in document.documentElement); ',
                 'window.document.getElementsByTagName("body")[0].className += isTouchDevice ? " touch" : " no-touch";'],
        page_header(response),
        navigation_bar(),
        content_area(response),
        [_.script(src=url + '?' + _current_git_commit())[''] for url in js_files(response)],
        flash_messages(response)
    ]

def css_files(response: mara_page.response.Response) -> [xml.XMLElement]:
    """The list of all css files to include in the page"""
    return [flask.url_for('mara_app.static', filename='bootstrap-4.0.0-alpha.6/bootstrap.min.css'),
            flask.url_for('mara_app.static', filename='font-awesome-4.7.0/css/font-awesome.min.css'),
            flask.url_for('mara_app.static', filename='tether-1.3.3/tether.min.css'),
            flask.url_for('mara_app.static', filename='mara.css')
            ] + response.css_files

def js_files(response: mara_page.response.Response):
    """The list of all js files to include in the page"""
    return [flask.url_for('mara_app.static', filename='jquery-3.1.1.min.js'),
            flask.url_for('mara_app.static', filename='tether-1.3.3/tether.min.js'),
            flask.url_for('mara_app.static', filename='bootstrap-4.0.0-alpha.6/bootstrap.min.js'),
            flask.url_for('mara_app.static', filename='mara.js')
            ] + response.js_files

def page_header(response: mara_page.response.Response):
    """Renders the fixed top part of the page"""
    return _.nav(id='mara-page-header', class_='navbar fixed-top')[
        _.a(class_='navigation-toggle-button fa fa-lg fa-reorder', onclick='toggleNavigation()')[' '],
        _.h1()[response.title],
        _.img(src=config.logo_url() + '?' + _current_git_commit()),
        _.span(class_='action-buttons')[map(action_button, response.action_buttons)]]

def action_button(button: mara_page.response.ActionButton):
    """Renders an action button"""
    return [_.a(class_='btn', href=button.action, title=button.title)[
                _.span(class_='fa fa-' + button.icon)[''], ' ',
                button.label]]

@functools.lru_cache(maxsize=None)
def navigation_bar() -> str:
    """Renders the navigation sidebar"""

    def render_entries(entries: [navigation.NavigationEntry] = [], level: int = 1):
        def render_entry(entry: navigation.NavigationEntry, level: int = 1):
            attrs = {}
            if entry.children:
                attrs.update({'onClick': 'toggleNavigationEntry(this)'})
            else:
                attrs.update({'onClick': 'highlightNavigationEntry(\'' + entry.uri_fn() + '\');',
                              'href': entry.uri_fn()})

            if entry.description:
                attrs.update({'title': entry.description, 'data-toggle': 'tooltip',
                              'data-container': 'body', 'data-placement': 'right'})
            return _.div(class_='mara-nav-entry level-' + str(level),
                         style='display:none' if level > 1 else '')[
                _.a(**attrs)[
                    _.div(class_='mara-nav-entry-icon fa fa-fw fa-' + entry.icon + (' fa-lg' if level == 1 else ''))[
                        ''] if entry.icon else '',
                    _.div(class_='mara-nav-entry-text')[entry.label.replace('_', '_<wbr>')],
                    _.div(class_='mara-caret fa fa-caret-down')[''] if entry.children else ''],
                render_entries(entry.children, level + 1)
            ]

        return [functools.partial(render_entry, level=level)(entry)
                for entry in sorted([entry for entry in entries if entry.visible], key=lambda x: x.rank)]

    return str(_.nav(id='mara-navigation', class_='nav')[render_entries(flask.current_app.navigation_root.children)])

def content_area(response: mara_page.response.Response) -> xml.XMLElement:
    """Renders the main content area"""
    return _.div(id='mara-main', class_='container-fluid')[response.response]

def flash_messages(response: mara_page.response.Response) -> xml.XMLElement:
    """Displays flask flash messages"""
    return [_.div(id='alerts'),
            _.script(type='text/javascript')[
                map(lambda m: 'showAlert("' + m[1].replace('"', '"') + '","' + (
                    m[0] if m[0] != 'message' else 'info') + '");',
                    flask.get_flashed_messages(True))

            ]]

@functools.lru_cache(maxsize=None)
def _current_git_commit():
    """Returns the current git commit of the mara application"""
    command = f'git --git-dir {flask.current_app.root_path}/.git rev-parse HEAD'
    return subprocess.getoutput(command)
<|endoftext|>


<commit_before>import flask
from flask import jsonify, abort, make_response

from models.models import *
from services import nlpService, fileService
from services.factService import FactService
from services.staticStrings import *

########################
# Conversation Handling
########################

def get_conversation(conversation_id):
    conversation = __get_conversation(conversation_id)

    return ConversationSchema().jsonify(conversation)

def init_conversation(name):
    conversation = Conversation(name=name)

    # Persist new conversation to DB
    db.session.add(conversation)
    db.session.commit()

    return jsonify(
        {
            'conversation_id': conversation.id
        }
    )

def receive_message(conversation_id, message):
    conversation = __get_conversation(conversation_id)

    response_text = None
    response_html = None
    file_request = None
    possible_answers = None
    additional_info = None
    enforce_possible_answer = False
    # First message in the conversation
    if len(conversation.messages) == 0:
        response_html = StaticStrings.chooseFrom(StaticStrings.disclaimer).format(name=conversation.name)
        possible_answers = ["Yes"]
        enforce_possible_answer = True
    else:
        # Add user's message
        user_message = Message(sender_type=SenderType.USER, text=message)
        conversation.messages.append(user_message)

        # Generate response text & optional parameters
        response = __generate_response(conversation, user_message.text)
        response_text = response.get('response_text')
        file_request = response.get('file_request')
        possible_answers = response.get('possible_answers')

    # Persist response message
    if response_text is not None:
        response = Message(
            sender_type=SenderType.BOT,
            text=response_text,
            possible_answers=str(possible_answers)
        )
    elif response_html is not None:
        response = Message(
            sender_type=SenderType.BOT,
            text=response_html,
            possible_answers=str(possible_answers)
        )
    else:
        return abort(make_response(jsonify(message="Response text not generated"), 400))

    # Create relationship between message and file request if present
    if file_request is not None:
        response.file_request = file_request

    conversation.messages.append(response)

    # Commit
    db.session.commit()

    # Build response dict
    response_dict = {'conversation_id': conversation.id}

    if response_text is not None:
        response_dict['message'] = response_text
    if response_html is not None:
        response_dict['html'] = response_html
    if file_request is not None:
        response_dict['file_request'] = FileRequestSchema().dump(file_request).data
    if possible_answers is not None:
        response_dict['possible_answers'] = possible_answers
        if enforce_possible_answer:
            response_dict['enforce_possible_answer'] = True

    return jsonify(response_dict)

################
# File Handling
################

def get_file_list(conversation_id):
    conversation = __get_conversation(conversation_id)

    return jsonify(
        {
            'files': [FileSchema().dump(file).data for file in conversation.files]
        }
    )

def upload_file(conversation_id, file):
    conversation = __get_conversation(conversation_id)

    # Check if the file has a filename
    if file.filename == '':
        abort(make_response(jsonify(message="No file selected"), 400))

    if fileService.is_accepted_format(file):
        # Create the file and commit it to generate id
        new_file = File(name=fileService.sanitize_name(file), type=file.content_type)
        conversation.files.append(new_file)
        db.session.commit()

        # Generate path information and upload file to disk
        new_file.path = fileService.generate_path(conversation.id, new_file.id)
        fileService.upload_file(file, new_file.path, new_file.name)
        db.session.commit()

        # Return the file info
        return FileSchema().jsonify(new_file)
    else:
        abort(make_response(
            jsonify(message="Filetype {} is not supported. Supported filetypes are {}.".format(
                fileService.get_file_extension(file), fileService.get_accepted_formats_string())), 400))

def get_file(conversation_id, file_id):
    conversation = __get_conversation(conversation_id)

    for file in conversation.files:
        if file.id == int(file_id):
            return flask.send_from_directory(file.path, file.name, mimetype=file.type, as_attachment=True)

    return abort(make_response(jsonify(message="File does not exist"), 404))

##################
# Private Methods
##################

def __get_conversation(conversation_id):
    conversation = Conversation.query.get(conversation_id)

    if conversation:
        return conversation

    abort(make_response(jsonify(message="Conversation does not exist"), 404))

def __generate_response(conversation, message):
    if __has_just_accepted_disclaimer(conversation):
        response_text = StaticStrings.chooseFrom(StaticStrings.welcome).format(name=conversation.name)
        return {'response_text': response_text}
    if conversation.person_type is None:
        return __determine_person_type(conversation, message)
    elif conversation.claim_category is None:
        return __determine_claim_category(conversation, message)
    elif conversation.current_fact is not None:
        # Assume it is an answer to the current fact
        nlp_request = nlpService.fact_extract([conversation.current_fact], message=message)

        for resolved_fact in nlp_request['facts']:
            for fact_name, fact_value in resolved_fact.items():
                new_fact = Fact(name=fact_name, value=fact_value)
                conversation.facts.append(new_fact)

        db.session.commit()
        question = __probe_facts(conversation)

        return {'response_text': question}

def __determine_person_type(conversation, message):
    person_type = None
    nlp_request = nlpService.fact_extract(['tenant_landlord'], message)

    for fact in nlp_request['facts']:
        person_type = fact['tenant_landlord']

    if person_type is not None:
        conversation.person_type = {
            'tenant': PersonType.TENANT,
            'landlord': PersonType.LANDLORD
        }[person_type]

        file_request = None
        if person_type == 'tenant':
            file_request = FileRequest(document_type=DocumentType.LEASE)

        db.session.commit()

        # Generate response based on person type
        response = None
        if person_type == 'tenant':
            response = StaticStrings.chooseFrom(StaticStrings.problem_inquiry_tenant).format(name=conversation.name)
        elif person_type == 'landlord':
            response = StaticStrings.chooseFrom(StaticStrings.problem_inquiry_landlord).format(name=conversation.name)

        return {'response_text': response, 'file_request': file_request}
    else:
        return {'response_text': StaticStrings.chooseFrom(StaticStrings.clarify)}

def __determine_claim_category(conversation, message):
    claim_category = None
    nlp_request = nlpService.problem_category(message)

    for fact in nlp_request['facts']:
        claim_category = fact['category']

    if claim_category is not None:
        conversation.claim_category = {
            'lease_termination': ClaimCategory.LEASE_TERMINATION,
            'rent_change': ClaimCategory.RENT_CHANGE,
            'nonpayment': ClaimCategory.NONPAYMENT,
            'deposits': ClaimCategory.DEPOSITS
        }[claim_category]

        db.session.commit()

        # Generate the first question
        first_question = __probe_facts(conversation)

        response = StaticStrings.chooseFrom(StaticStrings.category_acknowledge).format(
            claim_category=conversation.claim_category.value.lower().replace("_", " "), first_question=first_question)

        return {'response_text': response}
    else:
        return {'response_text': StaticStrings.chooseFrom(StaticStrings.clarify)}

def __probe_facts(conversation):
    resolved_facts = [fact.name for fact in conversation.facts]
    fact, question = FactService.get_question(conversation.claim_category.value.lower(), resolved_facts)

    if fact is not None:
        # Update fact being asked
        conversation.current_fact = fact
        db.session.commit()
    else:
        fact_dump = ''.join(repr(conversation.facts))
        question = "FACT DUMP: {}".format(fact_dump)

    return question

def __has_just_accepted_disclaimer(conversation):
    return len(conversation.messages) == 2
<commit_msg>[#5] Always returns possible_answers field within Message as a string<commit_after>import flask
from flask import jsonify, abort, make_response

from models.models import *
from services import nlpService, fileService
from services.factService import FactService
from services.staticStrings import *

########################
# Conversation Handling
########################

def get_conversation(conversation_id):
    conversation = __get_conversation(conversation_id)

    return ConversationSchema().jsonify(conversation)

def init_conversation(name):
    conversation = Conversation(name=name)

    # Persist new conversation to DB
    db.session.add(conversation)
    db.session.commit()

    return jsonify(
        {
            'conversation_id': conversation.id
        }
    )

def receive_message(conversation_id, message):
    conversation = __get_conversation(conversation_id)

    response_text = None
    response_html = None
    file_request = None
    possible_answers = None
    additional_info = None
    enforce_possible_answer = False
    # First message in the conversation
    if len(conversation.messages) == 0:
        response_html = StaticStrings.chooseFrom(StaticStrings.disclaimer).format(name=conversation.name)
        possible_answers = str(["Yes"])
        enforce_possible_answer = True
    else:
        # Add user's message
        user_message = Message(sender_type=SenderType.USER, text=message)
        conversation.messages.append(user_message)

        # Generate response text & optional parameters
        response = __generate_response(conversation, user_message.text)
        response_text = response.get('response_text')
        file_request = response.get('file_request')
        possible_answers = response.get('possible_answers')

    # Persist response message
    if response_text is not None:
        response = Message(
            sender_type=SenderType.BOT,
            text=response_text,
            possible_answers=possible_answers
        )
    elif response_html is not None:
        response = Message(
            sender_type=SenderType.BOT,
            text=response_html,
            possible_answers=possible_answers
        )
    else:
        return abort(make_response(jsonify(message="Response text not generated"), 400))

    # Create relationship between message and file request if present
    if file_request is not None:
        response.file_request = file_request

    conversation.messages.append(response)

    # Commit
    db.session.commit()

    # Build response dict
    response_dict = {'conversation_id': conversation.id}

    if response_text is not None:
        response_dict['message'] = response_text
    if response_html is not None:
        response_dict['html'] = response_html
    if file_request is not None:
        response_dict['file_request'] = FileRequestSchema().dump(file_request).data
    if possible_answers is not None:
        response_dict['possible_answers'] = possible_answers
        if enforce_possible_answer:
            response_dict['enforce_possible_answer'] = True

    return jsonify(response_dict)

################
# File Handling
################

def get_file_list(conversation_id):
    conversation = __get_conversation(conversation_id)

    return jsonify(
        {
            'files': [FileSchema().dump(file).data for file in conversation.files]
        }
    )

def upload_file(conversation_id, file):
    conversation = __get_conversation(conversation_id)

    # Check if the file has a filename
    if file.filename == '':
        abort(make_response(jsonify(message="No file selected"), 400))

    if fileService.is_accepted_format(file):
        # Create the file and commit it to generate id
        new_file = File(name=fileService.sanitize_name(file), type=file.content_type)
        conversation.files.append(new_file)
        db.session.commit()

        # Generate path information and upload file to disk
        new_file.path = fileService.generate_path(conversation.id, new_file.id)
        fileService.upload_file(file, new_file.path, new_file.name)
        db.session.commit()

        # Return the file info
        return FileSchema().jsonify(new_file)
    else:
        abort(make_response(
            jsonify(message="Filetype {} is not supported. Supported filetypes are {}.".format(
                fileService.get_file_extension(file), fileService.get_accepted_formats_string())), 400))

def get_file(conversation_id, file_id):
    conversation = __get_conversation(conversation_id)

    for file in conversation.files:
        if file.id == int(file_id):
            return flask.send_from_directory(file.path, file.name, mimetype=file.type, as_attachment=True)

    return abort(make_response(jsonify(message="File does not exist"), 404))

##################
# Private Methods
##################

def __get_conversation(conversation_id):
    conversation = Conversation.query.get(conversation_id)

    if conversation:
        return conversation

    abort(make_response(jsonify(message="Conversation does not exist"), 404))

def __generate_response(conversation, message):
    if __has_just_accepted_disclaimer(conversation):
        response_text = StaticStrings.chooseFrom(StaticStrings.welcome).format(name=conversation.name)
        return {'response_text': response_text}
    if conversation.person_type is None:
        return __determine_person_type(conversation, message)
    elif conversation.claim_category is None:
        return __determine_claim_category(conversation, message)
    elif conversation.current_fact is not None:
        # Assume it is an answer to the current fact
        nlp_request = nlpService.fact_extract([conversation.current_fact], message=message)

        for resolved_fact in nlp_request['facts']:
            for fact_name, fact_value in resolved_fact.items():
                new_fact = Fact(name=fact_name, value=fact_value)
                conversation.facts.append(new_fact)

        db.session.commit()
        question = __probe_facts(conversation)

        return {'response_text': question}

def __determine_person_type(conversation, message):
    person_type = None
    nlp_request = nlpService.fact_extract(['tenant_landlord'], message)

    for fact in nlp_request['facts']:
        person_type = fact['tenant_landlord']

    if person_type is not None:
        conversation.person_type = {
            'tenant': PersonType.TENANT,
            'landlord': PersonType.LANDLORD
        }[person_type]

        file_request = None
        if person_type == 'tenant':
            file_request = FileRequest(document_type=DocumentType.LEASE)

        db.session.commit()

        # Generate response based on person type
        response = None
        if person_type == 'tenant':
            response = StaticStrings.chooseFrom(StaticStrings.problem_inquiry_tenant).format(name=conversation.name)
        elif person_type == 'landlord':
            response = StaticStrings.chooseFrom(StaticStrings.problem_inquiry_landlord).format(name=conversation.name)

        return {'response_text': response, 'file_request': file_request}
    else:
        return {'response_text': StaticStrings.chooseFrom(StaticStrings.clarify)}

def __determine_claim_category(conversation, message):
    claim_category = None
    nlp_request = nlpService.problem_category(message)

    for fact in nlp_request['facts']:
        claim_category = fact['category']

    if claim_category is not None:
        conversation.claim_category = {
            'lease_termination': ClaimCategory.LEASE_TERMINATION,
            'rent_change': ClaimCategory.RENT_CHANGE,
            'nonpayment': ClaimCategory.NONPAYMENT,
            'deposits': ClaimCategory.DEPOSITS
        }[claim_category]

        db.session.commit()

        # Generate the first question
        first_question = __probe_facts(conversation)

        response = StaticStrings.chooseFrom(StaticStrings.category_acknowledge).format(
            claim_category=conversation.claim_category.value.lower().replace("_", " "), first_question=first_question)

        return {'response_text': response}
    else:
        return {'response_text': StaticStrings.chooseFrom(StaticStrings.clarify)}

def __probe_facts(conversation):
    resolved_facts = [fact.name for fact in conversation.facts]
    fact, question = FactService.get_question(conversation.claim_category.value.lower(), resolved_facts)

    if fact is not None:
        # Update fact being asked
        conversation.current_fact = fact
        db.session.commit()
    else:
        fact_dump = ''.join(repr(conversation.facts))
        question = "FACT DUMP: {}".format(fact_dump)

    return question

def __has_just_accepted_disclaimer(conversation):
    return len(conversation.messages) == 2
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2016 Ryan Collins <rcollins@chgr.mgh.harvard.edu>
# Distributed under terms of the MIT license.

"""
Calculates non-duplicate primary-aligned binned coverage of a chromosome from an input BAM file
"""

#Import libraries
import argparse
import sys
import pysam
import pybedtools

#Function to evaluate nucleotide coverage
def nuc_binCov(bam, chr, binsize, blacklist = None):
	"""
    Generates non-duplicate, primary-aligned nucleotide coverage in regular bin
    sizes on a specified chromosome from a coordinate-sorted bamfile

    Parameters
    ----------
    bam : pysam.AlignmentFile
        Input bam
    chr : string
        Chromosome to evaluate
    binsize : int
    	Size of bins in bp
    blacklist : string
    	Path to blacklist BED file

    Returns
    ------
    coverage : list
        chr, start, end, coverage

	"""

	#Define read filtering criteria
	#If True, read fails filtering
	def _filter(read):
		return (read.is_secondary or read.is_duplicate or
				read.is_supplementary or read.is_unmapped)

	#Subset bam for relevant reads and convert to BedTool
	subbam = pybedtools.BedTool(read for read in bam.fetch(str(chr)) if _filter(read) is False)

	#Instantiate coverage bins and convert to BedTool
	maxchrpos = {d['SN']: d['LN'] for d in bam.header['SQ']}[str(chr)]
	bin_starts = range(0, maxchrpos - binsize, binsize)
	bin_stops = range(binsize, maxchrpos, binsize)
	bins = []
	for i in range(0, len(bin_starts)-1):
		bins.append([chr, bin_starts[i], bin_stops[i]])
	bins = pybedtools.BedTool(bins)

	#Remove bins that have at least 5% overlap with blacklist by size
	bins_filtered = bins.intersect(blacklist, v=True, f=0.05)

	#Generate & return coverage
	coverage = bins_filtered.coverage(subbam, counts=True, sorted=True)
	return coverage

#Main function
def main():
    #Add arguments
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('bam', type=pysam.AlignmentFile,
                        help='Input bam')
    parser.add_argument('chr', help='Contig to evaluate')
    parser.add_argument('cov_out', help='Output bed file of raw coverage')
    parser.add_argument('-n', '--norm_out', nargs=1,
    					help='Output bed file of normalized coverage')
    parser.add_argument('-b', '--binsize', type=int, default=1000,
                        help='Bin size in bp (default: 1000)')
    parser.add_argument('-m', '--mode', default='nucleotide',
    					choices = ['nucleotide', 'physical'],
                        help='Evaluate nucleotide or physical coverage '
                             '(default: nucleotide)')
    parser.add_argument('-x', '--blacklist', nargs=1,
    	                help='BED file of regions to ignore')
    parser.add_argument('-v', '--overlap', nargs=1, type=float, default=0.05,
    	   				help='Maximum tolerated blacklist overlap before '
    	   				      'excluding bin')
    args = parser.parse_args()

    #Open outfiles
    fcovout = open(args.cov_out, 'w')
    if args.norm_out is not None:
	    fnormout = open(args.norm_out, 'w')

    #Get nucleotide coverage
    if args.mode == 'nucleotide':
    	coverage = nuc_binCov(args.bam, args.chr, args.binsize, args.blacklist)

    #Close outfiles
    fcovout.close()
    if args.norm_out is not None:
	    fnormout.close()

#Main block
if __name__ == '__main__':
    main()
<commit_msg>Update<commit_after>#!/usr/bin/env python

# Copyright (c) 2016 Ryan Collins <rcollins@chgr.mgh.harvard.edu>
# Distributed under terms of the MIT license.

"""
Calculates non-duplicate primary-aligned binned coverage of a chromosome from an input BAM file
"""

#Import libraries
import argparse
import sys
import pysam
import pybedtools

#Function to evaluate nucleotide coverage
def nuc_binCov(bam, chr, binsize, blacklist = None):
	"""
    Generates non-duplicate, primary-aligned nucleotide coverage in regular bin
    sizes on a specified chromosome from a coordinate-sorted bamfile

    Parameters
    ----------
    bam : pysam.AlignmentFile
        Input bam
    chr : string
        Chromosome to evaluate
    binsize : int
    	Size of bins in bp
    blacklist : string
    	Path to blacklist BED file

    Returns
    ------
    coverage : list
        chr, start, end, coverage
	"""

	#Define read filtering criteria
	#If True, read fails filtering
	def _filter(read):
		return (read.is_secondary or read.is_duplicate or
				read.is_supplementary or read.is_unmapped)

	#Subset bam for relevant reads and convert to BedTool
	subbam = pybedtools.BedTool(read for read in bam.fetch(str(chr)) if _filter(read) is False)

	#Instantiate coverage bins and convert to BedTool
	maxchrpos = {d['SN']: d['LN'] for d in bam.header['SQ']}[str(chr)]
	bin_starts = range(0, maxchrpos - binsize, binsize)
	bin_stops = range(binsize, maxchrpos, binsize)
	bins = []
	for i in range(0, len(bin_starts)-1):
		bins.append([chr, bin_starts[i], bin_stops[i]])
	bins = pybedtools.BedTool(bins)

	#Remove bins that have at least 5% overlap with blacklist by size
	bins_filtered = bins.intersect(blacklist, v=True, f=0.05)

	#Generate & return coverage
	coverage = bins_filtered.coverage(subbam, counts=True)
	return coverage

#Function to evaluate physical coverage
def nuc_binCov(bam, chr, binsize, blacklist = None):
	"""
    Generates non-duplicate, primary-aligned proper pair physical coverage
    in regular bin sizes on a specified chromosome from a coordinate-sorted
    bamfile

    Parameters
    ----------
    bam : pysam.AlignmentFile
        Input bam
    chr : string
        Chromosome to evaluate
    binsize : int
    	Size of bins in bp
    blacklist : string
    	Path to blacklist BED file

    Returns
    ------
    coverage : list
        chr, start, end, coverage
	"""

	#Define read filtering criteria
	#If True, read fails filtering
	def _filter(read):
		return (read.is_secondary or read.is_duplicate or
				read.is_supplementary or read.is_unmapped or
				read.mate_is_unmapped or (not read.is_proper_pair))

	#Subset bam for relevant reads and convert to BedTool
	subbam = (read for read in bam.fetch(str(chr)) if _filter(read) is False)
	nbam =
	subbam = pybedtools.BedTool(read for read in bam.fetch(str(chr)) if _filter(read) is False)

	# #Convert bam to bed of proper fragments
	# fragments =  subbam.bam_to_bed(bedpe=True)

	# #Instantiate coverage bins and convert to BedTool
	# maxchrpos = {d['SN']: d['LN'] for d in bam.header['SQ']}[str(chr)]
	# bin_starts = range(0, maxchrpos - binsize, binsize)
	# bin_stops = range(binsize, maxchrpos, binsize)
	# bins = []
	# for i in range(0, len(bin_starts)-1):
	# 	bins.append([chr, bin_starts[i], bin_stops[i]])
	# bins = pybedtools.BedTool(bins)

	# #Remove bins that have at least 5% overlap with blacklist by size
	# bins_filtered = bins.intersect(blacklist, v=True, f=0.05)

	# #Generate & return coverage
	# coverage = bins_filtered.coverage(subbam, counts=True, sorted=True)
	# return coverage

#Main function
def main():
    #Add arguments
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('bam', type=pysam.AlignmentFile,
                        help='Input bam')
    parser.add_argument('chr', help='Contig to evaluate')
    parser.add_argument('cov_out', help='Output bed file of raw coverage')
    parser.add_argument('-n', '--norm_out', nargs=1,
    					help='Output bed file of normalized coverage')
    parser.add_argument('-b', '--binsize', type=int, default=1000,
                        help='Bin size in bp (default: 1000)')
    parser.add_argument('-m', '--mode', default='nucleotide',
    					choices = ['nucleotide', 'physical'],
                        help='Evaluate nucleotide or physical coverage '
                             '(default: nucleotide)')
    parser.add_argument('-x', '--blacklist', nargs=1,
    	                help='BED file of regions to ignore')
    parser.add_argument('-v', '--overlap', nargs=1, type=float, default=0.05,
    	   				help='Maximum tolerated blacklist overlap before '
    	   				      'excluding bin')
    args = parser.parse_args()

    #Open outfiles
    fcovout = open(args.cov_out, 'w')
    if args.norm_out is not None:
	    fnormout = open(args.norm_out, 'w')

    #Get nucleotide coverage
    if args.mode == 'nucleotide':
    	coverage = nuc_binCov(args.bam, args.chr, args.binsize, args.blacklist)

    #Close outfiles
    fcovout.close()
    if args.norm_out is not None:
	    fnormout.close()

#Main block
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import idalink
import pysex
import logging
import loader

l = logging.getLogger("angr_binary")
l.setLevel(logging.DEBUG)

def ondemand(f):
	name = f.__name__
	def func(self, *args, **kwargs):
		if hasattr(self, "_" + name):
			return getattr(self, "_" + name)

		a = f(self, *args, **kwargs)
		setattr(self, "_" + name, a)
		return a
	func.__name__ = f.__name__
	return func

class Function(object):
	def __init__(self, func_start, ida):
		self.start = func_start
		self.ida = ida
		self.name = "sub_%x" % func_start

	@ondemand
	def range(self):
		starts, ends = [ ], [ ]
		l.debug("Getting range from IDA")

		f = self.ida.idaapi.get_func(self.start)
		r = (f.startEA, f.endEA)
		l.debug("Got range (%x, %x)." % r)
		return r

	@ondemand
	def ida_blocks(self):
		l.debug("Getting blocks from IDA")
		ida_blocks = { }

		flowchart = self.ida.idaapi.FlowChart(self.ida.idaapi.get_func(self.start))
		for block in flowchart:
			start, end = (block.startEA, block.endEA)
			block_bytes = self.ida.idaapi.get_many_bytes(start, end - start)

			if not block_bytes:
				l.warning("... empty block_bytes at %x" % start)
				continue

			ida_blocks[(start, end)] = block_bytes
		return ida_blocks

	@ondemand
	def bytes(self):
		start, end = self.range()
		return self.ida.idaapi.get_many_bytes(start, end - start)

	@ondemand
	def symbolic_translation(self):
                init = pysex.s_state.SymbolicState(memory=loader.load_binary(self.ida))
		return pysex.translate_bytes(self.start, self.bytes(), self.start, init)

	@ondemand
	def sym_vex_blocks(self):
		blocks = { }
		total_size = 0
		sblocks, exits_out, unsat_exits = self.symbolic_translation()

		for start,sirsb in sblocks:
			irsb = sirsb.irsb

			size = irsb.size()
			total_size += size
			# We return SymbolicIRSB instead of normal IRSB here
			blocks[start] = sirsb
			l.debug("Block at %x of size %d" % (start, irsb.size()))

		l.debug("Total VEX IRSB size, in bytes: %d" % total_size)
		return blocks

	@ondemand
	def exits(self):
		sblocks, exits_out, unsat_exits = self.symbolic_translation()
		exits = [ ]

		for exit in exits_out:
			try:
				exits.append(exit.concretize())
			except pysex.ConcretizingException:
				l.warning("Un-concrete exit.")

		return exits

class Binary(object):
	def __init__(self, filename):
		self.filename = filename
		self.ida = idalink.IDALink(filename)

	@ondemand
	def functions(self):
		functions = { }
		for f in self.ida.idautils.Functions():
			functions[f] = Function(f, self.ida)
		return functions

	@ondemand
	def our_functions(self):
		functions = { }
		remaining_exits = [ self.entry() ]

		while remaining_exits:
			current_exit = remaining_exits[0]
			remaining_exits = remaining_exits[1:]

			if current_exit not in functions:
				print "New function: %x" % current_exit
				f = Function(current_exit, self.ida)
				functions[current_exit] = f
				new_exits = f.exits()
				print "Exits from %x: %s" % (current_exit,[hex(i) for i in new_exits])
				remaining_exits += [ i for i in new_exits if i != 100 ]
		return functions

	# Gets the entry point of the binary.
	@ondemand
	def entry(self):
		return self.ida.idc.BeginEA()
<commit_msg>updated for pysex updates<commit_after>#!/usr/bin/env python

import idalink
import pysex
import logging
import loader

l = logging.getLogger("angr_binary")
l.setLevel(logging.DEBUG)

def ondemand(f):
	name = f.__name__
	def func(self, *args, **kwargs):
		if hasattr(self, "_" + name):
			return getattr(self, "_" + name)

		a = f(self, *args, **kwargs)
		setattr(self, "_" + name, a)
		return a
	func.__name__ = f.__name__
	return func

class Function(object):
	def __init__(self, func_start, ida):
		self.start = func_start
		self.ida = ida
		self.name = "sub_%x" % func_start

	@ondemand
	def range(self):
		starts, ends = [ ], [ ]
		l.debug("Getting range from IDA")

		f = self.ida.idaapi.get_func(self.start)
		r = (f.startEA, f.endEA)
		l.debug("Got range (%x, %x)." % r)
		return r

	@ondemand
	def ida_blocks(self):
		l.debug("Getting blocks from IDA")
		ida_blocks = { }

		flowchart = self.ida.idaapi.FlowChart(self.ida.idaapi.get_func(self.start))
		for block in flowchart:
			start, end = (block.startEA, block.endEA)
			block_bytes = self.ida.idaapi.get_many_bytes(start, end - start)

			if not block_bytes:
				l.warning("... empty block_bytes at %x" % start)
				continue

			ida_blocks[(start, end)] = block_bytes
		return ida_blocks

	@ondemand
	def bytes(self):
		start, end = self.range()
		return self.ida.idaapi.get_many_bytes(start, end - start)

	@ondemand
	def symbolic_translation(self):
                #init = pysex.s_state.SymbolicState(memory=loader.load_binary(self.ida))
		return pysex.translate_bytes(self.start, self.bytes(), self.start, None)

	@ondemand
	def sym_vex_blocks(self):
		blocks = { }
		total_size = 0
		sblocks, exits_out, unsat_exits = self.symbolic_translation()

		for exit_type in sblocks:
			for start, sirsb in sblocks[exit_type].iteritems():
				total_size += sirsb.irsb.size()
				blocks[start] = sirsb
				l.debug("Block at %x of size %d" % (start, sirsb.irsb.size()))

		l.debug("Total VEX IRSB size, in bytes: %d" % total_size)
		return blocks

	@ondemand
	def exits(self):
		sblocks, exits_out, unsat_exits = self.symbolic_translation()
		exits = [ ]

		for exit in exits_out:
			try:
				exits.append(exit.concretize())
			except pysex.ConcretizingException:
				l.warning("Un-concrete exit.")

		return exits

class Binary(object):
	def __init__(self, filename):
		self.filename = filename
		self.ida = idalink.IDALink(filename)

	@ondemand
	def functions(self):
		functions = { }
		for f in self.ida.idautils.Functions():
			functions[f] = Function(f, self.ida)
		return functions

	@ondemand
	def our_functions(self):
		functions = { }
		remaining_exits = [ self.entry() ]

		while remaining_exits:
			current_exit = remaining_exits[0]
			remaining_exits = remaining_exits[1:]

			if current_exit not in functions:
				print "New function: %x" % current_exit
				f = Function(current_exit, self.ida)
				functions[current_exit] = f
				new_exits = f.exits()
				print "Exits from %x: %s" % (current_exit,[hex(i) for i in new_exits])
				remaining_exits += [ i for i in new_exits if i != 100 ]
		return functions

	# Gets the entry point of the binary.
	@ondemand
	def entry(self):
		return self.ida.idc.BeginEA()
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

import os

import re
import requests
import sys

from colorclass import Color
from packaging import version
from packaging.utils import canonicalize_name
from pip.locations import site_config_files
from requests import HTTPError

try:
    from configparser import ConfigParser, NoOptionError
except ImportError:   # pragma: nocover
    from ConfigParser import ConfigParser, NoOptionError

try:
    from urllib.parse import urljoin
except ImportError:  # pragma: nocover
    from urlparse import urljoin

class PackagesStatusDetector(object):
    packages = []
    packages_status_map = {}
    PYPI_API_URL = None
    PYPI_API_TYPE = None
    pip_config_locations = [
        '~/.pip/pip.conf',
        '~/.pip/pip.ini',
        '~/.config/pip/pip.conf',
        '~/.config/pip/pip.ini',
    ]

    _prerelease = False

    def __init__(self, packages, use_default_index=False):
        self.packages = packages
        self.packages_status_map = {}
        self.PYPI_API_URL = 'https://pypi.python.org/pypi/{package}/json'
        self.PYPI_API_TYPE = 'pypi_json'

        if not use_default_index:
            self._update_index_url_from_configs()

        self._prerelease = False

    def _update_index_url_from_configs(self):
        """ Checks for alternative index-url in pip.conf """

        if 'VIRTUAL_ENV' in os.environ:
            self.pip_config_locations.append(os.path.join(os.environ['VIRTUAL_ENV'], 'pip.conf'))
            self.pip_config_locations.append(os.path.join(os.environ['VIRTUAL_ENV'], 'pip.ini'))

        if site_config_files:
            self.pip_config_locations.extend(site_config_files)

        index_url = None
        custom_config = None

        if 'PIP_INDEX_URL' in os.environ and os.environ['PIP_INDEX_URL']:
            # environ variable takes priority
            index_url = os.environ['PIP_INDEX_URL']
            custom_config = 'PIP_INDEX_URL environment variable'
        else:
            for pip_config_filename in self.pip_config_locations:
                if pip_config_filename.startswith('~'):
                    pip_config_filename = os.path.expanduser(pip_config_filename)

                if os.path.isfile(pip_config_filename):
                    config = ConfigParser()
                    config.read([pip_config_filename])
                    try:
                        index_url = config.get('global', 'index-url')
                        custom_config = pip_config_filename
                        break  # stop on first detected, because config locations have a priority
                    except NoOptionError:  # pragma: nocover
                        pass

        if index_url:
            self.PYPI_API_URL = self._prepare_api_url(index_url)
            print(Color('Setting API url to {{autoyellow}}{}{{/autoyellow}} as found in {{autoyellow}}{}{{/autoyellow}}'
                        '. Use --default-index-url to use pypi default index'.format(self.PYPI_API_URL, custom_config)))

    def _prepare_api_url(self, index_url):  # pragma: nocover
        if not index_url.endswith('/'):
            index_url += '/'

        if index_url.endswith('/simple/'):
            self.PYPI_API_TYPE = 'simple_html'
            return urljoin(index_url, '{package}')

        if index_url.endswith('/+simple/'):
            self.PYPI_API_TYPE = 'simple_html'
            return urljoin(index_url, '{package}')

        if '/pypi/' in index_url:
            base_url = index_url.split('/pypi/')[0]
            return urljoin(base_url, '/pypi/{package}/json')

        return urljoin(index_url, '/pypi/{package}/json')

    def detect_available_upgrades(self, options):
        self._prerelease = options.get('--prerelease', False)
        explicit_packages_lower = None
        if options['-p'] and options['-p'] != ['all']:
            explicit_packages_lower = [pack_name.lower() for pack_name in options['-p']]

        for i, package in enumerate(self.packages):
            try:
                package_name, pinned_version = self._expand_package(package)
                if not package_name or not pinned_version:  # pragma: nocover
                    # todo: treat <= or >= instead of ==
                    continue

                if explicit_packages_lower and package_name.lower() not in explicit_packages_lower:
                    # skip if explicit and not chosen
                    continue

                current_version = version.parse(pinned_version)

                if pinned_version and isinstance(current_version, version.Version):  # version parsing is correct
                    package_status, reason = self._fetch_index_package_info(package_name, current_version)
                    if not package_status:  # pragma: nocover
                        print(package, reason)
                        continue

                    print('{}/{}: {} ... '.format(i + 1, len(self.packages), package_name), end='')
                    sys.stdout.flush()

                    # compare versions
                    if current_version < package_status['latest_version']:
                        print('upgrade available: {} ==> {} (uploaded on {})'.format(current_version,
                                                                                package_status['latest_version'],
                                                                                package_status['upload_time']))
                    else:
                        print('up to date: {}'.format(current_version))
                    sys.stdout.flush()

                    self.packages_status_map[package_name] = package_status
            except Exception as e:  # noqa  # pragma: nocover
                print('Error while parsing package {} (skipping). \nException: '.format(package), e)

        return self.packages_status_map

    def _fetch_index_package_info(self, package_name, current_version):
        """
        :type package_name: str
        :type current_version: version.Version
        """

        try:
            package_canonical_name = package_name
            if self.PYPI_API_TYPE == 'simple_html':
                package_canonical_name = canonicalize_name(package_name)
            response = requests.get(self.PYPI_API_URL.format(package=package_canonical_name), timeout=15)
        except HTTPError as e:  # pragma: nocover
            return False, e.message

        if not response.ok:  # pragma: nocover
            return False, 'API error: {}'.format(response.reason)

        if self.PYPI_API_TYPE == 'pypi_json':
            return self._parse_pypi_json_package_info(package_name, current_version, response)
        elif self.PYPI_API_TYPE == 'simple_html':
            return self._parse_simple_html_package_info(package_name, current_version, response)
        else:  # pragma: nocover
            raise NotImplementedError('This type of PYPI_API_TYPE type is not supported')

    def _expand_package(self, package_line):
        if '==' in package_line:
            name, vers = package_line.split('==')

            if '[' in name and name.strip().endswith(']'):
                name = name.split('[')[0]

            return name, vers

        return None, None

    def _parse_pypi_json_package_info(self, package_name, current_version, response):
        """
        :type package_name: str
        :type current_version: version.Version
        :type response: requests.models.Response
        """

        data = response.json()
        all_versions = [version.parse(vers) for vers in data['releases'].keys()]
        filtered_versions = [vers for vers in all_versions if not vers.is_prerelease and not vers.is_postrelease]

        if not filtered_versions:  # pragma: nocover
            return False, 'error while parsing version'

        latest_version = max(filtered_versions)

        # even if user did not choose prerelease, if the package from requirements is pre/post release, use it
        if self._prerelease or current_version.is_postrelease or current_version.is_prerelease:
            prerelease_versions = [vers for vers in all_versions if vers.is_prerelease or vers.is_postrelease]
            if prerelease_versions:
                latest_version = max(prerelease_versions)
        try:
            try:
                latest_version_info = data['releases'][str(latest_version)][0]
            except KeyError:  # pragma: nocover
                # non-RFC versions, get the latest from pypi response
                latest_version = version.parse(data['info']['version'])
                latest_version_info = data['releases'][str(latest_version)][0]
        except Exception:  # pragma: nocover
            return False, 'error while parsing version'

        upload_time = latest_version_info['upload_time'].replace('T', ' ')

        return {
            'name': package_name,
            'current_version': current_version,
            'latest_version': latest_version,
            'upgrade_available': current_version < latest_version,
            'upload_time': upload_time
        }, 'success'

    def _parse_simple_html_package_info(self, package_name, current_version, response):
        """
        :type package_name: str
        :type current_version: version.Version
        :type response: requests.models.Response
        """
        pattern = r'<a.*>.*{name}-([A-z0-9\.-]*)(?:-py|\.tar).*<\/a>'.format(name=re.escape(package_name))
        versions_match = re.findall(pattern, response.content.decode('utf-8'), flags=re.IGNORECASE)

        all_versions = [version.parse(vers) for vers in versions_match]
        filtered_versions = [vers for vers in all_versions if not vers.is_prerelease and not vers.is_postrelease]

        if not filtered_versions:  # pragma: nocover
            return False, 'error while parsing version'

        latest_version = max(filtered_versions)

        # even if user did not choose prerelease, if the package from requirements is pre/post release, use it
        if self._prerelease or current_version.is_postrelease or current_version.is_prerelease:
            prerelease_versions = [vers for vers in all_versions if vers.is_prerelease or vers.is_postrelease]
            if prerelease_versions:
                latest_version = max(prerelease_versions)

        return {
           'name': package_name,
           'current_version': current_version,
           'latest_version': latest_version,
           'upgrade_available': current_version < latest_version,
           'upload_time': '-'
        }, 'success'
<commit_msg>fix issue #6<commit_after>from __future__ import print_function, unicode_literals

import os

import re
import requests
import sys

from colorclass import Color
from packaging import version
from packaging.utils import canonicalize_name
from pip.locations import site_config_files
from requests import HTTPError

try:
    from configparser import ConfigParser, NoOptionError, NoSectionError
except ImportError:   # pragma: nocover
    from ConfigParser import ConfigParser, NoOptionError

try:
    from urllib.parse import urljoin
except ImportError:  # pragma: nocover
    from urlparse import urljoin

class PackagesStatusDetector(object):
    packages = []
    packages_status_map = {}
    PYPI_API_URL = None
    PYPI_API_TYPE = None
    pip_config_locations = [
        '~/.pip/pip.conf',
        '~/.pip/pip.ini',
        '~/.config/pip/pip.conf',
        '~/.config/pip/pip.ini',
    ]

    _prerelease = False

    def __init__(self, packages, use_default_index=False):
        self.packages = packages
        self.packages_status_map = {}
        self.PYPI_API_URL = 'https://pypi.python.org/pypi/{package}/json'
        self.PYPI_API_TYPE = 'pypi_json'

        if not use_default_index:
            self._update_index_url_from_configs()

        self._prerelease = False

    def _update_index_url_from_configs(self):
        """ Checks for alternative index-url in pip.conf """

        if 'VIRTUAL_ENV' in os.environ:
            self.pip_config_locations.append(os.path.join(os.environ['VIRTUAL_ENV'], 'pip.conf'))
            self.pip_config_locations.append(os.path.join(os.environ['VIRTUAL_ENV'], 'pip.ini'))

        if site_config_files:
            self.pip_config_locations.extend(site_config_files)

        index_url = None
        custom_config = None

        if 'PIP_INDEX_URL' in os.environ and os.environ['PIP_INDEX_URL']:
            # environ variable takes priority
            index_url = os.environ['PIP_INDEX_URL']
            custom_config = 'PIP_INDEX_URL environment variable'
        else:
            for pip_config_filename in self.pip_config_locations:
                if pip_config_filename.startswith('~'):
                    pip_config_filename = os.path.expanduser(pip_config_filename)

                if os.path.isfile(pip_config_filename):
                    config = ConfigParser()
                    config.read([pip_config_filename])
                    try:
                        index_url = config.get('global', 'index-url')
                        custom_config = pip_config_filename
                        break  # stop on first detected, because config locations have a priority
                    except (NoOptionError, NoSectionError):  # pragma: nocover
                        pass

        if index_url:
            self.PYPI_API_URL = self._prepare_api_url(index_url)
            print(Color('Setting API url to {{autoyellow}}{}{{/autoyellow}} as found in {{autoyellow}}{}{{/autoyellow}}'
                        '. Use --default-index-url to use pypi default index'.format(self.PYPI_API_URL, custom_config)))

    def _prepare_api_url(self, index_url):  # pragma: nocover
        if not index_url.endswith('/'):
            index_url += '/'

        if index_url.endswith('/simple/'):
            self.PYPI_API_TYPE = 'simple_html'
            return urljoin(index_url, '{package}')

        if index_url.endswith('/+simple/'):
            self.PYPI_API_TYPE = 'simple_html'
            return urljoin(index_url, '{package}')

        if '/pypi/' in index_url:
            base_url = index_url.split('/pypi/')[0]
            return urljoin(base_url, '/pypi/{package}/json')

        return urljoin(index_url, '/pypi/{package}/json')

    def detect_available_upgrades(self, options):
        self._prerelease = options.get('--prerelease', False)
        explicit_packages_lower = None
        if options['-p'] and options['-p'] != ['all']:
            explicit_packages_lower = [pack_name.lower() for pack_name in options['-p']]

        for i, package in enumerate(self.packages):
            try:
                package_name, pinned_version = self._expand_package(package)
                if not package_name or not pinned_version:  # pragma: nocover
                    # todo: treat <= or >= instead of ==
                    continue

                if explicit_packages_lower and package_name.lower() not in explicit_packages_lower:
                    # skip if explicit and not chosen
                    continue

                current_version = version.parse(pinned_version)

                if pinned_version and isinstance(current_version, version.Version):  # version parsing is correct
                    package_status, reason = self._fetch_index_package_info(package_name, current_version)
                    if not package_status:  # pragma: nocover
                        print(package, reason)
                        continue

                    print('{}/{}: {} ... '.format(i + 1, len(self.packages), package_name), end='')
                    sys.stdout.flush()

                    # compare versions
                    if current_version < package_status['latest_version']:
                        print('upgrade available: {} ==> {} (uploaded on {})'.format(current_version,
                                                                                package_status['latest_version'],
                                                                                package_status['upload_time']))
                    else:
                        print('up to date: {}'.format(current_version))
                    sys.stdout.flush()

                    self.packages_status_map[package_name] = package_status
            except Exception as e:  # noqa  # pragma: nocover
                print('Error while parsing package {} (skipping). \nException: '.format(package), e)

        return self.packages_status_map

    def _fetch_index_package_info(self, package_name, current_version):
        """
        :type package_name: str
        :type current_version: version.Version
        """

        try:
            package_canonical_name = package_name
            if self.PYPI_API_TYPE == 'simple_html':
                package_canonical_name = canonicalize_name(package_name)
            response = requests.get(self.PYPI_API_URL.format(package=package_canonical_name), timeout=15)
        except HTTPError as e:  # pragma: nocover
            return False, e.message

        if not response.ok:  # pragma: nocover
            return False, 'API error: {}'.format(response.reason)

        if self.PYPI_API_TYPE == 'pypi_json':
            return self._parse_pypi_json_package_info(package_name, current_version, response)
        elif self.PYPI_API_TYPE == 'simple_html':
            return self._parse_simple_html_package_info(package_name, current_version, response)
        else:  # pragma: nocover
            raise NotImplementedError('This type of PYPI_API_TYPE type is not supported')

    def _expand_package(self, package_line):
        if '==' in package_line:
            name, vers = package_line.split('==')

            if '[' in name and name.strip().endswith(']'):
                name = name.split('[')[0]

            return name, vers

        return None, None

    def _parse_pypi_json_package_info(self, package_name, current_version, response):
        """
        :type package_name: str
        :type current_version: version.Version
        :type response: requests.models.Response
        """

        data = response.json()
        all_versions = [version.parse(vers) for vers in data['releases'].keys()]
        filtered_versions = [vers for vers in all_versions if not vers.is_prerelease and not vers.is_postrelease]

        if not filtered_versions:  # pragma: nocover
            return False, 'error while parsing version'

        latest_version = max(filtered_versions)

        # even if user did not choose prerelease, if the package from requirements is pre/post release, use it
        if self._prerelease or current_version.is_postrelease or current_version.is_prerelease:
            prerelease_versions = [vers for vers in all_versions if vers.is_prerelease or vers.is_postrelease]
            if prerelease_versions:
                latest_version = max(prerelease_versions)
        try:
            try:
                latest_version_info = data['releases'][str(latest_version)][0]
            except KeyError:  # pragma: nocover
                # non-RFC versions, get the latest from pypi response
                latest_version = version.parse(data['info']['version'])
                latest_version_info = data['releases'][str(latest_version)][0]
        except Exception:  # pragma: nocover
            return False, 'error while parsing version'

        upload_time = latest_version_info['upload_time'].replace('T', ' ')

        return {
            'name': package_name,
            'current_version': current_version,
            'latest_version': latest_version,
            'upgrade_available': current_version < latest_version,
            'upload_time': upload_time
        }, 'success'

    def _parse_simple_html_package_info(self, package_name, current_version, response):
        """
        :type package_name: str
        :type current_version: version.Version
        :type response: requests.models.Response
        """
        pattern = r'<a.*>.*{name}-([A-z0-9\.-]*)(?:-py|\.tar).*<\/a>'.format(name=re.escape(package_name))
        versions_match = re.findall(pattern, response.content.decode('utf-8'), flags=re.IGNORECASE)

        all_versions = [version.parse(vers) for vers in versions_match]
        filtered_versions = [vers for vers in all_versions if not vers.is_prerelease and not vers.is_postrelease]

        if not filtered_versions:  # pragma: nocover
            return False, 'error while parsing version'

        latest_version = max(filtered_versions)

        # even if user did not choose prerelease, if the package from requirements is pre/post release, use it
        if self._prerelease or current_version.is_postrelease or current_version.is_prerelease:
            prerelease_versions = [vers for vers in all_versions if vers.is_prerelease or vers.is_postrelease]
            if prerelease_versions:
                latest_version = max(prerelease_versions)

        return {
           'name': package_name,
           'current_version': current_version,
           'latest_version': latest_version,
           'upgrade_available': current_version < latest_version,
           'upload_time': '-'
        }, 'success'
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import requests
import time
from contextlib import contextmanager
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

base_url = 'https://www.rottentomatoes.com/browse/dvd-all/?services=amazon;amazon_prime;fandango_now;hbo_go;itunes;netflix_iw;vudu'

def parse_movie_urls():
    '''Creates dictionary of all movies under Browse All of RT using div class = "mb-movie"
    	and storing it as {title:url}'''

    #create dict
    movie_list = {}

    #start selenium driver, route to url
    driver = webdriver.Firefox()
    driver.get(base_url)

    #clicking loop: keep expanding page till it is showing all movies
    elem = driver.find_element_by_xpath('//*[@id="show-more-btn"]/button')

    #check current count vs total count
    soup = BeautifulSoup(driver.page_source,"lxml")
    show_count = soup.findAll('span',attrs={'id':'showing-count'})[0].text
    current_count = int(show_count.split(' ')[1])
    total_count = int(show_count.split(' ')[3])

    while(current_count <= total_count):

        elem.click()
        WebDriverWait(driver, 10)

        #check current count vs total count
        soup = BeautifulSoup(driver.page_source,"lxml")
        show_count = soup.findAll('span',attrs={'id':'showing-count'})[0].text
        current_count = int(show_count.split(' ')[1])
        total_count = int(show_count.split(' ')[3])
        print(str(current_count) +  ", " + str(total_count))

    driver.close()

if __name__ == '__main__':
	#run scraper and print completion time
	print('Running')
	start = time.time()
	parse_movie_urls()
	end = time.time() - start
	print("Completed, time: " + str(end) + " secs")
<commit_msg>clicker checks if show more can be clicked first<commit_after>from bs4 import BeautifulSoup
import requests
import time
from contextlib import contextmanager
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

base_url = 'https://www.rottentomatoes.com/browse/dvd-all/?services=amazon;amazon_prime;fandango_now;hbo_go;itunes;netflix_iw;vudu'

def parse_movie_urls():
    '''Creates dictionary of all movies under Browse All of RT using div class = "mb-movie"
    	and storing it as {title:url}'''

    #create dict
    movie_list = {}

    #start selenium driver, route to url
    driver = webdriver.Firefox()
    driver.get(base_url)

    #clicking loop: keep expanding page till it is showing all movies
    elem = driver.find_element_by_xpath('//*[@id="show-more-btn"]/button')

    #check current count vs total count
    soup = BeautifulSoup(driver.page_source,"lxml")
    show_count = soup.findAll('span',attrs={'id':'showing-count'})[0].text
    current_count = int(show_count.split(' ')[1])
    total_count = int(show_count.split(' ')[3])

    while(current_count <= total_count):

        elem.click()
        WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="show-more-btn"]/button')))

        #check current count vs total count
        soup = BeautifulSoup(driver.page_source,"lxml")
        show_count = soup.findAll('span',attrs={'id':'showing-count'})[0].text
        current_count = int(show_count.split(' ')[1])
        total_count = int(show_count.split(' ')[3])
        print(str(current_count) +  ", " + str(total_count))

    driver.close()

if __name__ == '__main__':
	#run scraper and print completion time
	print('Running')
	start = time.time()
	parse_movie_urls()
	end = time.time() - start
	print("Completed, time: " + str(end) + " secs")
<|endoftext|>