<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
Manage default email client and display help.

Usage:
    mailto.py <action> [<path>|<query>]

Possible actions:

config
    Show current settings

format
    Show current format

setformat
    Set new format

client [<query>]
    Show/choose email client

setclient <path>
    Set the email client to use to application at <path>.
    Called when an option in the above call is selected

help
    Display brief help (in Alfred)

openhelp
    Open the help.html file in the default application

openlog
    Open logfile in default app

dellog
    Delete the logfile.

delcache
    Delete the cache file containing the settings. Mostly for me when I fuck
    stuff up.

"""

from __future__ import print_function

import sys
import os
import json
from subprocess import check_output, check_call
from time import time

import alfred
from contacts import get_contacts
from log import logger, LOGFILE

log = logger(u'mailto')

__usage__ = u"""
Usage:
    mailto config
    mailto format
    mailto setformat [<format>]
    mailto client [<query>]
    mailto setclient [<path>]
    mailto help
    mailto openhelp
    mailto openlog
    mailto dellog
    mailto delcache
"""

__help__ = [
    (u"Open MailTo help file",
        u"In your browser"),
    (u"'mailtoconf' to view and change settings",
        u"View/change email client and format"),
    (u"Enter 'mailtohelp' for this message",
        u"Well, I guess you already figured this out …"),
]

SETTINGS_CACHE = os.path.join(alfred.work(False), u'mailto.json')

# Don't like names in mailto: URLs
KNOWN_BAD_CLIENTS = [u'Airmail']

class MailTo(object):

    def __init__(self):
        self._settings = self._load_settings()
        log.debug(u'Loaded settings : {!r}'.format(self._settings))
        self._all_apps = None
        self._contacts = None

    def get_use_contact_name(self):
        """Return bool re whether name should be used as well as address"""
        return self._settings.get(u'use_name', None)

    def set_use_contact_name(self, bool):
        """Set whether or not to use contact name as well as address"""
        self._settings[u'use_name'] = bool
        self._save_settings()

    use_contact_name = property(get_use_contact_name, set_use_contact_name)

    def format_recipient(self, email):
        """Return recipient formatted according to settings

        Returns:
            Bob Smith <bob.smith@example.com>
             - or -
            bob.smith@example.com
        """
        def _with_name(email):
            if self._contacts is None:
                self._contacts = dict(get_contacts()[0])  # email:name
            name = self._contacts.get(email)
            if name and name != email:
                return u'{} <{}>'.format(name, email)
            return email
        if self.use_contact_name is False:
            return email
        elif self.use_contact_name is True:
            return _with_name(email)
        else:
            if self.default_app[0] in KNOWN_BAD_CLIENTS:
                return email
            return _with_name(email)

    def get_default_app(self):
        """Return (appname, path) to default mail app

        Return (None, None) if no default is set.
        """
        path = self._settings.get(u'default_app')
        if path is None:
            return (None, None)
        return (self._appname(path), path)

    def set_default_app(self, path=None):
        """Set default app to use by path to application

        If path is None, the default will be deleted
        """
        self._settings[u'default_app'] = path
        self._save_settings()

    default_app = property(get_default_app, set_default_app)

    @property
    def all_apps(self):
        """Return list of apps in /Applications and ~/Applications

        Returns:
            list of tuples (appname, path)
        """
        t = time()
        if self._all_apps is None:
            command = [
                 u'mdfind',
                 u'-onlyin', u'/Applications',
                 u'-onlyin', os.path.expanduser(u'~/Applications'),
                 u'kind:application'
            ]
            lines = check_output(command).decode(u'utf-8').split(u'\n')
            app_paths = [s.strip() for s in lines if s.strip()]
            self._all_apps = [(self._appname(p), p) for p in app_paths]
        log.debug(u'All apps listed in {:0.4f} secs'.format(time() - t))
        return self._all_apps

    def _load_settings(self):
        if not os.path.exists(SETTINGS_CACHE):
            return dict(default_app=None, clients=[], use_name=None)
        with open(SETTINGS_CACHE) as file:
            return json.load(file)

    def _save_settings(self):
        log.debug(u'Saving settings : {}'.format(self._settings))
        with open(SETTINGS_CACHE, u'wb') as file:
            json.dump(self._settings, file, indent=2)

    def _appname(self, path):
        """Get app name from path"""
        return os.path.splitext(os.path.basename(path))[0]

def show_help():
    items = []
    for text, subtext in __help__:
        item = alfred.Item(
            {u'valid':u'yes'},
            text,
            subtext,
            icon=u'info.png'
        )
        items.append(item)
    print(alfred.xml(items, maxresults=25))

def open_help_file():
    """Open help.html with default app"""
    command = [u'open', os.path.join(os.path.dirname(__file__), u'help.html')]
    log.debug(u'help command : {}'.format(command))
    retcode = check_call(command)
    if retcode != 0:
        sys.exit(retcode)

def show_config():
    """Show current settings"""
    log.debug(u'show_config')
    mt = MailTo()
    use_name = mt.use_contact_name
    appname, path = mt.default_app
    # email client
    items = []
    if not appname:
        appname = u'System Default'
        path = u''
    items.append( alfred.Item(
        {u'valid':u'no'},
         u'Current Email Client: {}'.format(appname),
         path,
         icon=u'icon.png')
    )
    items.append(alfred.Item({u'valid':u'yes', u'arg':u'client'},
                              u'Change Client …',
                              u'', icon=u'icon.png'))
    # address format
    title = u'Current Format: Default (Name & Email)'
    subtitle = u'Email-only will be used with some problem clients'
    if use_name is True:
        title = u'Current Format: Name & Email'
        subtitle = u'E.g. Bob Smith <bob@example.com>, Joan Jones <joan@example.com>'
    elif use_name is False:
        title = u'Current Format: Email Only'
        subtitle = u'E.g. bob.smith@example.com, joan@example.com'
    items.append( alfred.Item(
        {u'valid':u'no'},
         title,
         subtitle,
         icon=u'icon.png')
    )
    items.append(alfred.Item({u'valid':u'yes', u'arg':u'format'},
                              u'Change Format …',
                              u'', icon=u'icon.png'))
    print(alfred.xml(items))

def choose_format():
    """Choose email format"""
    items = []
    mt = MailTo()
    use_name = mt.use_contact_name
    title = u'Current Setting: Default (Name & Email)'
    subtitle = u'Email-only will be used with some problem clients'
    if use_name is True:
        title = u'Current Setting: Name & Email'
        subtitle = u'E.g. Bob Smith <bob@example.com>, Joan Jones <joan@example.com>'
    elif use_name is False:
        title = u'Current Setting: Email Only'
        subtitle = u'E.g. bob.smith@example.com, joan@example.com'
    items.append( alfred.Item(
        {u'valid':u'no'},
         title,
         subtitle,
         icon=u'icon.png')
    )
    options = {
        u'email' : alfred.Item({u'valid':u'yes', u'arg':u'email'},
                               u'Call Email Client with Email Only',
                               u'e.g. bob.smith@example.com',
                               icon=u'icon.png'),
        u'name' : alfred.Item({u'valid':u'yes', u'arg':u'name'},
                               u'Call Email Client with Name and Email',
                               u'e.g. Bob Smith <bob.smith@example.com>',
                               icon=u'icon.png'),
        u'default' : alfred.Item({u'valid':u'yes', u'arg':u''},
                               u'Use Default Format',
                               u'Name and email except with known problem clients',
                               icon=u'icon.png')
    }
    if use_name is True:
        items.append(options[u'email'])
        items.append(options[u'default'])
    elif use_name is False:
        items.append(options[u'name'])
        items.append(options[u'default'])
    else:
        items.append(options[u'name'])
        items.append(options[u'email'])
    print(alfred.xml(items))

def choose_client(query):
    """Choose a new email client"""
    log.debug(u'choose_client : {!r}'.format(query))
    query = query.lower()
    mt = MailTo()
    current_app, current_path = mt.default_app
    apps = mt.all_apps
    hits = []
    items = []
    if query == u'':  # Show current setting
        if not current_app:
            appname = u'System Default'
            path = u''
        else:
            appname = current_app
            path = current_path
        items.append( alfred.Item(
            {u'valid':u'no'},
             u'Current Email Client: {}'.format(appname),
             path,
             icon=u'icon.png')
        )
    # Show other options
    if current_app is not None:
        items.append(alfred.Item(
            {u'valid':u'yes',
             u'arg':u'',
             u'autocomplete':u'System Default',
            },
            u'System Default Email Client',
            u'Use system default email client with MailTo',
            icon=u'icon.png')
        )
    for appname, path in apps:
        if appname.lower().startswith(query):
            hits.append((appname, path))
            # log.debug(u'Hit : {}'.format(appname))
    for appname, path in apps:
        if query in appname.lower() and (appname, path) not in hits:
            hits.append((appname, path))
            # log.debug(u'Hit : {}'.format(appname))
    log.debug(u"{} matches for '{}'".format(len(hits), query))
    if not hits and not items:
        return
    for appname, path in hits:
        items.append(alfred.Item(
            {u'valid':u'yes',
             u'arg':path,
             u'autocomplete':appname,
             # u'uid':path
            },
            appname,
            path,
            icon=(path, {u'type':u'fileicon'}))
        )
    print(alfred.xml(items))

def main():
    from docopt import docopt
    args = alfred.args()
    log.debug(u'alfred.args : {}'.format(args))
    args = docopt(__usage__, args)
    log.debug(u'docopt.args : {}'.format(args))

    # application actions
    if args.get(u'config'):  # show configuration
        show_config()

    # email client
    elif args.get(u'client'):  # choose client
        query = args.get(u'<query>')
        if not query:
            query = u''
        query = query.strip()
        choose_client(query)
    elif args.get(u'setclient'):  # set default mail client
        path = args.get(u'<path>')
        log.debug(u'setclient : {!r}'.format(path))
        mt = MailTo()
        if not path:
            mt.default_app = None
            print(u'System Default')
        else:
            mt.default_app = path
            appname = mt.default_app[0]
            print(appname)

    # email format
    elif args.get(u'format'):  # set email format
        choose_format()
    elif args.get(u'setformat'):  # set default format
        fmt = args.get(u'<format>')
        log.debug(u'setformat : {!r}'.format(fmt))
        mt = MailTo()
        if not fmt:
            mt.use_contact_name = None
            print(u'Default Format')
        elif fmt == u'name':
            mt.use_contact_name = True
            print(u'Name and Email Address')
        elif fmt == u'email':
            mt.use_contact_name = False
            print(u'Email Address Only')

    # other options
    elif args.get(u'delcache'):  # delete settings file
        if os.path.exists(SETTINGS_CACHE):
            os.unlink(SETTINGS_CACHE)
    elif args.get(u'dellog'):  # delete logfile
        if os.path.exists(LOGFILE):
            os.unlink(LOGFILE)
    elif args.get(u'openlog'):  # open logfile
        check_call([u'open', LOGFILE])
    elif args.get(u'help'):  # show help in Alfred
        show_help()
    elif args.get(u'openhelp'):  # open help file in browser
        open_help_file()
    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Indicate if app is currently configured and refuse to set it again<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
Manage default email client and display help.

Usage:
    mailto.py <action> [<path>|<query>]

Possible actions:

config
    Show current settings

format
    Show current format

setformat
    Set new format

client [<query>]
    Show/choose email client

setclient <path>
    Set the email client to use to application at <path>.
    Called when an option in the above call is selected

help
    Display brief help (in Alfred)

openhelp
    Open the help.html file in the default application

openlog
    Open logfile in default app

dellog
    Delete the logfile.

delcache
    Delete the cache file containing the settings. Mostly for me when I fuck
    stuff up.

"""

from __future__ import print_function

import sys
import os
import json
from subprocess import check_output, check_call
from time import time

import alfred
from contacts import get_contacts
from log import logger, LOGFILE

log = logger(u'mailto')

__usage__ = u"""
Usage:
    mailto config
    mailto format
    mailto setformat [<format>]
    mailto client [<query>]
    mailto setclient [<path>]
    mailto help
    mailto openhelp
    mailto openlog
    mailto dellog
    mailto delcache
"""

__help__ = [
    (u"Open MailTo help file",
        u"In your browser"),
    (u"'mailtoconf' to view and change settings",
        u"View/change email client and format"),
    (u"Enter 'mailtohelp' for this message",
        u"Well, I guess you already figured this out …"),
]

SETTINGS_CACHE = os.path.join(alfred.work(False), u'mailto.json')

# Don't like names in mailto: URLs
KNOWN_BAD_CLIENTS = [u'Airmail']

class MailTo(object):

    def __init__(self):
        self._settings = self._load_settings()
        log.debug(u'Loaded settings : {!r}'.format(self._settings))
        self._all_apps = None
        self._contacts = None

    def get_use_contact_name(self):
        """Return bool re whether name should be used as well as address"""
        return self._settings.get(u'use_name', None)

    def set_use_contact_name(self, bool):
        """Set whether or not to use contact name as well as address"""
        self._settings[u'use_name'] = bool
        self._save_settings()

    use_contact_name = property(get_use_contact_name, set_use_contact_name)

    def format_recipient(self, email):
        """Return recipient formatted according to settings

        Returns:
            Bob Smith <bob.smith@example.com>
             - or -
            bob.smith@example.com
        """
        def _with_name(email):
            if self._contacts is None:
                self._contacts = dict(get_contacts()[0])  # email:name
            name = self._contacts.get(email)
            if name and name != email:
                return u'{} <{}>'.format(name, email)
            return email
        if self.use_contact_name is False:
            return email
        elif self.use_contact_name is True:
            return _with_name(email)
        else:
            if self.default_app[0] in KNOWN_BAD_CLIENTS:
                return email
            return _with_name(email)

    def get_default_app(self):
        """Return (appname, path) to default mail app

        Return (None, None) if no default is set.
        """
        path = self._settings.get(u'default_app')
        if path is None:
            return (None, None)
        return (self._appname(path), path)

    def set_default_app(self, path=None):
        """Set default app to use by path to application

        If path is None, the default will be deleted
        """
        self._settings[u'default_app'] = path
        self._save_settings()

    default_app = property(get_default_app, set_default_app)

    @property
    def all_apps(self):
        """Return list of apps in /Applications and ~/Applications

        Returns:
            list of tuples (appname, path)
        """
        t = time()
        if self._all_apps is None:
            command = [
                 u'mdfind',
                 u'-onlyin', u'/Applications',
                 u'-onlyin', os.path.expanduser(u'~/Applications'),
                 u'kind:application'
            ]
            lines = check_output(command).decode(u'utf-8').split(u'\n')
            app_paths = [s.strip() for s in lines if s.strip()]
            self._all_apps = [(self._appname(p), p) for p in app_paths]
        log.debug(u'All apps listed in {:0.4f} secs'.format(time() - t))
        return self._all_apps

    def _load_settings(self):
        if not os.path.exists(SETTINGS_CACHE):
            return dict(default_app=None, clients=[], use_name=None)
        with open(SETTINGS_CACHE) as file:
            return json.load(file)

    def _save_settings(self):
        log.debug(u'Saving settings : {}'.format(self._settings))
        with open(SETTINGS_CACHE, u'wb') as file:
            json.dump(self._settings, file, indent=2)

    def _appname(self, path):
        """Get app name from path"""
        return os.path.splitext(os.path.basename(path))[0]

def show_help():
    items = []
    for text, subtext in __help__:
        item = alfred.Item(
            {u'valid':u'yes'},
            text,
            subtext,
            icon=u'info.png'
        )
        items.append(item)
    print(alfred.xml(items, maxresults=25))

def open_help_file():
    """Open help.html with default app"""
    command = [u'open', os.path.join(os.path.dirname(__file__), u'help.html')]
    log.debug(u'help command : {}'.format(command))
    retcode = check_call(command)
    if retcode != 0:
        sys.exit(retcode)

def show_config():
    """Show current settings"""
    log.debug(u'show_config')
    mt = MailTo()
    use_name = mt.use_contact_name
    appname, path = mt.default_app
    # email client
    items = []
    if not appname:
        appname = u'System Default'
        path = u''
    items.append( alfred.Item(
        {u'valid':u'no'},
         u'Current Email Client: {}'.format(appname),
         path,
         icon=u'icon.png')
    )
    items.append(alfred.Item({u'valid':u'yes', u'arg':u'client'},
                              u'Change Client …',
                              u'', icon=u'icon.png'))
    # address format
    title = u'Current Format: Default (Name & Email)'
    subtitle = u'Email-only will be used with some problem clients'
    if use_name is True:
        title = u'Current Format: Name & Email'
        subtitle = u'E.g. Bob Smith <bob@example.com>, Joan Jones <joan@example.com>'
    elif use_name is False:
        title = u'Current Format: Email Only'
        subtitle = u'E.g. bob.smith@example.com, joan@example.com'
    items.append( alfred.Item(
        {u'valid':u'no'},
         title,
         subtitle,
         icon=u'icon.png')
    )
    items.append(alfred.Item({u'valid':u'yes', u'arg':u'format'},
                              u'Change Format …',
                              u'', icon=u'icon.png'))
    print(alfred.xml(items))

def choose_format():
    """Choose email format"""
    items = []
    mt = MailTo()
    use_name = mt.use_contact_name
    title = u'Current Setting: Default (Name & Email)'
    subtitle = u'Email-only will be used with some problem clients'
    if use_name is True:
        title = u'Current Setting: Name & Email'
        subtitle = u'E.g. Bob Smith <bob@example.com>, Joan Jones <joan@example.com>'
    elif use_name is False:
        title = u'Current Setting: Email Only'
        subtitle = u'E.g. bob.smith@example.com, joan@example.com'
    items.append( alfred.Item(
        {u'valid':u'no'},
         title,
         subtitle,
         icon=u'icon.png')
    )
    options = {
        u'email' : alfred.Item({u'valid':u'yes', u'arg':u'email'},
                               u'Call Email Client with Email Only',
                               u'e.g. bob.smith@example.com',
                               icon=u'icon.png'),
        u'name' : alfred.Item({u'valid':u'yes', u'arg':u'name'},
                               u'Call Email Client with Name and Email',
                               u'e.g. Bob Smith <bob.smith@example.com>',
                               icon=u'icon.png'),
        u'default' : alfred.Item({u'valid':u'yes', u'arg':u''},
                               u'Use Default Format',
                               u'Name and email except with known problem clients',
                               icon=u'icon.png')
    }
    if use_name is True:
        items.append(options[u'email'])
        items.append(options[u'default'])
    elif use_name is False:
        items.append(options[u'name'])
        items.append(options[u'default'])
    else:
        items.append(options[u'name'])
        items.append(options[u'email'])
    print(alfred.xml(items))

def choose_client(query):
    """Choose a new email client"""
    log.debug(u'choose_client : {!r}'.format(query))
    query = query.lower()
    mt = MailTo()
    current_app, current_path = mt.default_app
    apps = mt.all_apps
    hits = []
    items = []
    if query == u'':  # Show current setting
        if not current_app:
            appname = u'System Default'
            path = u''
        else:
            appname = current_app
            path = current_path
        items.append( alfred.Item(
            {u'valid':u'no'},
             u'Current Email Client: {}'.format(appname),
             path,
             icon=u'icon.png')
        )
    # Show other options
    if current_app is not None:
        items.append(alfred.Item(
            {u'valid':u'yes',
             u'arg':u'',
             u'autocomplete':u'System Default',
            },
            u'System Default Email Client',
            u'Use system default email client with MailTo',
            icon=u'icon.png')
        )
    for appname, path in apps:
        if appname.lower().startswith(query):
            hits.append((appname, path))
            # log.debug(u'Hit : {}'.format(appname))
    for appname, path in apps:
        if query in appname.lower() and (appname, path) not in hits:
            hits.append((appname, path))
            # log.debug(u'Hit : {}'.format(appname))
    log.debug(u"{} matches for '{}'".format(len(hits), query))
    if not hits and not items:
        return
    for appname, path in hits:
        if path == current_path:
            items.append(alfred.Item(
                {u'valid':u'no'
                 # u'uid':path
                },
                u'{} (current client)'.format(appname),
                path,
                icon=(path, {u'type':u'fileicon'}))
            )
        else:
            items.append(alfred.Item(
                {u'valid':u'yes',
                 u'arg':path,
                 u'autocomplete':appname,
                 # u'uid':path
                },
                appname,
                path,
                icon=(path, {u'type':u'fileicon'}))
            )
    print(alfred.xml(items))

def main():
    from docopt import docopt
    args = alfred.args()
    log.debug(u'alfred.args : {}'.format(args))
    args = docopt(__usage__, args)
    log.debug(u'docopt.args : {}'.format(args))

    # application actions
    if args.get(u'config'):  # show configuration
        show_config()

    # email client
    elif args.get(u'client'):  # choose client
        query = args.get(u'<query>')
        if not query:
            query = u''
        query = query.strip()
        choose_client(query)
    elif args.get(u'setclient'):  # set default mail client
        path = args.get(u'<path>')
        log.debug(u'setclient : {!r}'.format(path))
        mt = MailTo()
        if not path:
            mt.default_app = None
            print(u'System Default')
        else:
            mt.default_app = path
            appname = mt.default_app[0]
            print(appname)

    # email format
    elif args.get(u'format'):  # set email format
        choose_format()
    elif args.get(u'setformat'):  # set default format
        fmt = args.get(u'<format>')
        log.debug(u'setformat : {!r}'.format(fmt))
        mt = MailTo()
        if not fmt:
            mt.use_contact_name = None
            print(u'Default Format')
        elif fmt == u'name':
            mt.use_contact_name = True
            print(u'Name and Email Address')
        elif fmt == u'email':
            mt.use_contact_name = False
            print(u'Email Address Only')

    # other options
    elif args.get(u'delcache'):  # delete settings file
        if os.path.exists(SETTINGS_CACHE):
            os.unlink(SETTINGS_CACHE)
    elif args.get(u'dellog'):  # delete logfile
        if os.path.exists(LOGFILE):
            os.unlink(LOGFILE)
    elif args.get(u'openlog'):  # open logfile
        check_call([u'open', LOGFILE])
    elif args.get(u'help'):  # show help in Alfred
        show_help()
    elif args.get(u'openhelp'):  # open help file in browser
        open_help_file()
    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import glob
import os
import os.path
import random
import shlex
import shutil
import subprocess
import sys
import tempfile
import urlparse
import logging

logger = logging.getLogger('docfu')

#
# Utils
#
def uri_parse(u):
    """ A provided url can be one of:
        * github repo: "feltnerm/foo"
        * git path: "http://github.com/feltnerm/foo.git"
        * filesystem path: "~/Projects/foo"

        This functions parses a provided URL string matching one of those
        three categories. """
    url = urlparse.urlparse(u)

    if not url.scheme:
        if u.count('/') == 1 and len(u.split('/')) == 2:
            # github url
            u = u.strip()
            u = "http://github.com/" + u
            url = urlparse.urlparse(u)
        else:
            u = "file://" + os.path.expanduser(u)
            url = urlparse.urlparse(u)

    return urlparse.urlunparse(url)

#
# Git / repository utils
#
def git_clone(git_url):
    """ Clone a github url into a temp directory. Set a global object to
    this class so it can be closed. """
    logger.debug("Cloning %s" % git_url)
    path = tmp_mk()
    git_clone_cmd = shlex.split('git clone %s %s' % (str(git_url), str(path)))
    logger.debug("%s"  % git_clone_cmd)
    retcode = subprocess.check_call(git_clone_cmd)
    return path

def git_checkout(git_repo_path, ref_type, ref_val):
    """ Checkout the code at git_repo_path. Ref is the specific branch or
    tag to use. """
    if ref_type == 'branch':
        git_checkout_cmd = shlex.split('git checkout %s' % str(ref_val))

    if ref_type == 'tag':
        git_checkout_cmd = shlex.split('git checkout -b %s' % str(tag))

    logger.debug("Checking out %s: %s" % (ref_type, ref_val))
    logger.debug("%s" % git_checkout_cmd)
    output = subprocess.Popen(git_checkout_cmd, cwd="%s" % str(git_repo_path)).communicate()
    logger.info(output)

def get_git_tag(git_repo_path):
    cmd = shlex.split("git describe --tags $(git rev-list --tags --max-count=1)")
    logger.debug("Getting git tag: %s" % cmd)
    p = subprocess.Popen(cmd, cwd="%s" % str(git_repo_path), stdout=subprocess.PIPE)
    out, err = p.communicate()
    if out and not err:
        logger.debug("Tag: %s" % out)
        return out
    logger.error("%s" % err)
    return ''

def get_git_branch(git_repo_path):
    cmd = shlex.split("git rev-parse --abbrev-ref HEAD")
    logger.debug("Getting git branch: %s" % cmd)
    p = subprocess.Popen(cmd, cwd="%s" % str(git_repo_path), stdout=subprocess.PIPE)
    out, err = p.communicate()
    if out and not err:
        logger.debug("Branch: %s" % out)
        return branch
    logger.error("%s" % err)
    return ''
#
# Temporary File / Directory Utilities
#

def tmp_mk():
    """ Make a temporary directory, already prefixed with `docfu-`,
    in `/tmp/`."""
    if not os.path.isdir(os.path.expanduser("~/tmp")):
        os.makedirs(os.path.expanduser("~/tmp"))
    path = tempfile.mkdtemp(prefix='docfu-', dir=os.path.expanduser('~/tmp'))
    logger.debug("Temporary path created at: %s" % path)
    return path

def tmp_close(path):
    """ Remove the directory denoted by `path`. """
    try:
        shutil.rmtree(path)
        logger.debug("Temporary path removed at: %s" % path)
    except Exception, e:
        if e.errno != 2:
            logger.error("Cannot remove temporary path %s. Error %s" % (path, e))
            raise

def tmp_cp(src):
    """ Copy the source directory to a tmp directory.

    @TODO: ignore version control and other things.
    """
    dest = tmp_mk()
    dest = os.path.join(os.path.expanduser("~"), 'tmp', 'docfu-%s' % random.randint(999, 10000))
    shutil.copytree(src, dest, ignore=shutil.ignore_patterns('*.git', 'node_modules'))
    logger.debug("Copied source files %s to tempdir %s" % (src, dest))
    return dest

def walk_files(path):
    """ Return a set of files found in `path`. """
    paths = set()
    logger.debug("Walking: %s" % path)
    for current, dirs, files in os.walk(path):
        for f in files:
            paths.add(os.path.join(current, f))
            logger.debug("Source file found: %s" % os.path.join(current, f))
    return paths

def list_refs(path):
    """ Return the list of branches/tags found in the root `path`.
    We assume the folder structure is like so:
        `<root>[/<ref_type>[/<ref_name>]]`
    {
        <ref_type> :
            {
                <path>: '',
                <refs>: [
                    {
                        <ref_val>: '',
                        <path>: ''
                    },
                ],
            },
    }
    """
    result = {}
    if os.path.isdir(path):
        logger.debug("Listing refs for root path: %s" % path)
        ref_types = [(x, os.path.basename(x)) for x in glob.glob(os.path.join(path, '*'))]

        for x in glob.glob(os.path.join(path, '*')):
            ref_type = os.path.basename(x)
            result[ref_type] = {}
            result[ref_type]['path'] = x
            result[ref_type]['refs'] = []
            for y in glob.glob(os.path.join(x, '*')):
                ref_value = os.path.basename(y)
                result[ref_type]['refs'].append({ 'ref_val': ref_value, 'path': y })

    return result

def list_doc_tree(root_path):
    """ Returns a 'tree-like' structure for navigation. """
    # (name, relpath, sub)
    doctree = []

    def recurse_doc_tree(item_path):
        result = []
        item_sub_dirs = []
        item_sub_files = []
        item_name = os.path.basename(item_path)
        item_dir = os.path.dirname(item_path)
        if os.path.isdir(item_path):
            for name in os.listdir(item_path):
                if not name.startswith(".") and name != 'index.html':
                    if os.path.isdir(os.path.join(item_path, name)):
                            item_sub_dirs.append(recurse_doc_tree(os.path.join(item_path, name)))

                    elif os.path.isfile(os.path.join(item_path, name)):
                        item_sub_files.append({ 'name': os.path.splitext(name)[0], 'path': item_path.replace(root_path, "")})

        result = { 'name': os.path.splitext(item_name)[0], 'path': item_path.replace(root_path, "") }
        if len(item_sub_dirs) > 0:
            result['dirs'] = item_sub_dirs
        if len(item_sub_files) > 0:
            result['files'] = item_sub_files
        return result

    for item_name in os.listdir(root_path):
        if not item_name.startswith(".") and item_name != 'index.html':
            doctree.append(recurse_doc_tree(os.path.join(root_path, item_name)))
    return doctree

    #matches = {}
    #for p, dirs, files in os.walk(path):
    #    name = os.path.basename(p)
    #    relpath = p.replace(path, "")
    #    files2 = []
    #    for f in files:
    #        if not f.startswith("."):
    #            fname = os.path.splitext(os.path.basename(f))[0]
    #            fpath = os.path.join(relpath, f)
    #            files2.append((fname, fpath))

    #    matches.setdefault(p,[]).append(os.path.join)
    #    result.append((name, relpath, files2))
    #return result
<commit_msg>fix: typo<commit_after>import glob
import os
import os.path
import random
import shlex
import shutil
import subprocess
import sys
import tempfile
import urlparse
import logging

logger = logging.getLogger('docfu')

#
# Utils
#
def uri_parse(u):
    """ A provided url can be one of:
        * github repo: "feltnerm/foo"
        * git path: "http://github.com/feltnerm/foo.git"
        * filesystem path: "~/Projects/foo"

        This functions parses a provided URL string matching one of those
        three categories. """
    url = urlparse.urlparse(u)

    if not url.scheme:
        if u.count('/') == 1 and len(u.split('/')) == 2:
            # github url
            u = u.strip()
            u = "http://github.com/" + u
            url = urlparse.urlparse(u)
        else:
            u = "file://" + os.path.expanduser(u)
            url = urlparse.urlparse(u)

    return urlparse.urlunparse(url)

#
# Git / repository utils
#
def git_clone(git_url):
    """ Clone a github url into a temp directory. Set a global object to
    this class so it can be closed. """
    logger.debug("Cloning %s" % git_url)
    path = tmp_mk()
    git_clone_cmd = shlex.split('git clone %s %s' % (str(git_url), str(path)))
    logger.debug("%s"  % git_clone_cmd)
    retcode = subprocess.check_call(git_clone_cmd)
    return path

def git_checkout(git_repo_path, ref_type, ref_val):
    """ Checkout the code at git_repo_path. Ref is the specific branch or
    tag to use. """
    if ref_type == 'branch':
        git_checkout_cmd = shlex.split('git checkout %s' % str(ref_val))

    if ref_type == 'tag':
        git_checkout_cmd = shlex.split('git checkout -b %s' % str(tag))

    logger.debug("Checking out %s: %s" % (ref_type, ref_val))
    logger.debug("%s" % git_checkout_cmd)
    output = subprocess.Popen(git_checkout_cmd, cwd="%s" % str(git_repo_path)).communicate()
    logger.info(output)

def get_git_tag(git_repo_path):
    cmd = shlex.split("git describe --tags $(git rev-list --tags --max-count=1)")
    logger.debug("Getting git tag: %s" % cmd)
    p = subprocess.Popen(cmd, cwd="%s" % str(git_repo_path), stdout=subprocess.PIPE)
    out, err = p.communicate()
    if out and not err:
        logger.debug("Tag: %s" % out)
        return out
    logger.error("%s" % err)
    return ''

def get_git_branch(git_repo_path):
    cmd = shlex.split("git rev-parse --abbrev-ref HEAD")
    logger.debug("Getting git branch: %s" % cmd)
    p = subprocess.Popen(cmd, cwd="%s" % str(git_repo_path), stdout=subprocess.PIPE)
    out, err = p.communicate()
    if out and not err:
        logger.debug("Branch: %s" % out)
        return out
    logger.error("%s" % err)
    return ''
#
# Temporary File / Directory Utilities
#

def tmp_mk():
    """ Make a temporary directory, already prefixed with `docfu-`,
    in `/tmp/`."""
    if not os.path.isdir(os.path.expanduser("~/tmp")):
        os.makedirs(os.path.expanduser("~/tmp"))
    path = tempfile.mkdtemp(prefix='docfu-', dir=os.path.expanduser('~/tmp'))
    logger.debug("Temporary path created at: %s" % path)
    return path

def tmp_close(path):
    """ Remove the directory denoted by `path`. """
    try:
        shutil.rmtree(path)
        logger.debug("Temporary path removed at: %s" % path)
    except Exception, e:
        if e.errno != 2:
            logger.error("Cannot remove temporary path %s. Error %s" % (path, e))
            raise

def tmp_cp(src):
    """ Copy the source directory to a tmp directory.

    @TODO: ignore version control and other things.
    """
    dest = tmp_mk()
    dest = os.path.join(os.path.expanduser("~"), 'tmp', 'docfu-%s' % random.randint(999, 10000))
    shutil.copytree(src, dest, ignore=shutil.ignore_patterns('*.git', 'node_modules'))
    logger.debug("Copied source files %s to tempdir %s" % (src, dest))
    return dest

def walk_files(path):
    """ Return a set of files found in `path`. """
    paths = set()
    logger.debug("Walking: %s" % path)
    for current, dirs, files in os.walk(path):
        for f in files:
            paths.add(os.path.join(current, f))
            logger.debug("Source file found: %s" % os.path.join(current, f))
    return paths

def list_refs(path):
    """ Return the list of branches/tags found in the root `path`.
    We assume the folder structure is like so:
        `<root>[/<ref_type>[/<ref_name>]]`
    {
        <ref_type> :
            {
                <path>: '',
                <refs>: [
                    {
                        <ref_val>: '',
                        <path>: ''
                    },
                ],
            },
    }
    """
    result = {}
    if os.path.isdir(path):
        logger.debug("Listing refs for root path: %s" % path)
        ref_types = [(x, os.path.basename(x)) for x in glob.glob(os.path.join(path, '*'))]

        for x in glob.glob(os.path.join(path, '*')):
            ref_type = os.path.basename(x)
            result[ref_type] = {}
            result[ref_type]['path'] = x
            result[ref_type]['refs'] = []
            for y in glob.glob(os.path.join(x, '*')):
                ref_value = os.path.basename(y)
                result[ref_type]['refs'].append({ 'ref_val': ref_value, 'path': y })

    return result

def list_doc_tree(root_path):
    """ Returns a 'tree-like' structure for navigation. """
    # (name, relpath, sub)
    doctree = []

    def recurse_doc_tree(item_path):
        result = []
        item_sub_dirs = []
        item_sub_files = []
        item_name = os.path.basename(item_path)
        item_dir = os.path.dirname(item_path)
        if os.path.isdir(item_path):
            for name in os.listdir(item_path):
                if not name.startswith(".") and name != 'index.html':
                    if os.path.isdir(os.path.join(item_path, name)):
                            item_sub_dirs.append(recurse_doc_tree(os.path.join(item_path, name)))

                    elif os.path.isfile(os.path.join(item_path, name)):
                        item_sub_files.append({ 'name': os.path.splitext(name)[0], 'path': item_path.replace(root_path, "")})

        result = { 'name': os.path.splitext(item_name)[0], 'path': item_path.replace(root_path, "") }
        if len(item_sub_dirs) > 0:
            result['dirs'] = item_sub_dirs
        if len(item_sub_files) > 0:
            result['files'] = item_sub_files
        return result

    for item_name in os.listdir(root_path):
        if not item_name.startswith(".") and item_name != 'index.html':
            doctree.append(recurse_doc_tree(os.path.join(root_path, item_name)))
    return doctree

    #matches = {}
    #for p, dirs, files in os.walk(path):
    #    name = os.path.basename(p)
    #    relpath = p.replace(path, "")
    #    files2 = []
    #    for f in files:
    #        if not f.startswith("."):
    #            fname = os.path.splitext(os.path.basename(f))[0]
    #            fpath = os.path.join(relpath, f)
    #            files2.append((fname, fpath))

    #    matches.setdefault(p,[]).append(os.path.join)
    #    result.append((name, relpath, files2))
    #return result
<|endoftext|>


<commit_before># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import collections
import os
import logging
import time
import cPickle as pickle
import task_history as history
logger = logging.getLogger("luigi.server")

from task_status import PENDING, FAILED, DONE, RUNNING, UNKNOWN

class Scheduler(object):
    ''' Abstract base class

    Note that the methods all take string arguments, not Task objects...
    '''
    add_task = NotImplemented
    get_work = NotImplemented
    ping = NotImplemented

UPSTREAM_RUNNING = 'UPSTREAM_RUNNING'
UPSTREAM_MISSING_INPUT = 'UPSTREAM_MISSING_INPUT'
UPSTREAM_FAILED = 'UPSTREAM_FAILED'

UPSTREAM_SEVERITY_ORDER = ('', UPSTREAM_RUNNING, UPSTREAM_MISSING_INPUT, UPSTREAM_FAILED)
UPSTREAM_SEVERITY_KEY = lambda st: UPSTREAM_SEVERITY_ORDER.index(st)
STATUS_TO_UPSTREAM_MAP = {FAILED: UPSTREAM_FAILED, RUNNING: UPSTREAM_RUNNING, PENDING: UPSTREAM_MISSING_INPUT}

class Task(object):
    def __init__(self, status, deps, resources):
        self.resources = resources
        self.stakeholders = set()  # workers that are somehow related to this task (i.e. don't prune while any of these workers are still active)
        self.workers = set()  # workers that can perform task - task is 'BROKEN' if none of these workers are active
        if deps is None:
            self.deps = set()
        else:
            self.deps = set(deps)
        self.status = status  # PENDING, RUNNING, FAILED or DONE
        self.time = time.time()  # Timestamp when task was first added
        self.retry = None
        self.remove = None
        self.worker_running = None  # the worker that is currently running the task or None
        self.expl = None

    def __repr__(self):
        return "Task(%r)" % vars(self)

class CentralPlannerScheduler(Scheduler):
    ''' Async scheduler that can handle multiple workers etc

    Can be run locally or on a server (using RemoteScheduler + server.Server).
    '''

    def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0,
                 state_path='/var/lib/luigi-server/state.pickle', task_history=None,
                 resources=None):
        '''
        (all arguments are in seconds)
        Keyword Arguments:
        retry_delay -- How long after a Task fails to try it again, or -1 to never retry
        remove_delay -- How long after a Task finishes to remove it from the scheduler
        state_path -- Path to state file (tasks and active workers)
        worker_disconnect_delay -- If a worker hasn't communicated for this long, remove it from active workers
        '''
        self._state_path = state_path
        self._tasks = {}
        self._retry_delay = retry_delay
        self._remove_delay = remove_delay
        self._worker_disconnect_delay = worker_disconnect_delay
        self._active_workers = {}  # map from id to timestamp (last updated)
        self._task_history = task_history or history.NopHistory()
        self._resources = resources
        # TODO: have a Worker object instead, add more data to it

    def dump(self):
        state = (self._tasks, self._active_workers)
        try:
            with open(self._state_path, 'w') as fobj:
                pickle.dump(state, fobj)
        except IOError:
            logger.warning("Failed saving scheduler state", exc_info=1)
        else:
            logger.info("Saved state in %s", self._state_path)

    def load(self):
        if os.path.exists(self._state_path):
            logger.info("Attempting to load state from %s", self._state_path)
            with open(self._state_path) as fobj:
                state = pickle.load(fobj)
            self._tasks, self._active_workers = state
        else:
            logger.info("No prior state file exists at %s. Starting with clean slate", self._state_path)

    def prune(self):
        logger.info("Starting pruning of task graph")
        # Delete workers that haven't said anything for a while (probably killed)
        delete_workers = []
        for worker in self._active_workers:
            if self._active_workers[worker] < time.time() - self._worker_disconnect_delay:
                logger.info("worker %r updated at %s timed out (no contact for >=%ss)", worker, self._active_workers[worker], self._worker_disconnect_delay)
                delete_workers.append(worker)

        for worker in delete_workers:
            self._active_workers.pop(worker)

        remaining_workers = set(self._active_workers.keys())

        # Mark tasks with no remaining active stakeholders for deletion
        for task_id, task in self._tasks.iteritems():
            if not task.stakeholders.intersection(remaining_workers):
                if task.remove is None:
                    logger.info("Task %r has stakeholders %r but none remain connected -> will remove task in %s seconds", task_id, task.stakeholders, self._remove_delay)
                    task.remove = time.time() + self._remove_delay

            if task.status == RUNNING and task.worker_running and task.worker_running not in remaining_workers:
                # If a running worker disconnects, tag all its jobs as FAILED and subject it to the same retry logic
                logger.info("Task %r is marked as running by disconnected worker %r -> marking as FAILED with retry delay of %rs", task_id, task.worker_running, self._retry_delay)
                task.worker_running = None
                task.status = FAILED
                task.retry = time.time() + self._retry_delay

        # Remove tasks that have no stakeholders
        remove_tasks = []
        for task_id, task in self._tasks.iteritems():
            if task.remove and time.time() > task.remove:
                logger.info("Removing task %r (no connected stakeholders)", task_id)
                remove_tasks.append(task_id)

        for task_id in remove_tasks:
            self._tasks.pop(task_id)

        # Reset FAILED tasks to PENDING if max timeout is reached, and retry delay is >= 0
        for task in self._tasks.values():
            if task.status == FAILED and self._retry_delay >= 0 and task.retry < time.time():
                task.status = PENDING
        logger.info("Done pruning task graph")

    def update(self, worker):
        # update timestamp so that we keep track
        # of whenever the worker was last active
        self._active_workers[worker] = time.time()

    def add_task(self, worker, task_id, status=PENDING, runnable=True, deps=None, expl=None,
                 resources=None):
        """
        * Add task identified by task_id if it doesn't exist
        * If deps is not None, update dependency list
        * Update status of task
        * Add additional workers/stakeholders
        """
        self.update(worker)

        task = self._tasks.setdefault(task_id, Task(status=PENDING, deps=deps, resources=resources))

        if task.remove is not None:
            task.remove = None  # unmark task for removal so it isn't removed after being added

        if not (task.status == RUNNING and status == PENDING):
            # don't allow re-scheduling of task while it is running, it must either fail or succeed first
            task.status = status
            if status == FAILED:
                task.retry = time.time() + self._retry_delay

        if deps is not None:
            task.deps = set(deps)

        task.stakeholders.add(worker)

        if runnable:
            task.workers.add(worker)

        if expl is not None:
            task.expl = expl
        self._update_task_history(task_id, status)

    def update_resources(self, **resources):
        if self._resources is None:
            self._resources = {}
        self._resources.update(resources)

    def _has_resources(self, needed_resources, used_resources):
        if self._resources is None or needed_resources is None:
            return True

        for resource, amount in needed_resources.items():
            if amount + used_resources[resource] > self._resources.get(resource, 0):
                return False
        return True

    def _used_resources(self):
        used_resources = collections.defaultdict(int)
        if self._resources is not None:
            for task in self._tasks.itervalues():
                if task.status == RUNNING and task.resources:
                    for resource, amount in task.resources.items():
                        used_resources[resource] += amount
        return used_resources

    def get_work(self, worker, host=None):
        # TODO: remove any expired nodes

        # Algo: iterate over all nodes, find first node with no dependencies and available resources

        # TODO: remove tasks that can't be done, figure out if the worker has absolutely
        # nothing it can wait for

        # Return remaining tasks that have no FAILED descendents
        self.update(worker)
        best_t = float('inf')
        best_task = None
        locally_pending_tasks = 0
        running_tasks = []
        used_resources = self._used_resources()

        for task_id, task in self._tasks.iteritems():
            if worker not in task.workers:
                continue

            if task.status == RUNNING:
                running_tasks.append({'task_id': task_id, 'worker': task.worker_running})

            if task.status != PENDING:
                continue

            if not self._has_resources(task.resources, used_resources):
                continue

            locally_pending_tasks += 1
            ok = True
            for dep in task.deps:
                if dep not in self._tasks:
                    ok = False
                elif self._tasks[dep].status != DONE:
                    ok = False

            if ok:
                if task.time < best_t:
                    best_t = task.time
                    best_task = task_id

        if best_task:
            t = self._tasks[best_task]
            t.status = RUNNING
            t.worker_running = worker
            self._update_task_history(best_task, RUNNING, host=host)

        return {'n_pending_tasks': locally_pending_tasks,
                'task_id': best_task,
                'running_tasks': running_tasks}

    def ping(self, worker):
        self.update(worker)

    def _upstream_status(self, task_id, upstream_status_table):
        if task_id in upstream_status_table:
            return upstream_status_table[task_id]
        elif task_id in self._tasks:
            task_stack = [task_id]

            while task_stack:
                dep_id = task_stack.pop()
                if dep_id in self._tasks:
                    dep = self._tasks[dep_id]
                    if dep_id not in upstream_status_table:
                        if dep.status == PENDING and dep.deps:
                            task_stack = task_stack + [dep_id] + list(dep.deps)
                            upstream_status_table[dep_id] = ''  # will be updated postorder
                        else:
                            dep_status = STATUS_TO_UPSTREAM_MAP.get(dep.status, '')
                            upstream_status_table[dep_id] = dep_status
                    elif upstream_status_table[dep_id] == '' and dep.deps:
                        # This is the postorder update step when we set the
                        # status based on the previously calculated child elements
                        upstream_status = [upstream_status_table.get(id, '') for id in dep.deps]
                        upstream_status.append('')  # to handle empty list
                        status = max(upstream_status, key=UPSTREAM_SEVERITY_KEY)
                        upstream_status_table[dep_id] = status
            return upstream_status_table[dep_id]

    def _serialize_task(self, task_id):
        task = self._tasks[task_id]
        return {
            'deps': list(task.deps),
            'status': task.status,
            'workers': list(task.workers),
            'start_time': task.time,
            'params': self._get_task_params(task_id),
            'name': self._get_task_name(task_id)
        }

    def _get_task_params(self, task_id):
        params = {}
        params_part = task_id.split('(')[1].strip(')')
        params_strings = params_part.split(", ")

        for param in params_strings:
            if not param:
                continue
            split_param = param.split('=')
            if len(split_param) != 2:
                return {'<complex parameters>': params_part}
            params[split_param[0]] = split_param[1]
        return params

    def _get_task_name(self, task_id):
        return task_id.split('(')[0]

    def graph(self):
        self.prune()
        serialized = {}
        for task_id, task in self._tasks.iteritems():
            serialized[task_id] = self._serialize_task(task_id)
        return serialized

    def _recurse_deps(self, task_id, serialized):
        if task_id not in serialized:
            task = self._tasks.get(task_id)
            if task is None:
                logger.warn('Missing task for id [%s]', task_id)
                serialized[task_id] = {
                    'deps': [],
                    'status': UNKNOWN,
                    'workers': [],
                    'start_time': UNKNOWN,
                    'params': self._get_task_params(task_id),
                    'name': self._get_task_name(task_id)
                }
            else:
                serialized[task_id] = self._serialize_task(task_id)
                for dep in task.deps:
                    self._recurse_deps(dep, serialized)

    def dep_graph(self, task_id):
        self.prune()
        serialized = {}
        if task_id in self._tasks:
            self._recurse_deps(task_id, serialized)
        return serialized

    def task_list(self, status, upstream_status):
        ''' query for a subset of tasks by status '''
        self.prune()
        result = {}
        upstream_status_table = {}  # used to memoize upstream status
        for task_id, task in self._tasks.iteritems():
            if not status or task.status == status:
                if (task.status != PENDING or not upstream_status or
                    upstream_status == self._upstream_status(task_id, upstream_status_table)):
                    serialized = self._serialize_task(task_id)
                    result[task_id] = serialized
        return result

    def fetch_error(self, task_id):
        if self._tasks[task_id].expl is not None:
            return {"taskId": task_id, "error": self._tasks[task_id].expl}
        else:
            return {"taskId": task_id, "error": ""}

    def _update_task_history(self, task_id, status, host=None):
        try:
            if status == DONE or status == FAILED:
                successful = (status == DONE)
                self._task_history.task_finished(task_id, successful)
            elif status == PENDING:
                self._task_history.task_scheduled(task_id)
            elif status == RUNNING:
                self._task_history.task_started(task_id, host)
        except:
            logger.warning("Error saving Task history", exc_info=1)

    @property
    def task_history(self):
        # Used by server.py to expose the calls
        return self._task_history
<commit_msg>Prefers jobs with more dependents when getting work from scheduler<commit_after># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import collections
import os
import logging
import time
import cPickle as pickle
import task_history as history
logger = logging.getLogger("luigi.server")

from task_status import PENDING, FAILED, DONE, RUNNING, UNKNOWN

class Scheduler(object):
    ''' Abstract base class

    Note that the methods all take string arguments, not Task objects...
    '''
    add_task = NotImplemented
    get_work = NotImplemented
    ping = NotImplemented

UPSTREAM_RUNNING = 'UPSTREAM_RUNNING'
UPSTREAM_MISSING_INPUT = 'UPSTREAM_MISSING_INPUT'
UPSTREAM_FAILED = 'UPSTREAM_FAILED'

UPSTREAM_SEVERITY_ORDER = ('', UPSTREAM_RUNNING, UPSTREAM_MISSING_INPUT, UPSTREAM_FAILED)
UPSTREAM_SEVERITY_KEY = lambda st: UPSTREAM_SEVERITY_ORDER.index(st)
STATUS_TO_UPSTREAM_MAP = {FAILED: UPSTREAM_FAILED, RUNNING: UPSTREAM_RUNNING, PENDING: UPSTREAM_MISSING_INPUT}

class Task(object):
    def __init__(self, status, deps, resources):
        self.resources = resources
        self.stakeholders = set()  # workers that are somehow related to this task (i.e. don't prune while any of these workers are still active)
        self.workers = set()  # workers that can perform task - task is 'BROKEN' if none of these workers are active
        if deps is None:
            self.deps = set()
        else:
            self.deps = set(deps)
        self.status = status  # PENDING, RUNNING, FAILED or DONE
        self.time = time.time()  # Timestamp when task was first added
        self.retry = None
        self.remove = None
        self.worker_running = None  # the worker that is currently running the task or None
        self.expl = None

    def __repr__(self):
        return "Task(%r)" % vars(self)

class CentralPlannerScheduler(Scheduler):
    ''' Async scheduler that can handle multiple workers etc

    Can be run locally or on a server (using RemoteScheduler + server.Server).
    '''

    def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0,
                 state_path='/var/lib/luigi-server/state.pickle', task_history=None,
                 resources=None):
        '''
        (all arguments are in seconds)
        Keyword Arguments:
        retry_delay -- How long after a Task fails to try it again, or -1 to never retry
        remove_delay -- How long after a Task finishes to remove it from the scheduler
        state_path -- Path to state file (tasks and active workers)
        worker_disconnect_delay -- If a worker hasn't communicated for this long, remove it from active workers
        '''
        self._state_path = state_path
        self._tasks = {}
        self._retry_delay = retry_delay
        self._remove_delay = remove_delay
        self._worker_disconnect_delay = worker_disconnect_delay
        self._active_workers = {}  # map from id to timestamp (last updated)
        self._task_history = task_history or history.NopHistory()
        self._resources = resources
        # TODO: have a Worker object instead, add more data to it

    def dump(self):
        state = (self._tasks, self._active_workers)
        try:
            with open(self._state_path, 'w') as fobj:
                pickle.dump(state, fobj)
        except IOError:
            logger.warning("Failed saving scheduler state", exc_info=1)
        else:
            logger.info("Saved state in %s", self._state_path)

    def load(self):
        if os.path.exists(self._state_path):
            logger.info("Attempting to load state from %s", self._state_path)
            with open(self._state_path) as fobj:
                state = pickle.load(fobj)
            self._tasks, self._active_workers = state
        else:
            logger.info("No prior state file exists at %s. Starting with clean slate", self._state_path)

    def prune(self):
        logger.info("Starting pruning of task graph")
        # Delete workers that haven't said anything for a while (probably killed)
        delete_workers = []
        for worker in self._active_workers:
            if self._active_workers[worker] < time.time() - self._worker_disconnect_delay:
                logger.info("worker %r updated at %s timed out (no contact for >=%ss)", worker, self._active_workers[worker], self._worker_disconnect_delay)
                delete_workers.append(worker)

        for worker in delete_workers:
            self._active_workers.pop(worker)

        remaining_workers = set(self._active_workers.keys())

        # Mark tasks with no remaining active stakeholders for deletion
        for task_id, task in self._tasks.iteritems():
            if not task.stakeholders.intersection(remaining_workers):
                if task.remove is None:
                    logger.info("Task %r has stakeholders %r but none remain connected -> will remove task in %s seconds", task_id, task.stakeholders, self._remove_delay)
                    task.remove = time.time() + self._remove_delay

            if task.status == RUNNING and task.worker_running and task.worker_running not in remaining_workers:
                # If a running worker disconnects, tag all its jobs as FAILED and subject it to the same retry logic
                logger.info("Task %r is marked as running by disconnected worker %r -> marking as FAILED with retry delay of %rs", task_id, task.worker_running, self._retry_delay)
                task.worker_running = None
                task.status = FAILED
                task.retry = time.time() + self._retry_delay

        # Remove tasks that have no stakeholders
        remove_tasks = []
        for task_id, task in self._tasks.iteritems():
            if task.remove and time.time() > task.remove:
                logger.info("Removing task %r (no connected stakeholders)", task_id)
                remove_tasks.append(task_id)

        for task_id in remove_tasks:
            self._tasks.pop(task_id)

        # Reset FAILED tasks to PENDING if max timeout is reached, and retry delay is >= 0
        for task in self._tasks.values():
            if task.status == FAILED and self._retry_delay >= 0 and task.retry < time.time():
                task.status = PENDING
        logger.info("Done pruning task graph")

    def update(self, worker):
        # update timestamp so that we keep track
        # of whenever the worker was last active
        self._active_workers[worker] = time.time()

    def add_task(self, worker, task_id, status=PENDING, runnable=True, deps=None, expl=None,
                 resources=None):
        """
        * Add task identified by task_id if it doesn't exist
        * If deps is not None, update dependency list
        * Update status of task
        * Add additional workers/stakeholders
        """
        self.update(worker)

        task = self._tasks.setdefault(task_id, Task(status=PENDING, deps=deps, resources=resources))

        if task.remove is not None:
            task.remove = None  # unmark task for removal so it isn't removed after being added

        if not (task.status == RUNNING and status == PENDING):
            # don't allow re-scheduling of task while it is running, it must either fail or succeed first
            task.status = status
            if status == FAILED:
                task.retry = time.time() + self._retry_delay

        if deps is not None:
            task.deps = set(deps)

        task.stakeholders.add(worker)

        if runnable:
            task.workers.add(worker)

        if expl is not None:
            task.expl = expl
        self._update_task_history(task_id, status)

    def update_resources(self, **resources):
        if self._resources is None:
            self._resources = {}
        self._resources.update(resources)

    def _has_resources(self, needed_resources, used_resources):
        if self._resources is None or needed_resources is None:
            return True

        for resource, amount in needed_resources.items():
            if amount + used_resources[resource] > self._resources.get(resource, 0):
                return False
        return True

    def _used_resources(self):
        used_resources = collections.defaultdict(int)
        if self._resources is not None:
            for task in self._tasks.itervalues():
                if task.status == RUNNING and task.resources:
                    for resource, amount in task.resources.items():
                        used_resources[resource] += amount
        return used_resources

    def get_work(self, worker, host=None):
        # TODO: remove any expired nodes

        # Algo: iterate over all nodes, find first node with no dependencies and available resources

        # TODO: remove tasks that can't be done, figure out if the worker has absolutely
        # nothing it can wait for

        # Return remaining tasks that have no FAILED descendents
        self.update(worker)
        best_t = (1, float('inf')) # first entry of these is always negative, so (1, x) is greater
        best_task = None
        locally_pending_tasks = 0
        running_tasks = []
        used_resources = self._used_resources()

        dependents = collections.defaultdict(int)
        for task_id, task in self._tasks.iteritems():
            if task.status != DONE:
                num_deps = float(len(task.deps))
                for dep in task.deps:
                    dependents[dep] += 1.0 / num_deps

        for task_id, task in self._tasks.iteritems():
            if worker not in task.workers:
                continue

            if task.status == RUNNING:
                running_tasks.append({'task_id': task_id, 'worker': task.worker_running})

            if task.status != PENDING:
                continue

            if not self._has_resources(task.resources, used_resources):
                continue

            locally_pending_tasks += 1
            ok = True
            for dep in task.deps:
                if dep not in self._tasks:
                    ok = False
                elif self._tasks[dep].status != DONE:
                    ok = False

            if ok:
                cur_t = (-dependents[task_id], task.time)
                if cur_t < best_t:
                    best_t = cur_t
                    best_task = task_id

        if best_task:
            t = self._tasks[best_task]
            t.status = RUNNING
            t.worker_running = worker
            self._update_task_history(best_task, RUNNING, host=host)

        return {'n_pending_tasks': locally_pending_tasks,
                'task_id': best_task,
                'running_tasks': running_tasks}

    def ping(self, worker):
        self.update(worker)

    def _upstream_status(self, task_id, upstream_status_table):
        if task_id in upstream_status_table:
            return upstream_status_table[task_id]
        elif task_id in self._tasks:
            task_stack = [task_id]

            while task_stack:
                dep_id = task_stack.pop()
                if dep_id in self._tasks:
                    dep = self._tasks[dep_id]
                    if dep_id not in upstream_status_table:
                        if dep.status == PENDING and dep.deps:
                            task_stack = task_stack + [dep_id] + list(dep.deps)
                            upstream_status_table[dep_id] = ''  # will be updated postorder
                        else:
                            dep_status = STATUS_TO_UPSTREAM_MAP.get(dep.status, '')
                            upstream_status_table[dep_id] = dep_status
                    elif upstream_status_table[dep_id] == '' and dep.deps:
                        # This is the postorder update step when we set the
                        # status based on the previously calculated child elements
                        upstream_status = [upstream_status_table.get(id, '') for id in dep.deps]
                        upstream_status.append('')  # to handle empty list
                        status = max(upstream_status, key=UPSTREAM_SEVERITY_KEY)
                        upstream_status_table[dep_id] = status
            return upstream_status_table[dep_id]

    def _serialize_task(self, task_id):
        task = self._tasks[task_id]
        return {
            'deps': list(task.deps),
            'status': task.status,
            'workers': list(task.workers),
            'start_time': task.time,
            'params': self._get_task_params(task_id),
            'name': self._get_task_name(task_id)
        }

    def _get_task_params(self, task_id):
        params = {}
        params_part = task_id.split('(')[1].strip(')')
        params_strings = params_part.split(", ")

        for param in params_strings:
            if not param:
                continue
            split_param = param.split('=')
            if len(split_param) != 2:
                return {'<complex parameters>': params_part}
            params[split_param[0]] = split_param[1]
        return params

    def _get_task_name(self, task_id):
        return task_id.split('(')[0]

    def graph(self):
        self.prune()
        serialized = {}
        for task_id, task in self._tasks.iteritems():
            serialized[task_id] = self._serialize_task(task_id)
        return serialized

    def _recurse_deps(self, task_id, serialized):
        if task_id not in serialized:
            task = self._tasks.get(task_id)
            if task is None:
                logger.warn('Missing task for id [%s]', task_id)
                serialized[task_id] = {
                    'deps': [],
                    'status': UNKNOWN,
                    'workers': [],
                    'start_time': UNKNOWN,
                    'params': self._get_task_params(task_id),
                    'name': self._get_task_name(task_id)
                }
            else:
                serialized[task_id] = self._serialize_task(task_id)
                for dep in task.deps:
                    self._recurse_deps(dep, serialized)

    def dep_graph(self, task_id):
        self.prune()
        serialized = {}
        if task_id in self._tasks:
            self._recurse_deps(task_id, serialized)
        return serialized

    def task_list(self, status, upstream_status):
        ''' query for a subset of tasks by status '''
        self.prune()
        result = {}
        upstream_status_table = {}  # used to memoize upstream status
        for task_id, task in self._tasks.iteritems():
            if not status or task.status == status:
                if (task.status != PENDING or not upstream_status or
                    upstream_status == self._upstream_status(task_id, upstream_status_table)):
                    serialized = self._serialize_task(task_id)
                    result[task_id] = serialized
        return result

    def fetch_error(self, task_id):
        if self._tasks[task_id].expl is not None:
            return {"taskId": task_id, "error": self._tasks[task_id].expl}
        else:
            return {"taskId": task_id, "error": ""}

    def _update_task_history(self, task_id, status, host=None):
        try:
            if status == DONE or status == FAILED:
                successful = (status == DONE)
                self._task_history.task_finished(task_id, successful)
            elif status == PENDING:
                self._task_history.task_scheduled(task_id)
            elif status == RUNNING:
                self._task_history.task_started(task_id, host)
        except:
            logger.warning("Error saving Task history", exc_info=1)

    @property
    def task_history(self):
        # Used by server.py to expose the calls
        return self._task_history
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name='neomodel',
    version='0.4.0',
    description='An object mapper for the neo4j graph database.',
    long_description=open('README.rst').read(),
    author='Robin Edwards',
    author_email='robin.ge@gmail.com',
    zip_safe=True,
    url='http://github.com/robinedwards/neomodel',
    license='MIT',
    packages=find_packages(),
    keywords='graph neo4j py2neo ORM',
    tests_require=['nose==1.1.2'],
    test_suite='nose.collector',
    install_requires=['py2neo==1.6.1', 'pytz==2013.8', 'lucene-querybuilder==0.2'],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3.3",
        "Topic :: Database",
    ])
<commit_msg>Bump py2neo to 1.6.2<commit_after>from setuptools import setup, find_packages

setup(
    name='neomodel',
    version='0.4.0',
    description='An object mapper for the neo4j graph database.',
    long_description=open('README.rst').read(),
    author='Robin Edwards',
    author_email='robin.ge@gmail.com',
    zip_safe=True,
    url='http://github.com/robinedwards/neomodel',
    license='MIT',
    packages=find_packages(),
    keywords='graph neo4j py2neo ORM',
    tests_require=['nose==1.1.2'],
    test_suite='nose.collector',
    install_requires=['py2neo==1.6.2', 'pytz==2013.8', 'lucene-querybuilder==0.2'],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3.3",
        "Topic :: Database",
    ])
<|endoftext|>


<commit_before>import os
import ssl

from . import errors
from .transport import SSLAdapter

class TLSConfig(object):
    """
    TLS configuration.

    Args:
        client_cert (tuple of str): Path to client cert, path to client key.
        ca_cert (str): Path to CA cert file.
        verify (bool or str): This can be ``False`` or a path to a CA cert
            file.
        ssl_version (int): A valid `SSL version`_.
        assert_hostname (bool): Verify the hostname of the server.

    .. _`SSL version`:
        https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
    """
    cert = None
    ca_cert = None
    verify = None
    ssl_version = None

    def __init__(self, client_cert=None, ca_cert=None, verify=None,
                 ssl_version=None, assert_hostname=None,
                 assert_fingerprint=None):
        # Argument compatibility/mapping with
        # https://docs.docker.com/engine/articles/https/
        # This diverges from the Docker CLI in that users can specify 'tls'
        # here, but also disable any public/default CA pool verification by
        # leaving tls_verify=False

        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint

        # TODO(dperny): according to the python docs, PROTOCOL_TLSvWhatever is
        # depcreated, and it's recommended to use OPT_NO_TLSvWhatever instead
        # to exclude versions. But I think that might require a bigger
        # architectural change, so I've opted not to pursue it at this time

        # If the user provides an SSL version, we should use their preference
        if ssl_version:
            self.ssl_version = ssl_version
        else:
            # If the user provides no ssl version, we should default to
            # TLSv1_2.  This option is the most secure, and will work for the
            # majority of users with reasonably up-to-date software. However,
            # before doing so, detect openssl version to ensure we can support
            # it.

            # ssl.OPENSSL_VERSION_INFO returns a tuple of 5 integers
            # representing version info. We want any OpenSSL version greater
            # than 1.0.1. Python compares tuples lexigraphically, which means
            # this comparison will work.
            if ssl.OPENSSL_VERSION_INFO > (1, 0, 1, 0, 0):
                # If this version is high enough to support TLSv1_2, then we
                # should use it.
                self.ssl_version = ssl.PROTOCOL_TLSv1_2
            else:
                # If we can't, use a differnent default. Before the commit
                # introducing this version detection, the comment read:
                # >>> TLS v1.0 seems to be the safest default; SSLv23 fails in
                # >>> mysterious ways:
                # >>> https://github.com/docker/docker-py/issues/963
                # Which is why we choose PROTOCOL_TLSv1
                self.ssl_version = ssl.PROTOCOL_TLSv1

        # "tls" and "tls_verify" must have both or neither cert/key files In
        # either case, Alert the user when both are expected, but any are
        # missing.

        if client_cert:
            try:
                tls_cert, tls_key = client_cert
            except ValueError:
                raise errors.TLSParameterError(
                    'client_config must be a tuple of'
                    ' (client certificate, key file)'
                )

            if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or
                                              not os.path.isfile(tls_key)):
                raise errors.TLSParameterError(
                    'Path to a certificate and key files must be provided'
                    ' through the client_config param'
                )
            self.cert = (tls_cert, tls_key)

        # If verify is set, make sure the cert exists
        self.verify = verify
        self.ca_cert = ca_cert
        if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert):
            raise errors.TLSParameterError(
                'Invalid CA certificate provided for `tls_ca_cert`.'
            )

    def configure_client(self, client):
        """
        Configure a client with these TLS options.
        """
        client.ssl_version = self.ssl_version

        if self.verify and self.ca_cert:
            client.verify = self.ca_cert
        else:
            client.verify = self.verify

        if self.cert:
            client.cert = self.cert

        client.mount('https://', SSLAdapter(
            ssl_version=self.ssl_version,
            assert_hostname=self.assert_hostname,
            assert_fingerprint=self.assert_fingerprint,
        ))
<commit_msg>Don't use PROTOCOL_TLSv1_2 directly to avoid ImportErrors<commit_after>import os
import ssl

from . import errors
from .transport import SSLAdapter

class TLSConfig(object):
    """
    TLS configuration.

    Args:
        client_cert (tuple of str): Path to client cert, path to client key.
        ca_cert (str): Path to CA cert file.
        verify (bool or str): This can be ``False`` or a path to a CA cert
            file.
        ssl_version (int): A valid `SSL version`_.
        assert_hostname (bool): Verify the hostname of the server.

    .. _`SSL version`:
        https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
    """
    cert = None
    ca_cert = None
    verify = None
    ssl_version = None

    def __init__(self, client_cert=None, ca_cert=None, verify=None,
                 ssl_version=None, assert_hostname=None,
                 assert_fingerprint=None):
        # Argument compatibility/mapping with
        # https://docs.docker.com/engine/articles/https/
        # This diverges from the Docker CLI in that users can specify 'tls'
        # here, but also disable any public/default CA pool verification by
        # leaving tls_verify=False

        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint

        # TODO(dperny): according to the python docs, PROTOCOL_TLSvWhatever is
        # depcreated, and it's recommended to use OPT_NO_TLSvWhatever instead
        # to exclude versions. But I think that might require a bigger
        # architectural change, so I've opted not to pursue it at this time

        # If the user provides an SSL version, we should use their preference
        if ssl_version:
            self.ssl_version = ssl_version
        else:
            # If the user provides no ssl version, we should default to
            # TLSv1_2.  This option is the most secure, and will work for the
            # majority of users with reasonably up-to-date software. However,
            # before doing so, detect openssl version to ensure we can support
            # it.
            if ssl.OPENSSL_VERSION_INFO[:3] >= (1, 0, 1) and hasattr(
                    ssl, 'PROTOCOL_TLSv1_2'):
                # If the OpenSSL version is high enough to support TLSv1_2,
                # then we should use it.
                self.ssl_version = getattr(ssl, 'PROTOCOL_TLSv1_2')
            else:
                # Otherwise, TLS v1.0 seems to be the safest default;
                # SSLv23 fails in mysterious ways:
                # https://github.com/docker/docker-py/issues/963
                self.ssl_version = ssl.PROTOCOL_TLSv1

        # "tls" and "tls_verify" must have both or neither cert/key files In
        # either case, Alert the user when both are expected, but any are
        # missing.

        if client_cert:
            try:
                tls_cert, tls_key = client_cert
            except ValueError:
                raise errors.TLSParameterError(
                    'client_config must be a tuple of'
                    ' (client certificate, key file)'
                )

            if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or
                                              not os.path.isfile(tls_key)):
                raise errors.TLSParameterError(
                    'Path to a certificate and key files must be provided'
                    ' through the client_config param'
                )
            self.cert = (tls_cert, tls_key)

        # If verify is set, make sure the cert exists
        self.verify = verify
        self.ca_cert = ca_cert
        if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert):
            raise errors.TLSParameterError(
                'Invalid CA certificate provided for `tls_ca_cert`.'
            )

    def configure_client(self, client):
        """
        Configure a client with these TLS options.
        """
        client.ssl_version = self.ssl_version

        if self.verify and self.ca_cert:
            client.verify = self.ca_cert
        else:
            client.verify = self.verify

        if self.cert:
            client.cert = self.cert

        client.mount('https://', SSLAdapter(
            ssl_version=self.ssl_version,
            assert_hostname=self.assert_hostname,
            assert_fingerprint=self.assert_fingerprint,
        ))
<|endoftext|>


<commit_before>from orpsoc.provider import Provider
import subprocess
import os.path
import sys
import tarfile

if sys.version_info[0] >= 3:
    import urllib.request as urllib
else:
    import urllib

class GitHub(Provider):
    def __init__(self, config):
        self.user   = config.get('user')
        self.repo   = config.get('repo')
        self.branch = config.get('branch')
        if 'version' in config:
            self.version = config.get('version')
        else:
            self.version = 'master'

    def fetch(self, local_dir, core_name):
        status = self.status(local_dir)
        if status == 'empty':
            self._checkout(local_dir)
        elif status == 'modified':
            self.clean_cache()
            self._checkout(local_dir)
        elif status == 'outofdate':
            self._update()
        elif status == 'downloaded':
            pass
        else:
            print("Something else is wrong")

    def _checkout(self, local_dir):
        #TODO : Sanitize URL
        url = 'https://github.com/{user}/{repo}/archive/{version}.tar.gz'.format(user=self.user, repo=self.repo, version=self.version)
        (filename, headers) = urllib.urlretrieve(url)

        t = tarfile.open(filename)
        (cache_root, core) = os.path.split(local_dir)

        #Ugly hack to get the first part of the directory name of the extracted files
        tmp = t.getnames()[0]
        t.extractall(cache_root)
        os.rename(os.path.join(cache_root, tmp),
                  os.path.join(cache_root, core))

    def status(self, local_dir):
        if not os.path.isdir(local_dir):
            return 'empty'
        else:
            return 'downloaded'
<commit_msg>Fix github provider fetch method.<commit_after>from orpsoc.provider import Provider
import subprocess
import os.path
import sys
import tarfile

if sys.version_info[0] >= 3:
    import urllib.request as urllib
else:
    import urllib

class GitHub(Provider):
    def __init__(self, config):
        self.user   = config.get('user')
        self.repo   = config.get('repo')
        self.branch = config.get('branch')
        if 'version' in config:
            self.version = config.get('version')
        else:
            self.version = 'master'

    def fetch(self, local_dir, core_name):
        status = self.status(local_dir)
        if status == 'empty':
            self._checkout(local_dir)
            return True
        elif status == 'modified':
            self.clean_cache()
            self._checkout(local_dir)
            return True
        elif status == 'outofdate':
            self._update()
            return True
        elif status == 'downloaded':
            pass
        else:
            print("Something else is wrong")
            return False
            #TODO: throw an exception here

    def _checkout(self, local_dir):
        #TODO : Sanitize URL
        url = 'https://github.com/{user}/{repo}/archive/{version}.tar.gz'.format(user=self.user, repo=self.repo, version=self.version)
        (filename, headers) = urllib.urlretrieve(url)

        t = tarfile.open(filename)
        (cache_root, core) = os.path.split(local_dir)

        #Ugly hack to get the first part of the directory name of the extracted files
        tmp = t.getnames()[0]
        t.extractall(cache_root)
        os.rename(os.path.join(cache_root, tmp),
                  os.path.join(cache_root, core))

    def status(self, local_dir):
        if not os.path.isdir(local_dir):
            return 'empty'
        else:
            return 'downloaded'
<|endoftext|>


<commit_before># encoding: utf-8
from setuptools import setup

# check availability of runtime dependencies
def check_any(*packages):
    """Issue a warning if none of the packages is available."""
    errors = []
    for package in packages:
        try:
            __import__(package)
            return True
        except ImportError:
            import sys
            errors.append(sys.exc_info()[1])
    if len(errors) == 1:
        print("Missing runtime dependency: %s" % errors[0])
    else:
        print("Missing runtime dependencies:")
        for err in errors:
            print("\t%s" % err)
    return False

check_any('dbus')
check_any('gobject')
check_any('pynotify', 'notify2')

# read long_description from README.rst
try:
    f = open('README.rst')
    long_description = f.read()
    f.close()
except IOError:
    long_description = None

setup(
    name='udiskie',
    version='0.5.1',
    description='Removable disk automounter for udisks',
    long_description=long_description,
    author='Byron Clark',
    author_email='byron@theclarkfamily.name',
    maintainer='Thomas Gläßle',
    maintainer_email='t_glaessle@gmx.de',
    url='https://github.com/coldfix/udiskie',
    license='MIT',
    packages=[
        'udiskie',
    ],
    namespace_packages=[
        'udiskie',
    ],
    entry_points={
        'console_scripts': [
            'udiskie = udiskie.cli:daemon',
            'udiskie-mount = udiskie.cli:mount',
            'udiskie-umount = udiskie.cli:umount',
        ],
    },
    extras_require={
        'notifications': ['notify2']
    },
    install_requires=[
        # Currently not building out of the box:
        # 'PyGObject',
        # 'dbus-python',
    ],
    tests_require=[
        'python-dbusmock>=0.7.2'
    ],
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Environment :: Console',
        'Environment :: X11 Applications',
        'Intended Audience :: Developers',
        'Intended Audience :: End Users/Desktop',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'License :: OSI Approved :: MIT License',
        'Topic :: Desktop Environment',
        'Topic :: Software Development',
        'Topic :: System :: Filesystems',
        'Topic :: System :: Hardware',
        'Topic :: Utilities',
    ],
)
<commit_msg>Bump version<commit_after># encoding: utf-8
from setuptools import setup

# check availability of runtime dependencies
def check_any(*packages):
    """Issue a warning if none of the packages is available."""
    errors = []
    for package in packages:
        try:
            __import__(package)
            return True
        except ImportError:
            import sys
            errors.append(sys.exc_info()[1])
    if len(errors) == 1:
        print("Missing runtime dependency: %s" % errors[0])
    else:
        print("Missing runtime dependencies:")
        for err in errors:
            print("\t%s" % err)
    return False

check_any('dbus')
check_any('gobject')
check_any('pynotify', 'notify2')

# read long_description from README.rst
try:
    f = open('README.rst')
    long_description = f.read()
    f.close()
except IOError:
    long_description = None

setup(
    name='udiskie',
    version='0.5.2',
    description='Removable disk automounter for udisks',
    long_description=long_description,
    author='Byron Clark',
    author_email='byron@theclarkfamily.name',
    maintainer='Thomas Gläßle',
    maintainer_email='t_glaessle@gmx.de',
    url='https://github.com/coldfix/udiskie',
    license='MIT',
    packages=[
        'udiskie',
    ],
    namespace_packages=[
        'udiskie',
    ],
    entry_points={
        'console_scripts': [
            'udiskie = udiskie.cli:daemon',
            'udiskie-mount = udiskie.cli:mount',
            'udiskie-umount = udiskie.cli:umount',
        ],
    },
    extras_require={
        'notifications': ['notify2']
    },
    install_requires=[
        # Currently not building out of the box:
        # 'PyGObject',
        # 'dbus-python',
    ],
    tests_require=[
        'python-dbusmock>=0.7.2'
    ],
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Environment :: Console',
        'Environment :: X11 Applications',
        'Intended Audience :: Developers',
        'Intended Audience :: End Users/Desktop',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'License :: OSI Approved :: MIT License',
        'Topic :: Desktop Environment',
        'Topic :: Software Development',
        'Topic :: System :: Filesystems',
        'Topic :: System :: Hardware',
        'Topic :: Utilities',
    ],
)
<|endoftext|>