<commit_before># This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Gets genres for imported music based on Last.fm tags.

Uses a provided whitelist file to determine which tags are valid genres.
The genre whitelist can be specified like so in .beetsconfig:

    [lastgenre]
    whitelist=/path/to/genres.txt

The included (default) genre list was produced by scraping Wikipedia.
The scraper script used is available here:
https://gist.github.com/1241307
"""
import logging
import pylast
import os
import yaml

from beets import plugins
from beets import ui
from beets.util import normpath
from beets import config
from beets import library

log = logging.getLogger('beets')

LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY)
C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

PYLAST_EXCEPTIONS = (
    pylast.WSError,
    pylast.MalformedResponseError,
    pylast.NetworkError,
)

def _tags_for(obj):
    """Given a pylast entity (album or track), returns a list of
    tag names for that entity. Returns an empty list if the entity is
    not found or another error occurs.
    """
    try:
        res = obj.get_top_tags()
    except PYLAST_EXCEPTIONS as exc:
        log.debug(u'last.fm error: %s' % unicode(exc))
        return []

    tags = []
    for el in res:
        if isinstance(el, pylast.TopItem):
            el = el.item
        tags.append(el.get_name())
    log.debug(u'last.fm tags: %s' % unicode(tags))
    return tags

def _tags_to_genre(tags):
    """Given a tag list, returns a genre. Returns the first tag that is
    present in the genre whitelist or None if no tag is suitable.
    """
    if not tags:
        return None
    elif not options['whitelist']:
        return tags[0].title()

    if options.get('c14n'):
        # Use the canonicalization tree.
        for tag in tags:
            genre = find_allowed(find_parents(tag, options['branches']))
            if genre:
                return genre
    else:
        # Just use the flat whitelist.
        return find_allowed(tags)

def flatten_tree(elem, path, branches):
    """Flatten nested lists/dictionaries into lists of strings
    (branches).
    """
    if not path:
        path = []

    if isinstance(elem, dict):
        for (k, v) in elem.items():
            flatten_tree(v, path + [k], branches)
    elif isinstance(elem, list):
        for sub in elem:
            flatten_tree(sub, path, branches)
    else:
        branches.append(path + [unicode(elem)])

def find_parents(candidate, branches):
    """Find parents genre of a given genre, ordered from the closest to
    the further parent.
    """
    for branch in branches:
        try:
            idx = branch.index(candidate.lower())
            return list(reversed(branch[:idx+1]))
        except ValueError:
            continue
    return [candidate]

def is_allowed(genre):
    """Returns True if the genre is present in the genre whitelist or
    False if not.
    """
    if genre is None:
        return False
    if genre.lower() in options['whitelist']:
        log.debug(u'verfied genre: %s' % genre)
        return True
    return False

def find_allowed(genres):
    """Returns the first genre that is present in the genre whitelist or
    None if no genre is suitable.
    """
    for genre in list(genres):
        if is_allowed(genre):
            return genre.title()
    return None

def fetch_genre(lastfm_obj):
    tags = []
    tags.extend(_tags_for(lastfm_obj))
    return _tags_to_genre(tags)

def fetch_album_genre(obj):
    lookup = u'{0}-{1}'.format(obj.albumartist, obj.album)
    if cache['album'].has_key(lookup):
        log.debug(u'using cache: %s = %s' % (lookup, cache['album'][lookup]))
        return cache['album'][lookup]
    cache['album'][lookup] = \
          fetch_genre(LASTFM.get_album(obj.albumartist, obj.album))
    log.debug(u'setting cache: %s = %s' % (lookup, cache['album'][lookup]))
    return cache['album'][lookup]

def fetch_album_artist_genre(obj):
    lookup = obj.albumartist
    if cache['artist'].has_key(lookup):
        log.debug(u'using cache: %s = %s' % (lookup, cache['artist'][lookup]))
        return cache['artist'][lookup]
    cache['artist'][lookup] = \
          fetch_genre(LASTFM.get_artist(obj.albumartist))
    log.debug(u'setting cache: %s = %s' % (lookup, cache['artist'][lookup]))
    return cache['artist'][lookup]

def fetch_artist_genre(obj):
    lookup = obj.artist
    if cache['artist'].has_key(lookup):
        log.debug(u'using cache: %s = %s' % (lookup, cache['artist'][lookup]))
        return cache['artist'][lookup]
    cache['artist'][lookup] = \
          fetch_genre(LASTFM.get_artist(obj.artist))
    log.debug(u'setting cache: %s = %s' % (lookup, cache['artist'][lookup]))
    return cache['artist'][lookup]

def fetch_track_genre(obj):
    lookup = u'{0}-{1}'.format(obj.artist, obj.title)
    if cache['track'].has_key(lookup):
        log.debug(u'using cache: %s = %s' % (lookup, cache['track'][lookup]))
        return cache['track'][lookup]
    cache['track'][lookup] = \
          fetch_genre(LASTFM.get_track(obj.artist, obj.title))
    log.debug(u'setting cache: %s = %s' % (lookup, cache['track'][lookup]))
    return cache['track'][lookup]

options = {
    'whitelist': None,
    'branches': None,
    'c14n': False,
}
sources = []
cache = {'artist':{}, 'album':{}, 'track':{}}
class LastGenrePlugin(plugins.BeetsPlugin):
    def __init__(self):
        super(LastGenrePlugin, self).__init__()
        self.import_stages = [self.imported]

        self.config.add({
            'whitelist': os.path.join(os.path.dirname(__file__), 'genres.txt'),
            'fallback': None,
            'canonical': None,
            'source': 'album',
            'force': False,
        })

        # Read the whitelist file.
        wl_filename = self.config['whitelist'].as_filename()
        whitelist = set()
        with open(wl_filename) as f:
            for line in f:
                line = line.decode('utf8').strip().lower()
                if line:
                    whitelist.add(line)
        options['whitelist'] = whitelist

        # Prepare sources
        source = self.config['source'].get()
        if source == 'track':
            sources.extend(['track', 'album', 'artist'])
        elif source == 'album':
            sources.extend(['album', 'artist'])
        elif source == 'artist':
            sources.extend(['artist'])

        # Read the genres tree for canonicalization if enabled.
        c14n_filename = self.config['canonical'].get()
        if c14n_filename is not None:
            c14n_filename = c14n_filename.strip()
            if not c14n_filename:
                c14n_filename = C14N_TREE
            c14n_filename = normpath(c14n_filename)

            genres_tree = yaml.load(open(c14n_filename, 'r'))
            branches = []
            flatten_tree(genres_tree, [], branches)
            options['branches'] = branches
            options['c14n'] = True

    def _get_album_genre(self, album, force, fallback_str):
        log.debug(u'_get_album_genre')
        if not force and is_allowed(album.genre):
            # already valid and no forced lookup
            log.debug(u"not fetching album genre. already valid")
            return album.genre
        result = None
        # no track lookup for album
        if 'album' in sources:
            result = fetch_album_genre(album)
            log.debug(u"last.fm album genre: %s" % result)
            if result:
                return result
        if 'artist' in sources:
            if not album.albumartist == 'Various Artists':
                # no artist lookup for Various Artists
                result = fetch_album_artist_genre(album)
                log.debug(u"last.fm album artist genre: %s" % result)
            if result:
                return result
        if is_allowed(album.genre):
            return album.genre
        if fallback_str:
            return fallback_str
        return None

    def _get_item_genre(self, item, force, fallback_str):
        if not force:
            if is_allowed(item.genre):
                # already valid and no forced lookup
                log.debug(u"not fetching item genre. already valid")
                return item.genre
            log.debug(u"replacing invalid item genre: %s" % item.genre)
        result = None
        if 'track' in sources:
            result = fetch_track_genre(item)
            if result:
                return result
            log.debug(u"no last.fm track genre")
        if 'album' in sources:
            if item.album:
                result = fetch_album_genre(item)
            if result:
                return result
            log.debug(u"no last.fm album genre")
        if 'artist' in sources:
            result = fetch_artist_genre(item)
            if result:
                return result
            log.debug(u"no last.fm artist genre")
        if is_allowed(item.genre):
            return item.genre
        if fallback_str:
            return fallback_str
        return result

    def commands(self):
        lastgenre_cmd = ui.Subcommand('lastgenre', help='fetch genres')
        def lastgenre_func(lib, opts, args):
            # The "write to files" option corresponds to the
            # import_write config value.
            write = config['import']['write'].get(bool)
            force = self.config['force'].get(bool)
            fallback_str = self.config['fallback'].get()
            for album in lib.albums(ui.decargs(args)):
                album.genre = self._get_album_genre(album, force, fallback_str)
                log.debug(u'adding last.fm album genre: %s' % album.genre)
                for item in album.items():
                    item.genre = self._get_item_genre(item, force,
                          fallback_str)
                    log.debug(u'adding last.fm item genre: %s' % item.genre)
                    if write:
                        item.write()

        lastgenre_cmd.func = lastgenre_func
        return [lastgenre_cmd]

    def imported(self, session, task):
        tags = []
        fallback_str = self.config['fallback'].get()
        if task.is_album:
            log.debug(u'imported: album')
            album = session.lib.get_album(task.album_id)
            album.genre = self._get_album_genre(album, True, fallback_str)
            log.debug(u'adding last.fm album genre: %s' % album.genre)
            for item in album.items():
                item.genre = self._get_item_genre(item, True, fallback_str)
                log.debug(u'adding last.fm item genre: %s' % item.genre)
        else:
            log.debug(u'imported: item')
            item = task.item
            item.genre = self._get_item_genre(item, True, fallback_str)
            log.debug(u'adding last.fm item genre: %s' % item.genre)
            session.lib.store(item)
<commit_msg>cleaned up, removed debug<commit_after># This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Gets genres for imported music based on Last.fm tags.

Uses a provided whitelist file to determine which tags are valid genres.
The genre whitelist can be specified like so in .beetsconfig:

    [lastgenre]
    whitelist=/path/to/genres.txt

The included (default) genre list was produced by scraping Wikipedia.
The scraper script used is available here:
https://gist.github.com/1241307
"""
import logging
import pylast
import os
import yaml

from beets import plugins
from beets import ui
from beets.util import normpath
from beets import config
from beets import library

log = logging.getLogger('beets')

LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY)
C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

PYLAST_EXCEPTIONS = (
    pylast.WSError,
    pylast.MalformedResponseError,
    pylast.NetworkError,
)

def _tags_for(obj):
    """Given a pylast entity (album or track), returns a list of
    tag names for that entity. Returns an empty list if the entity is
    not found or another error occurs.
    """
    try:
        res = obj.get_top_tags()
    except PYLAST_EXCEPTIONS as exc:
        log.debug(u'last.fm error: %s' % unicode(exc))
        return []

    tags = []
    for el in res:
        if isinstance(el, pylast.TopItem):
            el = el.item
        tags.append(el.get_name())
    log.debug(u'last.fm tags: %s' % unicode(tags))
    return tags

def _tags_to_genre(tags):
    """Given a tag list, returns a genre. Returns the first tag that is
    present in the genre whitelist or None if no tag is suitable.
    """
    if not tags:
        return None
    elif not options['whitelist']:
        return tags[0].title()

    if options.get('c14n'):
        # Use the canonicalization tree.
        for tag in tags:
            genre = find_allowed(find_parents(tag, options['branches']))
            if genre:
                return genre
    else:
        # Just use the flat whitelist.
        return find_allowed(tags)

def flatten_tree(elem, path, branches):
    """Flatten nested lists/dictionaries into lists of strings
    (branches).
    """
    if not path:
        path = []

    if isinstance(elem, dict):
        for (k, v) in elem.items():
            flatten_tree(v, path + [k], branches)
    elif isinstance(elem, list):
        for sub in elem:
            flatten_tree(sub, path, branches)
    else:
        branches.append(path + [unicode(elem)])

def find_parents(candidate, branches):
    """Find parents genre of a given genre, ordered from the closest to
    the further parent.
    """
    for branch in branches:
        try:
            idx = branch.index(candidate.lower())
            return list(reversed(branch[:idx+1]))
        except ValueError:
            continue
    return [candidate]

def is_allowed(genre):
    """Returns True if the genre is present in the genre whitelist or
    False if not.
    """
    if genre is None:
        return False
    if genre.lower() in options['whitelist']:
        return True
    return False

def find_allowed(genres):
    """Returns the first genre that is present in the genre whitelist or
    None if no genre is suitable.
    """
    for genre in list(genres):
        if is_allowed(genre):
            return genre.title()
    return None

def fetch_genre(lastfm_obj):
    """Returns the genre for this lastfm_obj.
    """
    return _tags_to_genre(_tags_for(lastfm_obj))

def fetch_album_genre(obj):
    """Returns the album genre for this obj.  Either performs a lookup in
    lastfm or returns the cached value.
    """
    lookup = u'album.{0}-{1}'.format(obj.albumartist, obj.album)
    if not cache.has_key(lookup):
        cache[lookup] = \
              fetch_genre(LASTFM.get_album(obj.albumartist, obj.album))
    return cache[lookup]

def fetch_album_artist_genre(obj):
    """Returns the album artists genre for this obj.  Either performs a lookup
    in lastfm or returns the cached value.
    """
    lookup = u'artist.${0}'.format(obj.albumartist)
    if not cache.has_key(lookup):
        cache[lookup] = \
              fetch_genre(LASTFM.get_artist(obj.albumartist))
    return cache[lookup]

def fetch_artist_genre(obj):
    """Returns the track artists genre for this obj.  Either performs a lookup
    in lastfm or returns the cached value.
    """
    lookup = u'artist.${0}'.format(obj.artist)
    if not cache.has_key(lookup):
        cache[lookup] = fetch_genre(LASTFM.get_artist(obj.artist))
    return cache[lookup]

def fetch_track_genre(obj):
    """Returns the track genre for this obj.  Either performs a lookup in
    lastfm or returns the cached value.  """
    lookup = u'track.{0}-{1}'.format(obj.artist, obj.title)
    if not cache.has_key(lookup):
        cache[lookup] = fetch_genre(LASTFM.get_track(obj.artist, obj.title))
    return cache[lookup]

options = {
    'whitelist': None,
    'branches': None,
    'c14n': False,
}
# simple cache to speed up artist and album lookups track or album mode.  it's
# probably not required to cache track lookups, but...
cache = {}
class LastGenrePlugin(plugins.BeetsPlugin):
    def __init__(self):
        super(LastGenrePlugin, self).__init__()
        self.import_stages = [self.imported]

        self.config.add({
            'whitelist': os.path.join(os.path.dirname(__file__), 'genres.txt'),
            'fallback': None,
            'canonical': None,
            'source': 'album',
            'force': False,
        })

        # Read the whitelist file.
        wl_filename = self.config['whitelist'].as_filename()
        whitelist = set()
        with open(wl_filename) as f:
            for line in f:
                line = line.decode('utf8').strip().lower()
                if line:
                    whitelist.add(line)
        options['whitelist'] = whitelist

        # Read the genres tree for canonicalization if enabled.
        c14n_filename = self.config['canonical'].get()
        if c14n_filename is not None:
            c14n_filename = c14n_filename.strip()
            if not c14n_filename:
                c14n_filename = C14N_TREE
            c14n_filename = normpath(c14n_filename)

            genres_tree = yaml.load(open(c14n_filename, 'r'))
            branches = []
            flatten_tree(genres_tree, [], branches)
            options['branches'] = branches
            options['c14n'] = True

    def _set_sources(self, source):
        """Prepare our internal represantation of valid sources we can use.
        """
        self.sources = []
        if source == 'track':
            self.sources.extend(['track', 'album', 'artist'])
        elif source == 'album':
            self.sources.extend(['album', 'artist'])
        elif source == 'artist':
            self.sources.extend(['artist'])

    def _get_album_genre(self, album, force, fallback_str):
        """Return the best candidate for album genre based on sources (see
        _set_sources).
        Going down from album -> artist -> original -> fallback -> None.
        """
        if not force and is_allowed(album.genre):
            return [album.genre, 'keep']
        result = None
        # no track lookup for album genre
        if 'album' in self.sources:
            result = fetch_album_genre(album)
            if result:
                return [result, 'album']
        if 'artist' in self.sources:
            # no artist lookup for Various Artists
            if not album.albumartist == 'Various Artists':
                result = fetch_album_artist_genre(album)
            if result:
                return [result, 'artist']
        if is_allowed(album.genre):
            return [album.genre, 'original']
        if fallback_str:
            return [fallback_str, 'fallback']
        return [None, None]

    def _get_item_genre(self, item, force, fallback_str):
        """Return the best candidate for item genre based on sources (see
        _set_sources).
        Going down from track -> album -> artist -> original -> fallback ->
        None.
        """
        if not force:
            if is_allowed(item.genre):
                return [item.genre, 'keep']
        result = None
        if 'track' in self.sources:
            result = fetch_track_genre(item)
            if result:
                return [result, 'track']
        if 'album' in self.sources:
            if item.album:
                result = fetch_album_genre(item)
            if result:
                return [result, 'album']
        if 'artist' in self.sources:
            result = fetch_artist_genre(item)
            if result:
                return [result, 'artist']
        if is_allowed(item.genre):
            return [item.genre, 'original']
        if fallback_str:
            return [fallback_str, 'fallback']
        return [None, None]

    def commands(self):
        lastgenre_cmd = ui.Subcommand('lastgenre', help='fetch genres')
        lastgenre_cmd.parser.add_option('-f', '--force', dest='force',
                              action='store_true',
                              default=self.config['force'].get(bool),
                              help='re-download genre when already present')
        lastgenre_cmd.parser.add_option('-v', '--verbose', dest='verbose',
                              action='store_true',
                              default=False,
                              help='be more verbose')
        lastgenre_cmd.parser.add_option('-s', '--source', dest='source',
                              type='string',
                              default=self.config['source'].get(),
                              help='set source, one of: artist / album / track')
        def lastgenre_func(lib, opts, args):
            # The "write to files" option corresponds to the
            # import_write config value.
            write = config['import']['write'].get(bool)
            force = opts.force
            self._set_sources(opts.source)
            for album in lib.albums(ui.decargs(args)):
                album.genre, src = self._get_album_genre(album, force, fallback_str)
                if opts.verbose:
                    log.info(u'LastGenre: Album({0} - {1}) > {2}({3})'.format(
                        album.albumartist, album.album, album.genre, src))
                for item in album.items():
                    item.genre, src = self._get_item_genre(item, force,
                          fallback_str)
                    lib.store(item)
                    if opts.verbose:
                        log.info(u'LastGenre: Item({0} - {1}) > {2}({3})'.format(
                            item.artist, item.title, item.genre, src))
                    if write:
                        item.write()

        lastgenre_cmd.func = lastgenre_func
        return [lastgenre_cmd]

    def imported(self, session, task):
        self._set_sources(self.config['source'].get())
        tags = []
        fallback_str = self.config['fallback'].get()
        if task.is_album:
            album = session.lib.get_album(task.album_id)
            album.genre, src = self._get_album_genre(album, True, fallback_str)
            log.debug(u'added last.fm album genre ({0}): {1}'.format(
                  src, album.genre))
            for item in album.items():
                item.genre, src = self._get_item_genre(item, True, fallback_str)
                log.debug(u'added last.fm item genre ({0}): {1}'.format(
                      src, item.genre))
        else:
            item = task.item
            item.genre, src = self._get_item_genre(item, True, fallback_str)
            log.debug(u'added last.fm item genre ({0}): {1}'.format(
                  src, item.genre))
            session.lib.store(item)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse, os, sys, time
VM_PATH = os.path.join(os.environ['HOME'], 'vm/images/base/')

def cmd(s):
    print s
    if os.system(s):
        raise RuntimeError("error executing '%s'"%s)

cmd("ls -lth %s|head -5"%VM_PATH)

parser = argparse.ArgumentParser(description="Create a new VM image.")
parser.add_argument("--prev", dest="prev", type=str, help="previous vm image name", default="")
parser.add_argument("--next", dest="next",  type=str, help="new vm image name", default="")
args = parser.parse_args()

prev = args.prev
next = args.next

if prev == "":
   # make it the most recent image
   prev = os.popen("ls -1t ~/vm/images/base/|head -1").read().strip().rstrip('.img')

if next == "":
   # make image name salvus-date
   next = time.strftime("salvus-%Y-%m-%d")

prev_path = os.path.join(VM_PATH, prev + '.img')
next_path = os.path.join(VM_PATH, next + '.img')

if not os.path.exists(prev_path):
    raise ValueError("previous vm image doesn't exist -- " + prev_path)

if os.path.exists(next_path):
    raise ValueError("next vm image already exists -- " + next_path)

cmd("qemu-img create -b %s -f qcow2 %s"%(prev_path, next_path))

cmd("virt-install --cpu host --network user,model=virtio --name %s --vcpus=16 --ram 32768 --import --disk %s,device=disk,bus=virtio,format=qcow2,cache=writeback --noautoconsole"%(next,next_path))

cmd("virsh -c qemu:///session qemu-monitor-command --hmp %s 'hostfwd_add ::2222-:22'"%next)

cmd("ssh localhost -p 2222")

print """
You probably want to do something like this:

    cd salvus/salvus
    . salvus-env
    git pull
    ./make_coffee
    # fix /etc/ssh/ssh_config
    sudo su
    apt-get update; apt-get upgrade
    reboot -h now
    sudo shutdown -h now

 Then

    virsh_undefine %s
    cd vm/images/base/
    ./push

"""%next
<commit_msg>make new_vm_image safer.<commit_after>#!/usr/bin/env python

import argparse, os, sys, time
VM_PATH = os.path.join(os.environ['HOME'], 'vm/images/base/')

def cmd(s):
    print s
    if os.system(s):
        raise RuntimeError("error executing '%s'"%s)

cmd("ls -lth %s|head -5"%VM_PATH)

parser = argparse.ArgumentParser(description="Create a new VM image.")
parser.add_argument("--prev", dest="prev", type=str, help="previous vm image name", default="")
parser.add_argument("--next", dest="next",  type=str, help="new vm image name", default="")
args = parser.parse_args()

prev = args.prev
next = args.next

if prev == "":
   # make it the most recent image
   prev = os.popen("ls -1t ~/vm/images/base/|head -1").read().strip().rstrip('.img')

if next == "":
   # make image name salvus-date
   next = time.strftime("salvus-%Y-%m-%d")

defined_machines = os.popen("virsh_list").read()

for machine in [prev, next]:
    if machine in defined_machines:
        print "%s is currently defined.  Please undefine it before proceeding further, in order to avoid any possible corruption."%machine
        sys.exit(1)

prev_path = os.path.join(VM_PATH, prev + '.img')
next_path = os.path.join(VM_PATH, next + '.img')

if not os.path.exists(prev_path):
    raise ValueError("previous vm image doesn't exist -- " + prev_path)

if os.path.exists(next_path):
    raise ValueError("next vm image already exists -- " + next_path)

cmd("qemu-img create -b %s -f qcow2 %s"%(prev_path, next_path))

cmd("virt-install --cpu host --network user,model=virtio --name %s --vcpus=16 --ram 32768 --import --disk %s,device=disk,bus=virtio,format=qcow2,cache=writeback --noautoconsole"%(next,next_path))

cmd("virsh -c qemu:///session qemu-monitor-command --hmp %s 'hostfwd_add ::2222-:22'"%next)

print "To connect, do\n\t\t ssh localhost -p 2222"

print """
You probably want to do something like this:

    cd salvus/salvus
    . salvus-env
    git pull
    ./make_coffee
    # fix /etc/ssh/ssh_config
    sudo su
    apt-get update; apt-get upgrade
    reboot -h now
    sudo shutdown -h now

 Then

    virsh_undefine %s
    cd vm/images/base/
    ./push

"""%next
<|endoftext|>


<commit_before>from django import forms

from manoria.models import Player, Settlement, SettlementBuilding

class PlayerCreateForm(forms.ModelForm):

    class Meta:
        model = Player
        fields = ["name"]

class SettlementCreateForm(forms.ModelForm):

    class Meta:
        model = Settlement
        fields = ["name"]

class BuildingCreateForm(forms.ModelForm):

    class Meta:
        model = SettlementBuilding
        fields = ["kind", "x", "y"]

    def __init__(self, settlement, *args, **kwargs):
        self.settlement = settlement
        super(BuildingCreateForm, self).__init__(*args, **kwargs)

    def clean(self):
        x = self.cleaned_data.get("x")
        y = self.cleaned_data.get("y")

        if all([x, y]):
            if not 1 <= x <= 20 or not 1 <= y <= 20:
                raise forms.ValidationError("Building is not within map range")

            if SettlementBuilding.objects.filter(settlement=self.settlement, x=x, y=y).exists():
                raise forms.ValidationError("A building exists at this location")

        return self.cleaned_data
<commit_msg>check for non-buildable terrain when creating buildings<commit_after>from django import forms

from manoria.models import Player, Settlement, SettlementBuilding

class PlayerCreateForm(forms.ModelForm):

    class Meta:
        model = Player
        fields = ["name"]

class SettlementCreateForm(forms.ModelForm):

    class Meta:
        model = Settlement
        fields = ["name"]

class BuildingCreateForm(forms.ModelForm):

    class Meta:
        model = SettlementBuilding
        fields = ["kind", "x", "y"]

    def __init__(self, settlement, *args, **kwargs):
        self.settlement = settlement
        super(BuildingCreateForm, self).__init__(*args, **kwargs)

    def clean(self):
        x = self.cleaned_data.get("x")
        y = self.cleaned_data.get("y")

        if all([x, y]):
            if not 1 <= x <= 20 or not 1 <= y <= 20:
                raise forms.ValidationError("Building is not within map range")

            if SettlementBuilding.objects.filter(settlement=self.settlement, x=x, y=y).exists():
                raise forms.ValidationError("A building exists at this location")

            non_buildable_terrain = SettlementTerrain.objects.filter(
                settlement=self.settlement, x=x, y=y, buildable_on=False
            )
            if non_buildable_terrain.exists():
                raise forms.ValidationError("Building cannot be placed on non-buildable terrain")

        return self.cleaned_data
<|endoftext|>


<commit_before>import math
import argparse
from datetime import datetime

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import cross_validation
from sklearn import tree
from sklearn import metrics

# plt.style.use('ggplot')

def preprocessing(X, y):
    X['weekday'] = X.index.weekday
    X['hour'] = X.index.hour
    X['year'] = X.index.year
    # Convert pandas dataframe to numpy arrays. At this point all columns
    # should have the same dtype
    X = X.values
    if y is not None:
        y = y.values
    return X, y

def scoring(y_real, y_predicted):
    y_real = y_real.astype(np.int)
    y_predicted = np.where(y_predicted > 0, y_predicted, 0).astype(np.int)
    return math.sqrt(
        metrics.mean_squared_error(np.log(y_predicted + 1), np.log(y_real + 1))
    )

def cv(estimator, X, y, n_folds=5, print_single=True):
    k_fold = cross_validation.KFold(n=len(X), n_folds=n_folds, indices=True)
    rshuffle = cross_validation.ShuffleSplit(n=len(X), n_iter=n_folds)
    scores = cross_validation.cross_val_score(estimator, X, y, cv=rshuffle,
                                              score_func=scoring)
    print scores
    print 'Average score: {:.3f} - std: {:.4f}'.format(np.mean(scores),
                                                       np.std(scores))

def loss_func(y_real, y_predicted):
    return math.sqrt(metrics.mean_squared_error(np.log(y_real + 1), np.log(y_predicted + 1)))

if __name__ == '__main__':
    # Command arguments
    parser = argparse.ArgumentParser(description='bike-sharing estimator')
    parser.add_argument('--no-cv', dest='cv', action='store_false',
                        default=True, help='Skip cross validation')
    parser.add_argument('--no-test', dest='test', action='store_false',
                        default=True, help='Skip test dataset')
    parser.add_argument('-k', dest='n_fold', type=int,
                        default=5, help='Number of cv folds')
    parser.add_argument('--no-single', dest='single', action='store_false',
                        default=True, help='Print only average cv score')
    parser.add_argument('-d', '--max-depth', dest='depth', type=int,
                        default=10, help='Max depth of decision tree')
    args = parser.parse_args()

    # Input
    common_input_options = {'delimiter': ',', 'index_col': 0, 'parse_dates': True}
    train_dataset = pd.read_csv('data/train.csv', **common_input_options)
    test_dataset = pd.read_csv('data/test.csv', usecols=(0,1,2,3,4,5,6,7,8),
                               **common_input_options)

    # Data preprocessing
    X_train, y_train = preprocessing(train_dataset.iloc[:,:-3],
                                     train_dataset.iloc[:,-1])
    X_test, y_test = preprocessing(test_dataset, None)

    # The interesting part
    estimator = tree.DecisionTreeRegressor(max_depth=args.depth)
    if args.cv:
        cv(estimator, X_train, y_train, args.n_fold, args.single)
    if args.test:
        results = estimator.fit(X_train, y_train).predict(X_test)
        results = np.where(results > 0, results, 0).astype(np.int)

        # Output
        output_dataframe = pd.DataFrame({'datetime': test_dataset.index})
        output_dataframe['count'] = results
        output_dataframe.to_csv('data/out.csv', index=False)
<commit_msg>bike-sharing: split test/train data in CV based on day, other minor changes.<commit_after>import math
import argparse
from datetime import datetime

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import cross_validation
from sklearn import tree
from sklearn import metrics

# plt.style.use('ggplot')

def preprocessing(X, y):
    X['weekday'] = X.index.weekday
    X['hour'] = X.index.hour
    X['year'] = X.index.year
    return X, y

def scoring(y_real, y_predicted):
    y_real = y_real.round().astype(np.int)
    y_predicted = np.where(y_predicted > 0, y_predicted, 0).round().astype(np.int)
    return math.sqrt(
        metrics.mean_squared_error(np.log(y_predicted + 1), np.log(y_real + 1))
    )

def cv(estimator, X, y, day_split_first=8, day_split_last=13):
    day_split = (a for a in
                 [np.where(X.index.day < i) + np.where(X.index.day >= i)
                  for i in xrange(day_split_first, day_split_last+1)])
    scores = cross_validation.cross_val_score(estimator, X, y, cv=day_split,
                                              score_func=scoring)
    print scores.round(3)
    print 'Avg: {1:.3f} | std: {2:.3f} | min: {3:.3f} | max: {6:.3f}'.format(
        *pd.Series(scores).describe()[:]
    )

if __name__ == '__main__':
    # Command arguments
    parser = argparse.ArgumentParser(description='bike-sharing estimator')
    parser.add_argument('--no-cv', dest='cv', action='store_false',
                        default=True, help='Skip cross validation')
    parser.add_argument('--no-test', dest='test', action='store_false',
                        default=True, help='Skip test dataset')
    parser.add_argument('-f', dest='first', type=int,
                        default=8, help='First starting day of test split')
    parser.add_argument('-l', dest='last', type=int,
                        default=13, help='Last starting day of test split')
    parser.add_argument('-d', '--max-depth', dest='depth', type=int,
                        default=10, help='Max depth of decision tree')
    args = parser.parse_args()

    # Input
    common_input_options = {'delimiter': ',', 'index_col': 0, 'parse_dates': True}
    train_dataset = pd.read_csv('data/train.csv', **common_input_options)
    test_dataset = pd.read_csv('data/test.csv', usecols=(0,1,2,3,4,5,6,7,8),
                               **common_input_options)

    # Data preprocessing
    X_train, y_train = preprocessing(train_dataset.iloc[:,:-3],
                                     train_dataset.iloc[:,-1])
    X_test, y_test = preprocessing(test_dataset, None)

    # The interesting part
    estimator = tree.DecisionTreeRegressor(max_depth=args.depth)
    if args.cv:
        cv(estimator, X_train, y_train, args.first, args.last)
    if args.test:
        results = estimator.fit(X_train, y_train).predict(X_test)
        results = np.where(results > 0, results, 0).round().astype(np.int)

        # Output
        output_dataframe = pd.DataFrame({'datetime': test_dataset.index})
        output_dataframe['count'] = results
        output_dataframe.to_csv('data/out.csv', index=False)
<|endoftext|>


<commit_before>class A(object):
    def method(self):
        self ## type A
        return 1

    def test(self, arg):
        self ## type A
        arg ## type bool|float|int

a = A()
a.method() ## type int

test("*confuse*")
a.test(1.1)
nonexisting.test("*disctract*")
a.test(1)
o = object()
o.test("*perplex*")

A.test(a, True)<commit_msg>new test case<commit_after>class A(object):
    def method(self):
        self ## type A
        return 1

    def test(self, arg):
        self ## type A
        arg ## type bool|float|int

a = A()
a.method() ## type int

test("*confuse*")
a.test(1.1)
nonexisting.test("*disctract*")
a.test(1)
o = object()
o.test("*perplex*")

A.test(a, True)

a.test ## type method<|endoftext|>


<commit_before># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Commit sha for bazelbuild/rules_docker."""

version = "d5b2ff91869bc6cb0464771bbb65629b3bc5c3ab"
<commit_msg>Project import generated by Copybara. (#597)<commit_after># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Commit sha for bazelbuild/rules_docker."""

version = "985cad97259764c1c9bce25b1e3e2ee92b906de4"
<|endoftext|>


<commit_before>import graphene
from django.db.models import Q
from django.utils import timezone
from graphene_django.registry import get_global_registry
from graphql.error import GraphQLError
from graphql_relay import from_global_id

from .core.enums import PermissionEnum, ReportingPeriod
from .core.types import PermissionDisplay

registry = get_global_registry()

def get_database_id(info, node_id, only_type):
    """Get a database ID from a node ID of given type."""
    _type, _id = graphene.relay.Node.from_global_id(node_id)
    graphene_type = info.schema.get_type(_type).graphene_type
    if graphene_type != only_type:
        raise AssertionError('Must receive a %s id.' % only_type._meta.name)
    return _id

def get_nodes(ids, graphene_type=None):
    """Return a list of nodes.

    If the `graphene_type` argument is provided, the IDs will be validated
    against this type. If the type was not provided, it will be looked up in
    the Graphene's registry. Raises an error if not all IDs are of the same
    type.
    """
    pks = []
    types = []
    invalid_ids = []
    error_msg = "Could not resolve to a nodes with the global id list of '%s'."
    for graphql_id in ids:
        if graphql_id:
            try:
                _type, _id = from_global_id(graphql_id)
            except Exception:
                invalid_ids.append(graphql_id)
            else:
                if graphene_type:
                    assert str(graphene_type) == _type, (
                        'Must receive an {} id.').format(
                            graphene_type._meta.name)
                pks.append(_id)
                types.append(_type)
    if invalid_ids:
        raise GraphQLError(
            error_msg % invalid_ids)

    # If `graphene_type` was not provided, check if all resolved types are
    # the same. This prevents from accidentally mismatching IDs of different
    # types.
    if types and not graphene_type:
        assert len(set(types)) == 1, 'Received IDs of more than one type.'
        # get type by name
        type_name = types[0]
        for model, _type in registry._registry.items():
            if _type._meta.name == type_name:
                graphene_type = _type
                break

    nodes = list(graphene_type._meta.model.objects.filter(pk__in=pks))
    if not nodes:
        raise GraphQLError(
            error_msg % ids)
    nodes_pk_list = [str(node.pk) for node in nodes]
    for pk in pks:
        assert pk in nodes_pk_list, (
            'There is no node of type {} with pk {}'.format(_type, pk))
    return nodes

def filter_by_query_param(queryset, query, search_fields):
    """Filter queryset according to given parameters.

    Keyword arguments:
    queryset - queryset to be filtered
    query - search string
    search_fields - fields considered in filtering
    """
    if query:
        query_by = {
            '{0}__{1}'.format(
                field, 'icontains'): query for field in search_fields}
        query_objects = Q()
        for q in query_by:
            query_objects |= Q(**{q: query_by[q]})
        return queryset.filter(query_objects).distinct()
    return queryset

def reporting_period_to_date(period):
    now = timezone.now()
    if period == ReportingPeriod.TODAY:
        start_date = now.replace(
            hour=0, minute=0, second=0, microsecond=0)
    elif period == ReportingPeriod.THIS_MONTH:
        start_date = now.replace(
            day=1, hour=0, minute=0, second=0, microsecond=0)
    else:
        raise ValueError('Unknown period: %s' % period)
    return start_date

def filter_by_period(queryset, period, field_name):
    start_date = reporting_period_to_date(period)
    return queryset.filter(**{'%s__gte' % field_name: start_date})

def generate_query_argument_description(search_fields):
    header = 'Supported filter parameters:\n'
    supported_list = ''
    for field in search_fields:
        supported_list += '* {0}\n'.format(field)
    return header + supported_list

def format_permissions_for_display(permissions):
    """Transform permissions queryset into PermissionDisplay list.

    Keyword arguments:
    permissions - queryset with permissions
    """
    formatted_permissions = []
    for permission in permissions:
        codename = '.'.join(
            [permission.content_type.app_label, permission.codename])
        formatted_permissions.append(
            PermissionDisplay(
                code=PermissionEnum.get(codename),
                name=permission.name))
    return formatted_permissions
<commit_msg>Preserve orders from ids for get_nodes<commit_after>import graphene
from django.db.models import Q
from django.utils import timezone
from graphene_django.registry import get_global_registry
from graphql.error import GraphQLError
from graphql_relay import from_global_id

from .core.enums import PermissionEnum, ReportingPeriod
from .core.types import PermissionDisplay

registry = get_global_registry()

def get_database_id(info, node_id, only_type):
    """Get a database ID from a node ID of given type."""
    _type, _id = graphene.relay.Node.from_global_id(node_id)
    graphene_type = info.schema.get_type(_type).graphene_type
    if graphene_type != only_type:
        raise AssertionError('Must receive a %s id.' % only_type._meta.name)
    return _id

def get_nodes(ids, graphene_type=None):
    """Return a list of nodes.

    If the `graphene_type` argument is provided, the IDs will be validated
    against this type. If the type was not provided, it will be looked up in
    the Graphene's registry. Raises an error if not all IDs are of the same
    type.
    """
    pks = []
    types = []
    invalid_ids = []
    error_msg = "Could not resolve to a nodes with the global id list of '%s'."
    for graphql_id in ids:
        if graphql_id:
            try:
                _type, _id = from_global_id(graphql_id)
            except Exception:
                invalid_ids.append(graphql_id)
            else:
                if graphene_type:
                    assert str(graphene_type) == _type, (
                        'Must receive an {} id.').format(
                            graphene_type._meta.name)
                pks.append(_id)
                types.append(_type)
    if invalid_ids:
        raise GraphQLError(
            error_msg % invalid_ids)

    # If `graphene_type` was not provided, check if all resolved types are
    # the same. This prevents from accidentally mismatching IDs of different
    # types.
    if types and not graphene_type:
        assert len(set(types)) == 1, 'Received IDs of more than one type.'
        # get type by name
        type_name = types[0]
        for model, _type in registry._registry.items():
            if _type._meta.name == type_name:
                graphene_type = _type
                break

    nodes = list(graphene_type._meta.model.objects.filter(pk__in=pks))
    nodes.sort(key=lambda e: pks.index(str(e.pk)))  # preserve order in pks
    if not nodes:
        raise GraphQLError(
            error_msg % ids)
    nodes_pk_list = [str(node.pk) for node in nodes]
    for pk in pks:
        assert pk in nodes_pk_list, (
            'There is no node of type {} with pk {}'.format(_type, pk))
    return nodes

def filter_by_query_param(queryset, query, search_fields):
    """Filter queryset according to given parameters.

    Keyword arguments:
    queryset - queryset to be filtered
    query - search string
    search_fields - fields considered in filtering
    """
    if query:
        query_by = {
            '{0}__{1}'.format(
                field, 'icontains'): query for field in search_fields}
        query_objects = Q()
        for q in query_by:
            query_objects |= Q(**{q: query_by[q]})
        return queryset.filter(query_objects).distinct()
    return queryset

def reporting_period_to_date(period):
    now = timezone.now()
    if period == ReportingPeriod.TODAY:
        start_date = now.replace(
            hour=0, minute=0, second=0, microsecond=0)
    elif period == ReportingPeriod.THIS_MONTH:
        start_date = now.replace(
            day=1, hour=0, minute=0, second=0, microsecond=0)
    else:
        raise ValueError('Unknown period: %s' % period)
    return start_date

def filter_by_period(queryset, period, field_name):
    start_date = reporting_period_to_date(period)
    return queryset.filter(**{'%s__gte' % field_name: start_date})

def generate_query_argument_description(search_fields):
    header = 'Supported filter parameters:\n'
    supported_list = ''
    for field in search_fields:
        supported_list += '* {0}\n'.format(field)
    return header + supported_list

def format_permissions_for_display(permissions):
    """Transform permissions queryset into PermissionDisplay list.

    Keyword arguments:
    permissions - queryset with permissions
    """
    formatted_permissions = []
    for permission in permissions:
        codename = '.'.join(
            [permission.content_type.app_label, permission.codename])
        formatted_permissions.append(
            PermissionDisplay(
                code=PermissionEnum.get(codename),
                name=permission.name))
    return formatted_permissions
<|endoftext|>


<commit_before>from datetime import datetime
import unittest

from hamcrest import *
import pytz
from werkzeug.datastructures import MultiDict
from freezegun import freeze_time

from backdrop.read.query import parse_request_args

class Test_parse_request_args(unittest.TestCase):
    def test_start_at_is_parsed(self):
        request_args = MultiDict([
            ("start_at", "2012-12-12T08:12:43+00:00")])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_first_start_at_is_used(self):
        request_args = MultiDict([
            ("start_at", "2012-12-12T08:12:43+00:00"),
            ("start_at", "2012-12-13T08:12:43+00:00"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_end_at_is_parsed(self):
        request_args = MultiDict([
            ("end_at", "2012-12-12T08:12:43+00:00")])

        args = parse_request_args(request_args)

        assert_that(args['end_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_first_end_at_is_used(self):
        request_args = MultiDict([
            ("end_at", "2012-12-12T08:12:43+00:00"),
            ("end_at", "2012-12-13T08:12:43+00:00"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['end_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_one_filter_by_is_parsed(self):
        request_args = MultiDict([
            ("filter_by", "foo:bar")])

        args = parse_request_args(request_args)

        assert_that(args['filter_by'], has_item(["foo", "bar"]))

    def test_many_filter_by_are_parsed(self):
        request_args = MultiDict([
            ("filter_by", "foo:bar"),
            ("filter_by", "bar:foo")
        ])

        args = parse_request_args(request_args)

        assert_that(args['filter_by'], has_item(["foo", "bar"]))
        assert_that(args['filter_by'], has_item(["bar", "foo"]))

    def test_build_query_with_boolean_value(self):
        request_args = MultiDict([
            ("filter_by", "planet:true"),
            ("filter_by", "star:false"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['filter_by'], has_item([ "planet", True ]))
        assert_that(args['filter_by'], has_item([ "star", False ]))

    def test_group_by_is_passed_through_untouched(self):
        request_args = MultiDict([("group_by", "foobar")])

        args = parse_request_args(request_args)

        assert_that(args['group_by'], is_('foobar'))

    def test_sort_is_parsed(self):
        request_args = MultiDict([
            ("sort_by", "foo:ascending")])

        args = parse_request_args(request_args)

        assert_that(args['sort_by'], is_(["foo", "ascending"]))

    def test_sort_will_use_first_argument_only(self):
        request_args = MultiDict([
            ("sort_by", "foo:descending"),
            ("sort_by", "foo:ascending"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['sort_by'], is_(["foo", "descending"]))

    def test_limit_is_parsed(self):
        request_args = MultiDict([
            ("limit", "123")
        ])

        args = parse_request_args(request_args)

        assert_that(args['limit'], is_(123))

    def test_one_collect_is_parsed_with_default_method(self):
        request_args = MultiDict([
            ("collect", "some_key")
        ])

        args = parse_request_args(request_args)

        assert_that(args['collect'], is_([("some_key", "default")]))

    def test_two_collects_are_parsed_with_default_methods(self):
        request_args = MultiDict([
            ("collect", "some_key"),
            ("collect", "some_other_key")
        ])

        args = parse_request_args(request_args)

        assert_that(args['collect'], is_([("some_key", "default"),
                                          ("some_other_key", "default")]))

    def test_one_collect_is_parsed_with_custom_method(self):
        request_args = MultiDict([
            ("collect", "some_key:mean")
        ])

        args = parse_request_args(request_args)

        assert_that(args['collect'], is_([("some_key", "mean")]))

    @freeze_time('2014-01-09 00:00:00')
    def test_now_and_positive_delta_sets_start_end_window(self):
        request_args = MultiDict([
            ('period', 'day'),
            ('delta', '3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 9, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 12, 0, 0, 0, tzinfo=pytz.UTC)))

    @freeze_time('2014-01-09 00:00:00')
    def test_now_and_negative_delta_sets_start_end_window(self):
        request_args = MultiDict([
            ('period', 'day'),
            ('delta', '-3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 6, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 9, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_on_boundary_and_positive_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-09T00:00:00+00:00'),
            ('period', 'day'),
            ('delta', '3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 9, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 12, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_on_boundary_and_negative_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-11T0:0:0+00:00'),
            ('period', 'day'),
            ('delta', '-3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 8, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 11, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_off_boundary_and_positive_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-09T01:02:03+00:00'),
            ('period', 'day'),
            ('delta', '3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 10, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 13, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_off_boundary_and_negative_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-11T23:58:57+00:00'),
            ('period', 'day'),
            ('delta', '-3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 8, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 11, 0, 0, 0, tzinfo=pytz.UTC)))
<commit_msg>Improving testing for when absent 'date' parameter<commit_after>from datetime import datetime
import unittest

from hamcrest import *
import pytz
from werkzeug.datastructures import MultiDict
from freezegun import freeze_time

from backdrop.read.query import parse_request_args

class Test_parse_request_args(unittest.TestCase):
    def test_start_at_is_parsed(self):
        request_args = MultiDict([
            ("start_at", "2012-12-12T08:12:43+00:00")])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_first_start_at_is_used(self):
        request_args = MultiDict([
            ("start_at", "2012-12-12T08:12:43+00:00"),
            ("start_at", "2012-12-13T08:12:43+00:00"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_end_at_is_parsed(self):
        request_args = MultiDict([
            ("end_at", "2012-12-12T08:12:43+00:00")])

        args = parse_request_args(request_args)

        assert_that(args['end_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_first_end_at_is_used(self):
        request_args = MultiDict([
            ("end_at", "2012-12-12T08:12:43+00:00"),
            ("end_at", "2012-12-13T08:12:43+00:00"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['end_at'], is_(
            datetime(2012, 12, 12, 8, 12, 43, tzinfo=pytz.UTC)))

    def test_one_filter_by_is_parsed(self):
        request_args = MultiDict([
            ("filter_by", "foo:bar")])

        args = parse_request_args(request_args)

        assert_that(args['filter_by'], has_item(["foo", "bar"]))

    def test_many_filter_by_are_parsed(self):
        request_args = MultiDict([
            ("filter_by", "foo:bar"),
            ("filter_by", "bar:foo")
        ])

        args = parse_request_args(request_args)

        assert_that(args['filter_by'], has_item(["foo", "bar"]))
        assert_that(args['filter_by'], has_item(["bar", "foo"]))

    def test_build_query_with_boolean_value(self):
        request_args = MultiDict([
            ("filter_by", "planet:true"),
            ("filter_by", "star:false"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['filter_by'], has_item([ "planet", True ]))
        assert_that(args['filter_by'], has_item([ "star", False ]))

    def test_group_by_is_passed_through_untouched(self):
        request_args = MultiDict([("group_by", "foobar")])

        args = parse_request_args(request_args)

        assert_that(args['group_by'], is_('foobar'))

    def test_sort_is_parsed(self):
        request_args = MultiDict([
            ("sort_by", "foo:ascending")])

        args = parse_request_args(request_args)

        assert_that(args['sort_by'], is_(["foo", "ascending"]))

    def test_sort_will_use_first_argument_only(self):
        request_args = MultiDict([
            ("sort_by", "foo:descending"),
            ("sort_by", "foo:ascending"),
        ])

        args = parse_request_args(request_args)

        assert_that(args['sort_by'], is_(["foo", "descending"]))

    def test_limit_is_parsed(self):
        request_args = MultiDict([
            ("limit", "123")
        ])

        args = parse_request_args(request_args)

        assert_that(args['limit'], is_(123))

    def test_one_collect_is_parsed_with_default_method(self):
        request_args = MultiDict([
            ("collect", "some_key")
        ])

        args = parse_request_args(request_args)

        assert_that(args['collect'], is_([("some_key", "default")]))

    def test_two_collects_are_parsed_with_default_methods(self):
        request_args = MultiDict([
            ("collect", "some_key"),
            ("collect", "some_other_key")
        ])

        args = parse_request_args(request_args)

        assert_that(args['collect'], is_([("some_key", "default"),
                                          ("some_other_key", "default")]))

    def test_one_collect_is_parsed_with_custom_method(self):
        request_args = MultiDict([
            ("collect", "some_key:mean")
        ])

        args = parse_request_args(request_args)

        assert_that(args['collect'], is_([("some_key", "mean")]))

    @freeze_time('2014-01-09 00:00:00')
    def test_no_date_means_now(self):
        request_args = MultiDict([
            ('period', 'day'),
            ('delta', '3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 9, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_on_boundary_and_positive_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-09T00:00:00+00:00'),
            ('period', 'day'),
            ('delta', '3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 9, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 12, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_on_boundary_and_negative_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-11T0:0:0+00:00'),
            ('period', 'day'),
            ('delta', '-3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 8, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 11, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_off_boundary_and_positive_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-09T01:02:03+00:00'),
            ('period', 'day'),
            ('delta', '3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 10, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 13, 0, 0, 0, tzinfo=pytz.UTC)))

    def test_date_off_boundary_and_negative_delta(self):
        request_args = MultiDict([
            ('date', '2014-01-11T23:58:57+00:00'),
            ('period', 'day'),
            ('delta', '-3'),
        ])

        args = parse_request_args(request_args)

        assert_that(args['start_at'], is_(
            datetime(2014, 1, 8, 0, 0, 0, tzinfo=pytz.UTC)))

        assert_that(args['end_at'], is_(
            datetime(2014, 1, 11, 0, 0, 0, tzinfo=pytz.UTC)))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Send events covering service status
'''

# Import Python Libs
from __future__ import absolute_import

import os
import logging

log = logging.getLogger(__name__)  # pylint: disable=invalid-name

last_status = {}

def validate(config):
    '''
    Validate the beacon configuration
    '''
    # Configuration for service beacon should be a list of dicts
    if not isinstance(config, dict):
        log.info('Configuration for service beacon must be a dictionary.')
        return False
    return True

def beacon(config):
    '''
    Scan for the configured services and fire events

    Example Config

    .. code-block:: yaml

        beacons:
          service:
            salt-master:
            mysql:

    The config above sets up beacons to check for
    the salt-master and mysql services.

    The config also supports two other parameters for each service:

    `onchangeonly`: when `onchangeonly` is True the beacon will fire
    events only when the service status changes.  Otherwise, it will fire an event
    at each beacon interval.  The default is False.

    `uncleanshutdown`: If `uncleanshutdown` is present it should point to the location
    of a pid file for the service.  Most services will not clean up this pid file
    if they are shutdown uncleanly (e.g. via `kill -9`) or if they are terminated
    through a crash such as a segmentation fault.  If the file is present, then
    the beacon will add `uncleanshutdown: True` to the event.  If not present,
    the field will be False.  The field is only added when the service is NOT running.
    Omitting the configuration variable altogether will turn this feature off.

    Here is an example that will fire an event whenever the state of nginx changes
    and report an uncleanshutdown.  This example is for Arch, which places nginx's
    pid file in `/run`.

    .. code-block:: yaml

        beacons:
          service:
            nginx:
              onchangeonly: True
              uncleanshutdown: /run/nginx.pid
    '''
    ret = []
    for service in config:
        ret_dict = {}
        ret_dict[service] = {'running': __salt__['service.status'](service)}

        # If no options is given to the service, we fall back to the defaults
        # assign a False value to oncleanshutdown and onchangeonly. Those
        # key:values are then added to the service dictionary.
        if config[service] is None:
            defaults = {
                    'oncleanshutdown':False,
                    'onchangeonly':False
                    }
            config[service] = defaults

        # We only want to report the nature of the shutdown
        # if the current running status is False
        # as well as if the config for the beacon asks for it
        if 'uncleanshutdown' in config[service] and not ret_dict[service]['running']:
            filename = config[service]['uncleanshutdown']
            if os.path.exists(filename):
                ret_dict[service]['uncleanshutdown'] = True
            else:
                ret_dict[service]['uncleanshutdown'] = False

        if 'onchangeonly' in config[service] and config[service]['onchangeonly'] is True:
            if service not in last_status:
                last_status[service] = ''
            if last_status[service] != ret_dict[service]:
                last_status[service] = ret_dict[service]
                ret.append(ret_dict)
        else:
            ret.append(ret_dict)

    return ret
<commit_msg>Added note about systemd and uncleanshutdown. Also fixed line lenght of comments to max 80 characters as per PEP0008<commit_after># -*- coding: utf-8 -*-
'''
Send events covering service status
'''

# Import Python Libs
from __future__ import absolute_import

import os
import logging

log = logging.getLogger(__name__)  # pylint: disable=invalid-name

last_status = {}

def validate(config):
    '''
    Validate the beacon configuration
    '''
    # Configuration for service beacon should be a list of dicts
    if not isinstance(config, dict):
        log.info('Configuration for service beacon must be a dictionary.')
        return False
    return True

def beacon(config):
    '''
    Scan for the configured services and fire events

    Example Config

    .. code-block:: yaml

        beacons:
          service:
            salt-master:
            mysql:

    The config above sets up beacons to check for
    the salt-master and mysql services.

    The config also supports two other parameters for each service:

    `onchangeonly`: when `onchangeonly` is True the beacon will fire
    events only when the service status changes.  Otherwise, it will fire an
    event at each beacon interval.  The default is False.

    `uncleanshutdown`: If `uncleanshutdown` is present it should point to the
    location of a pid file for the service.  Most services will not clean up
    this pid file if they are shutdown uncleanly (e.g. via `kill -9`) or if they
    are terminated through a crash such as a segmentation fault.  If the file is
    present, then the beacon will add `uncleanshutdown: True` to the event.  If
    not present, the field will be False.  The field is only added when the
    service is NOT running. Omitting the configuration variable altogether will
    turn this feature off.

    Please note that some init systems can remove the pid file if the service
    registers as crashed. One such example is nginx on CentOS 7, where the
    service unit removes the pid file when the service shuts down (IE: the pid
    file is observed as removed when kill -9 is sent to the nginx master
    process). The 'uncleanshutdown' option might not be of much use there,
    unless the unit file is modified.

    Here is an example that will fire an event whenever the state of nginx
    changes and report an uncleanshutdown.  This example is for Arch, which
    places nginx's pid file in `/run`.

    .. code-block:: yaml

        beacons:
          service:
            nginx:
              onchangeonly: True
              uncleanshutdown: /run/nginx.pid
    '''
    ret = []
    for service in config:
        ret_dict = {}
        ret_dict[service] = {'running': __salt__['service.status'](service)}

        # If no options is given to the service, we fall back to the defaults
        # assign a False value to oncleanshutdown and onchangeonly. Those
        # key:values are then added to the service dictionary.
        if config[service] is None:
            defaults = {
                    'oncleanshutdown':False,
                    'onchangeonly':False
                    }
            config[service] = defaults

        # We only want to report the nature of the shutdown
        # if the current running status is False
        # as well as if the config for the beacon asks for it
        if 'uncleanshutdown' in config[service] and not ret_dict[service]['running']:
            filename = config[service]['uncleanshutdown']
            if os.path.exists(filename):
                ret_dict[service]['uncleanshutdown'] = True
            else:
                ret_dict[service]['uncleanshutdown'] = False

        if 'onchangeonly' in config[service] and config[service]['onchangeonly'] is True:
            if service not in last_status:
                last_status[service] = ''
            if last_status[service] != ret_dict[service]:
                last_status[service] = ret_dict[service]
                ret.append(ret_dict)
        else:
            ret.append(ret_dict)

    return ret
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- mode: python; sh-basic-offset: 4; indent-tabs-mode: nil; coding: utf-8 -*-
# vim: tabstop=4 softtabstop=4 expandtab shiftwidth=4 fileencoding=utf-8
#
# Base script helper
# Copyright 2011, Jorge A Gallegos <kad@blegh.net>

import ConfigParser
import logging
import os
import sys

import clusto

levels = {
    'INFO': logging.INFO,
    'DEBUG': logging.DEBUG,
    'ERROR': logging.ERROR,
    'FATAL': logging.FATAL,
    'WARN': logging.WARN,
}

class CmdLineError(Exception):
    pass

class Script(object):

    logger = None
    config = None

    def __init__(self):
        pass

    def _get_description(self):
        if self.__class__.__doc__:
            help_string = self.__class__.__doc__
        else:
            help_string = '%s command help' % self.__class__.__name__.capitalize()
        return help_string

    def _setup_subparser(self, subparsers):
        parser = subparsers.add_parser(self.__class__.__name__.lower(),
            help=self._get_description())
        return parser

    def add_subparser(self, subparser):
        self._setup_subparser(subparser)

    def set_logger(self, logger):
        self.logger = logger

    def log(self, msg, level=logging.INFO):
        self.logger.log(level=level, msg=msg)

    def error(self, msg):
        self.log(msg, level=logging.ERROR)

    def warn(self, msg):
        self.log(msg, level=logging.WARN)

    def fatal(self, msg):
        self.log(msg, level=logging.FATAL)

    def debug(self, msg):
        self.log(msg, level=logging.DEBUG)

    def info(self, msg):
        self.log(msg)

    def run(self, *args, **kwargs):
        raise NotImplementedError()

    def load_config(self, filename, dsn=None):
        '''Find, parse, and return the configuration data needed by clusto.'''

        if filename:
            if not os.path.exists(os.path.realpath(filename)):
                msg = "Config file %s doesn't exist." % filename
                self.error(msg)
                raise CmdLineError(msg)
        else:
            msg = "Config file %s doesn't exist." % filename
            self.error(msg)
            raise CmdLineError(msg)

        self.config = ConfigParser.SafeConfigParser()
        self.debug('Reading %s' % filename)
        self.config.read([filename])

        if not self.config.has_section('clusto'):
            self.config.add_section('clusto')

        if 'CLUSTODSN' in os.environ:
            self.config.set('clusto', 'dsn', os.environ['CLUSTODSN'])

        if dsn:
            self.config.set('clusto', 'dsn', dsn)

        if not self.config.has_option('clusto', 'dsn'):
            raise CmdLineError("No database given for clusto data.")

    def get_config(self):
        '''Returns the config object'''

        return self.config

    def init_script(self, args, logger=None):
        '''Initialize the clusto environment for clusto scripts.

        Connects to the clusto database, returns a python SafeConfigParser and a
        logger.

        '''

        if logger:
            self.set_logger(logger)
        if 'CLUSTOCONFIG' in os.environ:
            filename = os.environ['CLUSTOCONFIG']
        else:
            filename = args.config
        self.load_config(filename, args.dsn)
        self.debug('Connecting to %s' % self.config.get('clusto', 'dsn'))
        clusto.connect(self.config)

        return self.config

def demodule(module):
    klass = ''.join([_.capitalize() for _ in module.split('_')])
    module = __import__('clusto.commands.%s' % module, fromlist=['clusto.commands'])
    klass = getattr(module, klass)
    return klass

def setup_base_parser(add_help=False):
    import argparse
    parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
        description='Clusto script', conflict_handler='resolve',
        add_help=add_help)
    parser.add_argument('--config', default='/etc/clusto/clusto.conf',
        help='Config file (defaults to /etc/clusto/clusto.conf)')
    parser.add_argument('--loglevel', default='INFO',
        help='Sets the logging level (defaults to INFO)')
    parser.add_argument('--dsn', help='Alternate DSN to use')
    return parser

def get_logger(loglevel='INFO'):
    format='%(name)s: %(levelname)-8s %(message)s'
    logging.basicConfig(format=format)
    log = logging.getLogger('clusto')
    log.setLevel(levels[loglevel])
    return log

def main():
    # These modules should only be imported if called from the cli
    from clusto import commands

    parser = setup_base_parser(add_help=True)
    subparsers = parser.add_subparsers(title='Available clusto commands', dest='subparser_name')
    for module in commands.__all__:
        try:
            klass = demodule(module)
            klass = klass()
            klass.add_subparser(subparsers)
            klass = None
        except Exception, e:
            print e
    args = parser.parse_args()

    log = get_logger(args.loglevel)
    if len(sys.argv) < 2:
        parser.print_help()
        return 0
    log.debug('Loading from clusto frontend: %s' % args.subparser_name)
    klass = demodule(args.subparser_name)
    klass = klass()
    klass.set_logger(log)
    try:
        klass.init_script(args=args, logger=log)
        klass.run(args)
    except Exception as e:
        log.error(str(e))

if __name__ == '__main__':
    sys.exit(main())

<commit_msg>Changes to the clusto frontend:  - Adding doc strings  - Account for commands with underscores to be translated to slashes  - Commands are now read from the FS items, not from __all__, this    so we can split commands in packages later without the need to    update the commands/__init__.py file<commit_after>#!/usr/bin/env python
# -*- mode: python; sh-basic-offset: 4; indent-tabs-mode: nil; coding: utf-8 -*-
# vim: tabstop=4 softtabstop=4 expandtab shiftwidth=4 fileencoding=utf-8
#
# Base script helper
# Copyright 2011, Jorge A Gallegos <kad@blegh.net>

import ConfigParser
import logging
import os
import sys

import clusto

levels = {
    'INFO': logging.INFO,
    'DEBUG': logging.DEBUG,
    'ERROR': logging.ERROR,
    'FATAL': logging.FATAL,
    'WARN': logging.WARN,
}

class CmdLineError(Exception):
    pass

class Script(object):
    '''
    Base script helper class
    '''

    logger = None
    config = None

    def __init__(self):
        pass

    def _get_description(self):
        '''
        Returns the docstring for the class if there's one,
        otherwise it returns a generic blurb
        '''

        if self.__class__.__doc__:
            help_string = self.__class__.__doc__
        else:
            help_string = '%s command help' % self.__class__.__name__.capitalize()
        return help_string

    def _setup_subparser(self, subparsers):
        '''
        Sets up the command subparser, this should (in theory at least)
        only be called from within add_subparser()
        '''

        command_name = self.__class__.__name__.lower().replace('_', '-')
        parser = subparsers.add_parser(command_name, help=self._get_description())
        return parser

    def add_subparser(self, subparser):
        '''
        Adds the command subparser to the base parser. This method should
        always call _setup_subparser so the command becomes available in
        the argparse namespace, otherwise it won't be listed
        '''

        self._setup_subparser(subparser)

    def set_logger(self, logger):
        '''
        Sets the class logger to whatever logging facility we provide
        '''
        self.logger = logger

    def log(self, msg, level=logging.INFO):
        '''
        I dislike having to type self.logger.log()
        '''
        self.logger.log(level=level, msg=msg)

    def error(self, msg):
        '''
        Passthrough to self.logger.error()
        '''
        self.log(msg, level=logging.ERROR)

    def warn(self, msg):
        '''
        Passthrough to self.logger.warn()
        '''
        self.log(msg, level=logging.WARN)

    def fatal(self, msg):
        '''
        Passthrough to self.logger.fatal()
        '''
        self.log(msg, level=logging.FATAL)

    def debug(self, msg):
        '''
        Passthrough to self.logger.debug()
        '''
        self.log(msg, level=logging.DEBUG)

    def info(self, msg):
        '''
        Passthrough to self.logger.info()
        '''
        self.log(msg)

    def run(self, *args, **kwargs):
        '''
        Main command loop, should be implemented in child classes
        '''
        raise NotImplementedError()

    def load_config(self, filename, dsn=None):
        '''
        Find, parse, and return the configuration data needed by clusto
        '''

        if filename:
            if not os.path.exists(os.path.realpath(filename)):
                msg = "Config file %s doesn't exist." % filename
                self.error(msg)
                raise CmdLineError(msg)
        else:
            msg = "Config file %s doesn't exist." % filename
            self.error(msg)
            raise CmdLineError(msg)

        self.config = ConfigParser.SafeConfigParser()
        self.debug('Reading %s' % filename)
        self.config.read([filename])

        if not self.config.has_section('clusto'):
            self.config.add_section('clusto')

        if 'CLUSTODSN' in os.environ:
            self.config.set('clusto', 'dsn', os.environ['CLUSTODSN'])

        if dsn:
            self.config.set('clusto', 'dsn', dsn)

        if not self.config.has_option('clusto', 'dsn'):
            raise CmdLineError("No database given for clusto data.")

    def get_config(self):
        '''
        Returns the config object
        '''

        return self.config

    def init_script(self, args, logger=None):
        '''
        Initialize the clusto environment for clusto scripts.

        Connects to the clusto database, returns a python SafeConfigParser
        '''

        if logger:
            self.set_logger(logger)
        if 'CLUSTOCONFIG' in os.environ:
            filename = os.environ['CLUSTOCONFIG']
        else:
            filename = args.config
        self.load_config(filename, args.dsn)
        self.debug('Connecting to %s' % self.config.get('clusto', 'dsn'))
        clusto.connect(self.config)

        return self.config

def demodule(module):
    '''
    Returns a class out of a given module name by doing:
    some_name_in_this_form => SomeNameInThisForm()
    '''

    klass = ''.join([_.capitalize() for _ in module.split('_')])
    module = __import__('clusto.commands.%s' % module, fromlist=['clusto.commands'])
    klass = getattr(module, klass)
    return klass

def setup_base_parser(add_help=False):
    '''
    Setups the base parser with common options, this is used here down below
    or can alternatively be used in the commands main() call for standalone
    script support
    '''

    import argparse
    parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
        description='Clusto script', conflict_handler='resolve',
        add_help=add_help)
    parser.add_argument('--config', default='/etc/clusto/clusto.conf',
        help='Config file (defaults to /etc/clusto/clusto.conf)')
    parser.add_argument('--loglevel', default='INFO',
        help='Sets the logging level (defaults to INFO)')
    parser.add_argument('--dsn', help='Alternate DSN to use')
    return parser

def get_logger(loglevel='INFO'):
    '''
    Returns a basic std{out,err} logger
    '''

    format='%(name)s: %(levelname)-8s %(message)s'
    logging.basicConfig(format=format)
    log = logging.getLogger('clusto')
    log.setLevel(levels[loglevel])
    return log

def main():
    # should only be imported if called from the cli
    from clusto import commands

    parser = setup_base_parser(add_help=True)
    subparsers = parser.add_subparsers(title='Available clusto commands', dest='subparser_name')
    help_subparser = subparsers.add_parser('help', help='Print clusto help')
    for dirpath, dirname, filenames in os.walk(commands.__path__[0]):
        for fn in filenames:
            if not fn.endswith('pyc') and not fn.startswith('__'):
                module = fn.split('.')[0]
                try:
                    klass = demodule(module)
                    klass = klass()
                    klass.add_subparser(subparsers)
                    klass = None
                except Exception, e:
                    pass
    args = parser.parse_args()

    log = get_logger(args.loglevel)
    if len(sys.argv) < 2:
        parser.print_help()
        return 0
    if args.subparser_name == 'help':
        parser.print_help()
        return 0
    log.debug('Loading from clusto frontend: %s' % args.subparser_name)
    klass = demodule(args.subparser_name)
    klass = klass()
    klass.set_logger(log)
    try:
        klass.init_script(args=args, logger=log)
        klass.run(args)
    except Exception as e:
        log.error(str(e))

if __name__ == '__main__':
    sys.exit(main())

<|endoftext|>


<commit_before>"""Views used by the feeds application."""

import datetime

from django import template
from django.http import Http404, HttpResponse
from django.conf import settings
from django.utils.feedgenerator import DefaultFeed

from cms.apps.pages.templatetags.pages import html

def index(request):
    """Generates a page of the latest articles."""
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the list of articles.
    articles = content.get_latest_articles()
    # Generate the context.
    context = {"articles": articles,
               "date": datetime.datetime.now()}
    # Render the template.
    template_names = ("%s/index.html" % article_model._meta.app_label,
                      "feeds/index.html")
    return page.render_to_response(request, template_names, context)

def year_archive(request, year):
    """Generates a page showing the articles in a given year."""
    year = int(year)
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the list of articles.
    articles = content.get_articles().filter(**{"%s__year" % article_model.date_field_name: year})
    # Generate the context.
    context = {"articles": articles,
               "date": datetime.datetime(year, 1, 1)}
    # Render the template.
    template_names = ("%s/year_archive.html" % article_model._meta.app_label,
                      "feeds/year_archive.html")
    return page.render_to_response(request, template_names, context)

def month_archive(request, year, month):
    """Generates a page showing the articles in a given month."""
    year = int(year)
    month = int(month)
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the list of articles.
    articles = content.get_articles().filter(**{"%s__year" % article_model.date_field_name: year,
                                                "%s__month" % article_model.date_field_name: month})
    # Generate the context.
    context = {"articles": articles,
               "date": datetime.datetime(year, month, 1)}
    # Render the template.
    template_names = ("%s/month_archive.html" % article_model._meta.app_label,
                      "feeds/month_archive.html")
    return page.render_to_response(request, template_names, context)

def article_detail(request, year, month, article_slug):
    """Dispatches to the article detail page."""
    year = int(year)
    month = int(month)
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the article.
    try:
        article = content.get_articles().get(**{"%s__year" % article_model.date_field_name: year,
                                                "%s__month" % article_model.date_field_name: month,
                                                "url_title": article_slug})
    except article_model.DoesNotExist:
        raise Http404, "That article does not exist"
    # Generate the context.
    context = {"article": article,
               "date": getattr(article, article_model.date_field_name)}
    # Render the template.
    template_names = ("%s/article_detail.html" % article_model._meta.app_label,
                      "feeds/article_detail.html")
    return article.render_to_response(request, template_names, context)

def rss(request):
    """Generates the RSS feed for this news feed."""
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    fullpath = "http://%s%%s" % page.site.domain
    homepage = page.homepage
    # Generate the feed title.
    title_context = {"title": page.browser_title or page.title,
                     "site_title": homepage.browser_title or homepage.title}
    title = template.loader.render_to_string("title.html", title_context, template.RequestContext(request))
    # Get the list of articles.
    all_articles = content.get_articles().order_by("-%s" % article_model.date_field_name)
    # Generate the head.
    feed = DefaultFeed(title=title,
                       link=fullpath % page.get_absolute_url(),
                       description=page.meta_description or homepage.meta_description,
                       language=settings.LANGUAGE_CODE)
    # Generate the feed body.
    for article in all_articles[:30]:
        feed.add_item(title=article.title,
                      link=fullpath % article.get_absolute_url(),
                      description=html(article.summary or article.content))
    # Generate the response.
    response = HttpResponse(mimetype=feed.mime_type)
    feed.write(response, settings.DEFAULT_CHARSET)
    return response

    <commit_msg>Empty month and year archives now throw 404.<commit_after>"""Views used by the feeds application."""

import datetime

from django import template
from django.http import Http404, HttpResponse
from django.conf import settings
from django.utils.feedgenerator import DefaultFeed

from cms.apps.pages.templatetags.pages import html

def index(request):
    """Generates a page of the latest articles."""
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the list of articles.
    articles = content.get_latest_articles()
    # Generate the context.
    context = {"articles": articles,
               "date": datetime.datetime.now()}
    # Render the template.
    template_names = ("%s/index.html" % article_model._meta.app_label,
                      "feeds/index.html")
    return page.render_to_response(request, template_names, context)

def year_archive(request, year):
    """Generates a page showing the articles in a given year."""
    year = int(year)
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the list of articles.
    articles = content.get_articles().filter(**{"%s__year" % article_model.date_field_name: year})
    if articles.count() == 0:
        raise Http404, "There are no articles to display."
    # Generate the context.
    context = {"articles": articles,
               "date": datetime.datetime(year, 1, 1)}
    # Render the template.
    template_names = ("%s/year_archive.html" % article_model._meta.app_label,
                      "feeds/year_archive.html")
    return page.render_to_response(request, template_names, context)

def month_archive(request, year, month):
    """Generates a page showing the articles in a given month."""
    year = int(year)
    month = int(month)
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the list of articles.
    articles = content.get_articles().filter(**{"%s__year" % article_model.date_field_name: year,
                                                "%s__month" % article_model.date_field_name: month})
    if articles.count() == 0:
        raise Http404, "There are no articles to display."
    # Generate the context.
    context = {"articles": articles,
               "date": datetime.datetime(year, month, 1)}
    # Render the template.
    template_names = ("%s/month_archive.html" % article_model._meta.app_label,
                      "feeds/month_archive.html")
    return page.render_to_response(request, template_names, context)

def article_detail(request, year, month, article_slug):
    """Dispatches to the article detail page."""
    year = int(year)
    month = int(month)
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    # Get the article.
    try:
        article = content.get_articles().get(**{"%s__year" % article_model.date_field_name: year,
                                                "%s__month" % article_model.date_field_name: month,
                                                "url_title": article_slug})
    except article_model.DoesNotExist:
        raise Http404, "That article does not exist"
    # Generate the context.
    context = {"article": article,
               "date": getattr(article, article_model.date_field_name)}
    # Render the template.
    template_names = ("%s/article_detail.html" % article_model._meta.app_label,
                      "feeds/article_detail.html")
    return article.render_to_response(request, template_names, context)

def rss(request):
    """Generates the RSS feed for this news feed."""
    # Get the model information.
    page = request.page
    content = page.content
    article_model = content.article_model
    fullpath = "http://%s%%s" % page.site.domain
    homepage = page.homepage
    # Generate the feed title.
    title_context = {"title": page.browser_title or page.title,
                     "site_title": homepage.browser_title or homepage.title}
    title = template.loader.render_to_string("title.html", title_context, template.RequestContext(request))
    # Get the list of articles.
    all_articles = content.get_articles().order_by("-%s" % article_model.date_field_name)
    # Generate the head.
    feed = DefaultFeed(title=title,
                       link=fullpath % page.get_absolute_url(),
                       description=page.meta_description or homepage.meta_description,
                       language=settings.LANGUAGE_CODE)
    # Generate the feed body.
    for article in all_articles[:30]:
        feed.add_item(title=article.title,
                      link=fullpath % article.get_absolute_url(),
                      description=html(article.summary or article.content))
    # Generate the response.
    response = HttpResponse(mimetype=feed.mime_type)
    feed.write(response, settings.DEFAULT_CHARSET)
    return response

    <|endoftext|>


<commit_before>#init for reconst aka the reconstruction module

# Test callable
from numpy.testing import Tester
test = Tester().test
bench = Tester().bench

del Tester

<commit_msg>Update code in dipy/reconst (PEP8)<commit_after># init for reconst aka the reconstruction module

# Test callable
from numpy.testing import Tester
test = Tester().test
bench = Tester().bench

del Tester
<|endoftext|>