<commit_before>from datetime import date

from django.http import Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from feincms.translations import short_language_code
# from tagging.models import Tag, TaggedItem

from elephantblog.models import Entry

from feincms.views.generic import list_detail

def entry(request, year, month, day, slug, language_code=None, **kwargs):
    context={}

    entry = get_object_or_404(Entry.objects.select_related(),
                              published_on__year=year,
                               published_on__month=month,
                               published_on__day=day,
                               slug=slug)

    if not entry.isactive() and not request.user.is_authenticated():
        raise Http404
    else:
        return render_to_response('blog/entry_detail.html', {'entry':entry, 'date': date(int(year),int(month),int(day))}, context_instance=RequestContext(request))

""" Date views use object_list generic view due to pagination """

def list(request, category=None, year=None, month=None, day=None, page=0, paginate_by=10,
         template_name='blog/entry_list.html', language_code=None, **kwargs):
    extra_context = {}
    if language_code:
        queryset = Entry.objects.active().filter(language=language_code)
    else:
        try:
            language_code = request._feincms_page.language
            queryset = Entry.objects.active().filter(language=language_code)
        except AttributeError:
            queryset = Entry.objects.active()
    if category:
        queryset = queryset.filter(categories__translations__title=category)
        extra_context.update({'category': category})
    if year:
        queryset = queryset.filter(published_on__year=int(year))
        extra_context.update({'drilldown_mode': 'year', 'title' : _('entries of the year')})
    else:
        year=1
    if month:
        queryset = queryset.filter(published_on__month=int(month))
        extra_context.update({'drilldown_mode': 'month', 'title' : _('entries of the month')})
    else:
        month=1
    if day:
        queryset = queryset.filter(published_on__day=int(day))
        extra_context.update({'drilldown_mode': 'day', 'title' : _('entries of the year')})
    else:
        day=1

    extra_context.update({'date':date(int(year), int(month), int(day))})

    return list_detail.object_list(
      request,
      queryset = queryset,
      paginate_by = paginate_by,
      page = page,
      template_name = template_name,
      extra_context = extra_context,
      **kwargs)

def headlines(request, *args, **kwargs):
    return list(request, *args, template_name='blog/entry_headlines.html', **kwargs)<commit_msg>change in views.<commit_after>from datetime import date

from django.http import Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from feincms.translations import short_language_code
# from tagging.models import Tag, TaggedItem

from elephantblog.models import Entry

from feincms.views.generic import list_detail

def entry(request, year, month, day, slug, language_code=None, **kwargs):
    context={}

    entry = get_object_or_404(Entry.objects.select_related(),
                              published_on__year=year,
                               published_on__month=month,
                               published_on__day=day,
                               slug=slug)

    if not entry.isactive() and not request.user.is_authenticated():
        raise Http404
    else:
        return render_to_response('blog/entry_detail.html', {'entry':entry, 'date': date(int(year),int(month),int(day))}, context_instance=RequestContext(request))

""" Date views use object_list generic view due to pagination """

def list(request, category=None, year=None, month=None, day=None, page=0, paginate_by=10,
         template_name='blog/entry_list.html', language_code=None, **kwargs):
    extra_context = {}
    if language_code:
        queryset = Entry.objects.active().filter(language=language_code)
    else:
        try:
            language_code = request._feincms_page.language
            queryset = Entry.objects.active().filter(language=language_code)
        except AttributeError:
            queryset = Entry.objects.active()
    if category:
        queryset = queryset.filter(categories__translations__title=category)
        extra_context.update({'category': category})
    if year:
        queryset = queryset.filter(published_on__year=int(year))
        extra_context.update({'drilldown_mode': 'year', 'title' : _('entries of the year')})
    else:
        year=1
    if month:
        queryset = queryset.filter(published_on__month=int(month))
        extra_context.update({'drilldown_mode': 'month', 'title' : _('entries of the month')})
    else:
        month=1
    if day:
        queryset = queryset.filter(published_on__day=int(day))
        extra_context.update({'drilldown_mode': 'day', 'title' : _('entries of the year')})
    else:
        day=1

    extra_context.update({'date':date(int(year), int(month), int(day))})

    return list_detail.object_list(
      request,
      queryset = queryset,
      paginate_by = paginate_by,
      page = page,
      template_name = template_name,
      extra_context = extra_context,
      **kwargs)

def headlines(request, *args, **kwargs):
    return list(request, *args, template_name='blog/entry_headlines.html', **kwargs)<|endoftext|>


<commit_before>import logging
from datetime import datetime
from threading import Lock

from pylons import session, tmpl_context as c

from event import Event, EventException
from types import *
import query as q
from store import EventStore
from rss import rss_feed
import notification
import queue

import adhocracy.model as model

log = logging.getLogger(__name__)
ep_lock = Lock()

def emit(event, agent, time=None, scopes=[], topics=[], **kwargs):
    e = Event(event, agent, time, scopes=scopes, topics=topics, **kwargs)
    es = EventStore(e)
    es.persist()

    if queue.has_queue():
        queue.post_event(e)
    else:
        log.warn("Queue failure.")
        process(e)

    log.debug("Event %s: %s" % (agent.name, unicode(e)))
    return e

def process(event):
    notification.notify(event)

def queue_process():
    not_blocked = ep_lock.acquire(blocking=False)
    if not_blocked:
        return
    try:
        queue.read_events(process)
    finally:
        ep_lock.release()

<commit_msg>[svn r432] next try<commit_after>import logging
from datetime import datetime
from threading import Lock

from pylons import session, tmpl_context as c

from event import Event, EventException
from types import *
import query as q
from store import EventStore
from rss import rss_feed
import notification
import queue

import adhocracy.model as model

log = logging.getLogger(__name__)
ep_lock = Lock()

def emit(event, agent, time=None, scopes=[], topics=[], **kwargs):
    e = Event(event, agent, time, scopes=scopes, topics=topics, **kwargs)
    es = EventStore(e)
    es.persist()

    if queue.has_queue():
        queue.post_event(e)
    else:
        log.warn("Queue failure.")
        process(e)

    log.debug("Event %s: %s" % (agent.name, unicode(e)))
    return e

def process(event):
    notification.notify(event)

def queue_process():
    available = ep_lock.acquire(blocking=False)
    if not available:
        return
    try:
        queue.read_events(process)
    finally:
        ep_lock.release()

<|endoftext|>


<commit_before>from __future__ import absolute_import
import argparse
import sys

from aetros import api

from aetros.utils import read_config, unpack_full_job_id, read_home_config

import aetros.const
import os

class StartCommand:
    def __init__(self, logger):
        self.logger = logger

    def main(self, args):
        from aetros.starter import start
        parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, prog=aetros.const.__prog__ + ' start')
        parser.add_argument('name', nargs='?', help='the model name, e.g. aetros/mnist-network to start new job, or job id, e.g. user/modelname/0db75a64acb74c27bd72c22e359de7a4c44a20e5 to start a pre-created job.')

        parser.add_argument('-i', '--image', help="Which Docker image to use for the command. Default read in aetros.yml. If not specified, command is executed on the host.")
        parser.add_argument('-l', '--local', action='store_true', help="Start the job immediately on the current machine.")
        parser.add_argument('-s', '--server', action='append', help="Limits the server pool to this server. Default not limitation or read in aetros.yml. Multiple --server allowed.")
        parser.add_argument('-b', '--branch', help="This overwrites the Git branch used when new job should be started.")
        parser.add_argument('--priority', help="Increases or decreases priority. Default is 0.")

        parser.add_argument('--cpu', help="How many CPU cores should be assigned to job. Docker only.")
        parser.add_argument('--memory', help="How much memory should be assigned to job. Docker only.")
        parser.add_argument('--gpu', help="How many GPU cards should be assigned to job. Docker only.")
        parser.add_argument('--gpu_memory', help="Memory requirement for the GPU. Docker only.")

        parser.add_argument('--rebuild-image', action='store_true', help="Makes sure the Docker image is re-built without cache.")

        parser.add_argument('--gpu-device', action='append', help="Which device id should be mapped into the NVIDIA docker container.")

        parser.add_argument('--max-time', help="Limit execution time in seconds. Sends SIGINT to the process group when reached.")
        parser.add_argument('--max-epochs', help="Limit execution epochs. Sends SIGINT to the process group when reached.")

        parser.add_argument('--insights', action='store_true', help="activates insights. Only for simple models.")
        parser.add_argument('--dataset', help="Dataset id when model has placeholders. Only for simple models with placeholders as input/output.")

        parser.add_argument('-p', '--param', action='append', help="Sets a hyperparameter, example '--param name=value'. Multiple --param allowed.")

        parsed_args = parser.parse_args(args)

        home_config = read_home_config()
        model_name = parsed_args.name

        if model_name.count('/') > 1:
            # start a concret job, by server command
            start(self.logger, model_name, cpus=int(parsed_args.cpu), memory=int(parsed_args.memory), gpu_devices=parsed_args.gpu_device)
            return

        # create a new job

        hyperparameter = {}
        if parsed_args.param:
            for param in parsed_args.param:
                if '=' not in param:
                    raise Exception('--param ' + param + ' does not contain a `=`. Please use "--param name=value"')

                name, value = param.split('=')
                hyperparameter[name] = value

        job_config = {'insights': parsed_args.insights}

        if parsed_args.image:
            job_config['image'] = parsed_args.image

        if parsed_args.branch:
            job_config['sourceGitTree'] = parsed_args.branch

        if parsed_args.max_epochs:
            job_config['maxEpochs'] = int(parsed_args.max_epochs)

        if parsed_args.max_time:
            job_config['maxTime'] = float(parsed_args.max_time)

        job_config['priority'] = 0
        if parsed_args.priority:
            job_config['priority'] = float(parsed_args.priority)

        if parsed_args.rebuild_image:
            job_config['config']['rebuild_image'] = True

        if 'resources' not in job_config:
            job_config['resources'] = {}

        if parsed_args.server:
            job_config['servers'] = []
            for name in parsed_args.server:
                job_config['servers'].append(name)

        job_config['resources'] = job_config.get('resources', {})
        resources = job_config['resources']
        resources['cpu'] = int(parsed_args.cpu or resources.get('cpu', 1))
        resources['memory'] = int(parsed_args.memory or resources.get('memory', 1))
        resources['gpu'] = int(parsed_args.gpu or resources.get('gpu', 0))
        resources['gpu_memory'] = int(parsed_args.gpu_memory or resources.get('gpu_memory', 0))

        try:
            self.logger.debug("Create job ...")
            created = api.create_job(model_name, parsed_args.local, hyperparameter, parsed_args.dataset, config=job_config)
        except api.ApiError as e:
            if 'Connection refused' in e.error:
                self.logger.error("You are offline")

            raise

        print("Job %s/%s created." % (model_name, created['id']))

        if parsed_args.local:
            cpus = job_config['resources']['cpu']
            memory = job_config['resources']['memory']

            if not parsed_args.gpu_device and job_config['resources']['gpu'] > 0:
                # if requested 2 GPUs and we have 3 GPUs with id [0,1,2], gpus should be [0,1]
                parsed_args.gpu_device = []
                for i in range(0, job_config['resources']['gpu']):
                    parsed_args.gpu_device.append(i)

            start(self.logger, model_name + '/' + created['id'], cpus=cpus, memory=memory, gpu_devices=parsed_args.gpu_device)
        else:
            print("Open http://%s/model/%s/job/%s to monitor it." % (home_config['host'], model_name, created['id']))
<commit_msg>fixed gpu monitoring<commit_after>from __future__ import absolute_import
import argparse
import sys

from aetros import api

from aetros.utils import read_config, unpack_full_job_id, read_home_config

import aetros.const
import os

class StartCommand:
    def __init__(self, logger):
        self.logger = logger

    def main(self, args):
        from aetros.starter import start
        parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, prog=aetros.const.__prog__ + ' start')
        parser.add_argument('name', nargs='?', help='the model name, e.g. aetros/mnist-network to start new job, or job id, e.g. user/modelname/0db75a64acb74c27bd72c22e359de7a4c44a20e5 to start a pre-created job.')

        parser.add_argument('-i', '--image', help="Which Docker image to use for the command. Default read in aetros.yml. If not specified, command is executed on the host.")
        parser.add_argument('-l', '--local', action='store_true', help="Start the job immediately on the current machine.")
        parser.add_argument('-s', '--server', action='append', help="Limits the server pool to this server. Default not limitation or read in aetros.yml. Multiple --server allowed.")
        parser.add_argument('-b', '--branch', help="This overwrites the Git branch used when new job should be started.")
        parser.add_argument('--priority', help="Increases or decreases priority. Default is 0.")

        parser.add_argument('--cpu', help="How many CPU cores should be assigned to job. Docker only.")
        parser.add_argument('--memory', help="How much memory should be assigned to job. Docker only.")
        parser.add_argument('--gpu', help="How many GPU cards should be assigned to job. Docker only.")
        parser.add_argument('--gpu_memory', help="Memory requirement for the GPU. Docker only.")

        parser.add_argument('--rebuild-image', action='store_true', help="Makes sure the Docker image is re-built without cache.")

        parser.add_argument('--gpu-device', action='append', help="Which device id should be mapped into the NVIDIA docker container.")

        parser.add_argument('--max-time', help="Limit execution time in seconds. Sends SIGINT to the process group when reached.")
        parser.add_argument('--max-epochs', help="Limit execution epochs. Sends SIGINT to the process group when reached.")

        parser.add_argument('--insights', action='store_true', help="activates insights. Only for simple models.")
        parser.add_argument('--dataset', help="Dataset id when model has placeholders. Only for simple models with placeholders as input/output.")

        parser.add_argument('-p', '--param', action='append', help="Sets a hyperparameter, example '--param name=value'. Multiple --param allowed.")

        parsed_args = parser.parse_args(args)

        home_config = read_home_config()
        model_name = parsed_args.name

        if model_name.count('/') > 1:
            # start a concret job, by server command
            gpu_devices = []
            if parsed_args.gpu_device:
                gpu_devices = [int(x) for x in parsed_args.gpu_device]

            start(self.logger, model_name, cpus=int(parsed_args.cpu), memory=int(parsed_args.memory),
                gpu_devices=gpu_devices)
            return

        # create a new job

        hyperparameter = {}
        if parsed_args.param:
            for param in parsed_args.param:
                if '=' not in param:
                    raise Exception('--param ' + param + ' does not contain a `=`. Please use "--param name=value"')

                name, value = param.split('=')
                hyperparameter[name] = value

        job_config = {'insights': parsed_args.insights}

        if parsed_args.image:
            job_config['image'] = parsed_args.image

        if parsed_args.branch:
            job_config['sourceGitTree'] = parsed_args.branch

        if parsed_args.max_epochs:
            job_config['maxEpochs'] = int(parsed_args.max_epochs)

        if parsed_args.max_time:
            job_config['maxTime'] = float(parsed_args.max_time)

        job_config['priority'] = 0
        if parsed_args.priority:
            job_config['priority'] = float(parsed_args.priority)

        if parsed_args.rebuild_image:
            job_config['config']['rebuild_image'] = True

        if 'resources' not in job_config:
            job_config['resources'] = {}

        if parsed_args.server:
            job_config['servers'] = []
            for name in parsed_args.server:
                job_config['servers'].append(name)

        job_config['resources'] = job_config.get('resources', {})
        resources = job_config['resources']
        resources['cpu'] = int(parsed_args.cpu or resources.get('cpu', 1))
        resources['memory'] = int(parsed_args.memory or resources.get('memory', 1))
        resources['gpu'] = int(parsed_args.gpu or resources.get('gpu', 0))
        resources['gpu_memory'] = int(parsed_args.gpu_memory or resources.get('gpu_memory', 0))

        try:
            self.logger.debug("Create job ...")
            created = api.create_job(model_name, parsed_args.local, hyperparameter, parsed_args.dataset, config=job_config)
        except api.ApiError as e:
            if 'Connection refused' in e.error:
                self.logger.error("You are offline")

            raise

        print("Job %s/%s created." % (model_name, created['id']))

        if parsed_args.local:
            cpus = job_config['resources']['cpu']
            memory = job_config['resources']['memory']

            if not parsed_args.gpu_device and job_config['resources']['gpu'] > 0:
                # if requested 2 GPUs and we have 3 GPUs with id [0,1,2], gpus should be [0,1]
                parsed_args.gpu_device = []
                for i in range(0, job_config['resources']['gpu']):
                    parsed_args.gpu_device.append(i)

            start(self.logger, model_name + '/' + created['id'], cpus=cpus, memory=memory, gpu_devices=parsed_args.gpu_device)
        else:
            print("Open http://%s/model/%s/job/%s to monitor it." % (home_config['host'], model_name, created['id']))
<|endoftext|>


<commit_before># Base Compiler Class

import os
import time
import subprocess

from glob import glob
from multiprocessing import Process, Queue

from buildit.database import Database

from buildit.utils import which, flatten, fix_strings, file_hash
from buildit.utils import format_options
from buildit.cprint import command as print_command

class Compiler(object):

    def __init__(self, project_name='PROJECT'):
        self._compile_steps = []
        self._type = 'binary'
        self._output_extension = '' # We normally won't have one.
        self._project_name = project_name
        self._compile_flags = ''
        self._link_flags = ''
        self._executable = which('echo')
        self._file_list = []
        self._link_list = []
        self._dependency_folder_list = []
        self._dependency_file_list = []
        self.user_extensions = []
        self.user_dependencies = []

        self._compile_steps.append(self.setup_files)
        self._compile_steps.append(self.compile_files)
        self._compile_steps.append(self.link_files)
        self._compile_steps.append(self.write_deps)

    def __str__(self):
        return 'Compiler'

    @property
    def run(self):
        self.database = Database(self._project_name)
        for function in self._compile_steps:
            return_value = function()
            if not return_value == 0:
                return return_value
        return 0

    def setup_files(self):
        return 0

    # Leave the implementation up to each compiler
    def compile_files(self):
        return 0

    def link_files(self):
        return 0

    def parse_deps(self):
        # We want this to run first
        self._compile_steps.insert(0, self.dependency_tracking)
        return 0

    def dependency_tracking(self):
        for folder in self._dependency_folder_list:
            if not os.path.isdir(folder):
                continue
            glob_list = []
            for root, dir, file_names in os.walk(folder):
                for extension in flatten(self.dependency_extensions):
                    glob_list += glob('{0}/*{1}'.format(root, extension))
            glob_list = fix_strings(glob_list)
            for file_name in glob_list:
                self._dependency_file_list.append(file_name)
        self._dependency_file_list.sort()
        return 0

    # TODO: Write the dependency list to disk
    # Just an FYI I hate this code, and there must be a
    # better way to handle it :L
    def write_deps(self):
        run_list = []
        run_queue = Queue(10)
        for file_name in self._file_list:
            file_parser = Process(target=self.read_deps, args=(file_name))
            file_parser.daemon = True
            run_list.append(file_parser)
        for file_parser in run_list:
            # Suspend operations until we can put more onto the stack :)
            while running_queue.full():
                time.sleep(0.1)
            file_parser.start()
            running_queue.put(file_parser)
        return 0

    # Implemented by each compiler/language.
    # (Is used by multiprocessing, however)
    def read_deps(self, file_name):
        pass

    def _percentage(self, counter, list_length):
        percentage = 100 * float(counter)/float(list_length)
        percentage = str(percentage).split('.')
        percentage = percentage.pop(0)
        return percentage

    def command(self, percentage, file_name):
        print_command('[{0:>3}%] {1}: {2}'.format(percentage,
            self.name.upper(), file_name))

    def add_flags(self, *flags):
        flags = flatten(list(flags))
        for flag in flags:
            self._compile_flags += format_options(flag)

    def add_compile_flags(self, *flags):
        flags = flatten(list(flags))
        for flag in flags:
            self._compile_flags += format_options(flag)

    def add_link_flags(self, *flags):
        flags = flatten(list(flags))
        for flag in flags:
            self._link_flags += format_options(flag)

    def add_dependency_folder(self, *folders):
        folders = flatten(list(folders))
        for folder in folders:
            self._dependency_folder_list.append(folder)

    def add_file_extension(self, *extensions):
        extensions = flatten(list(extensions))
        for extension in extensions:
            extension = extension.split('.')
            if len(extension) > 1:
                extension = '.'.join(extension)
            else:
                extension = extension.pop()
            self.user_extensions.append('.{0}'.format(extension))

    def add_dependency_extension(self, *extensions):
        extensions = flatten(list(extensions))
        for extension in extensions:
            extension = extension.split('.')
            if len(extension) > 1:
                extension = '.'.join(extension)
            else:
                extension = extension.pop()
            self.user_dependencies.append('.{0}'.format(extension))

    @property
    def output_extension(self):
        return self._output_extension

    @output_extension.setter
    def output_extension(self, value):
        self._output_extension = value

    @property
    def type(self):
        return self._type

    @type.setter
    def type(self, value):
        self._type = value

    @property
    def language(self):
        return self._language

    @property
    def executable(self):
        return self._executable

    @executable.setter
    def executable(self, value):
        if isinstance(value, basestring):
            self._executable = which(value)

    @property
    def build_directory(self):
        return self._build_directory

    @build_directory.setter
    def build_directory(self, value):
        self._build_directory = value

    @property
    def object_directory(self):
        return self._object_directory

    @object_directory.setter
    def object_directory(self, value):
        self._object_directory = value

    @property
    def extensions(self):
        return ['.txt'] + self.user_extensions

    @property
    def dependency_extensions(self):
        return ['.txt'] + self.user_dependencies

    @property
    def name(self):
        return self.__str__()
<commit_msg>Fixed a small syntax error sorry!<commit_after># Base Compiler Class

import os
import time
import subprocess

from glob import glob
from multiprocessing import Process, Queue

from buildit.database import Database

from buildit.utils import which, flatten, fix_strings, file_hash
from buildit.utils import format_options
from buildit.cprint import command as print_command

class Compiler(object):

    def __init__(self, project_name='PROJECT'):
        self._compile_steps = []
        self._type = 'binary'
        self._output_extension = '' # We normally won't have one.
        self._project_name = project_name
        self._compile_flags = ''
        self._link_flags = ''
        self._executable = which('echo')
        self._file_list = []
        self._link_list = []
        self._dependency_folder_list = []
        self._dependency_file_list = []
        self.user_extensions = []
        self.user_dependencies = []

        self._compile_steps.append(self.setup_files)
        self._compile_steps.append(self.compile_files)
        self._compile_steps.append(self.link_files)
        self._compile_steps.append(self.write_deps)

    def __str__(self):
        return 'Compiler'

    @property
    def run(self):
        self.database = Database(self._project_name)
        for function in self._compile_steps:
            return_value = function()
            if not return_value == 0:
                return return_value
        return 0

    def setup_files(self):
        return 0

    # Leave the implementation up to each compiler
    def compile_files(self):
        return 0

    def link_files(self):
        return 0

    def parse_deps(self):
        # We want this to run first
        self._compile_steps.insert(0, self.dependency_tracking)
        return 0

    def dependency_tracking(self):
        for folder in self._dependency_folder_list:
            if not os.path.isdir(folder):
                continue
            glob_list = []
            for root, dir, file_names in os.walk(folder):
                for extension in flatten(self.dependency_extensions):
                    glob_list += glob('{0}/*{1}'.format(root, extension))
            glob_list = fix_strings(glob_list)
            for file_name in glob_list:
                self._dependency_file_list.append(file_name)
        self._dependency_file_list.sort()
        return 0

    # TODO: Write the dependency list to disk
    # Just an FYI I hate this code, and there must be a
    # better way to handle it :L
    def write_deps(self):
        run_list = []
        run_queue = Queue(10)
        for file_name in self._file_list:
            file_parser = Process(target=self.read_deps, args=(file_name))
            file_parser.daemon = True
            run_list.append(file_parser)
        for file_parser in run_list:
            # Suspend operations until we can put more onto the stack :)
            while run_queue.full():
                time.sleep(0.1)
            file_parser.start()
            run_queue.put(file_parser)
        return 0

    # Implemented by each compiler/language.
    # (Is used by multiprocessing, however)
    def read_deps(self, file_name):
        pass

    def _percentage(self, counter, list_length):
        percentage = 100 * float(counter)/float(list_length)
        percentage = str(percentage).split('.')
        percentage = percentage.pop(0)
        return percentage

    def command(self, percentage, file_name):
        print_command('[{0:>3}%] {1}: {2}'.format(percentage,
            self.name.upper(), file_name))

    def add_flags(self, *flags):
        flags = flatten(list(flags))
        for flag in flags:
            self._compile_flags += format_options(flag)

    def add_compile_flags(self, *flags):
        flags = flatten(list(flags))
        for flag in flags:
            self._compile_flags += format_options(flag)

    def add_link_flags(self, *flags):
        flags = flatten(list(flags))
        for flag in flags:
            self._link_flags += format_options(flag)

    def add_dependency_folder(self, *folders):
        folders = flatten(list(folders))
        for folder in folders:
            self._dependency_folder_list.append(folder)

    def add_file_extension(self, *extensions):
        extensions = flatten(list(extensions))
        for extension in extensions:
            extension = extension.split('.')
            if len(extension) > 1:
                extension = '.'.join(extension)
            else:
                extension = extension.pop()
            self.user_extensions.append('.{0}'.format(extension))

    def add_dependency_extension(self, *extensions):
        extensions = flatten(list(extensions))
        for extension in extensions:
            extension = extension.split('.')
            if len(extension) > 1:
                extension = '.'.join(extension)
            else:
                extension = extension.pop()
            self.user_dependencies.append('.{0}'.format(extension))

    @property
    def output_extension(self):
        return self._output_extension

    @output_extension.setter
    def output_extension(self, value):
        self._output_extension = value

    @property
    def type(self):
        return self._type

    @type.setter
    def type(self, value):
        self._type = value

    @property
    def language(self):
        return self._language

    @property
    def executable(self):
        return self._executable

    @executable.setter
    def executable(self, value):
        if isinstance(value, basestring):
            self._executable = which(value)

    @property
    def build_directory(self):
        return self._build_directory

    @build_directory.setter
    def build_directory(self, value):
        self._build_directory = value

    @property
    def object_directory(self):
        return self._object_directory

    @object_directory.setter
    def object_directory(self, value):
        self._object_directory = value

    @property
    def extensions(self):
        return ['.txt'] + self.user_extensions

    @property
    def dependency_extensions(self):
        return ['.txt'] + self.user_dependencies

    @property
    def name(self):
        return self.__str__()
<|endoftext|>


<commit_before><commit_msg>python script V2<commit_after><|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.db import transaction
from django.template.defaultfilters import slugify

from rest_framework import serializers
from rest_framework import relations

from elastimorphic.serializers import ContentTypeField, PolymorphicSerializerMixin

from .models import Content, Tag, LogEntry, FeatureType, ObfuscatedUrlInfo

class ImageFieldSerializer(serializers.WritableField):
    """
    serializer field type for images
    """

    def __init__(self, caption_field=None, alt_field=None, **kwargs):
        """instantiates object

        :param caption_field: caption
        :param alt_field: alt value
        :param kwargs: keyword arguments (optional)
        :return: `serializers.WriteableField`
        """
        super(ImageFieldSerializer, self).__init__(**kwargs)
        self.caption_field = caption_field
        self.alt_field = alt_field

    def to_native(self, obj):
        if obj is None or obj.id is None:
            return None
        data = {
            "id": obj.id,
        }
        if self.caption_field:
            data["alt"] = obj.alt
            data["caption"] = obj.caption
        return data

    def from_native(self, data):
        if data is None:
            return None
        image_id = data.get("id")
        # Just in case a string gets passed in
        if image_id is not None:
            return int(image_id)
        return None

    def field_from_native(self, data, files, field_name, into):
        super(ImageFieldSerializer, self).field_from_native(data, files, field_name, into)
        image_data = data.get(field_name, {})
        if image_data is None:
            return

        if self.alt_field and "alt" in image_data:
            into[self.alt_field] = image_data["alt"]

        if self.caption_field and "caption" in image_data:
            into[self.caption_field] = image_data["caption"]

class TagSerializer(serializers.ModelSerializer):
    name = serializers.CharField(max_length=50)  # this is actually limited by the slug

    class Meta:
        model = Tag

    def to_native(self, obj):
        return {
            "id": obj.pk,
            "name": obj.name,
            "slug": obj.slug,
            "type": obj.get_mapping_type_name()
        }

class FeatureTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = FeatureType

    def to_native(self, obj):
        return {
            "id": obj.pk,
            "name": obj.name,
            "slug": obj.slug,
        }

class TagField(relations.RelatedField):
    """This is a relational field that handles the addition of tags to content
    objects. This field also allows the user to create tags in the db if they
    don't already exist."""

    read_only = False

    def to_native(self, obj):
        return {
            "id": obj.pk,
            "name": obj.name,
            "slug": obj.slug,
            "type": obj.get_mapping_type_name()
        }

    def from_native(self, value):
        """Basically, each tag dict must include a full dict with id,
        name and slug--or else you need to pass in a dict with just a name,
        which indicated that the Tag doesn't exist, and should be added."""

        if "id" in value:
            tag = Tag.objects.get(id=value["id"])
        else:
            if "name" not in value:
                raise ValidationError("Tags must include an ID or a name.")
            if len(slugify(value["name"])) > 50:
                raise ValidationError("Maximum tag length is 50 characters.")

            name = value["name"]
            slug = value.get("slug", slugify(name))
            try:
                tag = Tag.objects.get(slug=slug)
            except Tag.DoesNotExist:
                tag, created = Tag.objects.get_or_create(name=name, slug=slug)
        return tag

class FeatureTypeField(relations.RelatedField):
    """This is a relational field that handles the addition of feature_types to
    content objects. This field also allows the user to create feature_types in
    the db if they don't already exist."""

    read_only = False

    def to_native(self, obj):
        return obj.name

    def from_native(self, value):
        """Basically, each tag dict must include a full dict with id,
        name and slug--or else you need to pass in a dict with just a name,
        which indicated that the Tag doesn't exist, and should be added."""
        if value == "":
            return None

        slug = slugify(value)
        feature_type, created = FeatureType.objects.get_or_create(
            slug=slug,
            defaults={"name": value}
        )
        return feature_type

class DefaultUserSerializer(serializers.ModelSerializer):
    """Returns basic User fields"""

    class Meta:
        model = get_user_model()
        # model = settings.AUTH_USER_MODEL

    def to_native(self, obj):

        json = {
            "id": obj.pk,
            "username": obj.username,
            "email": obj.email,
            "first_name": obj.first_name,
            "last_name": obj.last_name,
            "full_name": obj.get_full_name(),
            "short_name": obj.get_short_name()
        }

        return json

    def from_native(self, data, files):
        """Basically, each author dict must include either a username or id."""
        # model = get_user_model()
        model = self.Meta.model

        if "id" in data:
            author = model.objects.get(id=data["id"])
        else:
            if "username" not in data:
                raise ValidationError("Authors must include an ID or a username.")
            username = data["username"]
            author = model.objects.get(username=username)
        return author

UserSerializer = getattr(settings, "BULBS_USER_SERIALIZER", DefaultUserSerializer)

class ContentSerializer(serializers.ModelSerializer):

    polymorphic_ctype = ContentTypeField(source="polymorphic_ctype_id", read_only=True)
    tags = TagField(many=True)
    feature_type = FeatureTypeField(required=False)
    authors = UserSerializer(many=True, required=False, allow_add_remove=True, read_only=False)
    thumbnail = ImageFieldSerializer(required=False, read_only=True)
    first_image = ImageFieldSerializer(required=False, read_only=True)
    thumbnail_override = ImageFieldSerializer(required=False)
    absolute_url = serializers.Field(source="get_absolute_url")
    status = serializers.Field(source="get_status")

    class Meta:
        model = Content

    @transaction.commit_on_success
    def save(self, *args, **kwargs):
        if not "index" in kwargs:
            kwargs["index"] = False
        return super(ContentSerializer, self).save(*args, **kwargs)

class LogEntrySerializer(serializers.ModelSerializer):

    class Meta:
        model = LogEntry

class PolymorphicContentSerializer(PolymorphicSerializerMixin, ContentSerializer):
    pass

class ObfuscatedUrlInfoSerializer(serializers.ModelSerializer):

    expire_date = serializers.DateTimeField()
    create_date = serializers.DateTimeField()
    url_uuid = serializers.CharField(min_length=32, max_length=32)

    def validate(self, attrs):
        if attrs["expire_date"] < attrs["create_date"]:
            raise serializers.ValidationError(
                "Start date must occur before expiration date.")

    class Meta:
        model = ObfuscatedUrlInfo
<commit_msg>Replaced transaction.commit_on_success with transaction.atomic (Django>=1.6)<commit_after>from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.db import transaction
from django.template.defaultfilters import slugify

from rest_framework import serializers
from rest_framework import relations

from elastimorphic.serializers import ContentTypeField, PolymorphicSerializerMixin

from .models import Content, Tag, LogEntry, FeatureType, ObfuscatedUrlInfo

class ImageFieldSerializer(serializers.WritableField):
    """
    serializer field type for images
    """

    def __init__(self, caption_field=None, alt_field=None, **kwargs):
        """instantiates object

        :param caption_field: caption
        :param alt_field: alt value
        :param kwargs: keyword arguments (optional)
        :return: `serializers.WriteableField`
        """
        super(ImageFieldSerializer, self).__init__(**kwargs)
        self.caption_field = caption_field
        self.alt_field = alt_field

    def to_native(self, obj):
        if obj is None or obj.id is None:
            return None
        data = {
            "id": obj.id,
        }
        if self.caption_field:
            data["alt"] = obj.alt
            data["caption"] = obj.caption
        return data

    def from_native(self, data):
        if data is None:
            return None
        image_id = data.get("id")
        # Just in case a string gets passed in
        if image_id is not None:
            return int(image_id)
        return None

    def field_from_native(self, data, files, field_name, into):
        super(ImageFieldSerializer, self).field_from_native(data, files, field_name, into)
        image_data = data.get(field_name, {})
        if image_data is None:
            return

        if self.alt_field and "alt" in image_data:
            into[self.alt_field] = image_data["alt"]

        if self.caption_field and "caption" in image_data:
            into[self.caption_field] = image_data["caption"]

class TagSerializer(serializers.ModelSerializer):
    name = serializers.CharField(max_length=50)  # this is actually limited by the slug

    class Meta:
        model = Tag

    def to_native(self, obj):
        return {
            "id": obj.pk,
            "name": obj.name,
            "slug": obj.slug,
            "type": obj.get_mapping_type_name()
        }

class FeatureTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = FeatureType

    def to_native(self, obj):
        return {
            "id": obj.pk,
            "name": obj.name,
            "slug": obj.slug,
        }

class TagField(relations.RelatedField):
    """This is a relational field that handles the addition of tags to content
    objects. This field also allows the user to create tags in the db if they
    don't already exist."""

    read_only = False

    def to_native(self, obj):
        return {
            "id": obj.pk,
            "name": obj.name,
            "slug": obj.slug,
            "type": obj.get_mapping_type_name()
        }

    def from_native(self, value):
        """Basically, each tag dict must include a full dict with id,
        name and slug--or else you need to pass in a dict with just a name,
        which indicated that the Tag doesn't exist, and should be added."""

        if "id" in value:
            tag = Tag.objects.get(id=value["id"])
        else:
            if "name" not in value:
                raise ValidationError("Tags must include an ID or a name.")
            if len(slugify(value["name"])) > 50:
                raise ValidationError("Maximum tag length is 50 characters.")

            name = value["name"]
            slug = value.get("slug", slugify(name))
            try:
                tag = Tag.objects.get(slug=slug)
            except Tag.DoesNotExist:
                tag, created = Tag.objects.get_or_create(name=name, slug=slug)
        return tag

class FeatureTypeField(relations.RelatedField):
    """This is a relational field that handles the addition of feature_types to
    content objects. This field also allows the user to create feature_types in
    the db if they don't already exist."""

    read_only = False

    def to_native(self, obj):
        return obj.name

    def from_native(self, value):
        """Basically, each tag dict must include a full dict with id,
        name and slug--or else you need to pass in a dict with just a name,
        which indicated that the Tag doesn't exist, and should be added."""
        if value == "":
            return None

        slug = slugify(value)
        feature_type, created = FeatureType.objects.get_or_create(
            slug=slug,
            defaults={"name": value}
        )
        return feature_type

class DefaultUserSerializer(serializers.ModelSerializer):
    """Returns basic User fields"""

    class Meta:
        model = get_user_model()
        # model = settings.AUTH_USER_MODEL

    def to_native(self, obj):

        json = {
            "id": obj.pk,
            "username": obj.username,
            "email": obj.email,
            "first_name": obj.first_name,
            "last_name": obj.last_name,
            "full_name": obj.get_full_name(),
            "short_name": obj.get_short_name()
        }

        return json

    def from_native(self, data, files):
        """Basically, each author dict must include either a username or id."""
        # model = get_user_model()
        model = self.Meta.model

        if "id" in data:
            author = model.objects.get(id=data["id"])
        else:
            if "username" not in data:
                raise ValidationError("Authors must include an ID or a username.")
            username = data["username"]
            author = model.objects.get(username=username)
        return author

UserSerializer = getattr(settings, "BULBS_USER_SERIALIZER", DefaultUserSerializer)

class ContentSerializer(serializers.ModelSerializer):

    polymorphic_ctype = ContentTypeField(source="polymorphic_ctype_id", read_only=True)
    tags = TagField(many=True)
    feature_type = FeatureTypeField(required=False)
    authors = UserSerializer(many=True, required=False, allow_add_remove=True, read_only=False)
    thumbnail = ImageFieldSerializer(required=False, read_only=True)
    first_image = ImageFieldSerializer(required=False, read_only=True)
    thumbnail_override = ImageFieldSerializer(required=False)
    absolute_url = serializers.Field(source="get_absolute_url")
    status = serializers.Field(source="get_status")

    class Meta:
        model = Content

    @transaction.atomic
    def save(self, *args, **kwargs):
        if not "index" in kwargs:
            kwargs["index"] = False
        return super(ContentSerializer, self).save(*args, **kwargs)

class LogEntrySerializer(serializers.ModelSerializer):

    class Meta:
        model = LogEntry

class PolymorphicContentSerializer(PolymorphicSerializerMixin, ContentSerializer):
    pass

class ObfuscatedUrlInfoSerializer(serializers.ModelSerializer):

    expire_date = serializers.DateTimeField()
    create_date = serializers.DateTimeField()
    url_uuid = serializers.CharField(min_length=32, max_length=32)

    def validate(self, attrs):
        if attrs["expire_date"] < attrs["create_date"]:
            raise serializers.ValidationError(
                "Start date must occur before expiration date.")

    class Meta:
        model = ObfuscatedUrlInfo
<|endoftext|>


<commit_before># pylint: disable=C0111,R0903

"""Displays the current song being played

Requires the following library:
    * python-dbus

Parameters:
"""

import bumblebee.input
import bumblebee.output
import bumblebee.engine

try:
    import dbus
except ImportError:
    pass

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        super(Module, self).__init__(engine, config,
                                     bumblebee.output.Widget(full_text=self.spotify)
                                     )
        self._song = ""

        cmd="dbus-send --session --type=method_call --dest=org.mpris.MediaPlayer2.spotify \
                /org/mpris/MediaPlayer2 org.mpris.MediaPlayer2.Player."
        engine.input.register_callback(self, button=bumblebee.input.LEFT_MOUSE,
            cmd=cmd + "Previous")
        engine.input.register_callback(self, button=bumblebee.input.RIGHT_MOUSE,
            cmd=cmd + "Next")
        engine.input.register_callback(self, button=bumblebee.input.MIDDLE_MOUSE,
            cmd=cmd + "PlayPause")

    def spotify(self, widget):
        return str(self._song)

    def update(self, widgets):
        try:
            bus = dbus.SessionBus()
            spotify = bus.get_object("org.mpris.MediaPlayer2.spotify", "/org/mpris/MediaPlayer2")
            spotify_iface = dbus.Interface(spotify, 'org.freedesktop.DBus.Properties')
            props = spotify_iface.Get('org.mpris.MediaPlayer2.Player', 'Metadata')
            self._song = (str(props['xesam:artist'][0]) + " - " + str(props['xesam:title']))
        except Exception:
            self._song = ""

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>[module/spotify] Add new format string parameter<commit_after># pylint: disable=C0111,R0903

"""Displays the current song being played

Requires the following library:
    * python-dbus

Parameters:
    * spotify.format: Format string (defaults to "{artist} - {title}")
                      Available values are: {album}, {title}, {artist}, {trackNumber}
"""

import bumblebee.input
import bumblebee.output
import bumblebee.engine

try:
    import dbus
except ImportError:
    pass

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        super(Module, self).__init__(engine, config,
                                     bumblebee.output.Widget(full_text=self.spotify)
                                     )
        self._song = ""
        self._format = self.parameter("format", "{artist} - {title}")

        cmd="dbus-send --session --type=method_call --dest=org.mpris.MediaPlayer2.spotify \
                /org/mpris/MediaPlayer2 org.mpris.MediaPlayer2.Player."
        engine.input.register_callback(self, button=bumblebee.input.LEFT_MOUSE,
            cmd=cmd + "Previous")
        engine.input.register_callback(self, button=bumblebee.input.RIGHT_MOUSE,
            cmd=cmd + "Next")
        engine.input.register_callback(self, button=bumblebee.input.MIDDLE_MOUSE,
            cmd=cmd + "PlayPause")

    def spotify(self, widget):
        return str(self._song)

    def update(self, widgets):
        try:
            bus = dbus.SessionBus()
            spotify = bus.get_object("org.mpris.MediaPlayer2.spotify", "/org/mpris/MediaPlayer2")
            spotify_iface = dbus.Interface(spotify, 'org.freedesktop.DBus.Properties')
            props = spotify_iface.Get('org.mpris.MediaPlayer2.Player', 'Metadata')
            self._song = self._format.format(album=str(props.get('xesam:album')),
                                             title=str(props.get('xesam:title')),
                                             artist=','.join(props.get('xesam:artist')),
                                             trackNumber=str(props.get('xesam:trackNumber')))
        except Exception:
            self._song = ""

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before># pylint: disable=C0111,R0903

"""Displays network IO for interfaces.

Parameters:
    * traffic.exclude: Comma-separated list of interface prefixes to exclude (defaults to "lo,virbr,docker,vboxnet,veth")
    * traffic.states: Comma-separated list of states to show (prefix with "^" to invert - i.e. ^down -> show all devices that are not in state down)
    * traffic.showname: If set to False, hide network interface name (defaults to True)
    * traffic.format: Format string for download/upload speeds.
                      Defaults to "{:.2f}"
    * traffic.graphlen: Graph lenth in seconds. Positive even integer. Each
                        char shows 2 seconds. If set, enables up/down traffic
                        graphs
"""

import re
import time
import psutil
import netifaces

import bumblebee.util
import bumblebee.input
import bumblebee.output
import bumblebee.engine

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        widgets = []
        super(Module, self).__init__(engine, config, widgets)
        self._exclude = tuple(filter(len, self.parameter("exclude", "lo,virbr,docker,vboxnet,veth").split(",")))
        self._status = ""

        self._showname = bumblebee.util.asbool(self.parameter("showname", True))
        self._format = self.parameter("format", "{:.2f}")
        self._prev = {}
        self._states = {}
        self._lastcheck = 0
        self._states["include"] = []
        self._states["exclude"] = []
        for state in tuple(filter(len, self.parameter("states", "").split(","))):
            if state[0] == "^":
                self._states["exclude"].append(state[1:])
            else:
                self._states["include"].append(state)
        self._graphlen = int(self.parameter("graphlen", 0))
        if self._graphlen > 0:
            self._graphdata = {}
        self._first_run = True
        self._update_widgets(widgets)

    def state(self, widget):
        if "traffic.rx" in widget.name:
            return "rx"
        if "traffic.tx" in widget.name:
            return "tx"
        return self._status

    def update(self, widgets):
        self._update_widgets(widgets)

    def create_widget(self, widgets, name, txt=None, attributes={}):
        widget = bumblebee.output.Widget(name=name)
        widget.full_text(txt)
        widgets.append(widget)

        for key in attributes:
            widget.set(key, attributes[key])

        return widget

    def get_addresses(self, intf):
        retval = []
        try:
            for ip in netifaces.ifaddresses(intf).get(netifaces.AF_INET, []):
                if ip.get("addr", "") != "":
                    retval.append(ip.get("addr"))
        except Exception:
            return []
        return retval

    def get_minwidth_str(self):
        """
            computes theme.minwidth string
            based on traffic.format and traffic.graphlen parameters
        """
        minwidth_str = ""
        if self._graphlen > 0:
            graph_len = int(self._graphlen / 2)
            graph_prefix = "0" * graph_len
            minwidth_str += graph_prefix
        minwidth_str += "1000"
        try:
            length = int(re.match("{:\.(\d+)f}", self._format).group(1))
            if length > 0:
                minwidth_str += "." + "0" * length
        except AttributeError:
            # return default value
            return "1000.00MB"
        finally:
            minwidth_str += "MB"
        return minwidth_str

    def _update_widgets(self, widgets):
        interfaces = [i for i in netifaces.interfaces() if not i.startswith(self._exclude)]

        del widgets[:]

        counters = psutil.net_io_counters(pernic=True)
        now = time.time()
        timediff = now - (self._lastcheck if self._lastcheck else now)
        if timediff <= 0: timediff = 1
        self._lastcheck = now
        for interface in interfaces:
            if self._graphlen > 0:
                import logging
                logging.info(self._graphdata)
                if interface not in self._graphdata:
                    self._graphdata[interface] = {
                        "rx": [0] * self._graphlen,
                        "tx": [0] * self._graphlen}
            if not interface: interface = "lo"
            state = "down"
            if len(self.get_addresses(interface)) > 0:
                state = "up"
            elif bumblebee.util.asbool(self.parameter("hide_down", True)):
                continue

            if len(self._states["exclude"]) > 0 and state in self._states["exclude"]: continue
            if len(self._states["include"]) > 0 and state not in self._states["include"]: continue

            data = {
                "rx": counters[interface].bytes_recv,
                "tx": counters[interface].bytes_sent,
            }

            name = "traffic-{}".format(interface)

            if self._showname:
               self.create_widget(widgets, name, interface)

            for direction in ["rx", "tx"]:
                name = "traffic.{}-{}".format(direction, interface)
                widget = self.create_widget(widgets, name, attributes={"theme.minwidth": self.get_minwidth_str()})
                prev = self._prev.get(name, 0)
                bspeed = (int(data[direction]) - int(prev))/timediff
                speed = bumblebee.util.bytefmt(bspeed, self._format)
                txtspeed = '{0}/s'.format(speed)
                if self._graphlen > 0:
                    # skip first value returned by psutil, because it is
                    # giant and ruins the grapth ratio until it gets pushed
                    # out of saved list
                    if self._first_run is True:
                        self._first_run = False
                    else:
                        self._graphdata[interface][direction] = self._graphdata[interface][direction][1:]
                        self._graphdata[interface][direction].append(bspeed)
                    txtspeed = "{}{}".format(bumblebee.output.bgraph(self._graphdata[interface][direction]), txtspeed)
                widget.full_text(txtspeed)
                self._prev[name] = data[direction]

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>[modules/traffic] remove forgotten debug logging<commit_after># pylint: disable=C0111,R0903

"""Displays network IO for interfaces.

Parameters:
    * traffic.exclude: Comma-separated list of interface prefixes to exclude (defaults to "lo,virbr,docker,vboxnet,veth")
    * traffic.states: Comma-separated list of states to show (prefix with "^" to invert - i.e. ^down -> show all devices that are not in state down)
    * traffic.showname: If set to False, hide network interface name (defaults to True)
    * traffic.format: Format string for download/upload speeds.
                      Defaults to "{:.2f}"
    * traffic.graphlen: Graph lenth in seconds. Positive even integer. Each
                        char shows 2 seconds. If set, enables up/down traffic
                        graphs
"""

import re
import time
import psutil
import netifaces

import bumblebee.util
import bumblebee.input
import bumblebee.output
import bumblebee.engine

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        widgets = []
        super(Module, self).__init__(engine, config, widgets)
        self._exclude = tuple(filter(len, self.parameter("exclude", "lo,virbr,docker,vboxnet,veth").split(",")))
        self._status = ""

        self._showname = bumblebee.util.asbool(self.parameter("showname", True))
        self._format = self.parameter("format", "{:.2f}")
        self._prev = {}
        self._states = {}
        self._lastcheck = 0
        self._states["include"] = []
        self._states["exclude"] = []
        for state in tuple(filter(len, self.parameter("states", "").split(","))):
            if state[0] == "^":
                self._states["exclude"].append(state[1:])
            else:
                self._states["include"].append(state)
        self._graphlen = int(self.parameter("graphlen", 0))
        if self._graphlen > 0:
            self._graphdata = {}
        self._first_run = True
        self._update_widgets(widgets)

    def state(self, widget):
        if "traffic.rx" in widget.name:
            return "rx"
        if "traffic.tx" in widget.name:
            return "tx"
        return self._status

    def update(self, widgets):
        self._update_widgets(widgets)

    def create_widget(self, widgets, name, txt=None, attributes={}):
        widget = bumblebee.output.Widget(name=name)
        widget.full_text(txt)
        widgets.append(widget)

        for key in attributes:
            widget.set(key, attributes[key])

        return widget

    def get_addresses(self, intf):
        retval = []
        try:
            for ip in netifaces.ifaddresses(intf).get(netifaces.AF_INET, []):
                if ip.get("addr", "") != "":
                    retval.append(ip.get("addr"))
        except Exception:
            return []
        return retval

    def get_minwidth_str(self):
        """
            computes theme.minwidth string
            based on traffic.format and traffic.graphlen parameters
        """
        minwidth_str = ""
        if self._graphlen > 0:
            graph_len = int(self._graphlen / 2)
            graph_prefix = "0" * graph_len
            minwidth_str += graph_prefix
        minwidth_str += "1000"
        try:
            length = int(re.match("{:\.(\d+)f}", self._format).group(1))
            if length > 0:
                minwidth_str += "." + "0" * length
        except AttributeError:
            # return default value
            return "1000.00MB"
        finally:
            minwidth_str += "MB"
        return minwidth_str

    def _update_widgets(self, widgets):
        interfaces = [i for i in netifaces.interfaces() if not i.startswith(self._exclude)]

        del widgets[:]

        counters = psutil.net_io_counters(pernic=True)
        now = time.time()
        timediff = now - (self._lastcheck if self._lastcheck else now)
        if timediff <= 0: timediff = 1
        self._lastcheck = now
        for interface in interfaces:
            if self._graphlen > 0:
                if interface not in self._graphdata:
                    self._graphdata[interface] = {
                        "rx": [0] * self._graphlen,
                        "tx": [0] * self._graphlen}
            if not interface: interface = "lo"
            state = "down"
            if len(self.get_addresses(interface)) > 0:
                state = "up"
            elif bumblebee.util.asbool(self.parameter("hide_down", True)):
                continue

            if len(self._states["exclude"]) > 0 and state in self._states["exclude"]: continue
            if len(self._states["include"]) > 0 and state not in self._states["include"]: continue

            data = {
                "rx": counters[interface].bytes_recv,
                "tx": counters[interface].bytes_sent,
            }

            name = "traffic-{}".format(interface)

            if self._showname:
               self.create_widget(widgets, name, interface)

            for direction in ["rx", "tx"]:
                name = "traffic.{}-{}".format(direction, interface)
                widget = self.create_widget(widgets, name, attributes={"theme.minwidth": self.get_minwidth_str()})
                prev = self._prev.get(name, 0)
                bspeed = (int(data[direction]) - int(prev))/timediff
                speed = bumblebee.util.bytefmt(bspeed, self._format)
                txtspeed = '{0}/s'.format(speed)
                if self._graphlen > 0:
                    # skip first value returned by psutil, because it is
                    # giant and ruins the grapth ratio until it gets pushed
                    # out of saved list
                    if self._first_run is True:
                        self._first_run = False
                    else:
                        self._graphdata[interface][direction] = self._graphdata[interface][direction][1:]
                        self._graphdata[interface][direction].append(bspeed)
                    txtspeed = "{}{}".format(bumblebee.output.bgraph(self._graphdata[interface][direction]), txtspeed)
                widget.full_text(txtspeed)
                self._prev[name] = data[direction]

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Functions related to plotting maps. """
from collections import namedtuple

try:
    import cartopy.crs as ccrs
    from cartopy.feature import NaturalEarthFeature, COLORS
except ImportError:
    raise ImportError('You have to install `cartopy` to use functions '
                      'located in `typhon.plots.maps`.')
else:
    from matplotlib import pyplot as plt

__all__ = [
    'worldmap',
    'get_cfeatures_at_scale',
]

def worldmap(lat, lon, var=None, fig=None, ax=None, projection=None,
             bg=False, **kwargs):
    """Plots the track of a variable on a worldmap.

    Args:
        lat: Array of latitudes.
        lon: Array of longitudes.
        var: Additional array for the variable to plot. If given,
            the track changes the color according to a color map.
        fig: A matplotlib figure object. If not given, the current
            figure is used.
        ax: A matplotlib axis object. If not given, a new axis
            object will be created in the current figure.
        projection: If no axis is given, specify here the cartopy projection.
        bg: If true, a background image will be drawn.
        **kwargs:

    Returns:
        Axis and scatter plot objects.
    """
    # Default keyword arguments to pass to hist2d().
    kwargs_defaults = {
        "cmap": "qualitative1",
        "s": 1,
    }
    kwargs_defaults.update(kwargs)

    if fig is None:
        fig = plt.gcf()

    if projection is None:
        if ax is None:

            projection = ccrs.PlateCarree()
        else:
            projection = ax.projection

    if ax is None:
        ax = fig.add_subplot(111, projection=projection)

    if bg:
        ax.stock_img()

    # It is counter-intuitive but if we want to plot our data with normal
    # latitudes and longitudes, we always have to set the transform to
    # PlateCarree (see https://github.com/SciTools/cartopy/issues/911)
    if var is None:
        scatter_plot = ax.scatter(
            lon, lat, transform=ccrs.PlateCarree(), **kwargs_defaults)
    else:
        scatter_plot = ax.scatter(
            lon, lat, c=var, transform=ccrs.PlateCarree(), **kwargs_defaults)
        ax.colorbar(scatter_plot)

    return ax, scatter_plot

def get_cfeatures_at_scale(scale='110m'):
    """Return a collection of `NaturalEarthFeature` at given scale.

    Parameters:
        scale (str): The dataset scale, i.e. one of ‘10m’, ‘50m’,
            or ‘110m’. Corresponding to 1:10,000,000, 1:50,000,000,
            and 1:110,000,000 respectively.

    Returns:
        collections.namedtuple:
            Collection of :class:`~cartopy.feature.NaturalEarthFeature`

    Examples:
        >>> features = get_cfeatures_at_scale('50m')
        >>> print(features.COASTLINE.scale)
        '50m'

    """
    d = {}

    d['BORDERS'] = NaturalEarthFeature(
        category='cultural',
        name='admin_0_boundary_lines_land',
        scale=scale,
        edgecolor='black',
        facecolor='none',
    )

    d['STATES'] = NaturalEarthFeature(
        category='cultural',
        name='admin_1_states_provinces_lakes',
        scale=scale,
        edgecolor='black',
        facecolor='none',
    )

    d['COASTLINE'] = NaturalEarthFeature(
        category='physical',
        name='coastline',
        scale=scale,
        edgecolor='black',
        facecolor='none',
    )

    d['LAKES'] = NaturalEarthFeature(
        category='physical',
        name='lakes',
        scale=scale,
        edgecolor='face',
        facecolor=COLORS['water'],
    )

    d['LAND'] = NaturalEarthFeature(
        category='physical',
        name='land',
        scale=scale,
        edgecolor='face',
        facecolor=COLORS['land'],
        zorder=-1,
    )

    d['OCEAN'] = NaturalEarthFeature(
        category='physical',
        name='ocean',
        scale=scale,
        edgecolor='face',
        facecolor=COLORS['water'],
        zorder=-1,
    )

    d['RIVERS'] = NaturalEarthFeature(
        category='physical',
        name='rivers_lake_centerlines',
        scale=scale,
        edgecolor=COLORS['water'],
        facecolor='none',
    )

    NaturalEarthFeatures = namedtuple(
        typename='NaturalEarthFeatures',
        field_names=(
            'BORDERS',
            'STATES',
            'COASTLINE',
            'LAKES',
            'LAND',
            'OCEAN',
            'RIVERS',
        ),
    )

    return NaturalEarthFeatures(**d)
<commit_msg>Fix bug in worldmap<commit_after># -*- coding: utf-8 -*-
"""Functions related to plotting maps. """
from collections import namedtuple

try:
    import cartopy.crs as ccrs
    from cartopy.feature import NaturalEarthFeature, COLORS
except ImportError:
    raise ImportError('You have to install `cartopy` to use functions '
                      'located in `typhon.plots.maps`.')
else:
    from matplotlib import pyplot as plt

__all__ = [
    'worldmap',
    'get_cfeatures_at_scale',
]

def worldmap(lat, lon, var=None, fig=None, ax=None, projection=None,
             bg=False, **kwargs):
    """Plots the track of a variable on a worldmap.

    Args:
        lat: Array of latitudes.
        lon: Array of longitudes.
        var: Additional array for the variable to plot. If given,
            the track changes the color according to a color map.
        fig: A matplotlib figure object. If not given, the current
            figure is used.
        ax: A matplotlib axis object. If not given, a new axis
            object will be created in the current figure.
        projection: If no axis is given, specify here the cartopy projection.
        bg: If true, a background image will be drawn.
        **kwargs:

    Returns:
        Scatter plot objects.
    """
    # Default keyword arguments to pass to hist2d().
    kwargs_defaults = {
        "cmap": "qualitative1",
        "s": 1,
    }
    kwargs_defaults.update(kwargs)

    if fig is None:
        fig = plt.gcf()

    if projection is None:
        if ax is None:

            projection = ccrs.PlateCarree()
        else:
            projection = ax.projection

    if ax is None:
        ax = fig.add_subplot(111, projection=projection)

    if bg:
        ax.stock_img()

    # It is counter-intuitive but if we want to plot our data with normal
    # latitudes and longitudes, we always have to set the transform to
    # PlateCarree (see https://github.com/SciTools/cartopy/issues/911)
    if var is None:
        scatter_plot = ax.scatter(
            lon, lat, transform=ccrs.PlateCarree(), **kwargs_defaults)
    else:
        scatter_plot = ax.scatter(
            lon, lat, c=var, transform=ccrs.PlateCarree(), **kwargs_defaults)

    return scatter_plot

def get_cfeatures_at_scale(scale='110m'):
    """Return a collection of `NaturalEarthFeature` at given scale.

    Parameters:
        scale (str): The dataset scale, i.e. one of ‘10m’, ‘50m’,
            or ‘110m’. Corresponding to 1:10,000,000, 1:50,000,000,
            and 1:110,000,000 respectively.

    Returns:
        collections.namedtuple:
            Collection of :class:`~cartopy.feature.NaturalEarthFeature`

    Examples:
        >>> features = get_cfeatures_at_scale('50m')
        >>> print(features.COASTLINE.scale)
        '50m'

    """
    d = {}

    d['BORDERS'] = NaturalEarthFeature(
        category='cultural',
        name='admin_0_boundary_lines_land',
        scale=scale,
        edgecolor='black',
        facecolor='none',
    )

    d['STATES'] = NaturalEarthFeature(
        category='cultural',
        name='admin_1_states_provinces_lakes',
        scale=scale,
        edgecolor='black',
        facecolor='none',
    )

    d['COASTLINE'] = NaturalEarthFeature(
        category='physical',
        name='coastline',
        scale=scale,
        edgecolor='black',
        facecolor='none',
    )

    d['LAKES'] = NaturalEarthFeature(
        category='physical',
        name='lakes',
        scale=scale,
        edgecolor='face',
        facecolor=COLORS['water'],
    )

    d['LAND'] = NaturalEarthFeature(
        category='physical',
        name='land',
        scale=scale,
        edgecolor='face',
        facecolor=COLORS['land'],
        zorder=-1,
    )

    d['OCEAN'] = NaturalEarthFeature(
        category='physical',
        name='ocean',
        scale=scale,
        edgecolor='face',
        facecolor=COLORS['water'],
        zorder=-1,
    )

    d['RIVERS'] = NaturalEarthFeature(
        category='physical',
        name='rivers_lake_centerlines',
        scale=scale,
        edgecolor=COLORS['water'],
        facecolor='none',
    )

    NaturalEarthFeatures = namedtuple(
        typename='NaturalEarthFeatures',
        field_names=(
            'BORDERS',
            'STATES',
            'COASTLINE',
            'LAKES',
            'LAND',
            'OCEAN',
            'RIVERS',
        ),
    )

    return NaturalEarthFeatures(**d)
<|endoftext|>


<commit_before>from django.contrib.admin import site
from django.apps import apps
from django.utils.text import capfirst
from django.core.urlresolvers import reverse, NoReverseMatch
from django.core.exceptions import ImproperlyConfigured
from django.utils import six
from django.conf import settings
from django import template
register = template.Library()

MAX_LENGTH_BOOTSTRAP_COLUMN = 12

def css_classes_for_field(field, custom_classes):
    required = 'required' if field.field.required else ''
    classes = field.css_classes('{} {}'.format(custom_classes, required))
    return classes

@register.filter()
def get_label(field, custom_classes=''):
    classes = css_classes_for_field(field, custom_classes)
    return field.label_tag(attrs={'class': classes}, label_suffix='')

@register.filter()
def add_class(field, custom_classes=''):
    classes = css_classes_for_field(field, custom_classes)
    try:
        # For widgets like SelectMultiple, checkboxselectmultiple
        field.field.widget.widget.attrs.update({'class': classes})
    except:
        field.field.widget.attrs.update({'class': classes})
    return field

@register.filter()
def widget_type(field):
    if isinstance(field, dict):
        return 'adminreadonlyfield'
    try:
        # For widgets like SelectMultiple, checkboxselectmultiple
        widget_type = field.field.widget.widget.__class__.__name__.lower()
    except:
        widget_type = field.field.widget.__class__.__name__.lower()
    return widget_type

@register.filter()
def placeholder(field, placeholder=''):
    field.field.widget.attrs.update({'placeholder': placeholder})
    return field

def sidebar_menu_setting():
    return getattr(settings, 'BOOTSTRAP_ADMIN_SIDEBAR_MENU', False)

@register.assignment_tag
def display_sidebar_menu(has_filters=False):
    if has_filters:
        # Always display the menu in change_list.html
        return True
    return sidebar_menu_setting()

@register.inclusion_tag('bootstrap_admin/sidebar_menu.html',
                        takes_context=True)
def render_menu_app_list(context):
    show_global_menu = sidebar_menu_setting()
    dependencie = 'django.core.context_processors.request'
    if not show_global_menu:
        return {'app_list': ''}

    if (dependencie not in settings.TEMPLATE_CONTEXT_PROCESSORS):
        raise ImproperlyConfigured(
            "bootstrap_admin: in order to use the 'sidebar menu' requires" +
            " the '%s' to be added to settings.TEMPLATE_CONTEXT_PROCESSORS."
            % dependencie
        )

    # Code adapted from django.contrib.admin.AdminSite
    app_dict = {}
    user = context.get('user')
    for model, model_admin in site._registry.items():
        app_label = model._meta.app_label
        has_module_perms = user.has_module_perms(app_label)

        if has_module_perms:
            perms = model_admin.get_model_perms(context.get('request'))

            # Check whether user has any perm for this module.
            # If so, add the module to the model_list.
            if True in perms.values():
                info = (app_label, model._meta.model_name)
                model_dict = {
                    'name': capfirst(model._meta.verbose_name_plural),
                    'object_name': model._meta.object_name,
                    'perms': perms,
                }
                if perms.get('change', False):
                    try:
                        model_dict['admin_url'] = reverse(
                            'admin:%s_%s_changelist' % info,
                            current_app=site.name
                        )
                    except NoReverseMatch:
                        pass
                if perms.get('add', False):
                    try:
                        model_dict['add_url'] = reverse(
                            'admin:%s_%s_add' % info,
                            current_app=site.name
                        )
                    except NoReverseMatch:
                        pass
                if app_label in app_dict:
                    app_dict[app_label]['models'].append(model_dict)
                else:
                    app_dict[app_label] = {
                        'name': apps.get_app_config(app_label).verbose_name,
                        'app_label': app_label,
                        'app_url': reverse(
                            'admin:app_list',
                            kwargs={'app_label': app_label},
                            current_app=site.name
                        ),
                        'has_module_perms': has_module_perms,
                        'models': [model_dict],
                    }

    # Sort the apps alphabetically.
    app_list = list(six.itervalues(app_dict))
    app_list.sort(key=lambda x: x['name'].lower())

    # Sort the models alphabetically within each sapp.
    for app in app_list:
        app['models'].sort(key=lambda x: x['name'])
    return {'app_list': app_list, 'current_url': context.get('request').path}

@register.filter()
def class_for_field_boxes(line):
    size_column = MAX_LENGTH_BOOTSTRAP_COLUMN / len(line.fields)
    return 'col-sm-{0}'.format(size_column or 1)  # if '0' replace with 1
<commit_msg>Don't overrride default widget classes fixes #55<commit_after>from django.contrib.admin import site
from django.apps import apps
from django.utils.text import capfirst
from django.core.urlresolvers import reverse, NoReverseMatch
from django.core.exceptions import ImproperlyConfigured
from django.utils import six
from django.conf import settings
from django import template
register = template.Library()

MAX_LENGTH_BOOTSTRAP_COLUMN = 12

def css_classes_for_field(field, custom_classes):
    orig_class = field.field.widget.attrs.get('class', '')
    required = 'required' if field.field.required else ''
    classes = field.css_classes(' '.join([orig_class, custom_classes, required]))
    return classes

@register.filter()
def get_label(field, custom_classes=''):
    classes = css_classes_for_field(field, custom_classes)
    return field.label_tag(attrs={'class': classes}, label_suffix='')

@register.filter()
def add_class(field, custom_classes=''):
    classes = css_classes_for_field(field, custom_classes)
    try:
        # For widgets like SelectMultiple, checkboxselectmultiple
        field.field.widget.widget.attrs.update({'class': classes})
    except:
        field.field.widget.attrs.update({'class': classes})
    return field

@register.filter()
def widget_type(field):
    if isinstance(field, dict):
        return 'adminreadonlyfield'
    try:
        # For widgets like SelectMultiple, checkboxselectmultiple
        widget_type = field.field.widget.widget.__class__.__name__.lower()
    except:
        widget_type = field.field.widget.__class__.__name__.lower()
    return widget_type

@register.filter()
def placeholder(field, placeholder=''):
    field.field.widget.attrs.update({'placeholder': placeholder})
    return field

def sidebar_menu_setting():
    return getattr(settings, 'BOOTSTRAP_ADMIN_SIDEBAR_MENU', False)

@register.assignment_tag
def display_sidebar_menu(has_filters=False):
    if has_filters:
        # Always display the menu in change_list.html
        return True
    return sidebar_menu_setting()

@register.inclusion_tag('bootstrap_admin/sidebar_menu.html',
                        takes_context=True)
def render_menu_app_list(context):
    show_global_menu = sidebar_menu_setting()
    dependencie = 'django.core.context_processors.request'
    if not show_global_menu:
        return {'app_list': ''}

    if (dependencie not in settings.TEMPLATE_CONTEXT_PROCESSORS):
        raise ImproperlyConfigured(
            "bootstrap_admin: in order to use the 'sidebar menu' requires" +
            " the '%s' to be added to settings.TEMPLATE_CONTEXT_PROCESSORS."
            % dependencie
        )

    # Code adapted from django.contrib.admin.AdminSite
    app_dict = {}
    user = context.get('user')
    for model, model_admin in site._registry.items():
        app_label = model._meta.app_label
        has_module_perms = user.has_module_perms(app_label)

        if has_module_perms:
            perms = model_admin.get_model_perms(context.get('request'))

            # Check whether user has any perm for this module.
            # If so, add the module to the model_list.
            if True in perms.values():
                info = (app_label, model._meta.model_name)
                model_dict = {
                    'name': capfirst(model._meta.verbose_name_plural),
                    'object_name': model._meta.object_name,
                    'perms': perms,
                }
                if perms.get('change', False):
                    try:
                        model_dict['admin_url'] = reverse(
                            'admin:%s_%s_changelist' % info,
                            current_app=site.name
                        )
                    except NoReverseMatch:
                        pass
                if perms.get('add', False):
                    try:
                        model_dict['add_url'] = reverse(
                            'admin:%s_%s_add' % info,
                            current_app=site.name
                        )
                    except NoReverseMatch:
                        pass
                if app_label in app_dict:
                    app_dict[app_label]['models'].append(model_dict)
                else:
                    app_dict[app_label] = {
                        'name': apps.get_app_config(app_label).verbose_name,
                        'app_label': app_label,
                        'app_url': reverse(
                            'admin:app_list',
                            kwargs={'app_label': app_label},
                            current_app=site.name
                        ),
                        'has_module_perms': has_module_perms,
                        'models': [model_dict],
                    }

    # Sort the apps alphabetically.
    app_list = list(six.itervalues(app_dict))
    app_list.sort(key=lambda x: x['name'].lower())

    # Sort the models alphabetically within each sapp.
    for app in app_list:
        app['models'].sort(key=lambda x: x['name'])
    return {'app_list': app_list, 'current_url': context.get('request').path}

@register.filter()
def class_for_field_boxes(line):
    size_column = MAX_LENGTH_BOOTSTRAP_COLUMN / len(line.fields)
    return 'col-sm-{0}'.format(size_column or 1)  # if '0' replace with 1
<|endoftext|>


<commit_before># Copyright (C) 2018 - TODAY, Pavlov Media
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Agreements Legal",
    "summary": "Manage Agreements, LOI and Contracts",
    "author": "Pavlov Media, "
    "Open Source Integrators, "
    "Yves Goldberg (Ygol Internetwork), "
    "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/contract",
    "category": "Partner",
    "license": "AGPL-3",
    "version": "14.0.2.4.0",
    "depends": ["contacts", "agreement", "product"],
    "data": [
        "data/ir_sequence.xml",
        "data/agreement_stage.xml",
        "data/agreement_type.xml",
        "security/res_groups.xml",
        "security/ir.model.access.csv",
        "report/agreement.xml",
        "views/res_config_settings.xml",
        "views/agreement_appendix.xml",
        "views/agreement_clause.xml",
        "views/agreement_recital.xml",
        "views/agreement_section.xml",
        "views/agreement_stages.xml",
        "views/agreement_type.xml",
        "views/agreement_subtype.xml",
        "views/res_partner.xml",
        "views/agreement.xml",
        "views/menu.xml",
        "views/assets.xml",
        "wizards/create_agreement_wizard.xml",
    ],
    "demo": ["demo/demo.xml"],
    "qweb": ["static/src/xml/agreement.xml"],
    "application": True,
    "development_status": "Beta",
    "maintainers": ["max3903", "ygol"],
}
<commit_msg>agreement_legal 14.0.2.4.1<commit_after># Copyright (C) 2018 - TODAY, Pavlov Media
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Agreements Legal",
    "summary": "Manage Agreements, LOI and Contracts",
    "author": "Pavlov Media, "
    "Open Source Integrators, "
    "Yves Goldberg (Ygol Internetwork), "
    "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/contract",
    "category": "Partner",
    "license": "AGPL-3",
    "version": "14.0.2.4.1",
    "depends": ["contacts", "agreement", "product"],
    "data": [
        "data/ir_sequence.xml",
        "data/agreement_stage.xml",
        "data/agreement_type.xml",
        "security/res_groups.xml",
        "security/ir.model.access.csv",
        "report/agreement.xml",
        "views/res_config_settings.xml",
        "views/agreement_appendix.xml",
        "views/agreement_clause.xml",
        "views/agreement_recital.xml",
        "views/agreement_section.xml",
        "views/agreement_stages.xml",
        "views/agreement_type.xml",
        "views/agreement_subtype.xml",
        "views/res_partner.xml",
        "views/agreement.xml",
        "views/menu.xml",
        "views/assets.xml",
        "wizards/create_agreement_wizard.xml",
    ],
    "demo": ["demo/demo.xml"],
    "qweb": ["static/src/xml/agreement.xml"],
    "application": True,
    "development_status": "Beta",
    "maintainers": ["max3903", "ygol"],
}
<|endoftext|>


<commit_before>from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator, CONTACT_DETAIL_TYPE_MAP

import re

COUNCIL_PAGE = 'http://www.chatham-kent.ca/Council/councilmembers/Pages/CouncilMembers.aspx'

class ChathamKentPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)

    wards = page.xpath('//table[@class="ms-rteTable-4"]')
    for ward in wards:
      district_info = ward.xpath('.//p')[0].text_content()
      if 'Mayor' in district_info:
        district = 'Chatham-Kent'
        role = 'Mayor'
      else:
        district = re.findall(r'(?<=Council Ward \d )(.*)(?= \()', district_info)[0]
        role = 'Councillor'

      councillors = ward.xpath('.//a')
      for councillor in councillors:
        name = councillor.text_content()
        url = councillor.attrib['href']
        page = lxmlize(url)

        p = Legislator(name=name, post_id=district, role=role)
        p.add_source(COUNCIL_PAGE)
        p.add_source(url)

        p.image = page.xpath('//div[@class="pageContent"]//img/@src')[0]

        address = page.xpath('//div[@id="div_contact_us_top_container_S"]//div[@class="div_contact_us_content_address"]')[0].text_content()
        p.add_contact('address', address, 'legislature')

        contacts = page.xpath('//div[@id="div_contact_us_top_container_S"]//div[@class="div_contact_us_content_kv"]/div')
        for contact in contacts:
          contact_type, contact = contact.text_content().split(':')
          contact_type = CONTACT_DETAIL_TYPE_MAP[contact_type.strip()]
          p.add_contact(contact_type, contact.strip(), None if contact_type == 'email' else 'legislature')
        yield p
<commit_msg>changed name district by wards number<commit_after>from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator, CONTACT_DETAIL_TYPE_MAP

import re

COUNCIL_PAGE = 'http://www.chatham-kent.ca/Council/councilmembers/Pages/CouncilMembers.aspx'

class ChathamKentPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)

    wards = page.xpath('//table[@class="ms-rteTable-4"]')
    for ward in wards:
      district_info = ward.xpath('.//p')[0].text_content()
      if 'Mayor' in district_info:
        district = 'Chatham-Kent'
        role = 'Mayor'
      else:
        district = re.findall(r'(Ward \d)', district_info)[0]
        role = 'Councillor'

      councillors = ward.xpath('.//a')
      for councillor in councillors:
        name = councillor.text_content()
        url = councillor.attrib['href']
        page = lxmlize(url)

        p = Legislator(name=name, post_id=district, role=role)
        p.add_source(COUNCIL_PAGE)
        p.add_source(url)

        p.image = page.xpath('//div[@class="pageContent"]//img/@src')[0]

        address = page.xpath('//div[@id="div_contact_us_top_container_S"]//div[@class="div_contact_us_content_address"]')[0].text_content()
        p.add_contact('address', address, 'legislature')

        contacts = page.xpath('//div[@id="div_contact_us_top_container_S"]//div[@class="div_contact_us_content_kv"]/div')
        for contact in contacts:
          contact_type, contact = contact.text_content().split(':')
          contact_type = CONTACT_DETAIL_TYPE_MAP[contact_type.strip()]
          p.add_contact(contact_type, contact.strip(), None if contact_type == 'email' else 'legislature')
        yield p
<|endoftext|>