<commit_before># coding=utf-8
from django.db import models
from django.core.urlresolvers import reverse
from user.models import User
from base.models import AbstractBaseModel
from django.conf import settings
from django.db.models import Q
from django.utils import timezone
from tag.models import Tag

import os
import math

class Event(AbstractBaseModel):
    # Numbers are arbitrary
    name = models.CharField(max_length=100)
    start_time = models.DateTimeField()
    end_time = models.DateTimeField()
    meeting_place = models.CharField(max_length=400)
    longitude = models.FloatField(default=35.7291)
    latitude = models.FloatField(default=139.7191)
    image = models.ImageField(upload_to='events/', null=True, blank=True)
    contact = models.CharField(max_length=200)
    details = models.TextField()
    notes = models.TextField(blank=True)
    private_notes = models.TextField(blank=True)
    hashtag = models.CharField(max_length=100, blank=True)
    share_message = models.CharField(max_length=100, blank=True)
    host_user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        related_name='host_event',
    )
    supporter = models.ManyToManyField(User, related_name="support", blank=True)

    # regionは都道府県で指定
    prefectures = settings.PREFECTURES
    region_list = [(key, value[0]) for key, value in sorted(prefectures.items(), key=lambda x:x[1][1])]
    region = models.CharField(max_length=10, choices=region_list)

    participant = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='participating_event',
        through='Participation',
        blank=True,
    )
    admin = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='admin_event',
        blank=True,
    )

    tag = models.ManyToManyField(Tag, blank=True)

    class Meta:
        ordering = ['-start_time']

    def __str__(self):
        return self.name + ": " + self.start_time.strftime("%Y-%m-%d") + " ~ " + self.end_time.strftime("%Y-%m-%d")

    def get_absolute_url(self):
        return reverse('event:detail', kwargs={'pk': self.id})

    def save(self, *args, **kwargs):
        return super(Event, self).save(*args, **kwargs)

    def get_image_url(self):
        if self.image:
            return self.image.url
        else:
            return os.path.join(
                settings.MEDIA_URL,
                'events/',
                "default_event_image.svg",
            )

    def get_tags_as_string(self):
        return "\n".join([tag.name for tag in self.tag.all()])

    def is_full(self):
        frames = Frame.objects.filter(event=self)
        for frame in frames:
            if not frame.is_full():
                return False
        return True

    def is_closed(self):
        frames = Frame.objects.filter(event=self)
        for frame in frames:
            if not frame.is_closed():
                return False
        return True

    def is_started(self):
        return timezone.now() > self.start_time

    def is_over(self):
        return timezone.now() > self.end_time

    def get_status(self):
        if self.is_over():
            return "終了"
        elif self.is_started():
            return "開催中"
        elif self.is_closed():
            return "締切済"
        elif self.is_full():
            return "満員"
        else:
            return "募集中"

    def get_region_kanji(self):
        region = self.prefectures.get(self.region)
        if not region:
            return '未設定'  # XXX: regionがこない場合は未設定でいいのか
        return region[0]

    def start_time_format(self):
        return self.start_time

    def end_time_format(self):
        return self.end_time

    def get_reserved_users(self):
        return [participation.user for participation in self.participation_set.filter(status="参加中")]

    def get_waiting_users(self):
        return [participation.user for participation in self.participation_set.filter(status="キャンセル待ち").order_by('created')]

    def get_host_user_as_list(self):
        return [self.host_user]

    @classmethod
    def get_events_in_range(cls, ne_lat, sw_lat, ne_lng, sw_lng):
        return cls.objects.filter(latitude__range=(ne_lat, sw_lat), longitude__range=(ne_lng, sw_lng))

class Frame(AbstractBaseModel):
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    description = models.TextField(default='通常参加枠')
    upper_limit = models.IntegerField(blank=True, null=True)
    deadline = models.DateTimeField()
    participant = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='participating_frame',
        through='Participation',
        blank=True,
    )

    def __str__(self):
        return "Frame #" + str(self.pk) + " in Event #" + str(self.event_id)

    def is_full(self):
        if self.upper_limit is None:
            return False
        else:
            participant_query = Q(frame=self)
            status_query = Q(status="参加中")
            num_participants = Participation.objects.filter(participant_query & status_query).count()

            return num_participants >= self.upper_limit

    def is_closed(self):
        return timezone.now() > self.deadline

    def num_participants(self):
        status_query = Q(status="参加中")
        return self.participation_set.filter(status_query).count()

    def deadline_format(self):
        return self.deadline

    def participant_id_list(self):
        return self.participation_set.all().values_list('user',flat=True)

    def reserved_id_list(self):
        return self.participation_set.filter(status="参加中").values_list('user',flat=True)

    def get_reserved_users(self):
        return [participation.user for participation in self.participation_set.filter(status="参加中")]

    def waiting_id_list(self):
        return self.participation_set.filter(status="キャンセル待ち").values_list('user',flat=True)

    def get_filled_rate(self):
        """Calculate participant capacity ratio.
        参加者上限数に対し、何%の応募があるかを返す。

        この関数は必ず何らかの整数値が返る事が期待されている。
        値が不明であることを明示する場合は、関数の呼び出し元で
        不明に対する処理を調整しなければならない。
        """
        participants = float(self.num_participants())
        limit = float(self.upper_limit)
        try:
            ratio = math.floor(participants * 100 / limit)
        except ZeroDivisionError:
            # XXX: zero division = 100%?
            ratio = 100

        return ratio

class Participation(AbstractBaseModel):
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    status = models.CharField(max_length=30)
    frame = models.ForeignKey(Frame, blank=True, null=True)

    class Meta:
        unique_together = (('event', 'user'),)

    def __str__(self):
        return "Participant:" + self.user.username +", Status: " + self.status

    def save(self, *args, **kwargs):
        return super(Participation, self).save(*args, **kwargs)

class Comment(AbstractBaseModel):
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    text = models.TextField()
    reply_to = models.ForeignKey(
        'self',
        on_delete=models.CASCADE,
        related_name='replies',
        null=True
    )

    def __str__(self):
        if self.reply_to:
            return ">> %s\n%s :\"%s" % (
                str(self.reply_to),
                self.user.username,
                self.text,
            )
        else:
            return "%s: \"%s\"" % (
                self.user.username,
                self.text,
            )

    def get_absolute_url(self):
        return reverse('event:detail', kwargs={'pk': self.event.id})

    def save(self, *args, **kwargs):
        return super(Comment, self).save(*args, **kwargs)

class Question(models.Model):
    event = models.ForeignKey(
        Event,
        on_delete=models.CASCADE,
        related_name='question',
    )
    question = models.TextField()

    def __str__(self):
        return self.question

class Answer(AbstractBaseModel):
    question = models.ForeignKey(
        Question,
        on_delete=models.CASCADE,
        related_name='answer',
    )
    participation = models.ForeignKey(
        Participation,
        on_delete=models.CASCADE,
        related_name='answer',
    )
    text = models.TextField()

    def __str__(self):
        return "Q: " + self.question.question + " -> " + "A: " + self.text

    def save(self, *args, **kwargs):
        return super(Answer, self).save(*args, **kwargs)
<commit_msg>[COSME] E501 line too long (80 > 79 characters)<commit_after># coding=utf-8
from django.db import models
from django.core.urlresolvers import reverse
from user.models import User
from base.models import AbstractBaseModel
from django.conf import settings
from django.db.models import Q
from django.utils import timezone
from tag.models import Tag

import os
import math

class Event(AbstractBaseModel):
    # Numbers are arbitrary
    name = models.CharField(max_length=100)
    start_time = models.DateTimeField()
    end_time = models.DateTimeField()
    meeting_place = models.CharField(max_length=400)
    longitude = models.FloatField(default=35.7291)
    latitude = models.FloatField(default=139.7191)
    image = models.ImageField(upload_to='events/', null=True, blank=True)
    contact = models.CharField(max_length=200)
    details = models.TextField()
    notes = models.TextField(blank=True)
    private_notes = models.TextField(blank=True)
    hashtag = models.CharField(max_length=100, blank=True)
    share_message = models.CharField(max_length=100, blank=True)
    host_user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        related_name='host_event',
    )
    supporter = models.ManyToManyField(User,
                                       related_name="support",
                                       blank=True)

    # regionは都道府県で指定
    prefectures = settings.PREFECTURES
    region_list = [(key, value[0]) for key, value in sorted(prefectures.items(), key=lambda x:x[1][1])]
    region = models.CharField(max_length=10, choices=region_list)

    participant = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='participating_event',
        through='Participation',
        blank=True,
    )
    admin = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='admin_event',
        blank=True,
    )

    tag = models.ManyToManyField(Tag, blank=True)

    class Meta:
        ordering = ['-start_time']

    def __str__(self):
        return self.name + ": " + self.start_time.strftime("%Y-%m-%d") + " ~ " + self.end_time.strftime("%Y-%m-%d")

    def get_absolute_url(self):
        return reverse('event:detail', kwargs={'pk': self.id})

    def save(self, *args, **kwargs):
        return super(Event, self).save(*args, **kwargs)

    def get_image_url(self):
        if self.image:
            return self.image.url
        else:
            return os.path.join(
                settings.MEDIA_URL,
                'events/',
                "default_event_image.svg",
            )

    def get_tags_as_string(self):
        return "\n".join([tag.name for tag in self.tag.all()])

    def is_full(self):
        frames = Frame.objects.filter(event=self)
        for frame in frames:
            if not frame.is_full():
                return False
        return True

    def is_closed(self):
        frames = Frame.objects.filter(event=self)
        for frame in frames:
            if not frame.is_closed():
                return False
        return True

    def is_started(self):
        return timezone.now() > self.start_time

    def is_over(self):
        return timezone.now() > self.end_time

    def get_status(self):
        if self.is_over():
            return "終了"
        elif self.is_started():
            return "開催中"
        elif self.is_closed():
            return "締切済"
        elif self.is_full():
            return "満員"
        else:
            return "募集中"

    def get_region_kanji(self):
        region = self.prefectures.get(self.region)
        if not region:
            return '未設定'  # XXX: regionがこない場合は未設定でいいのか
        return region[0]

    def start_time_format(self):
        return self.start_time

    def end_time_format(self):
        return self.end_time

    def get_reserved_users(self):
        return [participation.user for participation in self.participation_set.filter(status="参加中")]

    def get_waiting_users(self):
        return [participation.user for participation in self.participation_set.filter(status="キャンセル待ち").order_by('created')]

    def get_host_user_as_list(self):
        return [self.host_user]

    @classmethod
    def get_events_in_range(cls, ne_lat, sw_lat, ne_lng, sw_lng):
        return cls.objects.filter(latitude__range=(ne_lat, sw_lat), longitude__range=(ne_lng, sw_lng))

class Frame(AbstractBaseModel):
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    description = models.TextField(default='通常参加枠')
    upper_limit = models.IntegerField(blank=True, null=True)
    deadline = models.DateTimeField()
    participant = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        related_name='participating_frame',
        through='Participation',
        blank=True,
    )

    def __str__(self):
        return "Frame #" + str(self.pk) + " in Event #" + str(self.event_id)

    def is_full(self):
        if self.upper_limit is None:
            return False
        else:
            participant_query = Q(frame=self)
            status_query = Q(status="参加中")
            num_participants = Participation.objects.filter(participant_query & status_query).count()

            return num_participants >= self.upper_limit

    def is_closed(self):
        return timezone.now() > self.deadline

    def num_participants(self):
        status_query = Q(status="参加中")
        return self.participation_set.filter(status_query).count()

    def deadline_format(self):
        return self.deadline

    def participant_id_list(self):
        return self.participation_set.all().values_list('user',flat=True)

    def reserved_id_list(self):
        return self.participation_set.filter(status="参加中").values_list('user',flat=True)

    def get_reserved_users(self):
        return [participation.user for participation in self.participation_set.filter(status="参加中")]

    def waiting_id_list(self):
        return self.participation_set.filter(status="キャンセル待ち").values_list('user',flat=True)

    def get_filled_rate(self):
        """Calculate participant capacity ratio.
        参加者上限数に対し、何%の応募があるかを返す。

        この関数は必ず何らかの整数値が返る事が期待されている。
        値が不明であることを明示する場合は、関数の呼び出し元で
        不明に対する処理を調整しなければならない。
        """
        participants = float(self.num_participants())
        limit = float(self.upper_limit)
        try:
            ratio = math.floor(participants * 100 / limit)
        except ZeroDivisionError:
            # XXX: zero division = 100%?
            ratio = 100

        return ratio

class Participation(AbstractBaseModel):
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    status = models.CharField(max_length=30)
    frame = models.ForeignKey(Frame, blank=True, null=True)

    class Meta:
        unique_together = (('event', 'user'),)

    def __str__(self):
        return "Participant:" + self.user.username +", Status: " + self.status

    def save(self, *args, **kwargs):
        return super(Participation, self).save(*args, **kwargs)

class Comment(AbstractBaseModel):
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    text = models.TextField()
    reply_to = models.ForeignKey(
        'self',
        on_delete=models.CASCADE,
        related_name='replies',
        null=True
    )

    def __str__(self):
        if self.reply_to:
            return ">> %s\n%s :\"%s" % (
                str(self.reply_to),
                self.user.username,
                self.text,
            )
        else:
            return "%s: \"%s\"" % (
                self.user.username,
                self.text,
            )

    def get_absolute_url(self):
        return reverse('event:detail', kwargs={'pk': self.event.id})

    def save(self, *args, **kwargs):
        return super(Comment, self).save(*args, **kwargs)

class Question(models.Model):
    event = models.ForeignKey(
        Event,
        on_delete=models.CASCADE,
        related_name='question',
    )
    question = models.TextField()

    def __str__(self):
        return self.question

class Answer(AbstractBaseModel):
    question = models.ForeignKey(
        Question,
        on_delete=models.CASCADE,
        related_name='answer',
    )
    participation = models.ForeignKey(
        Participation,
        on_delete=models.CASCADE,
        related_name='answer',
    )
    text = models.TextField()

    def __str__(self):
        return "Q: " + self.question.question + " -> " + "A: " + self.text

    def save(self, *args, **kwargs):
        return super(Answer, self).save(*args, **kwargs)
<|endoftext|>


<commit_before>"""
    bmh_search.py

    This module implements bmh search to find a substring in a string

    BMH Search Overview:
    --------------------
    Uses a bad-character shift of the rightmost character of the window to
    compute shifts.

    The trick to this algorithm is `bmbc`, a lookup table with a default
    value equal to the length of the pattern to be searched, so that
    the algorithm can skip `len(pattern)` indices through the string
    for efficiency's sake. For example, if we're searching through the
    string "cotton milled paper" for the pattern "grumble", we look at
    the last letter "r" (BMH goes backwards through a string) and notices
    that it is not equal to "e". Thus, we can afford to jump our search
    index back a whole seven characters.

    However, not all the entries in `bmbc` are equal to `len(pattern)`.
    If we searched the string "adventure time" for "grumble", we'd find
    the "e" to match but mismatch the "m" and "l" in the string and
    pattern, respectively. In this case, we can only jump back six
    characters safely, which is why `bmbc` contains values that are not
    simply `len(pattern)`.

    Pre: a string > substring.

    Post: returns a list of indices where the substring was found.

    Time: Complexity: O(m + n), where m is the substring to be found.

    Space: Complexity: O(m), where m is the substring to be found.

    Psuedo Code: https://github.com/FooBarWidget/boyer-moore-horspool
                 (converted from C++)

    bmh_search.search(string, substring) -> list[integers]
    bmh_search.search(string, substring) -> list[empty]
"""

def search(text, pattern):
    pattern_length = len(pattern)
    text_length = len(text)
    offsets = []
    if pattern_length > text_length:
        return offsets
    bmbc = [pattern_length] * 256
    for index, char in enumerate(pattern[:-1]):
        bmbc[ord(char)] = pattern_length - index - 1
    bmbc = tuple(bmbc)
    search_index = pattern_length - 1
    while search_index < text_length:
        pattern_index = pattern_length - 1
        text_index = search_index
        while text_index >= 0 and \
              text[text_index] == pattern[pattern_index]:
            pattern_index -= 1
            text_index -= 1
        if pattern_index == -1:
            offsets.append(text_index + 1)
        search_index += bmbc[ord(text[search_index])]

    return offsets
<commit_msg>Line continuation follows PEP8<commit_after>"""
    bmh_search.py

    This module implements bmh search to find a substring in a string

    BMH Search Overview:
    --------------------
    Uses a bad-character shift of the rightmost character of the window to
    compute shifts.

    The trick to this algorithm is `bmbc`, a lookup table with a default
    value equal to the length of the pattern to be searched, so that
    the algorithm can skip `len(pattern)` indices through the string
    for efficiency's sake. For example, if we're searching through the
    string "cotton milled paper" for the pattern "grumble", we look at
    the last letter "r" (BMH goes backwards through a string) and notices
    that it is not equal to "e". Thus, we can afford to jump our search
    index back a whole seven characters.

    However, not all the entries in `bmbc` are equal to `len(pattern)`.
    If we searched the string "adventure time" for "grumble", we'd find
    the "e" to match but mismatch the "m" and "l" in the string and
    pattern, respectively. In this case, we can only jump back six
    characters safely, which is why `bmbc` contains values that are not
    simply `len(pattern)`.

    Pre: a string > substring.

    Post: returns a list of indices where the substring was found.

    Time: Complexity: O(m + n), where m is the substring to be found.

    Space: Complexity: O(m), where m is the substring to be found.

    Psuedo Code: https://github.com/FooBarWidget/boyer-moore-horspool
                 (converted from C++)

    bmh_search.search(string, substring) -> list[integers]
    bmh_search.search(string, substring) -> list[empty]
"""

def search(text, pattern):
    pattern_length = len(pattern)
    text_length = len(text)
    offsets = []
    if pattern_length > text_length:
        return offsets
    bmbc = [pattern_length] * 256
    for index, char in enumerate(pattern[:-1]):
        bmbc[ord(char)] = pattern_length - index - 1
    bmbc = tuple(bmbc)
    search_index = pattern_length - 1
    while search_index < text_length:
        pattern_index = pattern_length - 1
        text_index = search_index
        while text_index >= 0 and \
                text[text_index] == pattern[pattern_index]:
            pattern_index -= 1
            text_index -= 1
        if pattern_index == -1:
            offsets.append(text_index + 1)
        search_index += bmbc[ord(text[search_index])]

    return offsets
<|endoftext|>


<commit_before>import requests
import json
import util

ZHBELL_SNIPPET ="""

%s

"""

ZHBELL_RECEIVER = 'weiyi@papayamobile.com'

def get_latest():
    headers = {'User-agent': 'Mozilla/5.0'}
    r = requests.get('http://news-at.zhihu.com/api/4/news/latest', headers=headers)
    return json.loads(r.content)

def get_new_stories():
    r = get_latest()
    return r['stories']

def get_top_stories():
    r = get_latest()
    return r['top_stories']

def notify(title='[ZHBell]Latest', body='N/A'):
    util.send_email(title, body, ZHBELL_RECEIVER)

def wrap(items=None):
    if not items:
        return 'N/A'
    body = ''.join([ZHBELL_SNIPPET % (i['images'][0] if i.get('images') else i['image'], 'http://daily.zhihu.com/story/%s' % i['id'], i['title']) for i in items])
    return body

def task_top():
    r = get_top_stories()
    r = wrap(r)
    notify(title='[ZHBell]Top', body=r)

def task_new():
    r = get_new_stories()
    r = wrap(r)
    notify(title='[ZHBell]New', body=r)
<commit_msg>add zhihuhot<commit_after># -*- coding: utf-8 -*-
import requests
import json
import util
from bs4 import BeautifulSoup
import random

ZHH_NUM = 20

ZHBELL_SNIPPET ="""

%s

"""

ZHBELL_RECEIVER = 'weiyi@papayamobile.com'

ZHPIC = ['https://pic4.zhimg.com/7d8c15df29ce6900c80f9cf630992687_xl.jpg',
         'https://pic1.zhimg.com/e8b840284350f4a2d01ea6869926bfcc_xl.jpg',
         'https://pic1.zhimg.com/1f9c99e8aa4b0ab76d4885ffd0eb10c0_xl.jpg',
         'https://pic1.zhimg.com/5c5f5d55992d64aa8184b29c53e80af4_xl.jpg',
         'https://pic1.zhimg.com/a240a363b7d7e0681502017c57b0ba20_xl.jpg',
         'https://pic4.zhimg.com/5bda61313f8e3767edaa6134941e207b_xl.jpg',
         'https://pic3.zhimg.com/d8426d43e_xl.jpg',
         'https://pic3.zhimg.com/cf0156d3a_xl.jpg',
         'https://pic2.zhimg.com/ca0657b5e0ef154ee25c3e357a6c607d_xl.jpg',
         'https://pic4.zhimg.com/f07808da5625fef3607f8b75b770349f_xl.jpg',
         'https://pic1.zhimg.com/975baaf73fd76f48ce6f05e19b176878_xl.jpg',
         'https://pic3.zhimg.com/2bd2e49d3f849155376393085edc1d9e_xl.jpg',
         ]

def get_zh_pic():
    i = random.randrange(0, len(ZHPIC))
    return ZHPIC[i]

def get_latest():
    headers = {'User-agent': 'Mozilla/5.0'}
    r = requests.get('http://news-at.zhihu.com/api/4/news/latest', headers=headers)
    return json.loads(r.content)

def get_new_stories():
    r = get_latest()
    return r['stories']

def get_top_stories():
    r = get_latest()
    return r['top_stories']

def notify(title='[ZHBell]Latest', body='N/A'):
    util.send_email(title, body, ZHBELL_RECEIVER)

def wrap(items=None):
    if not items:
        return 'N/A'
    body = ''.join([ZHBELL_SNIPPET % (i['images'][0] if i.get('images') else i['image'], 'http://daily.zhihu.com/story/%s' % i['id'], i['title']) for i in items])
    return body

def task_top():
    r = get_top_stories()
    r = wrap(r)
    notify(title='[ZHBell]Top', body=r)
    task_zhihuhot()

def task_new():
    r = get_new_stories()
    r = wrap(r)
    notify(title='[ZHBell]New', body=r)

def task_zhihuhot():
    r = get_zhihuhot()
    r = wrap_zhihuhot(r)
    notify(title='[ZHBell]知乎热门', body=r)

def wrap_zhihuhot(items=None):
    if not items:
        return 'N/A'
    body = ZHBELL_SNIPPET % (get_zh_pic(), 'http://zhihuhot.sinaapp.com/', '知乎热门TOP500')
    body = body.decode('utf8')
    body += ''.join([ZHBELL_SNIPPET % (get_zh_pic(), i[1], i[0]) for i in items])
    return  body

def get_zhihuhot():
    r = requests.get('http://zhihuhot.sinaapp.com/')
    soup = BeautifulSoup(r.text)
    r = soup.findAll("div")[1:ZHH_NUM+1]
    r = [(a.get_text(), a.find('a')['href']) for a in r]
    return r

<|endoftext|>


<commit_before>import skimage.io  # bug. need to import this before tensorflow
import skimage.transform  # bug. need to import this before tensorflow
import tensorflow as tf
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.training import moving_averages

from config import Config

import datetime
import numpy as np
import os
import time

MOVING_AVERAGE_DECAY = 0.9997
BN_DECAY = MOVING_AVERAGE_DECAY
BN_EPSILON = 0.001
CONV_WEIGHT_DECAY = 0.00004
CONV_WEIGHT_STDDEV = 0.1
FC_WEIGHT_DECAY = 0.00004
FC_WEIGHT_STDDEV = 0.01
RESNET_VARIABLES = 'resnet_variables'
UPDATE_OPS_COLLECTION = 'resnet_update_ops'  # must be grouped with training op
IMAGENET_MEAN_BGR = [103.062623801, 115.902882574, 123.151630838, ]

tf.app.flags.DEFINE_integer('input_size', 224, "input image size")

activation = tf.nn.relu

def inference(x, is_training,
              num_classes=1000,
              num_blocks=[3, 4, 6, 3],  # defaults to 50-layer network
              use_bias=False, # defaults to using batch norm
              bottleneck=True):
    c = Config()
    c['bottleneck'] = bottleneck
    c['is_training'] = tf.convert_to_tensor(is_training,
                                            dtype='bool',
                                            name='is_training')
    c['ksize'] = 3
    c['stride'] = 1
    c['use_bias'] = use_bias
    c['fc_units_out'] = num_classes
    c['num_blocks'] = num_blocks
    c['stack_stride'] = 2

    with tf.variable_scope('scale1'):
        c['conv_filters_out'] = 64
        c['ksize'] = 7
        c['stride'] = 2
        x = _conv(x, c)
        x = _bn(x, c)
        x = activation(x)

    with tf.variable_scope('scale2'):
        x = _max_pool(x, ksize=3, stride=2)
        c['num_blocks'] = num_blocks[0]
        c['stack_stride'] = 1
        c['block_filters_internal'] = 64
        x = stack(x, c)

    with tf.variable_scope('scale3'):
        c['num_blocks'] = num_blocks[1]
        c['block_filters_internal'] = 128
        assert c['stack_stride'] == 2
        x = stack(x, c)

    with tf.variable_scope('scale4'):
        c['num_blocks'] = num_blocks[2]
        c['block_filters_internal'] = 256
        x = stack(x, c)

    with tf.variable_scope('scale5'):
        c['num_blocks'] = num_blocks[3]
        c['block_filters_internal'] = 512
        x = stack(x, c)

    # post-net
    x = tf.reduce_mean(x, reduction_indices=[1, 2], name="avg_pool")

    if num_classes != None:
        with tf.variable_scope('fc'):
            x = _fc(x, c)

    return x

# This is what they use for CIFAR-10 and 100.
# See Section 4.2 in http://arxiv.org/abs/1512.03385
def inference_small(x,
                    is_training,
                    num_blocks=3, # 6n+2 total weight layers will be used.
                    use_bias=False, # defaults to using batch norm
                    num_classes=10):
    c = Config()
    c['bottleneck'] = False
    c['is_training'] = tf.convert_to_tensor(is_training,
                                            dtype='bool',
                                            name='is_training')
    c['ksize'] = 3
    c['stride'] = 1
    c['use_bias'] = use_bias
    c['fc_units_out'] = num_classes
    c['num_blocks'] = num_blocks

    with tf.variable_scope('scale1'):
        c['conv_filters_out'] = 16
        c['block_filters_internal'] = 16
        c['stack_stride'] = 1
        x = _conv(x, c)
        x = _bn(x, c)
        x = activation(x)
        x = stack(x, c)

    with tf.variable_scope('scale2'):
        c['block_filters_internal'] = 32
        c['stack_stride'] = 2
        x = stack(x, c)

    with tf.variable_scope('scale3'):
        c['block_filters_internal'] = 64
        c['stack_stride'] = 2
        x = stack(x, c)

    # post-net
    x = tf.reduce_mean(x, reduction_indices=[1, 2], name="avg_pool")

    if num_classes != None:
        with tf.variable_scope('fc'):
            x = _fc(x, c)

    return x

def _imagenet_preprocess(rgb):
    """Changes RGB [0,1] valued image to BGR [0,255] with mean subtracted."""
    red, green, blue = tf.split(3, 3, rgb * 255.0)
    bgr = tf.concat(3, [blue, green, red])
    bgr -= IMAGENET_MEAN_BGR
    return bgr

def loss(logits, labels):
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels)
    cross_entropy_mean = tf.reduce_mean(cross_entropy)

    regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)

    loss_ = tf.add_n([cross_entropy_mean] + regularization_losses)
    tf.scalar_summary('loss', loss_)

    return loss_

def stack(x, c):
    for n in range(c['num_blocks']):
        s = c['stack_stride'] if n == 0 else 1
        c['block_stride'] = s
        with tf.variable_scope('block%d' % (n + 1)):
            x = block(x, c)
    return x

def block(x, c):
    filters_in = x.get_shape()[-1]

    # Note: filters_out isn't how many filters are outputed.
    # That is the case when bottleneck=False but when bottleneck is
    # True, filters_internal*4 filters are outputted. filters_internal is how many filters
    # the 3x3 convs output internally.
    m = 4 if c['bottleneck'] else 1
    filters_out = m * c['block_filters_internal']

    shortcut = x  # branch 1

    c['conv_filters_out'] = c['block_filters_internal']

    if c['bottleneck']:
        with tf.variable_scope('a'):
            c['ksize'] = 1
            c['stride'] = c['block_stride']
            x = _conv(x, c)
            x = _bn(x, c)
            x = activation(x)

        with tf.variable_scope('b'):
            x = _conv(x, c)
            x = _bn(x, c)
            x = activation(x)

        with tf.variable_scope('c'):
            c['conv_filters_out'] = filters_out
            c['ksize'] = 1
            assert c['stride'] == 1
            x = _conv(x, c)
            x = _bn(x, c)
    else:
        with tf.variable_scope('A'):
            c['stride'] = c['block_stride']
            assert c['ksize'] == 3
            x = _conv(x, c)
            x = _bn(x, c)
            x = activation(x)

        with tf.variable_scope('B'):
            c['conv_filters_out'] = filters_out
            assert c['ksize'] == 3
            assert c['stride'] == 1
            x = _conv(x, c)
            x = _bn(x, c)

    with tf.variable_scope('shortcut'):
        if filters_out != filters_in or c['block_stride'] != 1:
            c['ksize'] = 1
            c['stride'] = c['block_stride']
            c['conv_filters_out'] = filters_out
            shortcut = _conv(shortcut, c)
            shortcut = _bn(shortcut, c)

    return activation(x + shortcut)

def _bn(x, c):
    x_shape = x.get_shape()
    params_shape = x_shape[-1:]

    if c['use_bias']:
        bias = _get_variable('bias', params_shape,
                             initializer=tf.zeros_initializer)
        return x + bias

    axis = list(range(len(x_shape) - 1))

    beta = _get_variable('beta',
                         params_shape,
                         initializer=tf.zeros_initializer)
    gamma = _get_variable('gamma',
                          params_shape,
                          initializer=tf.ones_initializer)

    moving_mean = _get_variable('moving_mean',
                                params_shape,
                                initializer=tf.zeros_initializer,
                                trainable=False)
    moving_variance = _get_variable('moving_variance',
                                    params_shape,
                                    initializer=tf.ones_initializer,
                                    trainable=False)

    # These ops will only be preformed when training.
    mean, variance = tf.nn.moments(x, axis)
    update_moving_mean = moving_averages.assign_moving_average(moving_mean,
                                                               mean, BN_DECAY)
    update_moving_variance = moving_averages.assign_moving_average(
        moving_variance, variance, BN_DECAY)
    tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
    tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)

    mean, variance = control_flow_ops.cond(
        c['is_training'], lambda: (mean, variance),
        lambda: (moving_mean, moving_variance))

    x = tf.nn.batch_normalization(x, mean, variance, beta, gamma, BN_EPSILON)
    #x.set_shape(inputs.get_shape()) ??

    return x

def _fc(x, c):
    num_units_in = x.get_shape()[1]
    num_units_out = c['fc_units_out']
    weights_initializer = tf.truncated_normal_initializer(
        stddev=FC_WEIGHT_STDDEV)

    weights = _get_variable('weights',
                            shape=[num_units_in, num_units_out],
                            initializer=weights_initializer,
                            weight_decay=FC_WEIGHT_STDDEV)
    biases = _get_variable('biases',
                           shape=[num_units_out],
                           initializer=tf.zeros_initializer)
    x = tf.nn.xw_plus_b(x, weights, biases)
    return x

def _get_variable(name,
                  shape,
                  initializer,
                  weight_decay=0.0,
                  dtype='float',
                  trainable=True):
    "A little wrapper around tf.get_variable to do weight decay and add to"
    "resnet collection"
    if weight_decay > 0:
        regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
    else:
        regularizer = None
    collections = [tf.GraphKeys.VARIABLES, RESNET_VARIABLES]
    return tf.get_variable(name,
                           shape=shape,
                           initializer=initializer,
                           dtype=dtype,
                           regularizer=regularizer,
                           collections=collections,
                           trainable=trainable)

def _conv(x, c):
    ksize = c['ksize']
    stride = c['stride']
    filters_out = c['conv_filters_out']

    filters_in = x.get_shape()[-1]
    shape = [ksize, ksize, filters_in, filters_out]
    initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV)
    weights = _get_variable('weights',
                            shape=shape,
                            dtype='float',
                            initializer=initializer,
                            weight_decay=CONV_WEIGHT_DECAY)
    return tf.nn.conv2d(x, weights, [1, stride, stride, 1], padding='SAME')

def _max_pool(x, ksize=3, stride=2):
    return tf.nn.max_pool(x,
                          ksize=[1, ksize, ksize, 1],
                          strides=[1, stride, stride, 1],
                          padding='SAME')
<commit_msg>add API for using calling inference_small with a config<commit_after>import skimage.io  # bug. need to import this before tensorflow
import skimage.transform  # bug. need to import this before tensorflow
import tensorflow as tf
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.training import moving_averages

from config import Config

import datetime
import numpy as np
import os
import time

MOVING_AVERAGE_DECAY = 0.9997
BN_DECAY = MOVING_AVERAGE_DECAY
BN_EPSILON = 0.001
CONV_WEIGHT_DECAY = 0.00004
CONV_WEIGHT_STDDEV = 0.1
FC_WEIGHT_DECAY = 0.00004
FC_WEIGHT_STDDEV = 0.01
RESNET_VARIABLES = 'resnet_variables'
UPDATE_OPS_COLLECTION = 'resnet_update_ops'  # must be grouped with training op
IMAGENET_MEAN_BGR = [103.062623801, 115.902882574, 123.151630838, ]

tf.app.flags.DEFINE_integer('input_size', 224, "input image size")

activation = tf.nn.relu

def inference(x, is_training,
              num_classes=1000,
              num_blocks=[3, 4, 6, 3],  # defaults to 50-layer network
              use_bias=False, # defaults to using batch norm
              bottleneck=True):
    c = Config()
    c['bottleneck'] = bottleneck
    c['is_training'] = tf.convert_to_tensor(is_training,
                                            dtype='bool',
                                            name='is_training')
    c['ksize'] = 3
    c['stride'] = 1
    c['use_bias'] = use_bias
    c['fc_units_out'] = num_classes
    c['num_blocks'] = num_blocks
    c['stack_stride'] = 2

    with tf.variable_scope('scale1'):
        c['conv_filters_out'] = 64
        c['ksize'] = 7
        c['stride'] = 2
        x = conv(x, c)
        x = bn(x, c)
        x = activation(x)

    with tf.variable_scope('scale2'):
        x = _max_pool(x, ksize=3, stride=2)
        c['num_blocks'] = num_blocks[0]
        c['stack_stride'] = 1
        c['block_filters_internal'] = 64
        x = stack(x, c)

    with tf.variable_scope('scale3'):
        c['num_blocks'] = num_blocks[1]
        c['block_filters_internal'] = 128
        assert c['stack_stride'] == 2
        x = stack(x, c)

    with tf.variable_scope('scale4'):
        c['num_blocks'] = num_blocks[2]
        c['block_filters_internal'] = 256
        x = stack(x, c)

    with tf.variable_scope('scale5'):
        c['num_blocks'] = num_blocks[3]
        c['block_filters_internal'] = 512
        x = stack(x, c)

    # post-net
    x = tf.reduce_mean(x, reduction_indices=[1, 2], name="avg_pool")

    if num_classes != None:
        with tf.variable_scope('fc'):
            x = fc(x, c)

    return x

# This is what they use for CIFAR-10 and 100.
# See Section 4.2 in http://arxiv.org/abs/1512.03385
def inference_small(x,
                    is_training,
                    num_blocks=3, # 6n+2 total weight layers will be used.
                    use_bias=False, # defaults to using batch norm
                    num_classes=10):
    c = Config()
    c['is_training'] = tf.convert_to_tensor(is_training,
                                            dtype='bool',
                                            name='is_training')
    c['use_bias'] = use_bias
    c['fc_units_out'] = num_classes
    c['num_blocks'] = num_blocks
    c['num_classes'] = num_classes
    inference_small_config(x, c)

def inference_small_config(x, c):
    c['bottleneck'] = False
    c['ksize'] = 3
    c['stride'] = 1
    with tf.variable_scope('scale1'):
        c['conv_filters_out'] = 16
        c['block_filters_internal'] = 16
        c['stack_stride'] = 1
        x = conv(x, c)
        x = bn(x, c)
        x = activation(x)
        x = stack(x, c)

    with tf.variable_scope('scale2'):
        c['block_filters_internal'] = 32
        c['stack_stride'] = 2
        x = stack(x, c)

    with tf.variable_scope('scale3'):
        c['block_filters_internal'] = 64
        c['stack_stride'] = 2
        x = stack(x, c)

    # post-net
    x = tf.reduce_mean(x, reduction_indices=[1, 2], name="avg_pool")

    if c['num_classes'] != None:
        with tf.variable_scope('fc'):
            x = fc(x, c)

    return x

def _imagenet_preprocess(rgb):
    """Changes RGB [0,1] valued image to BGR [0,255] with mean subtracted."""
    red, green, blue = tf.split(3, 3, rgb * 255.0)
    bgr = tf.concat(3, [blue, green, red])
    bgr -= IMAGENET_MEAN_BGR
    return bgr

def loss(logits, labels):
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels)
    cross_entropy_mean = tf.reduce_mean(cross_entropy)

    regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)

    loss_ = tf.add_n([cross_entropy_mean] + regularization_losses)
    tf.scalar_summary('loss', loss_)

    return loss_

def stack(x, c):
    for n in range(c['num_blocks']):
        s = c['stack_stride'] if n == 0 else 1
        c['block_stride'] = s
        with tf.variable_scope('block%d' % (n + 1)):
            x = block(x, c)
    return x

def block(x, c):
    filters_in = x.get_shape()[-1]

    # Note: filters_out isn't how many filters are outputed.
    # That is the case when bottleneck=False but when bottleneck is
    # True, filters_internal*4 filters are outputted. filters_internal is how many filters
    # the 3x3 convs output internally.
    m = 4 if c['bottleneck'] else 1
    filters_out = m * c['block_filters_internal']

    shortcut = x  # branch 1

    c['conv_filters_out'] = c['block_filters_internal']

    if c['bottleneck']:
        with tf.variable_scope('a'):
            c['ksize'] = 1
            c['stride'] = c['block_stride']
            x = conv(x, c)
            x = bn(x, c)
            x = activation(x)

        with tf.variable_scope('b'):
            x = conv(x, c)
            x = bn(x, c)
            x = activation(x)

        with tf.variable_scope('c'):
            c['conv_filters_out'] = filters_out
            c['ksize'] = 1
            assert c['stride'] == 1
            x = conv(x, c)
            x = bn(x, c)
    else:
        with tf.variable_scope('A'):
            c['stride'] = c['block_stride']
            assert c['ksize'] == 3
            x = conv(x, c)
            x = bn(x, c)
            x = activation(x)

        with tf.variable_scope('B'):
            c['conv_filters_out'] = filters_out
            assert c['ksize'] == 3
            assert c['stride'] == 1
            x = conv(x, c)
            x = bn(x, c)

    with tf.variable_scope('shortcut'):
        if filters_out != filters_in or c['block_stride'] != 1:
            c['ksize'] = 1
            c['stride'] = c['block_stride']
            c['conv_filters_out'] = filters_out
            shortcut = conv(shortcut, c)
            shortcut = bn(shortcut, c)

    return activation(x + shortcut)

def bn(x, c):
    x_shape = x.get_shape()
    params_shape = x_shape[-1:]

    if c['use_bias']:
        bias = _get_variable('bias', params_shape,
                             initializer=tf.zeros_initializer)
        return x + bias

    axis = list(range(len(x_shape) - 1))

    beta = _get_variable('beta',
                         params_shape,
                         initializer=tf.zeros_initializer)
    gamma = _get_variable('gamma',
                          params_shape,
                          initializer=tf.ones_initializer)

    moving_mean = _get_variable('moving_mean',
                                params_shape,
                                initializer=tf.zeros_initializer,
                                trainable=False)
    moving_variance = _get_variable('moving_variance',
                                    params_shape,
                                    initializer=tf.ones_initializer,
                                    trainable=False)

    # These ops will only be preformed when training.
    mean, variance = tf.nn.moments(x, axis)
    update_moving_mean = moving_averages.assign_moving_average(moving_mean,
                                                               mean, BN_DECAY)
    update_moving_variance = moving_averages.assign_moving_average(
        moving_variance, variance, BN_DECAY)
    tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
    tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)

    mean, variance = control_flow_ops.cond(
        c['is_training'], lambda: (mean, variance),
        lambda: (moving_mean, moving_variance))

    x = tf.nn.batch_normalization(x, mean, variance, beta, gamma, BN_EPSILON)
    #x.set_shape(inputs.get_shape()) ??

    return x

def fc(x, c):
    num_units_in = x.get_shape()[1]
    num_units_out = c['fc_units_out']
    weights_initializer = tf.truncated_normal_initializer(
        stddev=FC_WEIGHT_STDDEV)

    weights = _get_variable('weights',
                            shape=[num_units_in, num_units_out],
                            initializer=weights_initializer,
                            weight_decay=FC_WEIGHT_STDDEV)
    biases = _get_variable('biases',
                           shape=[num_units_out],
                           initializer=tf.zeros_initializer)
    x = tf.nn.xw_plus_b(x, weights, biases)
    return x

def _get_variable(name,
                  shape,
                  initializer,
                  weight_decay=0.0,
                  dtype='float',
                  trainable=True):
    "A little wrapper around tf.get_variable to do weight decay and add to"
    "resnet collection"
    if weight_decay > 0:
        regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
    else:
        regularizer = None
    collections = [tf.GraphKeys.VARIABLES, RESNET_VARIABLES]
    return tf.get_variable(name,
                           shape=shape,
                           initializer=initializer,
                           dtype=dtype,
                           regularizer=regularizer,
                           collections=collections,
                           trainable=trainable)

def conv(x, c):
    ksize = c['ksize']
    stride = c['stride']
    filters_out = c['conv_filters_out']

    filters_in = x.get_shape()[-1]
    shape = [ksize, ksize, filters_in, filters_out]
    initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV)
    weights = _get_variable('weights',
                            shape=shape,
                            dtype='float',
                            initializer=initializer,
                            weight_decay=CONV_WEIGHT_DECAY)
    return tf.nn.conv2d(x, weights, [1, stride, stride, 1], padding='SAME')

def _max_pool(x, ksize=3, stride=2):
    return tf.nn.max_pool(x,
                          ksize=[1, ksize, ksize, 1],
                          strides=[1, stride, stride, 1],
                          padding='SAME')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2015: Alignak team, see AUTHORS.txt file for contributors
#
# This file is part of Alignak.
#
# Alignak is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Alignak is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Alignak.  If not, see <http://www.gnu.org/licenses/>.
#
#
# This file incorporates work covered by the following copyright and
# permission notice:
#
#  Copyright (C) 2009-2014:
#     Thibault Cohen, titilambert@gmail.com
#     Jean Gabes, naparuba@gmail.com
#     Zoran Zaric, zz@zoranzaric.de

#  This file is part of Shinken.
#
#  Shinken is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Shinken is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

"""
 This class is an application that launches actions like
 notifications or event handlers
 The reactionner listens to the Arbiter for the configuration sent through
 the given port as first argument.
 The configuration sent by the arbiter specifies from which schedulers the
 will take actions.
 When the reactionner is already launched and has its own conf, it keeps
 on listening the arbiter (one a timeout)
 In case the arbiter has a new conf to send, the reactionner forget its old
 schedulers (and the associated actions) and take the new ones instead.
"""

import sys
import os
import optparse

# Try to see if we are in an android device or not
import imp
try:
    imp.find_module('android')
    is_android = True
except ImportError:
    is_android = False

if is_android is not None:
    # Add our main script dir
    if os.path.exists('/sdcard/sl4a/scripts/'):
        sys.path.append('/sdcard/sl4a/scripts/')
        os.chdir('/sdcard/sl4a/scripts/')

from alignak.daemons.reactionnerdaemon import Reactionner
from alignak import __version__

def main():
    """Parse args and run main daemon function

    :return: None
    """
    parser = optparse.OptionParser(
        "%prog [options]", version="%prog " + __version__)
    parser.add_option('-c', '--config',
                      dest="config_file", metavar="INI-CONFIG-FILE",
                      help='Config file')
    parser.add_option('-d', '--daemon', action='store_true',
                      dest="is_daemon",
                      help="Run in daemon mode")
    parser.add_option('-r', '--replace', action='store_true',
                      dest="do_replace",
                      help="Replace previous running reactionner")
    parser.add_option('--debugfile', dest='debug_file',
                      help=("Debug file. Default: not used "
                            "(why debug a bug free program? :) )"))
    parser.add_option("-p", "--profile",
                      dest="profile",
                      help="Dump a profile file. Need the python cProfile librairy")

    opts, args = parser.parse_args()
    if args:
        parser.error("Does not accept any argument.")

    daemon = Reactionner(debug=opts.debug_file is not None, **opts.__dict__)
    daemon.main()

if __name__ == '__main__':
    main()
<commit_msg>Enh: Pylint - C0103 for constants names in alignak_reactionner.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2015: Alignak team, see AUTHORS.txt file for contributors
#
# This file is part of Alignak.
#
# Alignak is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Alignak is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Alignak.  If not, see <http://www.gnu.org/licenses/>.
#
#
# This file incorporates work covered by the following copyright and
# permission notice:
#
#  Copyright (C) 2009-2014:
#     Thibault Cohen, titilambert@gmail.com
#     Jean Gabes, naparuba@gmail.com
#     Zoran Zaric, zz@zoranzaric.de

#  This file is part of Shinken.
#
#  Shinken is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Shinken is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

"""
 This class is an application that launches actions like
 notifications or event handlers
 The reactionner listens to the Arbiter for the configuration sent through
 the given port as first argument.
 The configuration sent by the arbiter specifies from which schedulers the
 will take actions.
 When the reactionner is already launched and has its own conf, it keeps
 on listening the arbiter (one a timeout)
 In case the arbiter has a new conf to send, the reactionner forget its old
 schedulers (and the associated actions) and take the new ones instead.
"""

import sys
import os
import optparse

# Try to see if we are in an android device or not
import imp
try:
    imp.find_module('android')
    IS_ANDROID = True
except ImportError:
    IS_ANDROID = False

if IS_ANDROID is not None:
    # Add our main script dir
    if os.path.exists('/sdcard/sl4a/scripts/'):
        sys.path.append('/sdcard/sl4a/scripts/')
        os.chdir('/sdcard/sl4a/scripts/')

from alignak.daemons.reactionnerdaemon import Reactionner
from alignak import __version__

def main():
    """Parse args and run main daemon function

    :return: None
    """
    parser = optparse.OptionParser(
        "%prog [options]", version="%prog " + __version__)
    parser.add_option('-c', '--config',
                      dest="config_file", metavar="INI-CONFIG-FILE",
                      help='Config file')
    parser.add_option('-d', '--daemon', action='store_true',
                      dest="is_daemon",
                      help="Run in daemon mode")
    parser.add_option('-r', '--replace', action='store_true',
                      dest="do_replace",
                      help="Replace previous running reactionner")
    parser.add_option('--debugfile', dest='debug_file',
                      help=("Debug file. Default: not used "
                            "(why debug a bug free program? :) )"))
    parser.add_option("-p", "--profile",
                      dest="profile",
                      help="Dump a profile file. Need the python cProfile librairy")

    opts, args = parser.parse_args()
    if args:
        parser.error("Does not accept any argument.")

    daemon = Reactionner(debug=opts.debug_file is not None, **opts.__dict__)
    daemon.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__version__ = '0.1'

import sys
import os
import urllib
import urllib2
import feedparser
import xml.etree.ElementTree as xml
import traceback

def open_file(to_read):
    """ Open a text file for reading, and strip the newlines
        returns a list, one list item per line
    """
    try:
        with open(to_read, 'r') as opened:
            return [got_lines.rstrip('\n') for got_lines in opened.readlines()]
    except IOError:
        print "Couldn't read values from %s\nCan't continue." % to_read
        raise

def item_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing item data
    """
    # Shunt each 'content' block into a list
    item_parsed = [a['content'][0]['value'] for a in fp_object.entries]
    # Shunt each 'title' and item ID value into a list
    item_title = [t['title'] for t in fp_object.entries]
    item_id = [i['zapi_key'] for i in fp_object.entries]
    items = []
    for index, content in enumerate(item_parsed):
        elem = xml.fromstring(content.encode('utf-8'))
        keys = [e.text for e in elem.iter('th')]
        values = [v.text for v in elem.iter('td')]
        zipped = dict(zip(keys, values))
        # add the utf-8 encoded 'title' and ID data to the dict:
        zipped['Title'] = item_title[index].encode('utf-8')
        zipped['ID'] = item_id[index].encode('utf-8')
        items.append(zipped)
    return items

def collections_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing collection titles and IDs
    """
    collections = []
    collection_key = [k['zapi_key'] for k in fp_object.entries]
    collection_title = [t['title'] for t in fp_object.entries]
    for index, content in enumerate(collection_key):
        collection_data = {}
        collection_data['ID'] = collection_key[index].encode('utf-8')
        collection_data['title'] = collection_title[index].encode('utf-8')
        collections.append(collection_data)
    return collections

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
        Most of the methods return Atom feed documents, which can be parsed
        using feedparser (http://www.feedparser.org/docs/)

        Valid optional URL parameters in all modes:
        format: "atom" or "bib", default: "atom"
        key: string, default:  null
        version: integer, default: null

        Valid optional URL parameters for format=atom:
        content: "none", "html", "bib", default: "html"
        order: string, name of field to be used, default: "dateAdded"
        sort: "asc", "desc", default varies by "order"
        limit: integer 1 - 99, default 50
        start: integer, default: 0
        pprint: boolean, default: false
    """
    user_id = None
    user_key = None

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        # Some API methods, not exhaustive
        self.api_methods = {
        'all_items':'/users/{u}/items',
        'top_level_items':'/users/{u}/items/top',
        'specific_item':'/users/{u}/items/{item}',
        'child_items':'/users/{u}/items/{item}/children',
        'item_tags':'/users/{u}/items/{item}/tags',
        'user_tags':'/users/{u}/tags',
        'items_for_tag':'/users/{u}/tags/{tag}/items',
        'collections':'/users/{u}/collections',
        'collection_items':'/users/{u}/collections/{collection}',
        'user_groups': '/users/{u}/groups',
        'group_items':'/groups/{group}/items'
        }

    def retrieve_data(self,
        request, url_params = None, request_params = None):
        """ Method for retrieving Zotero items via the API
            returns a dict containing feed items and lists of entries
        """
        # Add request parameter(s) if required
        if request_params:
            try:
                request_params['u'] = self.user_id
                request = urllib.quote(
                self.api_methods[request].format(**request_params))
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Otherwise, just add the user ID
        else:
            try:
                request = self.api_methods[request].format(u = self.user_id)
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Add URL parameters if they're passed
        if url_params:
            url_params['key'] = self.user_key
            data = urllib.urlencode(url_params)
        # Otherwise, just add the user key
        else:
            url_params = {'key': self.user_key}
            data = urllib.urlencode(url_params)
        request = '%s%s%s' % (request, '?', data)
        full_url = '%s%s' % (self.endpoint, request)
        data = urllib2.urlopen(full_url).read()
        # parse the result into Python data structures
        return feedparser.parse(data)

def main():
    """ main function
    """
    # Read a file from your cwd. Expects user id on line 1, key on line 2, LF
    auth_values = open_file(os.path.join(os.path.expanduser('~'),
    'zotero_keys.txt'))
    zot_id = auth_values[0]
    zot_key = auth_values[1]
    zot = Zotero(zot_id, zot_key)
    # Pass optional request parameters in a dict
    # req = {'item': 'T4AH4RZA'}
    par = {'limit': 2}
    item = zot.retrieve_data('top_level_items', par)
    # We can now do whatever we like with the returned data, e.g.:
    # We can pass our feedparser object to a helper function
    print item_data(item)

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, SystemExit):
        # actually raise these, for a clean exit
        raise
    except Exception, error:
        # all other exceptions: display the error
        print error
        # print "Stack trace:\n", traceback.print_exc(file = sys.stdout)
    else:
        pass
    finally:
        # exit cleanly once we've done everything else
        sys.exit(0)<commit_msg>More docstring faffery<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__version__ = '0.1'

import sys
import os
import urllib
import urllib2
import feedparser
import xml.etree.ElementTree as xml
import traceback

def open_file(to_read):
    """ Open a text file for reading, and strip the newlines
        returns a list, one list item per line
    """
    try:
        with open(to_read, 'r') as opened:
            return [got_lines.rstrip('\n') for got_lines in opened.readlines()]
    except IOError:
        print "Couldn't read values from %s\nCan't continue." % to_read
        raise

def item_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing item data
    """
    # Shunt each 'content' block into a list
    item_parsed = [a['content'][0]['value'] for a in fp_object.entries]
    # Shunt each 'title' and item ID value into a list
    item_title = [t['title'] for t in fp_object.entries]
    item_id = [i['zapi_key'] for i in fp_object.entries]
    items = []
    for index, content in enumerate(item_parsed):
        elem = xml.fromstring(content.encode('utf-8'))
        keys = [e.text for e in elem.iter('th')]
        values = [v.text for v in elem.iter('td')]
        zipped = dict(zip(keys, values))
        # add the utf-8 encoded 'title' and ID data to the dict:
        zipped['Title'] = item_title[index].encode('utf-8')
        zipped['ID'] = item_id[index].encode('utf-8')
        items.append(zipped)
    return items

def collections_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing collection titles and IDs
    """
    collections = []
    collection_key = [k['zapi_key'] for k in fp_object.entries]
    collection_title = [t['title'] for t in fp_object.entries]
    for index, content in enumerate(collection_key):
        collection_data = {}
        collection_data['ID'] = collection_key[index].encode('utf-8')
        collection_data['title'] = collection_title[index].encode('utf-8')
        collections.append(collection_data)
    return collections

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
        Most of the methods return Atom feed documents, which can be parsed
        using feedparser (http://www.feedparser.org/docs/)

        Valid optional URL parameters for all calls: ('key' added by default)
        format: "atom" or "bib", default: "atom"
        version: integer, default: null

        Valid optional URL parameters for format=atom:
        content: "none", "html", "bib", default: "html"
        order: string, name of field to be used, default: "dateAdded"
        sort: "asc", "desc", default varies by "order"
        limit: integer 1 - 99, default 50
        start: integer, default: 0
        pprint: boolean, default: false
    """
    user_id = None
    user_key = None

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        # Some API methods, not exhaustive
        self.api_methods = {
        'all_items':'/users/{u}/items',
        'top_level_items':'/users/{u}/items/top',
        'specific_item':'/users/{u}/items/{item}',
        'child_items':'/users/{u}/items/{item}/children',
        'item_tags':'/users/{u}/items/{item}/tags',
        'user_tags':'/users/{u}/tags',
        'items_for_tag':'/users/{u}/tags/{tag}/items',
        'collections':'/users/{u}/collections',
        'collection_items':'/users/{u}/collections/{collection}',
        'user_groups': '/users/{u}/groups',
        'group_items':'/groups/{group}/items'
        }

    def retrieve_data(self,
        request, url_params = None, request_params = None):
        """ Method for retrieving Zotero items via the API
            returns a dict containing feed items and lists of entries
        """
        # Add request parameter(s) if required
        if request_params:
            try:
                request_params['u'] = self.user_id
                request = urllib.quote(
                self.api_methods[request].format(**request_params))
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Otherwise, just add the user ID
        else:
            try:
                request = self.api_methods[request].format(u = self.user_id)
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Add URL parameters if they're passed
        if url_params:
            url_params['key'] = self.user_key
            data = urllib.urlencode(url_params)
        # Otherwise, just add the user key
        else:
            url_params = {'key': self.user_key}
            data = urllib.urlencode(url_params)
        request = '%s%s%s' % (request, '?', data)
        full_url = '%s%s' % (self.endpoint, request)
        data = urllib2.urlopen(full_url).read()
        # parse the result into Python data structures
        return feedparser.parse(data)

def main():
    """ main function
    """
    # Read a file from your cwd. Expects user id on line 1, key on line 2, LF
    auth_values = open_file(os.path.join(os.path.expanduser('~'),
    'zotero_keys.txt'))
    zot_id = auth_values[0]
    zot_key = auth_values[1]
    zot = Zotero(zot_id, zot_key)
    # Pass optional request parameters in a dict
    # req = {'item': 'T4AH4RZA'}
    par = {'limit': 2}
    item = zot.retrieve_data('top_level_items', par)
    # We can now do whatever we like with the returned data, e.g.:
    # We can pass our feedparser object to a helper function
    print item_data(item)

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, SystemExit):
        # actually raise these, for a clean exit
        raise
    except Exception, error:
        # all other exceptions: display the error
        print error
        # print "Stack trace:\n", traceback.print_exc(file = sys.stdout)
    else:
        pass
    finally:
        # exit cleanly once we've done everything else
        sys.exit(0)<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import datetime
import os

from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from mezzanine.blog.models import BlogPost, BlogCategory
from mezzanine.core.models import CONTENT_STATUS_PUBLISHED
from mezzanine.generic.models import AssignedKeyword, Keyword
from transliterate import slugify, translit
from transliterate.exceptions import LanguageDetectionError

def parse():
    folder = '/home/warmonger/Develop/Groups/PyNSK/pynsk/content/article/'

    _files = []
    for dir, _, files in os.walk(folder):
        _files.extend([os.path.join(dir, x) for x in files if x and x.endswith('.md')])

    categories = []
    for x in _files:
        with open(x, 'r') as fio:
            head = [next(fio) for _ in range(4)]

            space = next(fio).strip()
            while not space:
                space = next(fio)

            categories.append(head[3].replace('Category: ', '').replace('-', ' ').replace(' ', ' ').strip())

    categories = list(set(categories))

    for x in categories:
        try:
            if not x:
                continue
            slug = translit(x.strip(), reversed=True).replace(' ', '-').lower()
            BlogCategory.objects.get_or_create(title=x, slug=slug)
        except LanguageDetectionError as e:
            print(e, x)

    for x in _files:
        with open(x, 'r') as fio:
            head = [next(fio) for _ in range(4)]

            space = next(fio).strip()
            while not space:
                space = next(fio)

            raw_title = head[0].replace('Title: ', '').strip()

            slug = slugify(raw_title)

            date = datetime.datetime.strptime(
                    head[1].replace('Date: ', '').strip(), "%Y-%m-%d %H:%M")
            tags = [x.strip() for x in head[2].replace('Tags: ', '').strip().split(',') if x and x.strip()]

            category = head[3].replace('Category: ', '').replace('-', ' ').replace(' ', ' ').strip()
            text = fio.read().replace('!embedlycard', '!embed')

            post, created = BlogPost.objects.get_or_create(
                    title=raw_title,
                    slug=slug,
                    publish_date=date,
                    status=CONTENT_STATUS_PUBLISHED,
                    user=User.objects.get(username='admin'),
                    content=text,
            )

            for tag in tags:
                keyword, _ = Keyword.objects.get_or_create(title=tag)
                post.keywords.add(AssignedKeyword(keyword=keyword))

            post.categories.add(BlogCategory.objects.get_or_create(title=category)[0])

class Command(BaseCommand):
    def handle(self, *args, **options):
        parse()
<commit_msg>К команде import_pynsk добавил аргумент - путь до папки с контентом<commit_after># -*- encoding: utf-8 -*-
import datetime
import os

from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from mezzanine.blog.models import BlogPost, BlogCategory
from mezzanine.core.models import CONTENT_STATUS_PUBLISHED
from mezzanine.generic.models import AssignedKeyword, Keyword
from transliterate import slugify, translit
from transliterate.exceptions import LanguageDetectionError

def parse(folder: str):
    _files = []
    for dir, _, files in os.walk(folder):
        _files.extend([os.path.join(dir, x) for x in files if x and x.endswith('.md')])

    categories = []
    for x in _files:
        with open(x, 'r') as fio:
            head = [next(fio) for _ in range(4)]

            space = next(fio).strip()
            while not space:
                space = next(fio)

            categories.append(head[3].replace('Category: ', '').replace('-', ' ').replace(' ', ' ').strip())

    categories = list(set(categories))

    for x in categories:
        try:
            if not x:
                continue
            slug = translit(x.strip(), reversed=True).replace(' ', '-').lower()
            BlogCategory.objects.get_or_create(title=x, slug=slug)
        except LanguageDetectionError as e:
            print(e, x)

    for x in _files:
        with open(x, 'r') as fio:
            head = [next(fio) for _ in range(4)]

            space = next(fio).strip()
            while not space:
                space = next(fio)

            raw_title = head[0].replace('Title: ', '').strip()

            slug = slugify(raw_title)

            date = datetime.datetime.strptime(
                    head[1].replace('Date: ', '').strip(), "%Y-%m-%d %H:%M")
            tags = [x.strip() for x in head[2].replace('Tags: ', '').strip().split(',') if x and x.strip()]

            category = head[3].replace('Category: ', '').replace('-', ' ').replace(' ', ' ').strip()
            text = fio.read().replace('!embedlycard', '!embed')

            post, created = BlogPost.objects.get_or_create(
                    title=raw_title,
                    slug=slug,
                    publish_date=date,
                    status=CONTENT_STATUS_PUBLISHED,
                    user=User.objects.get(username='admin'),
                    content=text,
            )

            for tag in tags:
                keyword, _ = Keyword.objects.get_or_create(title=tag)
                post.keywords.add(AssignedKeyword(keyword=keyword))

            post.categories.add(BlogCategory.objects.get_or_create(title=category)[0])

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('folder', type=str)

    def handle(self, *args, **options):
        # '/home/warmonger/Develop/Groups/PyNSK/pynsk/content/article/'
        if 'folder' in options:
            parse(options['folder'])
        else:
            print('Not found folder path')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    Alignak REST backend client library
    ===================================

    This module is a Python library used for connecting to an Alignak backend.
"""
# Application version and manifest
VERSION = (0, 6, 1)

__application__ = u"Alignak Backend client"
__short_version__ = '.'.join((str(each) for each in VERSION[:2]))
__version__ = '.'.join((str(each) for each in VERSION[:4]))
__author__ = u"Alignak team"
__copyright__ = u"(c) 2015-2016 - %s" % __author__
__license__ = u"GNU Affero General Public License, version 3"
__description__ = u"Alignak backend client library"
__releasenotes__ = u"""Alignak backend client library"""
__doc_url__ = "https://github.com/Alignak-monitoring-contrib/alignak-backend-client"
# Application manifest
manifest = {
    'name': __application__,
    'version': __version__,
    'author': __author__,
    'description': __description__,
    'copyright': __copyright__,
    'license': __license__,
    'release': __releasenotes__,
    'doc': __doc_url__
}
<commit_msg>Set version as 0.6.2<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    Alignak REST backend client library
    ===================================

    This module is a Python library used for connecting to an Alignak backend.
"""
# Application version and manifest
VERSION = (0, 6, 2)

__application__ = u"Alignak Backend client"
__short_version__ = '.'.join((str(each) for each in VERSION[:2]))
__version__ = '.'.join((str(each) for each in VERSION[:4]))
__author__ = u"Alignak team"
__copyright__ = u"(c) 2015-2016 - %s" % __author__
__license__ = u"GNU Affero General Public License, version 3"
__description__ = u"Alignak backend client library"
__releasenotes__ = u"""Alignak backend client library"""
__doc_url__ = "https://github.com/Alignak-monitoring-contrib/alignak-backend-client"
# Application manifest
manifest = {
    'name': __application__,
    'version': __version__,
    'author': __author__,
    'description': __description__,
    'copyright': __copyright__,
    'license': __license__,
    'release': __releasenotes__,
    'doc': __doc_url__
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__version__ = '0.1'

import sys
import os
import urllib
import urllib2
import feedparser
import xml.etree.ElementTree as xml
import traceback

def open_file(to_read):
    """ Open a text file for reading, and strip the newlines
        returns a list, one list item per line
    """
    try:
        with open(to_read, 'r') as opened:
            return [got_lines.rstrip('\n') for got_lines in opened.readlines()]
    except IOError:
        print "Couldn't read values from %s\nCan't continue." % to_read
        raise

def item_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing item data
    """
    # Shunt each 'content' block into a list
    item_parsed = [a['content'][0]['value'] for a in fp_object.entries]
    # Shunt each 'title' and item ID value into a list
    item_title = [t['title'] for t in fp_object.entries]
    item_id = [i['zapi_key'] for i in fp_object.entries]
    items = []
    for index, content in enumerate(item_parsed):
        elem = xml.fromstring(content.encode('utf-8'))
        keys = [e.text for e in elem.iter('th')]
        values = [v.text for v in elem.iter('td')]
        zipped = dict(zip(keys, values))
        # add the utf-8 encoded 'title' and ID data to the dict:
        zipped['Title'] = item_title[index].encode('utf-8')
        zipped['ID'] = item_id[index].encode('utf-8')
        items.append(zipped)
    return items

def collections_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing collection titles and IDs
    """
    collections = []
    collection_key = [k['zapi_key'] for k in fp_object.entries]
    collection_title = [t['title'] for t in fp_object.entries]
    for index, content in enumerate(collection_key):
        collection_data = {}
        collection_data['ID'] = collection_key[index].encode('utf-8')
        collection_data['title'] = collection_title[index].encode('utf-8')
        collections.append(collection_data)
    return collections

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
        Most of the methods return Atom feed documents, which can be parsed
        using feedparser (http://www.feedparser.org/docs/)

        Valid optional URL parameters in all modes:
        format: "atom" or "bib", default: "atom"
        key: string, default:  null
        version: integer, default: null

        Valid optional URL parameters for format=atom:
        content: "none", "html", "bib", default: "html"
        order: string, name of field to be used, default: "dateAdded"
        sort: "asc", "desc", default varies by "order"
        limit: integer 1 - 99, default 50
        start: integer, default: 0
        pprint: boolean, default: false
    """
    user_id = None
    user_key = None

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        # Some API methods, not exhaustive
        self.api_methods = {
        'all_items':'/users/{u}/items',
        'top_level_items':'/users/{u}/items/top',
        'specific_item':'/users/{u}/items/{item}',
        'child_items':'/users/{u}/items/{item}/children',
        'item_tags':'/users/{u}/items/{item}/tags',
        'user_tags':'/users/{u}/tags',
        'items_for_tag':'/users/{u}/tags/{tag}/items',
        'collections':'/users/{u}/collections',
        'collection_items':'/users/{u}/collections/{collection}',
        'user_groups': '/users/{u}/groups',
        'group_items':'/groups/{group}/items'
        }

    def retrieve_data(self,
        request, url_params = None, request_params = None):
        """ Method for retrieving Zotero items via the API
            returns a dict containing feed items and lists of entries
        """
        # Add request parameter(s) if required
        if request_params:
            try:
                request_params['u'] = self.user_id
                request = urllib.quote(
                self.api_methods[request].format(**request_params))
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Otherwise, just add the user ID
        else:
            try:
                request = self.api_methods[request].format(u = self.user_id)
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Add URL parameters if they're passed
        if url_params:
            url_params['key'] = self.user_key
            data = urllib.urlencode(url_params)
        # Otherwise, just add the user key
        else:
            url_params = {'key': self.user_key}
            data = urllib.urlencode(url_params)
        request = '%s%s%s' % (request, '?', data)
        full_url = '%s%s' % (self.endpoint, request)
        data = urllib2.urlopen(full_url).read()
        # parse the result into Python data structures
        return feedparser.parse(data)

def main():
    """ main function
    """
    # Read a file from your cwd. Expects user id on line 1, key on line 2, LF
    auth_values = open_file(os.path.join(os.path.expanduser('~'),
    'zotero_keys.txt'))
    zot_id = auth_values[0]
    zot_key = auth_values[1]
    zot = Zotero(zot_id, zot_key)
    # Pass optional request parameters in a dict
    # req = {'item': 'T4AH4RZA'}
    par = {'limit': 2}
    item = zot.retrieve_data('top_level_items', par)
    # We can now do whatever we like with the returned data, e.g.:
    """ title_id = [j for j in zip([t['title'] for t in item.entries],
    [z['zapi_key'] for z in item.entries])]
    for entry in title_id:
        print entry """
    # We can pass our feedparser object to this helper function
    useful = item_data(item)
    print useful

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, SystemExit):
        # actually raise these, for a clean exit
        raise
    except Exception, error:
        # all other exceptions: display the error
        print error
        # print "Stack trace:\n", traceback.print_exc(file = sys.stdout)
    else:
        pass
    finally:
        # exit cleanly once we've done everything else
        sys.exit(0)<commit_msg>Getting rid of more pointless variables<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__version__ = '0.1'

import sys
import os
import urllib
import urllib2
import feedparser
import xml.etree.ElementTree as xml
import traceback

def open_file(to_read):
    """ Open a text file for reading, and strip the newlines
        returns a list, one list item per line
    """
    try:
        with open(to_read, 'r') as opened:
            return [got_lines.rstrip('\n') for got_lines in opened.readlines()]
    except IOError:
        print "Couldn't read values from %s\nCan't continue." % to_read
        raise

def item_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing item data
    """
    # Shunt each 'content' block into a list
    item_parsed = [a['content'][0]['value'] for a in fp_object.entries]
    # Shunt each 'title' and item ID value into a list
    item_title = [t['title'] for t in fp_object.entries]
    item_id = [i['zapi_key'] for i in fp_object.entries]
    items = []
    for index, content in enumerate(item_parsed):
        elem = xml.fromstring(content.encode('utf-8'))
        keys = [e.text for e in elem.iter('th')]
        values = [v.text for v in elem.iter('td')]
        zipped = dict(zip(keys, values))
        # add the utf-8 encoded 'title' and ID data to the dict:
        zipped['Title'] = item_title[index].encode('utf-8')
        zipped['ID'] = item_id[index].encode('utf-8')
        items.append(zipped)
    return items

def collections_data(fp_object):
    """ Takes the result of a parse operation, and returns a list containing
        one or more dicts containing collection titles and IDs
    """
    collections = []
    collection_key = [k['zapi_key'] for k in fp_object.entries]
    collection_title = [t['title'] for t in fp_object.entries]
    for index, content in enumerate(collection_key):
        collection_data = {}
        collection_data['ID'] = collection_key[index].encode('utf-8')
        collection_data['title'] = collection_title[index].encode('utf-8')
        collections.append(collection_data)
    return collections

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
        Most of the methods return Atom feed documents, which can be parsed
        using feedparser (http://www.feedparser.org/docs/)

        Valid optional URL parameters in all modes:
        format: "atom" or "bib", default: "atom"
        key: string, default:  null
        version: integer, default: null

        Valid optional URL parameters for format=atom:
        content: "none", "html", "bib", default: "html"
        order: string, name of field to be used, default: "dateAdded"
        sort: "asc", "desc", default varies by "order"
        limit: integer 1 - 99, default 50
        start: integer, default: 0
        pprint: boolean, default: false
    """
    user_id = None
    user_key = None

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        # Some API methods, not exhaustive
        self.api_methods = {
        'all_items':'/users/{u}/items',
        'top_level_items':'/users/{u}/items/top',
        'specific_item':'/users/{u}/items/{item}',
        'child_items':'/users/{u}/items/{item}/children',
        'item_tags':'/users/{u}/items/{item}/tags',
        'user_tags':'/users/{u}/tags',
        'items_for_tag':'/users/{u}/tags/{tag}/items',
        'collections':'/users/{u}/collections',
        'collection_items':'/users/{u}/collections/{collection}',
        'user_groups': '/users/{u}/groups',
        'group_items':'/groups/{group}/items'
        }

    def retrieve_data(self,
        request, url_params = None, request_params = None):
        """ Method for retrieving Zotero items via the API
            returns a dict containing feed items and lists of entries
        """
        # Add request parameter(s) if required
        if request_params:
            try:
                request_params['u'] = self.user_id
                request = urllib.quote(
                self.api_methods[request].format(**request_params))
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Otherwise, just add the user ID
        else:
            try:
                request = self.api_methods[request].format(u = self.user_id)
            except KeyError:
                print 'There\'s a request parameter missing:'
                raise
        # Add URL parameters if they're passed
        if url_params:
            url_params['key'] = self.user_key
            data = urllib.urlencode(url_params)
        # Otherwise, just add the user key
        else:
            url_params = {'key': self.user_key}
            data = urllib.urlencode(url_params)
        request = '%s%s%s' % (request, '?', data)
        full_url = '%s%s' % (self.endpoint, request)
        data = urllib2.urlopen(full_url).read()
        # parse the result into Python data structures
        return feedparser.parse(data)

def main():
    """ main function
    """
    # Read a file from your cwd. Expects user id on line 1, key on line 2, LF
    auth_values = open_file(os.path.join(os.path.expanduser('~'),
    'zotero_keys.txt'))
    zot_id = auth_values[0]
    zot_key = auth_values[1]
    zot = Zotero(zot_id, zot_key)
    # Pass optional request parameters in a dict
    # req = {'item': 'T4AH4RZA'}
    par = {'limit': 2}
    item = zot.retrieve_data('top_level_items', par)
    # We can now do whatever we like with the returned data, e.g.:
    """ title_id = [j for j in zip([t['title'] for t in item.entries],
    [z['zapi_key'] for z in item.entries])]
    for entry in title_id:
        print entry """
    # We can pass our feedparser object to a helper function
    print item_data(item)

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, SystemExit):
        # actually raise these, for a clean exit
        raise
    except Exception, error:
        # all other exceptions: display the error
        print error
        # print "Stack trace:\n", traceback.print_exc(file = sys.stdout)
    else:
        pass
    finally:
        # exit cleanly once we've done everything else
        sys.exit(0)<|endoftext|>