<commit_before>import ssl

import logbook
import logbook.more
from aiohttp import web

from threema.gateway import (
    Connection,
    util,
)
from threema.gateway.e2e import (
    add_callback_route,
    create_application,
)

async def handle_message(message):
    print('Got message ({}): {}'.format(repr(message), message))

def serve():
    # Create connection instance
    connection = Connection(
        identity='*YOUR_GATEWAY_THREEMA_ID',
        secret='YOUR_GATEWAY_THREEMA_ID_SECRET',
        key='private:YOUR_PRIVATE_KEY'
    )

    # Create the application and register the handler for incoming messages
    application = create_application(connection)
    add_callback_route(connection, application, handle_message, path='/gateway_callback')

    # Create an SSL context to terminate TLS.
    # Note: It is usually advisable to use a reverse proxy instead in front of
    #       the server that terminates TLS, e.g. Nginx.
    ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    ssl_context.load_cert_chain(certfile='YOUR_CERTFILE', keyfile='YOUR_KEYFILE')

    # Run a server that listens on any interface via port 8443. It will
    # gracefully shut down when Ctrl+C has been pressed.
    web.run_app(application, port=8443, ssl_context=ssl_context)

if __name__ == '__main__':
    util.enable_logging(logbook.WARNING)
    log_handler = logbook.more.ColorizedStderrHandler()
    with log_handler.applicationbound():
        serve()
<commit_msg>Fix the callback server example<commit_after>import ssl

import logbook
import logbook.more
from aiohttp import web

from threema.gateway import (
    Connection,
    e2e,
    util,
)

async def handle_message(message):
    print('Got message ({}): {}'.format(repr(message), message))

async def create_application():
    # Create connection instance
    connection = Connection(
        identity='*YOUR_GATEWAY_THREEMA_ID',
        secret='YOUR_GATEWAY_THREEMA_ID_SECRET',
        key='private:YOUR_PRIVATE_KEY'
    )

    # Create the application and register the handler for incoming messages
    application = e2e.create_application(connection)
    e2e.add_callback_route(
        connection, application, handle_message, path='/gateway_callback')
    return application

def main():
    # Create an SSL context to terminate TLS.
    # Note: It is usually advisable to use a reverse proxy instead in front of
    #       the server that terminates TLS, e.g. Nginx.
    ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    ssl_context.load_cert_chain(certfile='YOUR_CERTFILE', keyfile='YOUR_KEYFILE')

    # Run a server that listens on any interface via port 8443. It will
    # gracefully shut down when Ctrl+C has been pressed.
    web.run_app(create_application(), port=8443, ssl_context=ssl_context)

if __name__ == '__main__':
    util.enable_logging(logbook.WARNING)
    log_handler = logbook.more.ColorizedStderrHandler()
    with log_handler.applicationbound():
        main()
<|endoftext|>


<commit_before>import time
import asyncore
import socket
import traceback
from cPickle import dumps, loads

from .queue import Queue

queue_dict = dict()

def get_queue(queue_id):
    try:
        return queue_dict[queue_id]
    except KeyError:
        pass

    q = queue_dict[queue_id] = Queue()
    return q

class DelayedResult(Exception): pass

class Session(asyncore.dispatcher):
    def __init__(self, session_id, sock):
        asyncore.dispatcher.__init__(self, sock)
        self.result_ready = False
        self.session_id = session_id

    def writable(self):
        return self.result_ready

    def handle_read(self):
        data_len = self.recv(10)
        if not data_len:
            return

        data = self.recv(int(data_len))

        args = loads(data)
        method = args[0]
        args = args[1:]
        #print self.session_id, method, args

        try:
            result = getattr(self, 'do_' + method)(*args)
        except DelayedResult:
            #print self.session_id, 'DL'
            pass
        except Exception, e:
            traceback.print_exc()
            self.send_result(False, str(e))
        else:
            self.send_result(True, result)

    def handle_write(self):
        self.result_ready = False
        self.sendall(self.result)

    def handle_close(self):
        print 'session %d closed' % self.session_id
        for q in queue_dict.itervalues():
            q.session_done(self.session_id)

        self.close()

    def send_result(self, status, result):
        #print self.session_id, 'R', status, result
        self.result = dumps((status, result))
        self.result = str(len(self.result)).ljust(10) + self.result
        self.result_ready = True

    def do_put(self, queue_id, message, message_id, priority, group):
        return get_queue(queue_id).put(message, message_id, priority, group)

    def do_wait_ack(self, queue_id, message_id):
        get_queue(queue_id).wait_for_ack(self.session_id, message_id, self.on_ack)
        raise DelayedResult()

    def on_ack(self, result):
        self.send_result(True, result)

    def do_get(self, queue_id, multi, block):
        queue = get_queue(queue_id)
        msg_id, message = queue.get(self.session_id, multi)
        if not msg_id and block:
            queue.wait_for_message(self.session_id, self.on_message)
            raise DelayedResult()

        return msg_id, message

    def on_message(self, msg_id, message):
        self.send_result(True, (msg_id, message))

    def do_ack(self, queue_id, message_id, result):
        get_queue(queue_id).ack(message_id, result)

    def do_reput(self, queue_id, message_id, delay):
        get_queue(queue_id).reput(message_id, delay)

    def do_dump(self):
        now = time.time()
        result = {}

        for queue_id, queue in queue_dict.iteritems():
            messages, pmessages = result[queue_id] = [], queue.pmessages
            for msg_id, message in queue.all_messages():
                messages.append((msg_id, message.message, message.priority,
                    int(now - message.enterdate), message.get))

        return result

    def do_size(self, queue_id):
        return len(get_queue(queue_id))

class Server(asyncore.dispatcher):
    def __init__(self, addr):
        asyncore.dispatcher.__init__(self)
        self.addr = addr
        self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.bind(addr)
        self.listen(10)
        self.session_counter = 0
        print "Queue server started", self.addr

    def handle_accept(self):
        channel, addr = self.accept()
        self.session_counter += 1
        Session(self.session_counter, channel)

    def run(self):
        asyncore.loop()

    def writable(self):
        return False<commit_msg>watch for timeouted jobs<commit_after>import os
import time
import asyncore
import socket
import traceback
import threading
from cPickle import dumps, loads

from .queue import Queue

queue_dict = dict()

def get_queue(queue_id):
    try:
        return queue_dict[queue_id]
    except KeyError:
        pass

    q = queue_dict[queue_id] = Queue()
    return q

class DelayedResult(Exception): pass

class Session(asyncore.dispatcher):
    def __init__(self, session_id, sock):
        asyncore.dispatcher.__init__(self, sock)
        self.result_ready = False
        self.session_id = session_id

    def writable(self):
        return self.result_ready

    def handle_read(self):
        data_len = self.recv(10)
        if not data_len:
            return

        data = self.recv(int(data_len))

        args = loads(data)
        method = args[0]
        args = args[1:]
        #print self.session_id, method, args

        try:
            result = getattr(self, 'do_' + method)(*args)
        except DelayedResult:
            #print self.session_id, 'DL'
            pass
        except Exception, e:
            traceback.print_exc()
            self.send_result(False, str(e))
        else:
            self.send_result(True, result)

    def handle_write(self):
        self.result_ready = False
        self.sendall(self.result)

    def handle_close(self):
        print 'session %d closed' % self.session_id
        for q in queue_dict.itervalues():
            q.session_done(self.session_id)

        self.close()

    def send_result(self, status, result):
        #print self.session_id, 'R', status, result
        self.result = dumps((status, result))
        self.result = str(len(self.result)).ljust(10) + self.result
        self.result_ready = True

    def do_put(self, queue_id, message, message_id, priority, group):
        return get_queue(queue_id).put(message, message_id, priority, group)

    def do_wait_ack(self, queue_id, message_id):
        get_queue(queue_id).wait_for_ack(self.session_id, message_id, self.on_ack)
        raise DelayedResult()

    def on_ack(self, result):
        self.send_result(True, result)

    def do_get(self, queue_id, multi, block):
        queue = get_queue(queue_id)
        msg_id, message = queue.get(self.session_id, multi)
        if not msg_id and block:
            queue.wait_for_message(self.session_id, self.on_message)
            raise DelayedResult()

        return msg_id, message

    def on_message(self, msg_id, message):
        self.send_result(True, (msg_id, message))

    def do_ack(self, queue_id, message_id, result):
        get_queue(queue_id).ack(message_id, result)

    def do_reput(self, queue_id, message_id, delay):
        get_queue(queue_id).reput(message_id, delay)

    def do_dump(self):
        now = time.time()
        result = {}

        for queue_id, queue in queue_dict.iteritems():
            messages, pmessages = result[queue_id] = [], queue.pmessages
            for msg_id, message in queue.all_messages():
                messages.append((msg_id, message.message, message.priority,
                    int(now - message.enterdate), message.get))

        return result

    def do_size(self, queue_id):
        return len(get_queue(queue_id))

class Timer(asyncore.file_dispatcher):
    def handle_read(self):
        self.recv(1)
        for q in queue_dict.itervalues():
            q.check_for_new_messages()

def timer_feeder(fd):
    while True:
        time.sleep(10)
        os.write(fd, '0')

class Server(asyncore.dispatcher):
    def __init__(self, addr):
        asyncore.dispatcher.__init__(self)
        self.addr = addr
        self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.bind(addr)
        self.listen(10)
        self.session_counter = 0
        print "Queue server started", self.addr

    def handle_accept(self):
        channel, addr = self.accept()
        self.session_counter += 1
        Session(self.session_counter, channel)

    def run(self):
        r, w = os.pipe()
        self.timer = Timer(r)
        t = threading.Thread(target=timer_feeder, args=(w,))
        t.daemon = True
        t.start()
        asyncore.loop()

    def writable(self):
        return False<|endoftext|>


<commit_before>from django.utils import http
from django.core import cache
import simplejson

MOCK_REDIS_CACHE = None
COUCH_CACHE_TIMEOUT = 43200

DEBUG_TRACE = False

def rcache():
    return MOCK_REDIS_CACHE or cache.get_cache('redis')

CACHED_VIEW_PREFIX = '#cached_view_'

#the actual payload of the cached_doc
CACHED_DOC_PREFIX = '#cached_doc_'

#for a given doc_id, a reverse relationship to see which views are touched by this cache
#value is a list of the cached_view keys
CACHED_VIEW_DOC_REVERSE = '#reverse_cached_doc_'

def key_doc_id(doc_id):
    """
    Redis cache key for a full couch document by doc_id
    """
    ret = "%s%s" % (CACHED_DOC_PREFIX, doc_id)
    return ret

def key_reverse_doc(doc_id, suffix):
    """
    a doc_id to tell you if a cached doc is in a view
    """
    ret = "%s%s_%s" % (CACHED_VIEW_DOC_REVERSE, doc_id, suffix)
    return ret

def key_view_full(view_name, params):
    cache_view_key = "%(prefix)s_%(view_name)s_%(params_str)s" % {
        "prefix": CACHED_VIEW_PREFIX,
        "view_name": view_name,
        "params_str": http.urlquote('|'.join(["%s:%s" % (k, v) for k, v in params.items()]))
    }
    return cache_view_key

def key_view_partial(view_name_suffix):
    """
    Given a partial view_key (just basically prefixing by view_name, don't care about params)
    Used for invalidating ALL views of a given view_name, or hand building a view cache key
    """
    cache_view_key = "%(prefix)s_%(view_name)s" % {
        "prefix": CACHED_VIEW_PREFIX,
        "view_name": view_name_suffix,
    }
    return cache_view_key

def purge_by_doc_id(doc_id):
    """
    For a given doc_id, delete it and all reverses.

    return: tuple of (true|false existed or not, last_version)
    """
    #first by just individual doc
    doc_key = key_doc_id(doc_id)
    exists = rcache().get(doc_key, None)

    if exists:
        rcache().delete(key_doc_id(doc_id))
        #then all the reverse indices by that doc_id
        rcache().delete_pattern(key_reverse_doc(doc_id, '*'))
        return True, exists
    else:
        return False, None

    #todo:
    #walk all views that are seen by that doc_id and blow away too?
    #if the pillow was not running this would def. be the case.

def purge_view(view_key):
    """
    view_name: you want to put in your key, does not necessarily have to be the right one...wildcard it yo
    """
    cached_view = rcache().get(view_key, None)
    if cached_view:
        results = simplejson.loads(cached_view)

        def gen_doc_ids(results):
            if isinstance(results, dict):
                for x in results['row_stubs']:
                    yield x['id']
            elif isinstance(results, list):
                for x in results:
                    if 'id' in x:
                        yield x['id']

        for doc_id in gen_doc_ids(results):
            reverse_doc_key = key_reverse_doc(doc_id, cached_view)
            rcache().delete(reverse_doc_key)

    #then purge this key
    rcache().delete(view_key)

def cached_open_doc(db, doc_id, cache_expire=COUCH_CACHE_TIMEOUT, **params):
    """
    Main wrapping function to open up a doc. Replace db.open_doc(doc_id)
    """
    cached_doc = _get_cached_doc_only(doc_id)
    if not cached_doc:
        doc = db.open_doc(doc_id, **params)
        do_cache_doc(doc, cache_expire=cache_expire)
        return doc
    else:
        return cached_doc

def _get_cached_doc_only(doc_id):
    """
    helper cache retrieval method for open_doc - for use by views in retrieving their docs.
    """
    doc = rcache().get(key_doc_id(doc_id), None)
    if doc is not None:
        return simplejson.loads(doc)
    else:
        #uh oh, how do i retrieve this doc now?
        return None

def do_cache_doc(doc, cache_expire=COUCH_CACHE_TIMEOUT):
    rcache().set(key_doc_id(doc['_id']), simplejson.dumps(doc), timeout=cache_expire)

def cache_view_doc(doc_or_docid, view_key, cache_expire=COUCH_CACHE_TIMEOUT):
    """
    Cache by doc_id a reverse lookup of views that it is mutually dependent on
    """
    id = doc_or_docid['_id'] if isinstance(doc_or_docid, dict) else doc_or_docid
    key = key_reverse_doc(id, view_key)
    rcache().set(key, 1, timeout=cache_expire)

    if isinstance(doc_or_docid, dict):
        do_cache_doc(doc_or_docid)

def cached_view(db, view_name, wrapper=None, cache_expire=COUCH_CACHE_TIMEOUT, **params):
    """
    Call a view and cache the results, return cached if it's a cache hit.

    db: couch database to do query on
    view_name, params: couch view call parameters

    Note, a view call with include_docs=True will not be wrapped, you must wrap it on your own.

    """
    include_docs = params.get('include_docs', False)

    cache_view_key = key_view_full(view_name, params)
    if include_docs:
        #include_docs=True results in couchdbkit remove the 'rows' result and returns just the actual rows in an array
        cached_view = rcache().get(cache_view_key, None)
        if cached_view:
            results = simplejson.loads(cached_view)
            final_results = {}

            if include_docs:
                row_stubs = results['row_stubs']

                final_results['total_rows'] = results['total_rows']
                final_results['offset'] = results['offset']

                rows = []
                for stub in row_stubs:
                    row = {
                        "id": stub['id'],
                        "value": None,
                        "key": stub["key"],
                        "doc": _get_cached_doc_only(stub["id"])
                    }
                    rows.append(row)
                if wrapper:
                    final_results = [wrapper(x['doc']) for x in rows]
                else:
                    #final_results['rows'] = rows
                    final_results = rows
            return final_results
        else:
            #cache miss, get view, cache it and all docs
            view_obj = db.view(view_name, **params)
            view_obj._fetch_if_needed()
            view_results = view_obj._result_cache
            row_stubs = []

            for row in view_results["rows"]:
                stub = {
                    "id": row['id'],
                    "value": None,
                    "key": row["key"],
                }
                cache_view_doc(row["doc"], cache_view_key, cache_expire=cache_expire)
                row_stubs.append(stub)

            cached_results = {
                "total_rows": view_results['total_rows'],
                "offset": view_results['offset'],
                "row_stubs": row_stubs
            }

            if wrapper:
                retval = [wrapper(x['doc']) for x in view_results['rows']]
            else:
                retval = view_results['rows']
            rcache().set(cache_view_key, simplejson.dumps(cached_results), timeout=cache_expire)
            return retval

    else:
        ###########################
        # include_docs=False just returns the entire view verbatim
        cached_view = rcache().get(cache_view_key, None)
        if cached_view:
            results = simplejson.loads(cached_view)
            return results
        else:
            view_results = db.view(view_name, **params).all()
            rcache().set(cache_view_key, simplejson.dumps(view_results), timeout=cache_expire)
            for row in view_results:
                doc_id = row.get('id', None)
                if doc_id:
                    #a non reduce view will have doc_ids on each row, we want to reverse index these
                    #to know when to invalidate
                    cache_view_doc(doc_id, cache_view_key)
            return view_results
<commit_msg>adjusting cached_view to also do a doc_get if the doc isn't cached<commit_after>from django.utils import http
from django.core import cache
import simplejson

MOCK_REDIS_CACHE = None
COUCH_CACHE_TIMEOUT = 43200

DEBUG_TRACE = False

def rcache():
    return MOCK_REDIS_CACHE or cache.get_cache('redis')

CACHED_VIEW_PREFIX = '#cached_view_'

#the actual payload of the cached_doc
CACHED_DOC_PREFIX = '#cached_doc_'

#for a given doc_id, a reverse relationship to see which views are touched by this cache
#value is a list of the cached_view keys
CACHED_VIEW_DOC_REVERSE = '#reverse_cached_doc_'

def key_doc_id(doc_id):
    """
    Redis cache key for a full couch document by doc_id
    """
    ret = "%s%s" % (CACHED_DOC_PREFIX, doc_id)
    return ret

def key_reverse_doc(doc_id, suffix):
    """
    a doc_id to tell you if a cached doc is in a view
    """
    ret = "%s%s_%s" % (CACHED_VIEW_DOC_REVERSE, doc_id, suffix)
    return ret

def key_view_full(view_name, params):
    cache_view_key = "%(prefix)s_%(view_name)s_%(params_str)s" % {
        "prefix": CACHED_VIEW_PREFIX,
        "view_name": view_name,
        "params_str": http.urlquote('|'.join(["%s:%s" % (k, v) for k, v in params.items()]))
    }
    return cache_view_key

def key_view_partial(view_name_suffix):
    """
    Given a partial view_key (just basically prefixing by view_name, don't care about params)
    Used for invalidating ALL views of a given view_name, or hand building a view cache key
    """
    cache_view_key = "%(prefix)s_%(view_name)s" % {
        "prefix": CACHED_VIEW_PREFIX,
        "view_name": view_name_suffix,
    }
    return cache_view_key

def purge_by_doc_id(doc_id):
    """
    For a given doc_id, delete it and all reverses.

    return: tuple of (true|false existed or not, last_version)
    """
    #first by just individual doc
    doc_key = key_doc_id(doc_id)
    exists = rcache().get(doc_key, None)

    if exists:
        rcache().delete(key_doc_id(doc_id))
        #then all the reverse indices by that doc_id
        rcache().delete_pattern(key_reverse_doc(doc_id, '*'))
        return True, exists
    else:
        return False, None

    #todo:
    #walk all views that are seen by that doc_id and blow away too?
    #if the pillow was not running this would def. be the case.

def purge_view(view_key):
    """
    view_name: you want to put in your key, does not necessarily have to be the right one...wildcard it yo
    """
    cached_view = rcache().get(view_key, None)
    if cached_view:
        results = simplejson.loads(cached_view)

        def gen_doc_ids(results):
            if isinstance(results, dict):
                for x in results['row_stubs']:
                    yield x['id']
            elif isinstance(results, list):
                for x in results:
                    if 'id' in x:
                        yield x['id']

        for doc_id in gen_doc_ids(results):
            reverse_doc_key = key_reverse_doc(doc_id, cached_view)
            rcache().delete(reverse_doc_key)

    #then purge this key
    rcache().delete(view_key)

def cached_open_doc(db, doc_id, cache_expire=COUCH_CACHE_TIMEOUT, **params):
    """
    Main wrapping function to open up a doc. Replace db.open_doc(doc_id)
    """
    cached_doc = _get_cached_doc_only(doc_id)
    if not cached_doc:
        doc = db.open_doc(doc_id, **params)
        do_cache_doc(doc, cache_expire=cache_expire)
        return doc
    else:
        return cached_doc

def _get_cached_doc_only(doc_id):
    """
    helper cache retrieval method for open_doc - for use by views in retrieving their docs.
    """
    doc = rcache().get(key_doc_id(doc_id), None)
    if doc is not None:
        return simplejson.loads(doc)
    else:
        return None

def do_cache_doc(doc, cache_expire=COUCH_CACHE_TIMEOUT):
    rcache().set(key_doc_id(doc['_id']), simplejson.dumps(doc), timeout=cache_expire)

def cache_view_doc(doc_or_docid, view_key, cache_expire=COUCH_CACHE_TIMEOUT):
    """
    Cache by doc_id a reverse lookup of views that it is mutually dependent on
    """
    id = doc_or_docid['_id'] if isinstance(doc_or_docid, dict) else doc_or_docid
    key = key_reverse_doc(id, view_key)
    rcache().set(key, 1, timeout=cache_expire)

    if isinstance(doc_or_docid, dict):
        do_cache_doc(doc_or_docid)

def cached_view(db, view_name, wrapper=None, cache_expire=COUCH_CACHE_TIMEOUT, **params):
    """
    Call a view and cache the results, return cached if it's a cache hit.

    db: couch database to do query on
    view_name, params: couch view call parameters

    Note, a view call with include_docs=True will not be wrapped, you must wrap it on your own.

    """
    include_docs = params.get('include_docs', False)

    cache_view_key = key_view_full(view_name, params)
    if include_docs:
        #include_docs=True results in couchdbkit remove the 'rows' result and returns just the actual rows in an array
        cached_view = rcache().get(cache_view_key, None)
        if cached_view:
            results = simplejson.loads(cached_view)
            final_results = {}

            if include_docs:
                row_stubs = results['row_stubs']

                final_results['total_rows'] = results['total_rows']
                final_results['offset'] = results['offset']

                rows = []
                for stub in row_stubs:
                    row = {
                        "id": stub['id'],
                        "value": None,
                        "key": stub["key"],
                        #this feels hacky, but for some reason other views are squashing the master cached doc.
                        #a more true invalidation scheme should have this more readily address this, but for now
                        #do a db call here and cache it. Should be a _cached_doc_only call here
                        "doc": cached_open_doc(db, stub['id'])
                    }
                    rows.append(row)
                if wrapper:
                    final_results = [wrapper(x['doc']) for x in rows]
                else:
                    #final_results['rows'] = rows
                    final_results = rows
            return final_results
        else:
            #cache miss, get view, cache it and all docs
            view_obj = db.view(view_name, **params)
            view_obj._fetch_if_needed()
            view_results = view_obj._result_cache
            row_stubs = []

            for row in view_results["rows"]:
                stub = {
                    "id": row['id'],
                    "value": None,
                    "key": row["key"],
                }
                cache_view_doc(row["doc"], cache_view_key, cache_expire=cache_expire)
                row_stubs.append(stub)

            cached_results = {
                "total_rows": view_results['total_rows'],
                "offset": view_results['offset'],
                "row_stubs": row_stubs
            }

            if wrapper:
                retval = [wrapper(x['doc']) for x in view_results['rows']]
            else:
                retval = view_results['rows']
            rcache().set(cache_view_key, simplejson.dumps(cached_results), timeout=cache_expire)
            return retval

    else:
        ###########################
        # include_docs=False just returns the entire view verbatim
        cached_view = rcache().get(cache_view_key, None)
        if cached_view:
            results = simplejson.loads(cached_view)
            return results
        else:
            view_results = db.view(view_name, **params).all()
            rcache().set(cache_view_key, simplejson.dumps(view_results), timeout=cache_expire)
            for row in view_results:
                doc_id = row.get('id', None)
                if doc_id:
                    #a non reduce view will have doc_ids on each row, we want to reverse index these
                    #to know when to invalidate
                    cache_view_doc(doc_id, cache_view_key)
            return view_results
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import JsonResponse
from django.shortcuts import get_object_or_404
from django.views.generic import TemplateView

from digest.models import Issue, Item
from frontend.models import EditorMaterial

class Sitemap(TemplateView):
    content_type = 'text/xml'
    template_name = 'sitemap.html'

    def get_context_data(self, **kwargs):
        ctx = super(Sitemap, self).get_context_data(**kwargs)
        items = [
            {'loc': '',
             'changefreq': 'weekly',},
            {'loc': reverse('digest:issues'),
             'changefreq': 'weekly',},
            {'loc': reverse('digest:feed'),
             'changefreq': 'daily',},
        ]

        for issue in Issue.objects.filter(status='active'):
            items.append({'loc': issue.link, 'changefreq': 'weekly',})

        for item in Item.objects.filter(status='active',
                                        activated_at__lte=datetime.datetime.now()):
            items.append(
                    {'loc': '/view/%s' % item.pk,
                     'changefreq': 'never',})

        ctx.update(
                {'records': items,
                 'domain': 'http://%s' % settings.BASE_DOMAIN})
        return ctx

class Index(TemplateView):
    """Главная страница."""
    template_name = 'index.html'
    model = Issue
    context_object_name = 'index'

    def get_context_data(self, **kwargs):
        context = super(Index, self).get_context_data(**kwargs)
        issue = False
        try:
            issue = self.model.objects.filter(status='active').latest(
                    'published_at')
        except Issue.DoesNotExist:
            pass

        items = []
        if issue:
            qs = issue.item_set.filter(status='active')
            items = qs.order_by('-section__priority', '-priority')

        context.update(
                {'issue': issue,
                 'items': items,
                 'active_menu_item': 'home',})
        return context

class ViewEditorMaterial(TemplateView):
    template_name = 'old/editor_material_view.html'

    def get_context_data(self, **kwargs):
        section = kwargs.get('section', 'landing')
        slug = kwargs.get('slug')

        material = get_object_or_404(EditorMaterial,
                                     slug=slug,
                                     section=section,
                                     status='active')

        return {'material': material}

def get_items_json(request, year, month, day):
    result = {}
    items = Item.objects.filter(
            status='active',
            is_editors_choice=True,
            activated_at__year=int(year),
            activated_at__month=int(month),
            activated_at__day=int(day),
    )
    result['ok'] = bool(items)
    if items:
        keys = [
            'title',
            'description',
            'section',
            'link',
            'language',
        ]
        result['items'] = list(items.values(*keys))
    return JsonResponse(result)
<commit_msg>EDIT: доработал прошлый коммит<commit_after># -*- coding: utf-8 -*-
import datetime

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import JsonResponse
from django.shortcuts import get_object_or_404
from django.views.generic import TemplateView

from digest.models import Issue, Item
from frontend.models import EditorMaterial

class Sitemap(TemplateView):
    content_type = 'text/xml'
    template_name = 'sitemap.html'

    def get_context_data(self, **kwargs):
        ctx = super(Sitemap, self).get_context_data(**kwargs)
        items = [
            {'loc': '',
             'changefreq': 'weekly',},
            {'loc': reverse('digest:issues'),
             'changefreq': 'weekly',},
            {'loc': reverse('digest:feed'),
             'changefreq': 'daily',},
        ]

        for issue in Issue.objects.filter(status='active'):
            items.append({'loc': issue.link, 'changefreq': 'weekly',})

        for item in Item.objects.filter(status='active',
                                        activated_at__lte=datetime.datetime.now()):
            items.append(
                    {'loc': '/view/%s' % item.pk,
                     'changefreq': 'never',})

        ctx.update(
                {'records': items,
                 'domain': 'http://%s' % settings.BASE_DOMAIN})
        return ctx

class Index(TemplateView):
    """Главная страница."""
    template_name = 'index.html'
    model = Issue
    context_object_name = 'index'

    def get_context_data(self, **kwargs):
        context = super(Index, self).get_context_data(**kwargs)
        issue = False
        try:
            issue = self.model.objects.filter(status='active').latest(
                    'published_at')
        except Issue.DoesNotExist:
            pass

        items = []
        if issue:
            qs = issue.item_set.filter(status='active')
            items = qs.order_by('-section__priority', '-priority')

        context.update(
                {'issue': issue,
                 'items': items,
                 'active_menu_item': 'home',})
        return context

class ViewEditorMaterial(TemplateView):
    template_name = 'old/editor_material_view.html'

    def get_context_data(self, **kwargs):
        section = kwargs.get('section', 'landing')
        slug = kwargs.get('slug')

        material = get_object_or_404(EditorMaterial,
                                     slug=slug,
                                     section=section,
                                     status='active')

        return {'material': material}

def get_items_json(request, year, month, day):
    result = {}
    items = Item.objects.filter(
            status='active',
            is_editors_choice=True,
            activated_at__year=int(year),
            activated_at__month=int(month),
            activated_at__day=int(day),
    )
    result['ok'] = bool(items)
    if items:
        keys = [
            'title',
            'description',
            'section__title',
            'link',
            'language',
        ]
        result['items'] = list(items.values(*keys))
    return JsonResponse(result)
<|endoftext|>


<commit_before>import tensorflow as tf
import functools

from mathcv.config import config

def lazy_property(function):
    attribute = '_cache_' + function.__name__

    @property
    @functools.wraps(function)
    def decorator(self):
        if not hasattr(self, attribute):
            setattr(self, attribute, function(self))
        return getattr(self, attribute)

    return decorator

class Model:
    model_count = 0

    def __init__(self, images, labels, vocab_size):
        self.id = Model.model_count
        Model.model_count += 1
        self.images = images
        self.labels = labels
        self.vocab_size = vocab_size
        self.prediction
        self.loss
        self.accuracy

    @lazy_property
    def prediction(self):
        # Convolution layers
        conv1 = self.build_convolution_layer(self.images, [5, 5, 1, 32], 1, 'conv_layer_1')
        conv2 = self.build_convolution_layer(conv1, [5, 5, 32, 64], 2, 'conv_layer_2')
        conv3 = self.build_convolution_layer(conv2, [5, 5, 64, 128], 1, 'conv_layer_3')
        conv4 = self.build_convolution_layer(conv3, [5, 5, 128, 256], 2, 'conv_layer_4')
        conv5 = self.build_convolution_layer(conv4, [5, 5, 256, 384], 1, 'conv_layer_5')

        # Encoder - Fully connected layer
        with tf.variable_scope('encoder') as scope:
            fc_out = tf.contrib.layers.fully_connected(tf.contrib.layers.flatten(conv5), config['label_length'], scope='fcl')

        # Decoder
        with tf.variable_scope('decoder') as scope:
            lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(config['decoder_memory_dim'], state_is_tuple=True)
            rnn_outputs, rnn_states = tf.nn.dynamic_rnn(lstm_fw_cell, tf.expand_dims(fc_out, -1), dtype=tf.float32, scope='rnn')
            out = tf.contrib.layers.fully_connected(rnn_outputs, self.vocab_size, scope='fcl')
        return out

    @lazy_property
    def loss(self):
        assert self.labels is not None
        assert self.prediction is not None
        with tf.variable_scope('loss') as scope:
            out = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.prediction, labels=self.labels))
            tf.summary.scalar('clone_loss', out)
            tf.add_to_collection(tf.GraphKeys.LOSSES, out)
        return out

    @lazy_property
    def accuracy(self):
        assert self.labels is not None
        assert self.prediction is not None
        with tf.variable_scope('accuracy') as scope:
            int_predictions = tf.argmax(self.prediction, axis=2)
            out = tf.reduce_mean(tf.cast(tf.equal(self.labels, int_predictions), tf.float32), name='accuracy')
            tf.summary.scalar('clone_accuracy', out)
        return out

    @staticmethod
    def build_convolution_layer(input_feed, shape, k, scope_name, dropout=config['dropout_prob'], activation=tf.nn.relu):
        with tf.variable_scope(scope_name) as scope:
            weights = tf.get_variable('weights', shape, initializer=tf.random_normal_initializer())
            bias = tf.get_variable('bias', shape[3], initializer=tf.constant_initializer(0))
            conv = tf.nn.bias_add(tf.nn.conv2d(input_feed, weights, strides=[1, 1, 1, 1], padding='SAME'), bias)
            conv = activation(conv)
            pool = tf.nn.max_pool(conv, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')
            out = tf.nn.dropout(pool, dropout)
        return out
<commit_msg>increase tensor size<commit_after>import tensorflow as tf
import functools

from mathcv.config import config

def lazy_property(function):
    attribute = '_cache_' + function.__name__

    @property
    @functools.wraps(function)
    def decorator(self):
        if not hasattr(self, attribute):
            setattr(self, attribute, function(self))
        return getattr(self, attribute)

    return decorator

class Model:
    model_count = 0

    def __init__(self, images, labels, vocab_size):
        self.id = Model.model_count
        Model.model_count += 1
        self.images = images
        self.labels = labels
        self.vocab_size = vocab_size
        self.prediction
        self.loss
        self.accuracy

    @lazy_property
    def prediction(self):
        # Convolution layers
        conv1 = self.build_convolution_layer(self.images, [5, 5, 1, 32], 1, 'conv_layer_1')
        conv2 = self.build_convolution_layer(conv1, [5, 5, 32, 64], 2, 'conv_layer_2')
        conv3 = self.build_convolution_layer(conv2, [5, 5, 64, 128], 1, 'conv_layer_3')
        conv4 = self.build_convolution_layer(conv3, [5, 5, 128, 256], 2, 'conv_layer_4')
        conv5 = self.build_convolution_layer(conv4, [5, 5, 256, 384], 1, 'conv_layer_5')

        # Encoder - Fully connected layer
        with tf.variable_scope('encoder') as scope:
            fc_out = tf.contrib.layers.fully_connected(tf.contrib.layers.flatten(conv5), 2048, scope='fcl')

        # Decoder
        with tf.variable_scope('decoder') as scope:
            lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(config['decoder_memory_dim'], state_is_tuple=True)
            rnn_outputs, rnn_states = tf.nn.dynamic_rnn(lstm_fw_cell, tf.expand_dims(fc_out, -1), dtype=tf.float32, scope='rnn')
            out = tf.contrib.layers.fully_connected(rnn_outputs, self.vocab_size, scope='fcl')
        return out

    @lazy_property
    def loss(self):
        assert self.labels is not None
        assert self.prediction is not None
        with tf.variable_scope('loss') as scope:
            out = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.prediction, labels=self.labels))
            tf.summary.scalar('clone_loss', out)
            tf.add_to_collection(tf.GraphKeys.LOSSES, out)
        return out

    @lazy_property
    def accuracy(self):
        assert self.labels is not None
        assert self.prediction is not None
        with tf.variable_scope('accuracy') as scope:
            int_predictions = tf.argmax(self.prediction, axis=2)
            out = tf.reduce_mean(tf.cast(tf.equal(self.labels, int_predictions), tf.float32), name='accuracy')
            tf.summary.scalar('clone_accuracy', out)
        return out

    @staticmethod
    def build_convolution_layer(input_feed, shape, k, scope_name, dropout=config['dropout_prob'], activation=tf.nn.relu):
        with tf.variable_scope(scope_name) as scope:
            weights = tf.get_variable('weights', shape, initializer=tf.random_normal_initializer())
            bias = tf.get_variable('bias', shape[3], initializer=tf.constant_initializer(0))
            conv = tf.nn.bias_add(tf.nn.conv2d(input_feed, weights, strides=[1, 1, 1, 1], padding='SAME'), bias)
            conv = activation(conv)
            pool = tf.nn.max_pool(conv, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')
            out = tf.nn.dropout(pool, dropout)
        return out
<|endoftext|>


<commit_before>from __future__ import print_function
import httplib2
import os

from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage

import datetime
import rfc3339      # for date object -> date string
import iso8601      # for date string -> date object

import ConfigParser
import time
import pytz

from enum import Enum

#import paho.mqtt.client as mqtt
from threading import Thread
from Queue import Queue

#TODO unused
class EcsCommand(Enum):
    OFF         = 1
    ON          = 2

class EcsHeatProfile(Enum):
    LOW         = 1
    MEDIUM      = 2
    HIGH        = 3

#defines
ECS_STATE_OFF = "OFF"
ECS_STATE_ON = "ON"

#Globals
nextEventfromCalendar = None

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/calendar-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Client ECS'

HEAT_MANAGER_PERIOD         = 1
DELAY_BETWEEN_SCHED_CHECK   = 1
DELAY_BETWEEN_AGENDA_CHECK  = 10  # in multiples of SCHED check
NB_EVENTS_TO_GET_FROM_CALENDAR  = 1

class ThermoEvent:
        def __init__(self, title, value, start, end):
                self.title = title
                self.value = value
                self.start = start
                self.end = end

#=========================================================================#
#                      callback definition for MQTT                       #
#=========================================================================#

# The callback for when the client receives a CONNACK response from the server.
def on_connect(client, userdata, flags, rc):
    print("Connected with result code "+str(rc) + "\n")

def on_message(client, userdata, msg):
    print(msg.topic+" "+str(msg.payload) + "\n")

#=========================================================================#
#           helper functions for timezone and format conversion           #
#=========================================================================#
def get_date_object(date_string):
  return iso8601.parse_date(date_string)

def get_date_string(date_object):
  return rfc3339.rfc3339(date_object)

#=========================================================================#
# Functions for Google calendar data retrieval                            #
# Coming from Google API example                                          #
# https://developers.google.com/google-apps/calendar/quickstart/python    #
#=========================================================================#

def get_credentials():
    """Gets valid user credentials from storage.

    If nothing has been stored, or if the stored credentials are invalid,
    the OAuth2 flow is completed to obtain the new credentials.

    Returns:
        Credentials, the obtained credential.
    """
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'calendar-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        print('Storing credentials to ' + credential_path)
    return credentials

def turnEcsOn(): print ("Turning ECS ON :", time.time())

def getEventsFromCalendar(calendarId):
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
    service = discovery.build('calendar', 'v3', http=http)

    now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time

    mynow  = datetime.datetime.utcnow()
    myweek = datetime.timedelta(days=7)

    start_date = mynow.isoformat() + 'Z'
    end_date = (mynow + myweek).isoformat() + 'Z'

    print('Getting the upcoming events')
    eventsResult = service.events().list(
        calendarId=calendarId, timeMin=start_date, timeMax=end_date, maxResults=NB_EVENTS_TO_GET_FROM_CALENDAR, singleEvents=True,
        orderBy='startTime').execute()
    events = eventsResult.get('items', [])

    return events

#=========================================================================#
#           Heat profile to temperature conversion                        #
#=========================================================================#

def getTargetTemperature(profile):
    if (profile == EcsHeatProfile.HIGH):
        return 55
    elif (profile == EcsHeatProfile.MEDIUM):
        return 50
    elif (profile == EcsHeatProfile.LOW):
        return 45
    else:
        return 50

#=========================================================================#
# Heat manager thread body                                                #
# processes messages from Queue :                                         #
#       - ECS state change requests (from Ecs Scheduler thread)           #
#       - ECS force state (from MQTTLoop thread)                          #
#       - temperature(s)  (from MQTTLoop thread)                          #
#=========================================================================#
def heatManager(msqQueue, mqttClient):
    ecsState = ECS_STATE_OFF
    ecsStateForced = False
    ecsTemperature = 0
    heatProfile  = EcsHeatProfile.MEDIUM

    while True:
        print ("HeatManager waiting for message")
        msg = msqQueue.get()
        msgType = msg[0]
        msgValue = msg[1]
        print ("HeatManager waking up. message received")

        #process messages
        if(msgType == "ECS_STATE_CHANGE"):
            if (ecsStateForced is False):
                if (msgValue == ECS_STATE_OFF):
                    print("Heat Manager : turning ECS OFF")
                    ecsState = ECS_STATE_OFF
                    #mqttClient.publish("ECS/state", payload='1', qos=0, retain=False)

                elif (msgValue == ECS_STATE_ON):
                    heatProfile = msg[2]
                    ecsState = ECS_STATE_ON
                    if(ecsTemperature < getTargetTemperature(heatProfile)):
                        print("Heat Manager : turning ECS ON")
                        #mqttClient.publish("ECS/state", payload='2', qos=0, retain=False)
                    else:
                        print("Heat Manager : No ECS ON despite calendar, due to target temperature already reached")
                else:
                    print("Heat Manager : Error : unknown EcsState %s in received message" % msgValue)

        elif(msgType == "ECS_TEMPERATURE"):
            ecsTemperature = msgValue
            print ("updating temperature : %i", msgValue)
            #Check against temperature target when ECS is ON and not in forced mode
            if ((ecsState == ECS_STATE_ON) and (ecsStateForced is False)):
                if (ecsTemperature < getTargetTemperature(heatProfile)):
                    print("Heat Manager : Switching ECS OFF despite calendar, due to target temperature reached")
                    ecsState = ECS_STATE_OFF
                    #mqttClient.publish("ECS/state", payload='1', qos=0, retain=False)

        elif(msgType == "ECS_FORCE"):
            if (msgValue == ECS_STATE_OFF):
                print("Heat Manager : Forcing ECS OFF")
                ecsStateForced = True
                if (ecsState == ECS_STATE_ON):
                    print("\tHeat Manager : Switching ECS OFF")
                    #mqttClient.publish("ECS/state", payload='1', qos=0, retain=False)
                    ecsState = ECS_STATE_OFF
            elif (msgValue == ECS_STATE_ON):
                print("Heat Manager : Forcing ECS ON")
                ecsStateForced = True
                if (ecsState == ECS_STATE_OFF):
                    print("\tHeat Manager : Switching ECS ON")
                    #mqttClient.publish("ECS/state", payload='2', qos=0, retain=False)
                    ecsState = ECS_STATE_ON

            else:
                print("Heat Manager : Unknown message value %s " % msgValue)

        else:
            print("Heat Manager : Unknown message type %s " % msgType)

#=========================================================================#
# Calendar monitor thread body                                            #
# Periodically checks for events defined in cloud                         #
#=========================================================================#
def calendarMonitor(calendarId):
    global nextEventfromCalendar
    while True:
        events = getEventsFromCalendar(calendarId)
        nextEventfromCalendar = None
        if not events:
            print('No upcoming events found.')
        else:
            myevent = events[0]
            title = myevent['summary']
            content = ""
            start = get_date_object(myevent['start'].get('dateTime'))
            end   = get_date_object(myevent['end'].get('dateTime'))
            nextEventfromCalendar = ThermoEvent(title, content, start, end)

        time.sleep(DELAY_BETWEEN_AGENDA_CHECK)

#=========================================================================#
# MQTT Loop thread body                                                   #
# Subscribes to MQTT topics, and then send messages to heatManager        #
# upon MQTT message reception                                             #
#=========================================================================#
def mqttLoop(heatMgrQueue, mqttClient):
    count = 0
    while True:
        heatMgrQueue.put(("ECS_TEMPERATURE" , 22 + count))
        if(count == 30):
            heatMgrQueue.put(("ECS_FORCE" , ECS_STATE_OFF))
        if(count == 75):
            heatMgrQueue.put(("ECS_FORCE" , ECS_STATE_ON))
        count += 1
        time.sleep(1)

#=========================================================================#
# ECS State scheduler                                                     #
# Sends commands to heatManager based on periodical checks of             #
# next event defined by Calendar monitor                                  #
#=========================================================================#
def ecsStateScheduler(heatMgrQueue):
    logNoEventDisplayed     = False
    ecsState = ECS_STATE_OFF
    nextStartTime = ""
    while True:
        if(nextEventfromCalendar):
            parisTz = pytz.timezone('Europe/Paris')
            now = parisTz.localize(datetime.datetime.now())
            #print ("START : ", nextEventfromCalendar.start)
            #print ("END : ", nextEventfromCalendar.end)
            #print ("NOW : ", now)
            nextStartTime = nextEventfromCalendar.start - now
            if(nextEventfromCalendar.start < now):
                if(logNoEventDisplayed == False):
                    print("No event Scheduled")
                    logNoEventDisplayed = True
            else:
                print("Next Start in :", nextStartTime)

            if (now > nextEventfromCalendar.start) and  (now < nextEventfromCalendar.end):
                nextEndTime = nextEventfromCalendar.end - now
                print("Next End in :", nextEndTime)
                if(ecsState == ECS_STATE_OFF):
                    ecsState = ECS_STATE_ON
                    if(nextEventfromCalendar.title == "HIGH"):
                        heatProfile = "HIGH"
                    elif (nextEventfromCalendar.title == "LOW"):
                        heatProfile = "LOW"
                    else:
                        heatProfile = "MEDIUM"
                    heatMgrQueue.put(("ECS_STATE_CHANGE", ECS_STATE_ON, heatProfile))
                    print("Switching ECS STATE :", ecsState)

            elif(ecsState != ECS_STATE_OFF):
                ecsState = ECS_STATE_OFF
                heatMgrQueue.put(("ECS_STATE_CHANGE", ECS_STATE_OFF))
                print("Switching ECS STATE :", ecsState)
                logNextEndDisplayed = False
                logNoEventDisplayed = False

        else:
            if(ecsState == ECS_STATE_ON):
                logNoEventDisplayed = False
                ecsState = ECS_STATE_OFF
                heatMgrQueue.put(("ECS_STATE_CHANGE", ECS_STATE_OFF))
                print("Switching ECS STATE :", ecsState)

            if(logNoEventDisplayed == False):
                print("No event Scheduled")
                logNoEventDisplayed = True

        time.sleep(DELAY_BETWEEN_SCHED_CHECK)

#=========================================================================#
# Main...                                                                 #
#=========================================================================#
def main():
    config = ConfigParser.ConfigParser()
    config.read('myconf.conf')
    calendarId = config.get('Calendar', 'calendarId')
    mqttAddress = config.get('MQTT', 'mqttAddress')

    heatMgrQueue = Queue()

    mqttClient = ""
 #   mqttClient = mqtt.Client()
 #   mqttClient.on_connect = on_connect
 #   mqttClient.on_message = on_message
 #   mqttClient.connect(MQTT_ADDRESS)
 #   mqttClient.loop_start()

    CalendarMonitorThread = Thread(target=calendarMonitor, args=(calendarId,))
    CalendarMonitorThread.start()
    EcsStateSchedulerThread = Thread(target=ecsStateScheduler, args=(heatMgrQueue,))
    EcsStateSchedulerThread.start()
    HeatManagerThread = Thread(target=heatManager, args=(heatMgrQueue,mqttClient,))
    HeatManagerThread.start()
    MqttLoopThread = Thread(target=mqttLoop, args=(heatMgrQueue,mqttClient,))
    MqttLoopThread.start()

if __name__ == '__main__':
    main()
<commit_msg>small change on status<commit_after>from __future__ import print_function
import httplib2
import os

from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage

import datetime
import rfc3339      # for date object -> date string
import iso8601      # for date string -> date object

import ConfigParser
import time
import pytz

from enum import Enum

#import paho.mqtt.client as mqtt
from threading import Thread
from Queue import Queue

#TODO unused
class EcsCommand(Enum):
    OFF         = 1
    ON          = 2

class EcsHeatProfile(Enum):
    LOW         = 1
    MEDIUM      = 2
    HIGH        = 3

#defines
ECS_STATE_OFF = "OFF"
ECS_STATE_ON = "ON"

#Globals
nextEventfromCalendar = None

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/calendar-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Client ECS'

HEAT_MANAGER_PERIOD         = 1
DELAY_BETWEEN_SCHED_CHECK   = 1
DELAY_BETWEEN_AGENDA_CHECK  = 10  # in multiples of SCHED check
NB_EVENTS_TO_GET_FROM_CALENDAR  = 1

class ThermoEvent:
        def __init__(self, title, value, start, end):
                self.title = title
                self.value = value
                self.start = start
                self.end = end

#=========================================================================#
#                      callback definition for MQTT                       #
#=========================================================================#

# The callback for when the client receives a CONNACK response from the server.
def on_connect(client, userdata, flags, rc):
    print("Connected with result code "+str(rc) + "\n")

def on_message(client, userdata, msg):
    print(msg.topic+" "+str(msg.payload) + "\n")

#=========================================================================#
#           helper functions for timezone and format conversion           #
#=========================================================================#
def get_date_object(date_string):
  return iso8601.parse_date(date_string)

def get_date_string(date_object):
  return rfc3339.rfc3339(date_object)

#=========================================================================#
# Functions for Google calendar data retrieval                            #
# Coming from Google API example                                          #
# https://developers.google.com/google-apps/calendar/quickstart/python    #
#=========================================================================#

def get_credentials():
    """Gets valid user credentials from storage.

    If nothing has been stored, or if the stored credentials are invalid,
    the OAuth2 flow is completed to obtain the new credentials.

    Returns:
        Credentials, the obtained credential.
    """
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'calendar-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        print('Storing credentials to ' + credential_path)
    return credentials

def turnEcsOn(): print ("Turning ECS ON :", time.time())

def getEventsFromCalendar(calendarId):
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
    service = discovery.build('calendar', 'v3', http=http)

    now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time

    mynow  = datetime.datetime.utcnow()
    myweek = datetime.timedelta(days=7)

    start_date = mynow.isoformat() + 'Z'
    end_date = (mynow + myweek).isoformat() + 'Z'

    print('Getting the upcoming events')
    eventsResult = service.events().list(
        calendarId=calendarId, timeMin=start_date, timeMax=end_date, maxResults=NB_EVENTS_TO_GET_FROM_CALENDAR, singleEvents=True,
        orderBy='startTime').execute()
    events = eventsResult.get('items', [])

    return events

#=========================================================================#
#           Heat profile to temperature conversion                        #
#=========================================================================#

def getTargetTemperature(profile):
    if (profile == EcsHeatProfile.HIGH):
        return 55
    elif (profile == EcsHeatProfile.MEDIUM):
        return 50
    elif (profile == EcsHeatProfile.LOW):
        return 45
    else:
        return 50

#=========================================================================#
# Heat manager thread body                                                #
# processes messages from Queue :                                         #
#       - ECS state change requests (from Ecs Scheduler thread)           #
#       - ECS force state (from MQTTLoop thread)                          #
#       - temperature(s)  (from MQTTLoop thread)                          #
#=========================================================================#
def heatManager(msqQueue, mqttClient):
    ecsState = ECS_STATE_OFF
    ecsStateForced = False
    ecsTemperature = 0
    heatProfile  = EcsHeatProfile.MEDIUM

    while True:
        print ("HeatManager waiting for message")
        msg = msqQueue.get()
        msgType = msg[0]
        msgValue = msg[1]
        print ("HeatManager waking up. message received")

        #process messages
        if(msgType == "ECS_STATE_CHANGE"):
            if (ecsStateForced is False):
                if (msgValue == ECS_STATE_OFF):
                    print("Heat Manager : turning ECS OFF")
                    ecsState = ECS_STATE_OFF
                    #mqttClient.publish("ECS/state", payload='1', qos=0, retain=False)

                elif (msgValue == ECS_STATE_ON):
                    heatProfile = msg[2]
                    ecsState = ECS_STATE_ON
                    if(ecsTemperature < getTargetTemperature(heatProfile)):
                        print("Heat Manager : turning ECS ON")
                        #mqttClient.publish("ECS/state", payload='2', qos=0, retain=False)
                    else:
                        print("Heat Manager : No ECS ON despite calendar, due to target temperature already reached")
                else:
                    print("Heat Manager : Error : unknown EcsState %s in received message" % msgValue)

        elif(msgType == "ECS_TEMPERATURE"):
            ecsTemperature = msgValue
            print ("updating temperature : %i", msgValue)
            #Check against temperature target when ECS is ON and not in forced mode
            if ((ecsState == ECS_STATE_ON) and (ecsStateForced is False)):
                if (ecsTemperature < getTargetTemperature(heatProfile)):
                    print("Heat Manager : Switching ECS OFF despite calendar, due to target temperature reached")
                    ecsState = ECS_STATE_OFF
                    #mqttClient.publish("ECS/state", payload='1', qos=0, retain=False)

        elif(msgType == "ECS_FORCE"):
            if (msgValue == ECS_STATE_OFF):
                print("Heat Manager : Forcing ECS OFF")
                ecsStateForced = True
                if (ecsState == ECS_STATE_ON):
                    print("\tHeat Manager : Switching ECS OFF")
                    #mqttClient.publish("ECS/state", payload='1', qos=0, retain=False)
                    ecsState = ECS_STATE_OFF
            elif (msgValue == ECS_STATE_ON):
                print("Heat Manager : Forcing ECS ON")
                ecsStateForced = True
                if (ecsState == ECS_STATE_OFF):
                    print("\tHeat Manager : Switching ECS ON")
                    #mqttClient.publish("ECS/state", payload='2', qos=0, retain=False)
                    ecsState = ECS_STATE_ON

            else:
                print("Heat Manager : Unknown message value %s " % msgValue)

        else:
            print("Heat Manager : Unknown message type %s " % msgType)

#=========================================================================#
# Calendar monitor thread body                                            #
# Periodically checks for events defined in cloud                         #
#=========================================================================#
def calendarMonitor(calendarId):
    global nextEventfromCalendar
    while True:
        events = getEventsFromCalendar(calendarId)
        nextEventfromCalendar = None
        if not events:
            print('No upcoming events found.')
        else:
            myevent = events[0]
            title = myevent['summary']
            content = ""
            start = get_date_object(myevent['start'].get('dateTime'))
            end   = get_date_object(myevent['end'].get('dateTime'))
            nextEventfromCalendar = ThermoEvent(title, content, start, end)

        time.sleep(DELAY_BETWEEN_AGENDA_CHECK)

#=========================================================================#
# MQTT Loop thread body                                                   #
# Subscribes to MQTT topics, and then send messages to heatManager        #
# upon MQTT message reception                                             #
#=========================================================================#
def mqttLoop(heatMgrQueue, mqttClient):
    count = 0
    while True:
        heatMgrQueue.put(("ECS_TEMPERATURE" , 22 + count))
        if(count == 30):
            heatMgrQueue.put(("ECS_FORCE" , ECS_STATE_OFF))
        if(count == 75):
            heatMgrQueue.put(("ECS_FORCE" , ECS_STATE_ON))
        count += 1
        time.sleep(1)

#=========================================================================#
# ECS State scheduler                                                     #
# Sends commands to heatManager based on periodical checks of             #
# next event defined by Calendar monitor                                  #
#=========================================================================#
def ecsStateScheduler(heatMgrQueue):
    logNoEventDisplayed     = False
    ecsState = ECS_STATE_OFF
    nextStartTime = ""
    while True:
        if(nextEventfromCalendar):
            parisTz = pytz.timezone('Europe/Paris')
            now = parisTz.localize(datetime.datetime.now())
            #print ("START : ", nextEventfromCalendar.start)
            #print ("END : ", nextEventfromCalendar.end)
            #print ("NOW : ", now)
            nextStartTime = nextEventfromCalendar.start - now
            if(nextEventfromCalendar.start < now):
                if(logNoEventDisplayed == False):
                    print("No event Scheduled")
                    logNoEventDisplayed = True
            else:
                print("Next Start in :", nextStartTime)

            if (now > nextEventfromCalendar.start) and  (now < nextEventfromCalendar.end):
                nextEndTime = nextEventfromCalendar.end - now
                print("Next End in :", nextEndTime)
                if(ecsState == ECS_STATE_OFF):
                    ecsState = ECS_STATE_ON
                    if(nextEventfromCalendar.title == "HIGH"):
                        heatProfile = "HIGH"
                    elif (nextEventfromCalendar.title == "LOW"):
                        heatProfile = "LOW"
                    else:
                        heatProfile = "MEDIUM"
                    heatMgrQueue.put(("ECS_STATE_CHANGE", ECS_STATE_ON, heatProfile))
                    print("Switching ECS STATE :", ecsState)

            elif(ecsState != ECS_STATE_OFF):
                ecsState = ECS_STATE_OFF
                heatMgrQueue.put(("ECS_STATE_CHANGE", ECS_STATE_OFF))
                print("Switching ECS STATE :", ecsState)
                logNextEndDisplayed = False
                logNoEventDisplayed = False

        else:
            if(ecsState == ECS_STATE_ON):
                logNoEventDisplayed = False
                ecsState = ECS_STATE_OFF
                heatMgrQueue.put(("ECS_STATE_CHANGE", ECS_STATE_OFF))
                print("Switching ECS STATE :", ecsState)

            if(logNoEventDisplayed == False):
                print("No event Scheduled")
                logNoEventDisplayed = True

        time.sleep(DELAY_BETWEEN_SCHED_CHECK)

#=========================================================================#
# Main...                                                                 #
#=========================================================================#
def main():
    config = ConfigParser.ConfigParser()
    config.read('myconf.conf')
    calendarId = config.get('Calendar', 'calendarId')
    mqttAddress = config.get('MQTT', 'mqttAddress')

    heatMgrQueue = Queue()

    mqttClient = ""
 #   mqttClient = mqtt.Client()
 #   mqttClient.on_connect = on_connect
 #   mqttClient.on_message = on_message
 #   mqttClient.connect(MQTT_ADDRESS)
 #   mqttClient.loop_start()

    CalendarMonitorThread = Thread(target=calendarMonitor, args=(calendarId,))
    CalendarMonitorThread.start()
    EcsStateSchedulerThread = Thread(target=ecsStateScheduler, args=(heatMgrQueue,))
    EcsStateSchedulerThread.start()
    HeatManagerThread = Thread(target=heatManager, args=(heatMgrQueue,mqttClient,))
    HeatManagerThread.start()
    MqttLoopThread = Thread(target=mqttLoop, args=(heatMgrQueue,mqttClient,))
    MqttLoopThread.start()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.utils.pdf import get_pdf
from frappe.email.smtp import get_outgoing_email_account
from frappe.utils import get_url, scrub_urls, strip, expand_relative_urls, cint, split_emails, to_markdown, markdown
import email.utils

def get_email(recipients, sender='', msg='', subject='[No Subject]',
	text_content = None, footer=None, print_html=None, formatted=None, attachments=None,
	content=None, reply_to=None, cc=(), email_account=None):
	"""send an html email as multipart with attachments and all"""
	content = content or msg
	emailobj = EMail(sender, recipients, subject, reply_to=reply_to, cc=cc, email_account=None)

	if not content.strip().startswith("<"):
		content = markdown(content)

	emailobj.set_html(content, text_content, footer=footer, print_html=print_html, formatted=formatted)

	if isinstance(attachments, dict):
		attachments = [attachments]

	for attach in (attachments or []):
		emailobj.add_attachment(**attach)

	return emailobj

class EMail:
	"""
	Wrapper on the email module. Email object represents emails to be sent to the client.
	Also provides a clean way to add binary `FileData` attachments
	Also sets all messages as multipart/alternative for cleaner reading in text-only clients
	"""
	def __init__(self, sender='', recipients=(), subject='', alternative=0, reply_to=None, cc=(), email_account=None):
		from email.mime.multipart import MIMEMultipart
		from email import Charset
		Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8')

		if isinstance(recipients, basestring):
			recipients = recipients.replace(';', ',').replace('\n', '')
			recipients = split_emails(recipients)

		# remove null
		recipients = filter(None, (strip(r) for r in recipients))

		self.sender = sender
		self.reply_to = reply_to or sender
		self.recipients = recipients
		self.subject = subject

		self.msg_root = MIMEMultipart('mixed')
		self.msg_multipart = MIMEMultipart('alternative')
		self.msg_root.attach(self.msg_multipart)
		self.cc = cc or []
		self.html_set = False

		self.email_account = email_account

	def set_html(self, message, text_content = None, footer=None, print_html=None, formatted=None):
		"""Attach message in the html portion of multipart/alternative"""
		if not formatted:
			formatted = get_formatted_html(self.subject, message, footer, print_html, email_account=self.email_account)

		# this is the first html part of a multi-part message,
		# convert to text well
		if not self.html_set:
			if text_content:
				self.set_text(expand_relative_urls(text_content))
			else:
				self.set_html_as_text(expand_relative_urls(formatted))

		self.set_part_html(formatted)
		self.html_set = True

	def set_text(self, message):
		"""
			Attach message in the text portion of multipart/alternative
		"""
		from email.mime.text import MIMEText
		part = MIMEText(message, 'plain', 'utf-8')
		self.msg_multipart.attach(part)

	def set_part_html(self, message):
		from email.mime.text import MIMEText
		part = MIMEText(message, 'html', 'utf-8')
		self.msg_multipart.attach(part)

	def set_html_as_text(self, html):
		"""return html2text"""
		self.set_text(to_markdown(html))

	def set_message(self, message, mime_type='text/html', as_attachment=0, filename='attachment.html'):
		"""Append the message with MIME content to the root node (as attachment)"""
		from email.mime.text import MIMEText

		maintype, subtype = mime_type.split('/')
		part = MIMEText(message, _subtype = subtype)

		if as_attachment:
			part.add_header('Content-Disposition', 'attachment', filename=filename)

		self.msg_root.attach(part)

	def attach_file(self, n):
		"""attach a file from the `FileData` table"""
		from frappe.utils.file_manager import get_file
		res = get_file(n)
		if not res:
			return

		self.add_attachment(res[0], res[1])

	def add_attachment(self, fname, fcontent, content_type=None):
		"""add attachment"""
		from email.mime.audio import MIMEAudio
		from email.mime.base import MIMEBase
		from email.mime.image import MIMEImage
		from email.mime.text import MIMEText

		import mimetypes
		if not content_type:
			content_type, encoding = mimetypes.guess_type(fname)

		if content_type is None:
			# No guess could be made, or the file is encoded (compressed), so
			# use a generic bag-of-bits type.
			content_type = 'application/octet-stream'

		maintype, subtype = content_type.split('/', 1)
		if maintype == 'text':
			# Note: we should handle calculating the charset
			if isinstance(fcontent, unicode):
				fcontent = fcontent.encode("utf-8")
			part = MIMEText(fcontent, _subtype=subtype, _charset="utf-8")
		elif maintype == 'image':
			part = MIMEImage(fcontent, _subtype=subtype)
		elif maintype == 'audio':
			part = MIMEAudio(fcontent, _subtype=subtype)
		else:
			part = MIMEBase(maintype, subtype)
			part.set_payload(fcontent)
			# Encode the payload using Base64
			from email import encoders
			encoders.encode_base64(part)

		# Set the filename parameter
		if fname:
			part.add_header(b'Content-Disposition',
				("attachment; filename=\"%s\"" % fname).encode('utf-8'))

		self.msg_root.attach(part)

	def add_pdf_attachment(self, name, html, options=None):
		self.add_attachment(name, get_pdf(html, options), 'application/octet-stream')

	def get_default_sender(self):
		return  get_outgoing_email_account().default_sender

	def validate(self):
		"""validate the email ids"""
		from frappe.utils import validate_email_add

		if not self.sender:
			self.sender = self.get_default_sender()

		validate_email_add(strip(self.sender), True)
		self.reply_to = validate_email_add(strip(self.reply_to) or self.sender, True)

		self.recipients = [strip(r) for r in self.recipients]
		self.cc = [strip(r) for r in self.cc]

		for e in self.recipients + (self.cc or []):
			validate_email_add(e, True)

	def set_message_id(self, message_id):
		self.msg_root["Message-Id"] = "<{0}@{1}>".format(message_id, frappe.local.site)

	def set_in_reply_to(self, in_reply_to):
		"""Used to send the Message-Id of a received email back as In-Reply-To"""
		self.msg_root["In-Reply-To"] = in_reply_to

	def make(self):
		"""build into msg_root"""
		headers = {
			"Subject":        strip(self.subject).encode("utf-8"),
			"From":           self.sender.encode("utf-8"),
			"To":             ', '.join(self.recipients).encode("utf-8"),
			"Date":           email.utils.formatdate(),
			"Reply-To":       self.reply_to.encode("utf-8") if self.reply_to else None,
			"CC":             ', '.join(self.cc).encode("utf-8") if self.cc else None,
			b'X-Frappe-Site': get_url().encode('utf-8'),
		}

		# reset headers as values may be changed.
		for key, val in headers.iteritems():
			if self.msg_root.has_key(key):
				del self.msg_root[key]
			self.msg_root[key] = val

		# call hook to enable apps to modify msg_root before sending
		for hook in frappe.get_hooks("make_email_body_message"):
			frappe.get_attr(hook)(self)

	def as_string(self):
		"""validate, build message and convert to string"""
		self.validate()
		self.make()
		return self.msg_root.as_string()

def get_formatted_html(subject, message, footer=None, print_html=None, email_account=None):
	message = scrub_urls(message)

	if not email_account:
		email_account = get_outgoing_email_account(False)

	rendered_email = frappe.get_template("templates/emails/standard.html").render({
		"content": message,
		"signature": get_signature(email_account),
		"footer": get_footer(email_account, footer),
		"title": subject,
		"print_html": print_html,
		"subject": subject
	})

	return rendered_email

def get_signature(email_account):
	if email_account and email_account.add_signature and email_account.signature:
		return "

" + email_account.signature
	else:
		return ""

def get_footer(email_account, footer=None):
	"""append a footer (signature)"""
	footer = footer or ""

	if email_account and email_account.footer:
		footer += '
{0}
'.format(email_account.footer)

	footer += ""

	company_address = frappe.db.get_default("email_footer_address")

	if company_address:
		footer += '
{0}
'\
			.format(company_address.replace("\n", "
"))

	if not cint(frappe.db.get_default("disable_standard_email_footer")):
		for default_mail_footer in frappe.get_hooks("default_mail_footer"):
			footer += '
{0}
'.format(default_mail_footer)

	return footer
<commit_msg>[fix] email should have fully qualified URLs<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.utils.pdf import get_pdf
from frappe.email.smtp import get_outgoing_email_account
from frappe.utils import get_url, scrub_urls, strip, expand_relative_urls, cint, split_emails, to_markdown, markdown
import email.utils

def get_email(recipients, sender='', msg='', subject='[No Subject]',
	text_content = None, footer=None, print_html=None, formatted=None, attachments=None,
	content=None, reply_to=None, cc=(), email_account=None):
	"""send an html email as multipart with attachments and all"""
	content = content or msg
	emailobj = EMail(sender, recipients, subject, reply_to=reply_to, cc=cc, email_account=None)

	if not content.strip().startswith("<"):
		content = markdown(content)

	emailobj.set_html(content, text_content, footer=footer, print_html=print_html, formatted=formatted)

	if isinstance(attachments, dict):
		attachments = [attachments]

	for attach in (attachments or []):
		emailobj.add_attachment(**attach)

	return emailobj

class EMail:
	"""
	Wrapper on the email module. Email object represents emails to be sent to the client.
	Also provides a clean way to add binary `FileData` attachments
	Also sets all messages as multipart/alternative for cleaner reading in text-only clients
	"""
	def __init__(self, sender='', recipients=(), subject='', alternative=0, reply_to=None, cc=(), email_account=None):
		from email.mime.multipart import MIMEMultipart
		from email import Charset
		Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8')

		if isinstance(recipients, basestring):
			recipients = recipients.replace(';', ',').replace('\n', '')
			recipients = split_emails(recipients)

		# remove null
		recipients = filter(None, (strip(r) for r in recipients))

		self.sender = sender
		self.reply_to = reply_to or sender
		self.recipients = recipients
		self.subject = subject

		self.msg_root = MIMEMultipart('mixed')
		self.msg_multipart = MIMEMultipart('alternative')
		self.msg_root.attach(self.msg_multipart)
		self.cc = cc or []
		self.html_set = False

		self.email_account = email_account

	def set_html(self, message, text_content = None, footer=None, print_html=None, formatted=None):
		"""Attach message in the html portion of multipart/alternative"""
		if not formatted:
			formatted = get_formatted_html(self.subject, message, footer, print_html, email_account=self.email_account)

		# this is the first html part of a multi-part message,
		# convert to text well
		if not self.html_set:
			if text_content:
				self.set_text(expand_relative_urls(text_content))
			else:
				self.set_html_as_text(expand_relative_urls(formatted))

		self.set_part_html(formatted)
		self.html_set = True

	def set_text(self, message):
		"""
			Attach message in the text portion of multipart/alternative
		"""
		from email.mime.text import MIMEText
		part = MIMEText(message, 'plain', 'utf-8')
		self.msg_multipart.attach(part)

	def set_part_html(self, message):
		from email.mime.text import MIMEText
		part = MIMEText(message, 'html', 'utf-8')
		self.msg_multipart.attach(part)

	def set_html_as_text(self, html):
		"""return html2text"""
		self.set_text(to_markdown(html))

	def set_message(self, message, mime_type='text/html', as_attachment=0, filename='attachment.html'):
		"""Append the message with MIME content to the root node (as attachment)"""
		from email.mime.text import MIMEText

		maintype, subtype = mime_type.split('/')
		part = MIMEText(message, _subtype = subtype)

		if as_attachment:
			part.add_header('Content-Disposition', 'attachment', filename=filename)

		self.msg_root.attach(part)

	def attach_file(self, n):
		"""attach a file from the `FileData` table"""
		from frappe.utils.file_manager import get_file
		res = get_file(n)
		if not res:
			return

		self.add_attachment(res[0], res[1])

	def add_attachment(self, fname, fcontent, content_type=None):
		"""add attachment"""
		from email.mime.audio import MIMEAudio
		from email.mime.base import MIMEBase
		from email.mime.image import MIMEImage
		from email.mime.text import MIMEText

		import mimetypes
		if not content_type:
			content_type, encoding = mimetypes.guess_type(fname)

		if content_type is None:
			# No guess could be made, or the file is encoded (compressed), so
			# use a generic bag-of-bits type.
			content_type = 'application/octet-stream'

		maintype, subtype = content_type.split('/', 1)
		if maintype == 'text':
			# Note: we should handle calculating the charset
			if isinstance(fcontent, unicode):
				fcontent = fcontent.encode("utf-8")
			part = MIMEText(fcontent, _subtype=subtype, _charset="utf-8")
		elif maintype == 'image':
			part = MIMEImage(fcontent, _subtype=subtype)
		elif maintype == 'audio':
			part = MIMEAudio(fcontent, _subtype=subtype)
		else:
			part = MIMEBase(maintype, subtype)
			part.set_payload(fcontent)
			# Encode the payload using Base64
			from email import encoders
			encoders.encode_base64(part)

		# Set the filename parameter
		if fname:
			part.add_header(b'Content-Disposition',
				("attachment; filename=\"%s\"" % fname).encode('utf-8'))

		self.msg_root.attach(part)

	def add_pdf_attachment(self, name, html, options=None):
		self.add_attachment(name, get_pdf(html, options), 'application/octet-stream')

	def get_default_sender(self):
		return  get_outgoing_email_account().default_sender

	def validate(self):
		"""validate the email ids"""
		from frappe.utils import validate_email_add

		if not self.sender:
			self.sender = self.get_default_sender()

		validate_email_add(strip(self.sender), True)
		self.reply_to = validate_email_add(strip(self.reply_to) or self.sender, True)

		self.recipients = [strip(r) for r in self.recipients]
		self.cc = [strip(r) for r in self.cc]

		for e in self.recipients + (self.cc or []):
			validate_email_add(e, True)

	def set_message_id(self, message_id):
		self.msg_root["Message-Id"] = "<{0}@{1}>".format(message_id, frappe.local.site)

	def set_in_reply_to(self, in_reply_to):
		"""Used to send the Message-Id of a received email back as In-Reply-To"""
		self.msg_root["In-Reply-To"] = in_reply_to

	def make(self):
		"""build into msg_root"""
		headers = {
			"Subject":        strip(self.subject).encode("utf-8"),
			"From":           self.sender.encode("utf-8"),
			"To":             ', '.join(self.recipients).encode("utf-8"),
			"Date":           email.utils.formatdate(),
			"Reply-To":       self.reply_to.encode("utf-8") if self.reply_to else None,
			"CC":             ', '.join(self.cc).encode("utf-8") if self.cc else None,
			b'X-Frappe-Site': get_url().encode('utf-8'),
		}

		# reset headers as values may be changed.
		for key, val in headers.iteritems():
			if self.msg_root.has_key(key):
				del self.msg_root[key]
			self.msg_root[key] = val

		# call hook to enable apps to modify msg_root before sending
		for hook in frappe.get_hooks("make_email_body_message"):
			frappe.get_attr(hook)(self)

	def as_string(self):
		"""validate, build message and convert to string"""
		self.validate()
		self.make()
		return self.msg_root.as_string()

def get_formatted_html(subject, message, footer=None, print_html=None, email_account=None):
	if not email_account:
		email_account = get_outgoing_email_account(False)

	rendered_email = frappe.get_template("templates/emails/standard.html").render({
		"content": message,
		"signature": get_signature(email_account),
		"footer": get_footer(email_account, footer),
		"title": subject,
		"print_html": print_html,
		"subject": subject
	})

	return scrub_urls(rendered_email)

def get_signature(email_account):
	if email_account and email_account.add_signature and email_account.signature:
		return "

" + email_account.signature
	else:
		return ""

def get_footer(email_account, footer=None):
	"""append a footer (signature)"""
	footer = footer or ""

	if email_account and email_account.footer:
		footer += '
{0}
'.format(email_account.footer)

	footer += ""

	company_address = frappe.db.get_default("email_footer_address")

	if company_address:
		footer += '
{0}
'\
			.format(company_address.replace("\n", "
"))

	if not cint(frappe.db.get_default("disable_standard_email_footer")):
		for default_mail_footer in frappe.get_hooks("default_mail_footer"):
			footer += '
{0}
'.format(default_mail_footer)

	return footer
<|endoftext|>


<commit_before>"""
USA-specific Form helpers
"""

from django.forms import ValidationError
from django.forms.fields import Field, RegexField, Select, EMPTY_VALUES, CharField
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext_lazy as _
import re

phone_digits_re = re.compile(r'^(?:1-?)?(\d{3})[-\.]?(\d{3})[-\.]?(\d{4})$')
ssn_re = re.compile(r"^(?P<area>\d{3})[-\ ]?(?P<group>\d{2})[-\ ]?(?P<serial>\d{4})$")

class USZipCodeField(RegexField):
    default_error_messages = {
        'invalid': _('Enter a zip code in the format XXXXX or XXXXX-XXXX.'),
    }

    def __init__(self, *args, **kwargs):
        super(USZipCodeField, self).__init__(r'^\d{5}(?:-\d{4})?$',
            max_length=None, min_length=None, *args, **kwargs)

class USPhoneNumberField(CharField):
    default_error_messages = {
        'invalid': u'Phone numbers must be in XXX-XXX-XXXX format.',
    }

    def clean(self, value):
        super(USPhoneNumberField, self).clean(value)
        if value in EMPTY_VALUES:
            return u''
        value = re.sub('(\(|\)|\s+)', '', smart_unicode(value))
        m = phone_digits_re.search(value)
        if m:
            return u'%s-%s-%s' % (m.group(1), m.group(2), m.group(3))
        raise ValidationError(self.error_messages['invalid'])

class USSocialSecurityNumberField(Field):
    """
    A United States Social Security number.

    Checks the following rules to determine whether the number is valid:

        * Conforms to the XXX-XX-XXXX format.
        * No group consists entirely of zeroes.
        * The leading group is not "666" (block "666" will never be allocated).
        * The number is not in the promotional block 987-65-4320 through
          987-65-4329, which are permanently invalid.
        * The number is not one known to be invalid due to otherwise widespread
          promotional use or distribution (e.g., the Woolworth's number or the
          1962 promotional number).
    """
    default_error_messages = {
        'invalid': _('Enter a valid U.S. Social Security number in XXX-XX-XXXX format.'),
    }

    def clean(self, value):
        super(USSocialSecurityNumberField, self).clean(value)
        if value in EMPTY_VALUES:
            return u''
        match = re.match(ssn_re, value)
        if not match:
            raise ValidationError(self.error_messages['invalid'])
        area, group, serial = match.groupdict()['area'], match.groupdict()['group'], match.groupdict()['serial']

        # First pass: no blocks of all zeroes.
        if area == '000' or \
           group == '00' or \
           serial == '0000':
            raise ValidationError(self.error_messages['invalid'])

        # Second pass: promotional and otherwise permanently invalid numbers.
        if area == '666' or \
           (area == '987' and group == '65' and 4320 <= int(serial) <= 4329) or \
           value == '078-05-1120' or \
           value == '219-09-9999':
            raise ValidationError(self.error_messages['invalid'])
        return u'%s-%s-%s' % (area, group, serial)

class USStateField(Field):
    """
    A form field that validates its input is a U.S. state name or abbreviation.
    It normalizes the input to the standard two-leter postal service
    abbreviation for the given state.
    """
    default_error_messages = {
        'invalid': u'Enter a U.S. state or territory.',
    }

    def clean(self, value):
        from us_states import STATES_NORMALIZED
        super(USStateField, self).clean(value)
        if value in EMPTY_VALUES:
            return u''
        try:
            value = value.strip().lower()
        except AttributeError:
            pass
        else:
            try:
                return STATES_NORMALIZED[value.strip().lower()].decode('ascii')
            except KeyError:
                pass
        raise ValidationError(self.error_messages['invalid'])

class USStateSelect(Select):
    """
    A Select widget that uses a list of U.S. states/territories as its choices.
    """
    def __init__(self, attrs=None):
        from us_states import STATE_CHOICES
        super(USStateSelect, self).__init__(attrs, choices=STATE_CHOICES)
<commit_msg>Fixed #12013 - Marked some default error messages for translation.<commit_after>"""
USA-specific Form helpers
"""

from django.forms import ValidationError
from django.forms.fields import Field, RegexField, Select, EMPTY_VALUES, CharField
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext_lazy as _
import re

phone_digits_re = re.compile(r'^(?:1-?)?(\d{3})[-\.]?(\d{3})[-\.]?(\d{4})$')
ssn_re = re.compile(r"^(?P<area>\d{3})[-\ ]?(?P<group>\d{2})[-\ ]?(?P<serial>\d{4})$")

class USZipCodeField(RegexField):
    default_error_messages = {
        'invalid': _('Enter a zip code in the format XXXXX or XXXXX-XXXX.'),
    }

    def __init__(self, *args, **kwargs):
        super(USZipCodeField, self).__init__(r'^\d{5}(?:-\d{4})?$',
            max_length=None, min_length=None, *args, **kwargs)

class USPhoneNumberField(CharField):
    default_error_messages = {
        'invalid': _('Phone numbers must be in XXX-XXX-XXXX format.'),
    }

    def clean(self, value):
        super(USPhoneNumberField, self).clean(value)
        if value in EMPTY_VALUES:
            return u''
        value = re.sub('(\(|\)|\s+)', '', smart_unicode(value))
        m = phone_digits_re.search(value)
        if m:
            return u'%s-%s-%s' % (m.group(1), m.group(2), m.group(3))
        raise ValidationError(self.error_messages['invalid'])

class USSocialSecurityNumberField(Field):
    """
    A United States Social Security number.

    Checks the following rules to determine whether the number is valid:

        * Conforms to the XXX-XX-XXXX format.
        * No group consists entirely of zeroes.
        * The leading group is not "666" (block "666" will never be allocated).
        * The number is not in the promotional block 987-65-4320 through
          987-65-4329, which are permanently invalid.
        * The number is not one known to be invalid due to otherwise widespread
          promotional use or distribution (e.g., the Woolworth's number or the
          1962 promotional number).
    """
    default_error_messages = {
        'invalid': _('Enter a valid U.S. Social Security number in XXX-XX-XXXX format.'),
    }

    def clean(self, value):
        super(USSocialSecurityNumberField, self).clean(value)
        if value in EMPTY_VALUES:
            return u''
        match = re.match(ssn_re, value)
        if not match:
            raise ValidationError(self.error_messages['invalid'])
        area, group, serial = match.groupdict()['area'], match.groupdict()['group'], match.groupdict()['serial']

        # First pass: no blocks of all zeroes.
        if area == '000' or \
           group == '00' or \
           serial == '0000':
            raise ValidationError(self.error_messages['invalid'])

        # Second pass: promotional and otherwise permanently invalid numbers.
        if area == '666' or \
           (area == '987' and group == '65' and 4320 <= int(serial) <= 4329) or \
           value == '078-05-1120' or \
           value == '219-09-9999':
            raise ValidationError(self.error_messages['invalid'])
        return u'%s-%s-%s' % (area, group, serial)

class USStateField(Field):
    """
    A form field that validates its input is a U.S. state name or abbreviation.
    It normalizes the input to the standard two-leter postal service
    abbreviation for the given state.
    """
    default_error_messages = {
        'invalid': _('Enter a U.S. state or territory.'),
    }

    def clean(self, value):
        from us_states import STATES_NORMALIZED
        super(USStateField, self).clean(value)
        if value in EMPTY_VALUES:
            return u''
        try:
            value = value.strip().lower()
        except AttributeError:
            pass
        else:
            try:
                return STATES_NORMALIZED[value.strip().lower()].decode('ascii')
            except KeyError:
                pass
        raise ValidationError(self.error_messages['invalid'])

class USStateSelect(Select):
    """
    A Select widget that uses a list of U.S. states/territories as its choices.
    """
    def __init__(self, attrs=None):
        from us_states import STATE_CHOICES
        super(USStateSelect, self).__init__(attrs, choices=STATE_CHOICES)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vi:ts=4:et

import unittest
import pycurl

from . import util
from . import appmanager

setup_module, teardown_module = appmanager.setup(('app', 8380))

class SockoptFunctionTest(unittest.TestCase):
    def setUp(self):
        self.curl = pycurl.Curl()
        self.curl.setopt(self.curl.URL, 'http://localhost:8380/success')

    def tearDown(self):
        self.curl.close()

    def test_sockoptfunction_ok(self):
        called = {}

        def sockoptfunction(curlfd, purpose):
            called['called'] = True
            return 0

        self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction)

        self.curl.perform()
        assert called['called']

    def test_sockoptfunction_fail(self):
        called = {}

        def sockoptfunction(curlfd, purpose):
            called['called'] = True
            return 1

        self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction)

        try:
            self.curl.perform()
            self.fail('should have raised')
        except pycurl.error as e:
            self.assertEqual(pycurl.E_ABORTED_BY_CALLBACK, e.args[0])
        assert called['called']

    def test_sockoptfunction_bogus_return(self):
        called = {}

        def sockoptfunction(curlfd, purpose):
            called['called'] = True
            return 'bogus'

        self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction)

        try:
            self.curl.perform()
            self.fail('should have raised')
        except pycurl.error as e:
            self.assertEqual(pycurl.E_ABORTED_BY_CALLBACK, e.args[0])
        assert called['called']

    @util.min_libcurl(7, 28, 0)
    def test_socktype_accept(self):
        assert hasattr(pycurl, 'SOCKTYPE_ACCEPT')
        assert hasattr(self.curl, 'SOCKTYPE_ACCEPT')

    def test_socktype_ipcxn(self):
        assert hasattr(pycurl, 'SOCKTYPE_IPCXN')
        assert hasattr(self.curl, 'SOCKTYPE_IPCXN')

class SshKeyfunctionUnsetTest(unittest.TestCase):
    def setUp(self):
        self.curl = pycurl.Curl()

    def test_sockoptfunction_none(self):
        self.curl.setopt(pycurl.SOCKOPTFUNCTION, None)

    def test_sockoptfunction_unset(self):
        self.curl.unsetopt(pycurl.SOCKOPTFUNCTION)
<commit_msg>Older libcurls return E_COULDNT_CONNECT<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vi:ts=4:et

import unittest
import pycurl

from . import util
from . import appmanager

setup_module, teardown_module = appmanager.setup(('app', 8380))

class SockoptFunctionTest(unittest.TestCase):
    def setUp(self):
        self.curl = pycurl.Curl()
        self.curl.setopt(self.curl.URL, 'http://localhost:8380/success')

    def tearDown(self):
        self.curl.close()

    def test_sockoptfunction_ok(self):
        called = {}

        def sockoptfunction(curlfd, purpose):
            called['called'] = True
            return 0

        self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction)

        self.curl.perform()
        assert called['called']

    def test_sockoptfunction_fail(self):
        called = {}

        def sockoptfunction(curlfd, purpose):
            called['called'] = True
            return 1

        self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction)

        try:
            self.curl.perform()
            self.fail('should have raised')
        except pycurl.error as e:
            assert e.args[0] in [pycurl.E_ABORTED_BY_CALLBACK, pycurl.E_COULDNT_CONNECT], \
                'Unexpected pycurl error code %s' % e.args[0]
        assert called['called']

    def test_sockoptfunction_bogus_return(self):
        called = {}

        def sockoptfunction(curlfd, purpose):
            called['called'] = True
            return 'bogus'

        self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction)

        try:
            self.curl.perform()
            self.fail('should have raised')
        except pycurl.error as e:
            assert e.args[0] in [pycurl.E_ABORTED_BY_CALLBACK, pycurl.E_COULDNT_CONNECT], \
                'Unexpected pycurl error code %s' % e.args[0]
        assert called['called']

    @util.min_libcurl(7, 28, 0)
    def test_socktype_accept(self):
        assert hasattr(pycurl, 'SOCKTYPE_ACCEPT')
        assert hasattr(self.curl, 'SOCKTYPE_ACCEPT')

    def test_socktype_ipcxn(self):
        assert hasattr(pycurl, 'SOCKTYPE_IPCXN')
        assert hasattr(self.curl, 'SOCKTYPE_IPCXN')

class SshKeyfunctionUnsetTest(unittest.TestCase):
    def setUp(self):
        self.curl = pycurl.Curl()

    def test_sockoptfunction_none(self):
        self.curl.setopt(pycurl.SOCKOPTFUNCTION, None)

    def test_sockoptfunction_unset(self):
        self.curl.unsetopt(pycurl.SOCKOPTFUNCTION)
<|endoftext|>