<commit_before>import logging, os, redis, string, subprocess
from apscheduler.scheduler import Scheduler
from datetime import datetime, timedelta
from multiprocessing import Pool
from plumbum import SshMachine
from rpyc.utils.factory import ssh_connect
from utilities import distance, threaded_map

process_pool_size = 60
sched = Scheduler(standalone=True)
logging.basicConfig()

all_dns = redis.Redis(connection_pool=redis.ConnectionPool(host='localhost', port=6379, db=0))
open_resolvers = redis.Redis(connection_pool=redis.ConnectionPool(host='localhost', port=6379, db=1))
geoip = redis.Redis(connection_pool=redis.ConnectionPool(host='localhost', port=6379, db=2))
pl_hosts = [line.split(' ')[0:4] for line in map(string.strip,open('pl-host-list-geo').readlines())]

from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, PickleType
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base

engine = create_engine('sqlite:///latency.db', echo=True)
Base = declarative_base(bind=engine)

class DataPoint(Base):
    __tablename__ = 'data'

    id = Column(Integer, primary_key=True)
    target1 = Column(PickleType)
    target2 = Column(PickleType)
    start = Column(PickleType)
    end = Column(PickleType)
    pings = Column(PickleType)
    address = Column(PickleType)

    def __init__(self, target1, target2, start, end, pings, address):
        self.target1 = target1
        self.target2 = target2
        self.start = start
        self.end = end
        self.pings = pings
        self.address = address

Base.metadata.create_all()
Session = sessionmaker(bind=engine)
s = Session()

class PlanetLabNode(object):
    def __init__(self, host, ip, lat, lon):
        self.host = host
        self.ip = ip
        self.lat = float(lat)
        self.lon = float(lon)
        self.connected = False

    def connect(self):
        try:
            self.connectPL()
        except Exception, e:
            try:
                self.restartPL()
                self.connectPL()
            except Exception, e:
                self.connected = False

    def connectPL(self):
        rem = SshMachine(self.host, user='ucb_268_measure', keyfile='~/.ssh/id_rsa',
                         ssh_opts=["-o StrictHostKeyChecking no",
                                   "-o UserKnownHostsFile=/dev/null"])
        conn = ssh_connect(rem, 18861, config={'allow_pickle' : True})
        assert conn.root.test() == 1, "%s: RPC Failure" % self.host
        self.conn = conn
        self.connected = True

    def restartPL(self):
        FNULL = open(os.devnull, 'w')
        subprocess.call(["ssh", "-t", "-i", "~/.ssh/id_rsa", "-o StrictHostKeyChecking no",
                         "-o UserKnownHostsFile=/dev/null", "ucb_268_measure@%s" % self.host,
                         "sudo tking-server stop; sudo tking-server start;"],
                         stdout=FNULL, stderr=FNULL)

    def handleConnExceptions(fn):
        def wrapped(self, *args, **kwargs):
            if not self.connected:
                self.connect()

            if self.connected:
                try:
                    return fn(self, *args, **kwargs)
                except Exception, e:
                    self.connected = False
                    try:
                        self.connect()
                        return fn(self, *args, **kwargs)
                    except Exception, e:
                        self.connected = False
                        return None
            else:
                return None
        return wrapped

    @handleConnExceptions
    def get_latency(self, target1, ip1, target2, ip2):
        return self.conn.root.get_latency(target1, ip1, target2, ip2)

    @handleConnExceptions
    def get_k(self, target, ip):
        return self.conn.get_k(target, ip)

    @handleConnExceptions
    def get_distance(self, lat, lon):
        return distance((self.lat, self.lon), (lat, lon))

def select_random_points():
    target1 = open_resolvers.randomkey()
    target2 = geoip.randomkey()
    while not geoip.exists(target1):
        target1 = open_resolvers.randomkey()

    ip1, coord1 = list(all_dns.smembers(target1))[0], eval(list(geoip.smembers(target1))[0])[1:]
    ip2, coord2 = list(all_dns.smembers(target2))[0], eval(list(geoip.smembers(target2))[0])[1:]

    return (target1, ip1, coord1), (target2, ip2, coord2)

def query_latency(target1, target2):
    name1, ip1, coord1 = target1
    name2, ip2, coord2 = target2

    # Get closest 4 PL Nodes
    distances = map(lambda node: (distance(coord1, (node.lat, node.lon)), node), pl_nodes)
    distances.sort()
    distances = distances[:4]

    results = threaded_map(lambda (dist, node): (node.host, node.get_latency(name1, ip1, name2, ip2)), distances, timeout=10.0)
    return results

def one_round(x):
    target1, target2 = select_random_points()
    results = query_latency(target1, target2)
    return target1, target2, results

pl_nodes = map(lambda args: PlanetLabNode(*args), pl_hosts)

def task():
    print 'Task'
    p = Pool(process_pool_size)
    results = p.map(one_round, range(process_pool_size))
    for target1, target2, result_set in filter(None,results):
        for host, info in filter(None,result_set):
            if info:
                try:
                    (end_time, start_time, ping_times, address) = info
                    point = DataPoint(target1, target2, start_time, end_time, ping_times, address)
                    s.add(point)
                except Exception, e:
                    pass
    p.close()
    s.commit()

sched.add_interval_job(task, minutes=1, misfire_grace_time=15)
sched.start()
<commit_msg>added more exception handling<commit_after>import logging, os, redis, string, subprocess
from apscheduler.scheduler import Scheduler
from datetime import datetime, timedelta
from multiprocessing import Pool
from plumbum import SshMachine
from rpyc.utils.factory import ssh_connect
from utilities import distance, threaded_map

process_pool_size = 60
sched = Scheduler(standalone=True)
logging.basicConfig()

all_dns = redis.Redis(connection_pool=redis.ConnectionPool(host='localhost', port=6379, db=0))
open_resolvers = redis.Redis(connection_pool=redis.ConnectionPool(host='localhost', port=6379, db=1))
geoip = redis.Redis(connection_pool=redis.ConnectionPool(host='localhost', port=6379, db=2))
pl_hosts = [line.split(' ')[0:4] for line in map(string.strip,open('pl-host-list-geo').readlines())]

from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, PickleType
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base

engine = create_engine('sqlite:///latency.db', echo=True)
Base = declarative_base(bind=engine)

class DataPoint(Base):
    __tablename__ = 'data'

    id = Column(Integer, primary_key=True)
    target1 = Column(PickleType)
    target2 = Column(PickleType)
    start = Column(PickleType)
    end = Column(PickleType)
    pings = Column(PickleType)
    address = Column(PickleType)

    def __init__(self, target1, target2, start, end, pings, address):
        self.target1 = target1
        self.target2 = target2
        self.start = start
        self.end = end
        self.pings = pings
        self.address = address

Base.metadata.create_all()
Session = sessionmaker(bind=engine)
s = Session()

class PlanetLabNode(object):
    def __init__(self, host, ip, lat, lon):
        self.host = host
        self.ip = ip
        self.lat = float(lat)
        self.lon = float(lon)
        self.connected = False

    def connect(self):
        try:
            self.connectPL()
        except Exception, e:
            try:
                self.restartPL()
                self.connectPL()
            except Exception, e:
                self.connected = False

    def connectPL(self):
        rem = SshMachine(self.host, user='ucb_268_measure', keyfile='~/.ssh/id_rsa',
                         ssh_opts=["-o StrictHostKeyChecking no",
                                   "-o UserKnownHostsFile=/dev/null"])
        conn = ssh_connect(rem, 18861, config={'allow_pickle' : True})
        assert conn.root.test() == 1, "%s: RPC Failure" % self.host
        self.conn = conn
        self.connected = True

    def restartPL(self):
        FNULL = open(os.devnull, 'w')
        subprocess.call(["ssh", "-t", "-i", "~/.ssh/id_rsa", "-o StrictHostKeyChecking no",
                         "-o UserKnownHostsFile=/dev/null", "ucb_268_measure@%s" % self.host,
                         "sudo tking-server stop; sudo tking-server start;"],
                         stdout=FNULL, stderr=FNULL)

    def handleConnExceptions(fn):
        def wrapped(self, *args, **kwargs):
            if not self.connected:
                self.connect()

            if self.connected:
                try:
                    return fn(self, *args, **kwargs)
                except Exception, e:
                    self.connected = False
                    try:
                        self.connect()
                        return fn(self, *args, **kwargs)
                    except Exception, e:
                        self.connected = False
                        return None
            else:
                return None
        return wrapped

    @handleConnExceptions
    def get_latency(self, target1, ip1, target2, ip2):
        return self.conn.root.get_latency(target1, ip1, target2, ip2)

    @handleConnExceptions
    def get_k(self, target, ip):
        return self.conn.get_k(target, ip)

    @handleConnExceptions
    def get_distance(self, lat, lon):
        return distance((self.lat, self.lon), (lat, lon))

def select_random_points():
    target1 = open_resolvers.randomkey()
    target2 = geoip.randomkey()
    while not geoip.exists(target1):
        target1 = open_resolvers.randomkey()

    ip1, coord1 = list(all_dns.smembers(target1))[0], eval(list(geoip.smembers(target1))[0])[1:]
    ip2, coord2 = list(all_dns.smembers(target2))[0], eval(list(geoip.smembers(target2))[0])[1:]

    return (target1, ip1, coord1), (target2, ip2, coord2)

def query_latency(target1, target2):
    name1, ip1, coord1 = target1
    name2, ip2, coord2 = target2

    # Get closest 4 PL Nodes
    distances = map(lambda node: (distance(coord1, (node.lat, node.lon)), node), pl_nodes)
    distances.sort()
    distances = distances[:4]

    results = threaded_map(lambda (dist, node): (node.host, node.get_latency(name1, ip1, name2, ip2)), distances, timeout=10.0)
    return results

def one_round(x):
    try:
        target1, target2 = select_random_points()
        results = query_latency(target1, target2)
        return target1, target2, results
    except Exception, e:
        return None

pl_nodes = map(lambda args: PlanetLabNode(*args), pl_hosts)

def task():
    print 'Task'
    p = Pool(process_pool_size)
    results = p.map(one_round, range(process_pool_size))
    for target1, target2, result_set in filter(None,results):
        for host, info in filter(None,result_set):
            if info:
                try:
                    (end_time, start_time, ping_times, address) = info
                    point = DataPoint(target1, target2, start_time, end_time, ping_times, address)
                    s.add(point)
                except Exception, e:
                    pass
    p.close()
    s.commit()

sched.add_interval_job(task, minutes=1, misfire_grace_time=15)
sched.start()
<|endoftext|>


<commit_before>from rpyc.utils.factory import ssh_connect
from plumbum import SshMachine
from termcolor import colored
import string, subprocess, StringIO
from time import sleep
from threading import Thread

hosts = map(string.strip,open('pl-host-list').readlines())

def testHost(host, buff):
    try:
        try:
            conn = rpyc.connect(host, 18861)
            print >> buff, colored('Service Is Exposed Externally', 'red')
        except:
            pass
            #print >> buff, colored("Service Not Exposed Exterally",'green')

        rem = SshMachine(host, user='ucb_268_measure', keyfile='~/.ssh/id_rsa')
        #print >> buff, colored('SSH Connected', 'green')
        try:
            conn = ssh_connect(rem, 18861)
            #print >> buff, conn.root.test()
            conn.root.test()
        except Exception, e:
            print >> buff, colored('Service is not Running', 'red')
            print >> buff, 'Attempting to Start Service...'
            subprocess.call(["ssh", "-t", "-i", "~/.ssh/id_rsa", "ucb_268_measure@%s" % host, "sudo tking-server stop"])
            subprocess.call(["ssh", "-t", "-i", "~/.ssh/id_rsa", "ucb_268_measure@%s" % host, "sudo tking-server start"])
            try:
                sleep(4)
                conn = ssh_connect(rem, 18861)
                #print >> buff, conn.root.test()
                conn.root.test()
            except:
                #print >> buff, colored('Could not start service', 'red')
                return

        responses = []
        for i in range(3):
            try:
                response = conn.root.get_latency('ns.nwt.cz','217.197.152.132','ns1.google.com','216.239.32.10')
                if type(response) == type(''):
                    responses.append(response)

                end_time, start_time, ping_time, address = response

                latency = end_time - start_time - ping_time
                responses.append((latency, address))
            except:
                response.append(None)

        a = responses[0]
        b = responses[1]
        c = responses[2]
        k = conn.root.exposed_get_k('google.com', '8.8.8.8')

        print "K:", k

        if type(a[0]) == type('') and type(b[0]) == type('') and type(c[0]) == type(''):
            print >> buff, colored('Amazon NS maybe inactive, or Node DNS server cannot bind', 'red')
            print >> buff, a, '\n', b, '\n', c
        elif a and b and c:
            print >> buff, colored('Recieved Latencies','green'), '\n', a, '\n', b, '\n', c
        else:
            print >> buff, colored ('Other Issue with latencies', 'red'), a, b, c
    except Exception, e:
        pass
        #print >> buff, colored('Could not connect', 'red')

threads = []
for host in hosts:
    buff = StringIO.StringIO()
    t = Thread(target=testHost, args=(host,buff))
    t.start()
    threads.append((host, t, buff))

for host, t, buff in threads:
    t.join(timeout=60.0)

for host, t, buff in threads:
    print '-----------------'
    print host
    print buff.getvalue(),
<commit_msg>print out K<commit_after>from rpyc.utils.factory import ssh_connect
from plumbum import SshMachine
from termcolor import colored
import string, subprocess, StringIO
from time import sleep
from threading import Thread

hosts = map(string.strip,open('pl-host-list').readlines())

def testHost(host, buff):
    try:
        try:
            conn = rpyc.connect(host, 18861)
            print >> buff, colored('Service Is Exposed Externally', 'red')
        except:
            pass
            #print >> buff, colored("Service Not Exposed Exterally",'green')

        rem = SshMachine(host, user='ucb_268_measure', keyfile='~/.ssh/id_rsa')
        #print >> buff, colored('SSH Connected', 'green')
        try:
            conn = ssh_connect(rem, 18861)
            #print >> buff, conn.root.test()
            conn.root.test()
        except Exception, e:
            print >> buff, colored('Service is not Running', 'red')
            print >> buff, 'Attempting to Start Service...'
            subprocess.call(["ssh", "-t", "-i", "~/.ssh/id_rsa", "ucb_268_measure@%s" % host, "sudo tking-server stop"])
            subprocess.call(["ssh", "-t", "-i", "~/.ssh/id_rsa", "ucb_268_measure@%s" % host, "sudo tking-server start"])
            try:
                sleep(4)
                conn = ssh_connect(rem, 18861)
                #print >> buff, conn.root.test()
                conn.root.test()
            except:
                #print >> buff, colored('Could not start service', 'red')
                return

        responses = []
        for i in range(3):
            try:
                response = conn.root.get_latency('ns.nwt.cz','217.197.152.132','ns1.google.com','216.239.32.10')
                if type(response) == type(''):
                    responses.append(response)

                end_time, start_time, ping_time, address = response

                latency = end_time - start_time - ping_time
                responses.append((latency, address))
            except:
                response.append(None)

        a = responses[0]
        b = responses[1]
        c = responses[2]
        k = conn.root.exposed_get_k('google.com', '8.8.8.8')

        print >> buff, "K:", k

        if type(a[0]) == type('') and type(b[0]) == type('') and type(c[0]) == type(''):
            print >> buff, colored('Amazon NS maybe inactive, or Node DNS server cannot bind', 'red')
            print >> buff, a, '\n', b, '\n', c
        elif a and b and c:
            print >> buff, colored('Recieved Latencies','green'), '\n', a, '\n', b, '\n', c
        else:
            print >> buff, colored ('Other Issue with latencies', 'red'), a, b, c
    except Exception, e:
        pass
        #print >> buff, colored('Could not connect', 'red')

threads = []
for host in hosts:
    buff = StringIO.StringIO()
    t = Thread(target=testHost, args=(host,buff))
    t.start()
    t.daemon = True
    threads.append((host, t, buff))

for host, t, buff in threads:
    t.join(timeout=60.0)

for host, t, buff in threads:
    print '-----------------'
    print host
    print buff.getvalue(),
<|endoftext|>


<commit_before>"""Kintaro extension for integrating Kintaro into Grow sites."""

import datetime
import json
import logging
import os
import re
import grow
import httplib2
from googleapiclient import discovery
from googleapiclient import errors
from grow.common import oauth
from grow.common import utils
from grow.pods import documents
from jinja2.ext import Extension
from protorpc import messages

KINTARO_HOST = 'kintaro-content-server.appspot.com'
KINTARO_API_ROOT = 'https://{host}/_ah/api'.format(host=KINTARO_HOST)
DISCOVERY_URL = (
    KINTARO_API_ROOT + '/discovery/v1/apis/{api}/{apiVersion}/rest')
KINTARO_EDIT_PATH_FORMAT = (
    'https://{host}'
    '/app#/project/{project}'
    '/repo/{repo}'
    '/collection/{collection}'
    '/document/{document}/edit')
PARTIAL_CONVERSION = re.compile(r'([A-Z])')
STORAGE_KEY = 'Grow SDK - Kintaro'

# Silence extra logging from googleapiclient.
discovery.logger.setLevel(logging.WARNING)

OAUTH_SCOPES = ('https://www.googleapis.com/auth/userinfo.email',
                'https://www.googleapis.com/auth/kintaro')

class BindingMessage(messages.Message):
    collection = messages.StringField(1)
    kintaro_collection = messages.StringField(2)

class _GoogleServicePreprocessor(grow.Preprocessor):
    _last_run = None

    def create_service(self, host):
        credentials = oauth.get_or_create_credentials(
            scope=OAUTH_SCOPES, storage_key=STORAGE_KEY)
        http = httplib2.Http(ca_certs=utils.get_cacerts_path())
        http = credentials.authorize(http)
        # Kintaro's server doesn't seem to be able to refresh expired tokens
        # properly (responds with a "Stateless token expired" error). So we
        # manage state ourselves and refresh slightly more often than once
        # per hour.
        now = datetime.datetime.now()
        if self._last_run is None \
                or now - self._last_run >= datetime.timedelta(minutes=50):
            credentials.refresh(http)
            self._last_run = now
        url = DISCOVERY_URL.replace('{host}', host)
        return discovery.build('content', 'v1', http=http,
                               discoveryServiceUrl=url)

class KintaroPreprocessor(_GoogleServicePreprocessor):
    KIND = 'kintaro'

    class Config(messages.Message):
        bind = messages.MessageField(BindingMessage, 1, repeated=True)
        repo = messages.StringField(2)
        project = messages.StringField(3)
        host = messages.StringField(4, default=KINTARO_HOST)
        use_index = messages.BooleanField(5, default=True)

    def __init__(self, *args, **kwargs):
        super(KintaroPreprocessor, self).__init__(*args, **kwargs)
        self._service = None

    @property
    def service(self):
        if not self._service:
            self._service = self.create_service(host=self.config.host)
        return self._service

    def bind_collection(self, entries, collection_pod_path):
        collection = self.pod.get_collection(collection_pod_path)
        existing_pod_paths = [
            doc.pod_path
            for doc in collection.docs(recursive=False, inject=False)]
        new_pod_paths = []
        for i, entry in enumerate(entries):
            # TODO: Ensure `create_doc` doesn't die if the file doesn't exist.
            basename = self._get_basename_from_entry(entry)
            path = os.path.join(collection.pod_path, basename)
            if not self.pod.file_exists(path):
                self.pod.write_yaml(path, {})
            doc_pod_path = os.path.join(collection.pod_path, basename)
            doc = collection.get_doc(doc_pod_path)
            fields, unused_body, basename = self._parse_entry(doc, entry)
            doc = collection.create_doc(basename, fields=fields, body='')
            new_pod_paths.append(doc.pod_path)
            self.pod.logger.info('Saved -> {}'.format(doc.pod_path))

        pod_paths_to_delete = set(existing_pod_paths) - set(new_pod_paths)
        for pod_path in pod_paths_to_delete:
            self.pod.delete_file(pod_path)
            self.pod.logger.info('Deleted -> {}'.format(pod_path))

    def _regroup_schema(self, schema):
        names_to_fields = {}
        for field in schema:
            names_to_fields[field['name']] = field
        return names_to_fields

    def _parse_field(self, key, value, field_data, locale=None):
        # Convert Kintaro keys to Grow built-ins.
        if hasattr(documents, 'BUILT_IN_FIELDS'):
            built_in_fields = documents.BUILT_IN_FIELDS
        else:
            # Support older versions of grow.
            built_in_fields = ['title', 'order']
        if key in built_in_fields:
            key = '${}'.format(key)
        key = self._parse_field_key(key, field_data)
        value = self._parse_field_deep(value, field_data, locale=locale)
        return key, value

    def _parse_field_deep(self, value, field_data, locale=None):
        single_field = not isinstance(value, list)
        if single_field:
            value = [value]

        # Handle ReferenceField as doc reference.
        if field_data['type'] == 'ReferenceField':
            for idx in range(len(value)):
                if not value[idx]:
                    continue
                for binding in self.config.bind:
                    if binding.kintaro_collection == value[idx]['collection_id']:
                        filename = '{}.yaml'.format(
                            value[idx]['document_id'])
                        content_path = os.path.join(
                            binding.collection, filename)
                        value[idx] = self.pod.get_doc(content_path, locale=locale)
                        break
        elif 'schema_fields' in field_data:
            names_to_schema_fields = self._regroup_schema(
                field_data['schema_fields'])
            for idx in range(len(value)):
                value[idx] = self._parse_field_value(
                    value[idx], names_to_schema_fields, locale=locale)
        if single_field:
            value = value[0]
        return value

    def _parse_field_key(self, key, field_data):
        if field_data['translatable']:
            key = '{}@'.format(key)
        return key

    def _parse_field_value(self, value, names_to_schema_fields, locale=None):
        clean_value = {}
        for sub_key, sub_value in value.iteritems():
            new_key = self._parse_field_key(
                sub_key, names_to_schema_fields[sub_key])
            clean_value[new_key] = self._parse_field_deep(
                sub_value, names_to_schema_fields[sub_key], locale=locale)
        return clean_value

    def _get_basename_from_entry(self, entry):
        return '{}.yaml'.format(entry['document_id'])

    def _parse_entry(self, doc, entry):
        basename = self._get_basename_from_entry(entry)
        schema = entry.get('schema', {})
        schema_fields = schema.get('schema_fields', [])
        names_to_schema_fields = self._regroup_schema(schema_fields)
        fields = entry.get('content_json', '{}')
        fields = json.loads(fields)
        clean_fields = {}
        # Preserve existing built-in fields prefixed with $.
        front_matter_data = doc.format.front_matter.data
        if front_matter_data:
            for key, value in front_matter_data.iteritems():
                if not key.startswith('$'):
                    continue
                clean_fields[key] = value
        # Overwrite with data from CMS.
        for name, value in fields.iteritems():
            field_data = names_to_schema_fields[name]
            key, value = self._parse_field(name, value, field_data, locale=doc.locale)
            clean_fields[key] = value
        # Populate $meta.
        if schema:
            # Strip modified info from schema.
            schema.pop('mod_info', None)
            clean_fields['$meta'] = {}
            clean_fields['$meta']['schema'] = schema
        body = ''
        return clean_fields, body, basename

    def _get_documents_from_search(self, repo_id, collection_id, project_id, documents):
        results = []
        for document in documents:
            document_id = document['document_id']
            entry = self.download_entry(
                document_id, collection_id, repo_id, project_id)
            results.append(entry)
        return results
        # TODO: Upgrade Grow's google api python client, use it to batch
        # requests.
        service = self.create_service(host=self.config.host)
        batch = service.new_batch_http_request()
        results = []

        def _add(entry):
            results.append(entry)
        for document in documents:
            document_id = document['document_id']
            req = service.documents().getDocument(
                document_id=document_id,
                collection_id=collection_id,
                project_id=project_id,
                repo_id=repo_id,
                include_schema=True,
                use_json=True)
            batch.add(req, callback=_add)
        batch.execute()
        return results

    def download_entries(self, repo_id, collection_id, project_id):
        body = {
            'repo_id': repo_id,
            'collection_id': collection_id,
            'project_id': project_id,
            'result_options': {
                'return_json': True,
                'return_schema': True,
            }
        }
        resp = self.service.documents().searchDocuments(body=body).execute()
        documents = resp.get('document_list', {}).get('documents', [])
        if not self.config.use_index:
            documents_from_get = self._get_documents_from_search(
                repo_id, collection_id, project_id, documents)
            return documents_from_get
        # Reformat document response to include schema.
        schema = resp.get('schema', {})
        for document in documents:
            document['schema'] = schema
        return documents

    def download_entry(self, document_id, collection_id, repo_id, project_id):
        resp = self.service.documents().getDocument(
            document_id=document_id,
            collection_id=collection_id,
            project_id=project_id,
            repo_id=repo_id,
            include_schema=True,
            use_json=True).execute()
        return resp

    def _normalize(self, path):
        return path.rstrip('/') if path else None

    def get_edit_url(self, doc=None):
        if not doc:
            return
        kintaro_collection = ''
        kintaro_document = doc.base
        doc_pod_path = self._normalize(doc.collection.pod_path)
        for binding in self.config.bind:
            if self._normalize(binding.collection) == doc_pod_path:
                kintaro_collection = binding.kintaro_collection
        if kintaro_collection:
            return KINTARO_EDIT_PATH_FORMAT.format(
                host=self.config.host,
                project=self.config.project,
                repo=self.config.repo,
                collection=kintaro_collection,
                document=kintaro_document)

    def can_inject(self, doc=None, collection=None):
        if not self.injected:
            return False
        if doc:
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    return True
        return False

    def inject(self, doc=None, collection=None):
        if doc:
            document_id = doc.base
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    entry = self.download_entry(
                        document_id=document_id,
                        collection_id=binding.kintaro_collection,
                        repo_id=self.config.repo,
                        project_id=self.config.project)
                    fields, _, _ = self._parse_entry(doc, entry)
                    doc.inject(fields, body='')
                    return doc

    def run(self, *args, **kwargs):
        for binding in self.config.bind:
            collection_pod_path = binding.collection
            kintaro_collection = binding.kintaro_collection
            entries = self.download_entries(
                repo_id=self.config.repo,
                collection_id=kintaro_collection,
                project_id=self.config.project)
            self.bind_collection(entries, collection_pod_path)

def schema_name_to_partial(value, sep='-', directory='views/partials',
                           use_sub_directory=False, prefix='partial'):
    """Parse a kintaro schema name to determine if it is a partial."""
    if value.lower().startswith(prefix):
        basename = value[len(prefix):]
        basename = PARTIAL_CONVERSION.sub(r'{}\1'.format(sep), basename)[1:]
        if use_sub_directory:
            directory = '{}/{}'.format(directory, basename.lower())
        return '/{}/{}.html'.format(directory, basename.lower())
    return None

class KintaroExtension(Extension):
    """Add a filter for jinja2 that assists with kintaro transformations."""

    def __init__(self, environment):
        super(KintaroExtension, self).__init__(environment)
        environment.filters[
            'kintaro.schema_name_to_partial'] = schema_name_to_partial
<commit_msg>Adding the ability to tag the environment when importing.<commit_after>"""Kintaro extension for integrating Kintaro into Grow sites."""

import datetime
import json
import logging
import os
import re
import grow
import httplib2
from googleapiclient import discovery
from googleapiclient import errors
from grow.common import oauth
from grow.common import utils
from grow.pods import documents
from jinja2.ext import Extension
from protorpc import messages

KINTARO_HOST = 'kintaro-content-server.appspot.com'
KINTARO_API_ROOT = 'https://{host}/_ah/api'.format(host=KINTARO_HOST)
DISCOVERY_URL = (
    KINTARO_API_ROOT + '/discovery/v1/apis/{api}/{apiVersion}/rest')
KINTARO_EDIT_PATH_FORMAT = (
    'https://{host}'
    '/app#/project/{project}'
    '/repo/{repo}'
    '/collection/{collection}'
    '/document/{document}/edit')
PARTIAL_CONVERSION = re.compile(r'([A-Z])')
STORAGE_KEY = 'Grow SDK - Kintaro'

# Silence extra logging from googleapiclient.
discovery.logger.setLevel(logging.WARNING)

OAUTH_SCOPES = ('https://www.googleapis.com/auth/userinfo.email',
                'https://www.googleapis.com/auth/kintaro')

class BindingMessage(messages.Message):
    collection = messages.StringField(1)
    kintaro_collection = messages.StringField(2)

class _GoogleServicePreprocessor(grow.Preprocessor):
    _last_run = None

    def create_service(self, host):
        credentials = oauth.get_or_create_credentials(
            scope=OAUTH_SCOPES, storage_key=STORAGE_KEY)
        http = httplib2.Http(ca_certs=utils.get_cacerts_path())
        http = credentials.authorize(http)
        # Kintaro's server doesn't seem to be able to refresh expired tokens
        # properly (responds with a "Stateless token expired" error). So we
        # manage state ourselves and refresh slightly more often than once
        # per hour.
        now = datetime.datetime.now()
        if self._last_run is None \
                or now - self._last_run >= datetime.timedelta(minutes=50):
            credentials.refresh(http)
            self._last_run = now
        url = DISCOVERY_URL.replace('{host}', host)
        return discovery.build('content', 'v1', http=http,
                               discoveryServiceUrl=url)

class KintaroPreprocessor(_GoogleServicePreprocessor):
    KIND = 'kintaro'

    class Config(messages.Message):
        bind = messages.MessageField(BindingMessage, 1, repeated=True)
        repo = messages.StringField(2)
        project = messages.StringField(3)
        host = messages.StringField(4, default=KINTARO_HOST)
        use_index = messages.BooleanField(5, default=True)

    def __init__(self, *args, **kwargs):
        super(KintaroPreprocessor, self).__init__(*args, **kwargs)
        self._service = None
        self._env_regex = None
        self._env_regex_replace = r'@env.\1'

    @property
    def service(self):
        if not self._service:
            self._service = self.create_service(host=self.config.host)
        return self._service

    def bind_collection(self, entries, collection_pod_path):
        collection = self.pod.get_collection(collection_pod_path)
        existing_pod_paths = [
            doc.pod_path
            for doc in collection.docs(recursive=False, inject=False)]
        new_pod_paths = []
        for i, entry in enumerate(entries):
            # TODO: Ensure `create_doc` doesn't die if the file doesn't exist.
            basename = self._get_basename_from_entry(entry)
            path = os.path.join(collection.pod_path, basename)
            if not self.pod.file_exists(path):
                self.pod.write_yaml(path, {})
            doc_pod_path = os.path.join(collection.pod_path, basename)
            doc = collection.get_doc(doc_pod_path)
            fields, unused_body, basename = self._parse_entry(doc, entry)
            doc = collection.create_doc(basename, fields=fields, body='')
            new_pod_paths.append(doc.pod_path)
            self.pod.logger.info('Saved -> {}'.format(doc.pod_path))

        pod_paths_to_delete = set(existing_pod_paths) - set(new_pod_paths)
        for pod_path in pod_paths_to_delete:
            self.pod.delete_file(pod_path)
            self.pod.logger.info('Deleted -> {}'.format(pod_path))

    def _regroup_schema(self, schema):
        names_to_fields = {}
        for field in schema:
            names_to_fields[field['name']] = field
        return names_to_fields

    def _parse_field(self, key, value, field_data, locale=None):
        # Convert Kintaro keys to Grow built-ins.
        if hasattr(documents, 'BUILT_IN_FIELDS'):
            built_in_fields = documents.BUILT_IN_FIELDS
        else:
            # Support older versions of grow.
            built_in_fields = ['title', 'order']
        if key in built_in_fields:
            key = '${}'.format(key)
        key = self._parse_field_key(key, field_data)
        value = self._parse_field_deep(value, field_data, locale=locale)
        return key, value

    def _parse_field_deep(self, value, field_data, locale=None):
        single_field = not isinstance(value, list)
        if single_field:
            value = [value]

        # Handle ReferenceField as doc reference.
        if field_data['type'] == 'ReferenceField':
            for idx in range(len(value)):
                if not value[idx]:
                    continue
                for binding in self.config.bind:
                    if binding.kintaro_collection == value[idx]['collection_id']:
                        filename = '{}.yaml'.format(
                            value[idx]['document_id'])
                        content_path = os.path.join(
                            binding.collection, filename)
                        value[idx] = self.pod.get_doc(
                            content_path, locale=locale)
                        break
        elif 'schema_fields' in field_data:
            names_to_schema_fields = self._regroup_schema(
                field_data['schema_fields'])
            for idx in range(len(value)):
                value[idx] = self._parse_field_value(
                    value[idx], names_to_schema_fields,
                    locale=locale)
        if single_field:
            value = value[0]
        return value

    def _parse_field_key(self, key, field_data):
        if field_data['translatable']:
            key = '{}@'.format(key)
        # Handle environment tagging.
        key = re.sub(self._env_regex, self._env_regex_replace, key)
        return key

    def _parse_field_value(self, value, names_to_schema_fields, locale=None):
        clean_value = {}
        for sub_key, sub_value in value.iteritems():
            new_key = self._parse_field_key(
                sub_key, names_to_schema_fields[sub_key])
            clean_value[new_key] = self._parse_field_deep(
                sub_value, names_to_schema_fields[sub_key],
                locale=locale)
        return clean_value

    def _get_basename_from_entry(self, entry):
        return '{}.yaml'.format(entry['document_id'])

    def _parse_entry(self, doc, entry):
        deployments = doc.pod.yaml.get('deployments', {}).keys()
        if not self._env_regex:
            self._env_regex = re.compile(
                r'_env_({})$'.format('|'.join(deployments)))
        basename = self._get_basename_from_entry(entry)
        schema = entry.get('schema', {})
        schema_fields = schema.get('schema_fields', [])
        names_to_schema_fields = self._regroup_schema(schema_fields)
        fields = entry.get('content_json', '{}')
        fields = json.loads(fields)
        clean_fields = {}
        # Preserve existing built-in fields prefixed with $.
        front_matter_data = doc.format.front_matter.data
        if front_matter_data:
            for key, value in front_matter_data.iteritems():
                if not key.startswith('$'):
                    continue
                clean_fields[key] = value
        # Overwrite with data from CMS.
        for name, value in fields.iteritems():
            field_data = names_to_schema_fields[name]
            key, value = self._parse_field(
                name, value, field_data, locale=doc.locale)
            clean_fields[key] = value
        # Populate $meta.
        if schema:
            # Strip modified info from schema.
            schema.pop('mod_info', None)
            clean_fields['$meta'] = {}
            clean_fields['$meta']['schema'] = schema
        body = ''
        return clean_fields, body, basename

    def _get_documents_from_search(self, repo_id, collection_id, project_id, documents):
        results = []
        for document in documents:
            document_id = document['document_id']
            entry = self.download_entry(
                document_id, collection_id, repo_id, project_id)
            results.append(entry)
        return results
        # TODO: Upgrade Grow's google api python client, use it to batch
        # requests.
        service = self.create_service(host=self.config.host)
        batch = service.new_batch_http_request()
        results = []

        def _add(entry):
            results.append(entry)
        for document in documents:
            document_id = document['document_id']
            req = service.documents().getDocument(
                document_id=document_id,
                collection_id=collection_id,
                project_id=project_id,
                repo_id=repo_id,
                include_schema=True,
                use_json=True)
            batch.add(req, callback=_add)
        batch.execute()
        return results

    def download_entries(self, repo_id, collection_id, project_id):
        body = {
            'repo_id': repo_id,
            'collection_id': collection_id,
            'project_id': project_id,
            'result_options': {
                'return_json': True,
                'return_schema': True,
            }
        }
        resp = self.service.documents().searchDocuments(body=body).execute()
        documents = resp.get('document_list', {}).get('documents', [])
        if not self.config.use_index:
            documents_from_get = self._get_documents_from_search(
                repo_id, collection_id, project_id, documents)
            return documents_from_get
        # Reformat document response to include schema.
        schema = resp.get('schema', {})
        for document in documents:
            document['schema'] = schema
        return documents

    def download_entry(self, document_id, collection_id, repo_id, project_id):
        resp = self.service.documents().getDocument(
            document_id=document_id,
            collection_id=collection_id,
            project_id=project_id,
            repo_id=repo_id,
            include_schema=True,
            use_json=True).execute()
        return resp

    def _normalize(self, path):
        return path.rstrip('/') if path else None

    def get_edit_url(self, doc=None):
        if not doc:
            return
        kintaro_collection = ''
        kintaro_document = doc.base
        doc_pod_path = self._normalize(doc.collection.pod_path)
        for binding in self.config.bind:
            if self._normalize(binding.collection) == doc_pod_path:
                kintaro_collection = binding.kintaro_collection
        if kintaro_collection:
            return KINTARO_EDIT_PATH_FORMAT.format(
                host=self.config.host,
                project=self.config.project,
                repo=self.config.repo,
                collection=kintaro_collection,
                document=kintaro_document)

    def can_inject(self, doc=None, collection=None):
        if not self.injected:
            return False
        if doc:
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    return True
        return False

    def inject(self, doc=None, collection=None):
        if doc:
            document_id = doc.base
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    entry = self.download_entry(
                        document_id=document_id,
                        collection_id=binding.kintaro_collection,
                        repo_id=self.config.repo,
                        project_id=self.config.project)
                    fields, _, _ = self._parse_entry(doc, entry)
                    doc.inject(fields, body='')
                    return doc

    def run(self, *args, **kwargs):
        for binding in self.config.bind:
            collection_pod_path = binding.collection
            kintaro_collection = binding.kintaro_collection
            entries = self.download_entries(
                repo_id=self.config.repo,
                collection_id=kintaro_collection,
                project_id=self.config.project)
            self.bind_collection(entries, collection_pod_path)

def schema_name_to_partial(value, sep='-', directory='views/partials',
                           use_sub_directory=False, prefix='partial'):
    """Parse a kintaro schema name to determine if it is a partial."""
    if value.lower().startswith(prefix):
        basename = value[len(prefix):]
        basename = PARTIAL_CONVERSION.sub(r'{}\1'.format(sep), basename)[1:]
        if use_sub_directory:
            directory = '{}/{}'.format(directory, basename.lower())
        return '/{}/{}.html'.format(directory, basename.lower())
    return None

class KintaroExtension(Extension):
    """Add a filter for jinja2 that assists with kintaro transformations."""

    def __init__(self, environment):
        super(KintaroExtension, self).__init__(environment)
        environment.filters[
            'kintaro.schema_name_to_partial'] = schema_name_to_partial
<|endoftext|>


<commit_before>"""Kintaro extension for integrating Kintaro into Grow sites."""

import datetime
import json
import logging
import os
import re
import grow
import httplib2
from googleapiclient import discovery
from googleapiclient import errors
from grow.common import oauth
from grow.common import utils
from grow.pods import documents
from jinja2.ext import Extension
from protorpc import messages

KINTARO_HOST = 'kintaro-content-server.appspot.com'
KINTARO_API_ROOT = 'https://{host}/_ah/api'.format(host=KINTARO_HOST)
DISCOVERY_URL = (
    KINTARO_API_ROOT + '/discovery/v1/apis/{api}/{apiVersion}/rest')
KINTARO_EDIT_PATH_FORMAT = (
    'https://{host}'
    '/app#/project/{project}'
    '/repo/{repo}'
    '/collection/{collection}'
    '/document/{document}/edit')
PARTIAL_CONVERSION = re.compile(r'([A-Z])')
STORAGE_KEY = 'Grow SDK - Kintaro'

# Silence extra logging from googleapiclient.
discovery.logger.setLevel(logging.WARNING)

OAUTH_SCOPES = ('https://www.googleapis.com/auth/userinfo.email',
                'https://www.googleapis.com/auth/kintaro')

class BindingMessage(messages.Message):
    collection = messages.StringField(1)
    kintaro_collection = messages.StringField(2)

class _GoogleServicePreprocessor(grow.Preprocessor):
    _last_run = None

    def create_service(self, host):
        credentials = oauth.get_or_create_credentials(
            scope=OAUTH_SCOPES, storage_key=STORAGE_KEY)
        http = httplib2.Http(ca_certs=utils.get_cacerts_path())
        http = credentials.authorize(http)
        # Kintaro's server doesn't seem to be able to refresh expired tokens
        # properly (responds with a "Stateless token expired" error). So we
        # manage state ourselves and refresh slightly more often than once
        # per hour.
        now = datetime.datetime.now()
        if self._last_run is None \
                or now - self._last_run >= datetime.timedelta(minutes=50):
            credentials.refresh(http)
            self._last_run = now
        url = DISCOVERY_URL.replace('{host}', host)
        return discovery.build('content', 'v1', http=http,
                               discoveryServiceUrl=url)

class KintaroPreprocessor(_GoogleServicePreprocessor):
    KIND = 'kintaro'

    class Config(messages.Message):
        bind = messages.MessageField(BindingMessage, 1, repeated=True)
        repo = messages.StringField(2)
        project = messages.StringField(3)
        host = messages.StringField(4, default=KINTARO_HOST)
        use_index = messages.BooleanField(5, default=True)

    def __init__(self, *args, **kwargs):
        super(KintaroPreprocessor, self).__init__(*args, **kwargs)
        self._service = None
        self._env_regex = None
        self._env_regex_match = None
        self._env_regex_replace = r'@env.\1'

    @property
    def service(self):
        if not self._service:
            self._service = self.create_service(host=self.config.host)
        return self._service

    def bind_collection(self, entries, collection_pod_path):
        collection = self.pod.get_collection(collection_pod_path)
        existing_pod_paths = [
            doc.pod_path
            for doc in collection.docs(recursive=False, inject=False)]
        new_pod_paths = []
        for i, entry in enumerate(entries):
            # TODO: Ensure `create_doc` doesn't die if the file doesn't exist.
            basename = self._get_basename_from_entry(entry)
            path = os.path.join(collection.pod_path, basename)
            if not self.pod.file_exists(path):
                self.pod.write_yaml(path, {})
            doc_pod_path = os.path.join(collection.pod_path, basename)
            doc = collection.get_doc(doc_pod_path)
            fields, unused_body, basename = self._parse_entry(doc, entry)
            doc = collection.create_doc(basename, fields=fields, body='')
            new_pod_paths.append(doc.pod_path)
            self.pod.logger.info('Saved -> {}'.format(doc.pod_path))

        pod_paths_to_delete = set(existing_pod_paths) - set(new_pod_paths)
        for pod_path in pod_paths_to_delete:
            self.pod.delete_file(pod_path)
            self.pod.logger.info('Deleted -> {}'.format(pod_path))

    def _fix_path_none(self, key, value):
        if self._env_regex_match.search(key):
            if key.startswith(('path', '$path')) and value is None:
                return ''
        return value

    def _regroup_schema(self, schema):
        names_to_fields = {}
        for field in schema:
            names_to_fields[field['name']] = field
        return names_to_fields

    def _parse_field(self, key, value, field_data, locale=None):
        # Convert Kintaro keys to Grow built-ins.
        if hasattr(documents, 'BUILT_IN_FIELDS'):
            built_in_fields = documents.BUILT_IN_FIELDS
        else:
            # Support older versions of grow.
            built_in_fields = ['title', 'order']
        if key in built_in_fields:
            key = '${}'.format(key)
        key = self._parse_field_key(key, field_data)
        value = self._parse_field_deep(key, value, field_data, locale=locale)
        value = self._fix_path_none(key, value)
        return key, value

    def _parse_field_deep(self, key, value, field_data, locale=None):
        single_field = not isinstance(value, list)
        if single_field:
            value = [value]

        # Handle ReferenceField as doc reference.
        if field_data['type'] == 'ReferenceField':
            for idx in range(len(value)):
                if not value[idx]:
                    continue
                for binding in self.config.bind:
                    if binding.kintaro_collection == value[idx]['collection_id']:
                        filename = '{}.yaml'.format(
                            value[idx]['document_id'])
                        content_path = os.path.join(
                            binding.collection, filename)
                        value[idx] = self.pod.get_doc(
                            content_path, locale=locale)
                        break
        elif 'schema_fields' in field_data:
            names_to_schema_fields = self._regroup_schema(
                field_data['schema_fields'])
            for idx in range(len(value)):
                value[idx] = self._parse_field_value(
                    value[idx], names_to_schema_fields,
                    locale=locale)
        if single_field:
            value = value[0]
        value = self._fix_path_none(key, value)

        return value

    def _parse_field_key(self, key, field_data):
        if field_data['translatable']:
            key = '{}@'.format(key)
        # Handle environment tagging.
        if self._env_regex:
            key = re.sub(self._env_regex, self._env_regex_replace, key)
        return key

    def _parse_field_value(self, value, names_to_schema_fields, locale=None):
        clean_value = {}
        for sub_key, sub_value in value.iteritems():
            new_key = self._parse_field_key(
                sub_key, names_to_schema_fields[sub_key])
            clean_value[new_key] = self._parse_field_deep(
                new_key, sub_value, names_to_schema_fields[sub_key],
                locale=locale)
        return clean_value

    def _get_basename_from_entry(self, entry):
        return '{}.yaml'.format(entry['document_id'])

    def _parse_entry(self, doc, entry):
        deployments = doc.pod.yaml.get('deployments', {}).keys()
        if deployments and not self._env_regex:
            self._env_regex = re.compile(
                r'_env_({})$'.format('|'.join(deployments)))
            self._env_regex_match = re.compile(
                r'@env.({})$'.format('|'.join(deployments)))
        basename = self._get_basename_from_entry(entry)
        schema = entry.get('schema', {})
        schema_fields = schema.get('schema_fields', [])
        names_to_schema_fields = self._regroup_schema(schema_fields)
        fields = entry.get('content_json', '{}')
        fields = json.loads(fields)
        clean_fields = {}
        # Preserve existing built-in fields prefixed with $.
        front_matter_data = doc.format.front_matter.data
        if front_matter_data:
            for key, value in front_matter_data.iteritems():
                if not key.startswith('$'):
                    continue
                clean_fields[key] = value
        # Overwrite with data from CMS.
        for name, value in fields.iteritems():
            field_data = names_to_schema_fields[name]
            key, value = self._parse_field(
                name, value, field_data, locale=doc.locale)
            clean_fields[key] = value
        # Populate $meta.
        if schema:
            # Strip modified info from schema.
            schema.pop('mod_info', None)
            clean_fields['$meta'] = {}
            clean_fields['$meta']['schema'] = schema
        body = ''
        return clean_fields, body, basename

    def _get_documents_from_search(self, repo_id, collection_id, project_id, documents):
        results = []
        for document in documents:
            document_id = document['document_id']
            entry = self.download_entry(
                document_id, collection_id, repo_id, project_id)
            results.append(entry)
        return results
        # TODO: Upgrade Grow's google api python client, use it to batch
        # requests.
        service = self.create_service(host=self.config.host)
        batch = service.new_batch_http_request()
        results = []

        def _add(entry):
            results.append(entry)
        for document in documents:
            document_id = document['document_id']
            req = service.documents().getDocument(
                document_id=document_id,
                collection_id=collection_id,
                project_id=project_id,
                repo_id=repo_id,
                include_schema=True,
                use_json=True)
            batch.add(req, callback=_add)
        batch.execute()
        return results

    def download_entries(self, repo_id, collection_id, project_id):
        body = {
            'repo_id': repo_id,
            'collection_id': collection_id,
            'project_id': project_id,
            'result_options': {
                'return_json': True,
                'return_schema': True,
            }
        }
        resp = self.service.documents().searchDocuments(body=body).execute()
        documents = resp.get('document_list', {}).get('documents', [])
        if not self.config.use_index:
            documents_from_get = self._get_documents_from_search(
                repo_id, collection_id, project_id, documents)
            return documents_from_get
        # Reformat document response to include schema.
        schema = resp.get('schema', {})
        for document in documents:
            document['schema'] = schema
        return documents

    def download_entry(self, document_id, collection_id, repo_id, project_id):
        resp = self.service.documents().getDocument(
            document_id=document_id,
            collection_id=collection_id,
            project_id=project_id,
            repo_id=repo_id,
            include_schema=True,
            use_json=True).execute()
        return resp

    def _normalize(self, path):
        return path.rstrip('/') if path else None

    def get_edit_url(self, doc=None):
        if not doc:
            return
        kintaro_collection = ''
        kintaro_document = doc.base
        doc_pod_path = self._normalize(doc.collection.pod_path)
        for binding in self.config.bind:
            if self._normalize(binding.collection) == doc_pod_path:
                kintaro_collection = binding.kintaro_collection
        if kintaro_collection:
            return KINTARO_EDIT_PATH_FORMAT.format(
                host=self.config.host,
                project=self.config.project,
                repo=self.config.repo,
                collection=kintaro_collection,
                document=kintaro_document)

    def can_inject(self, doc=None, collection=None):
        if not self.injected:
            return False
        if doc:
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    return True
        return False

    def inject(self, doc=None, collection=None):
        if doc:
            document_id = doc.base
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    entry = self.download_entry(
                        document_id=document_id,
                        collection_id=binding.kintaro_collection,
                        repo_id=self.config.repo,
                        project_id=self.config.project)
                    fields, _, _ = self._parse_entry(doc, entry)
                    doc.inject(fields, body='')
                    return doc

    def run(self, *args, **kwargs):
        for binding in self.config.bind:
            collection_pod_path = binding.collection
            kintaro_collection = binding.kintaro_collection
            entries = self.download_entries(
                repo_id=self.config.repo,
                collection_id=kintaro_collection,
                project_id=self.config.project)
            self.bind_collection(entries, collection_pod_path)

def schema_name_to_partial(value, sep='-', directory='views/partials',
                           use_sub_directory=False, prefix='partial'):
    """Parse a kintaro schema name to determine if it is a partial."""
    if value.lower().startswith(prefix):
        basename = value[len(prefix):]
        basename = PARTIAL_CONVERSION.sub(r'{}\1'.format(sep), basename)[1:]
        if use_sub_directory:
            directory = '{}/{}'.format(directory, basename.lower())
        return '/{}/{}.html'.format(directory, basename.lower())
    return None

class KintaroExtension(Extension):
    """Add a filter for jinja2 that assists with kintaro transformations."""

    def __init__(self, environment):
        super(KintaroExtension, self).__init__(environment)
        environment.filters[
            'kintaro.schema_name_to_partial'] = schema_name_to_partial
<commit_msg>Making it work with no regex.<commit_after>"""Kintaro extension for integrating Kintaro into Grow sites."""

import datetime
import json
import logging
import os
import re
import grow
import httplib2
from googleapiclient import discovery
from googleapiclient import errors
from grow.common import oauth
from grow.common import utils
from grow.pods import documents
from jinja2.ext import Extension
from protorpc import messages

KINTARO_HOST = 'kintaro-content-server.appspot.com'
KINTARO_API_ROOT = 'https://{host}/_ah/api'.format(host=KINTARO_HOST)
DISCOVERY_URL = (
    KINTARO_API_ROOT + '/discovery/v1/apis/{api}/{apiVersion}/rest')
KINTARO_EDIT_PATH_FORMAT = (
    'https://{host}'
    '/app#/project/{project}'
    '/repo/{repo}'
    '/collection/{collection}'
    '/document/{document}/edit')
PARTIAL_CONVERSION = re.compile(r'([A-Z])')
STORAGE_KEY = 'Grow SDK - Kintaro'

# Silence extra logging from googleapiclient.
discovery.logger.setLevel(logging.WARNING)

OAUTH_SCOPES = ('https://www.googleapis.com/auth/userinfo.email',
                'https://www.googleapis.com/auth/kintaro')

class BindingMessage(messages.Message):
    collection = messages.StringField(1)
    kintaro_collection = messages.StringField(2)

class _GoogleServicePreprocessor(grow.Preprocessor):
    _last_run = None

    def create_service(self, host):
        credentials = oauth.get_or_create_credentials(
            scope=OAUTH_SCOPES, storage_key=STORAGE_KEY)
        http = httplib2.Http(ca_certs=utils.get_cacerts_path())
        http = credentials.authorize(http)
        # Kintaro's server doesn't seem to be able to refresh expired tokens
        # properly (responds with a "Stateless token expired" error). So we
        # manage state ourselves and refresh slightly more often than once
        # per hour.
        now = datetime.datetime.now()
        if self._last_run is None \
                or now - self._last_run >= datetime.timedelta(minutes=50):
            credentials.refresh(http)
            self._last_run = now
        url = DISCOVERY_URL.replace('{host}', host)
        return discovery.build('content', 'v1', http=http,
                               discoveryServiceUrl=url)

class KintaroPreprocessor(_GoogleServicePreprocessor):
    KIND = 'kintaro'

    class Config(messages.Message):
        bind = messages.MessageField(BindingMessage, 1, repeated=True)
        repo = messages.StringField(2)
        project = messages.StringField(3)
        host = messages.StringField(4, default=KINTARO_HOST)
        use_index = messages.BooleanField(5, default=True)

    def __init__(self, *args, **kwargs):
        super(KintaroPreprocessor, self).__init__(*args, **kwargs)
        self._service = None
        self._env_regex = None
        self._env_regex_match = None
        self._env_regex_replace = r'@env.\1'

    @property
    def service(self):
        if not self._service:
            self._service = self.create_service(host=self.config.host)
        return self._service

    def bind_collection(self, entries, collection_pod_path):
        collection = self.pod.get_collection(collection_pod_path)
        existing_pod_paths = [
            doc.pod_path
            for doc in collection.docs(recursive=False, inject=False)]
        new_pod_paths = []
        for i, entry in enumerate(entries):
            # TODO: Ensure `create_doc` doesn't die if the file doesn't exist.
            basename = self._get_basename_from_entry(entry)
            path = os.path.join(collection.pod_path, basename)
            if not self.pod.file_exists(path):
                self.pod.write_yaml(path, {})
            doc_pod_path = os.path.join(collection.pod_path, basename)
            doc = collection.get_doc(doc_pod_path)
            fields, unused_body, basename = self._parse_entry(doc, entry)
            doc = collection.create_doc(basename, fields=fields, body='')
            new_pod_paths.append(doc.pod_path)
            self.pod.logger.info('Saved -> {}'.format(doc.pod_path))

        pod_paths_to_delete = set(existing_pod_paths) - set(new_pod_paths)
        for pod_path in pod_paths_to_delete:
            self.pod.delete_file(pod_path)
            self.pod.logger.info('Deleted -> {}'.format(pod_path))

    def _fix_path_none(self, key, value):
        if self._env_regex_match and self._env_regex_match.search(key):
            if key.startswith(('path', '$path')) and value is None:
                return ''
        return value

    def _regroup_schema(self, schema):
        names_to_fields = {}
        for field in schema:
            names_to_fields[field['name']] = field
        return names_to_fields

    def _parse_field(self, key, value, field_data, locale=None):
        # Convert Kintaro keys to Grow built-ins.
        if hasattr(documents, 'BUILT_IN_FIELDS'):
            built_in_fields = documents.BUILT_IN_FIELDS
        else:
            # Support older versions of grow.
            built_in_fields = ['title', 'order']
        if key in built_in_fields:
            key = '${}'.format(key)
        key = self._parse_field_key(key, field_data)
        value = self._parse_field_deep(key, value, field_data, locale=locale)
        value = self._fix_path_none(key, value)
        return key, value

    def _parse_field_deep(self, key, value, field_data, locale=None):
        single_field = not isinstance(value, list)
        if single_field:
            value = [value]

        # Handle ReferenceField as doc reference.
        if field_data['type'] == 'ReferenceField':
            for idx in range(len(value)):
                if not value[idx]:
                    continue
                for binding in self.config.bind:
                    if binding.kintaro_collection == value[idx]['collection_id']:
                        filename = '{}.yaml'.format(
                            value[idx]['document_id'])
                        content_path = os.path.join(
                            binding.collection, filename)
                        value[idx] = self.pod.get_doc(
                            content_path, locale=locale)
                        break
        elif 'schema_fields' in field_data:
            names_to_schema_fields = self._regroup_schema(
                field_data['schema_fields'])
            for idx in range(len(value)):
                value[idx] = self._parse_field_value(
                    value[idx], names_to_schema_fields,
                    locale=locale)
        if single_field:
            value = value[0]
        value = self._fix_path_none(key, value)

        return value

    def _parse_field_key(self, key, field_data):
        if field_data['translatable']:
            key = '{}@'.format(key)
        # Handle environment tagging.
        if self._env_regex:
            key = re.sub(self._env_regex, self._env_regex_replace, key)
        return key

    def _parse_field_value(self, value, names_to_schema_fields, locale=None):
        clean_value = {}
        for sub_key, sub_value in value.iteritems():
            new_key = self._parse_field_key(
                sub_key, names_to_schema_fields[sub_key])
            clean_value[new_key] = self._parse_field_deep(
                new_key, sub_value, names_to_schema_fields[sub_key],
                locale=locale)
        return clean_value

    def _get_basename_from_entry(self, entry):
        return '{}.yaml'.format(entry['document_id'])

    def _parse_entry(self, doc, entry):
        deployments = doc.pod.yaml.get('deployments', {}).keys()
        if deployments and not self._env_regex:
            self._env_regex = re.compile(
                r'_env_({})$'.format('|'.join(deployments)))
            self._env_regex_match = re.compile(
                r'@env.({})$'.format('|'.join(deployments)))
        basename = self._get_basename_from_entry(entry)
        schema = entry.get('schema', {})
        schema_fields = schema.get('schema_fields', [])
        names_to_schema_fields = self._regroup_schema(schema_fields)
        fields = entry.get('content_json', '{}')
        fields = json.loads(fields)
        clean_fields = {}
        # Preserve existing built-in fields prefixed with $.
        front_matter_data = doc.format.front_matter.data
        if front_matter_data:
            for key, value in front_matter_data.iteritems():
                if not key.startswith('$'):
                    continue
                clean_fields[key] = value
        # Overwrite with data from CMS.
        for name, value in fields.iteritems():
            field_data = names_to_schema_fields[name]
            key, value = self._parse_field(
                name, value, field_data, locale=doc.locale)
            clean_fields[key] = value
        # Populate $meta.
        if schema:
            # Strip modified info from schema.
            schema.pop('mod_info', None)
            clean_fields['$meta'] = {}
            clean_fields['$meta']['schema'] = schema
        body = ''
        return clean_fields, body, basename

    def _get_documents_from_search(self, repo_id, collection_id, project_id, documents):
        results = []
        for document in documents:
            document_id = document['document_id']
            entry = self.download_entry(
                document_id, collection_id, repo_id, project_id)
            results.append(entry)
        return results
        # TODO: Upgrade Grow's google api python client, use it to batch
        # requests.
        service = self.create_service(host=self.config.host)
        batch = service.new_batch_http_request()
        results = []

        def _add(entry):
            results.append(entry)
        for document in documents:
            document_id = document['document_id']
            req = service.documents().getDocument(
                document_id=document_id,
                collection_id=collection_id,
                project_id=project_id,
                repo_id=repo_id,
                include_schema=True,
                use_json=True)
            batch.add(req, callback=_add)
        batch.execute()
        return results

    def download_entries(self, repo_id, collection_id, project_id):
        body = {
            'repo_id': repo_id,
            'collection_id': collection_id,
            'project_id': project_id,
            'result_options': {
                'return_json': True,
                'return_schema': True,
            }
        }
        resp = self.service.documents().searchDocuments(body=body).execute()
        documents = resp.get('document_list', {}).get('documents', [])
        if not self.config.use_index:
            documents_from_get = self._get_documents_from_search(
                repo_id, collection_id, project_id, documents)
            return documents_from_get
        # Reformat document response to include schema.
        schema = resp.get('schema', {})
        for document in documents:
            document['schema'] = schema
        return documents

    def download_entry(self, document_id, collection_id, repo_id, project_id):
        resp = self.service.documents().getDocument(
            document_id=document_id,
            collection_id=collection_id,
            project_id=project_id,
            repo_id=repo_id,
            include_schema=True,
            use_json=True).execute()
        return resp

    def _normalize(self, path):
        return path.rstrip('/') if path else None

    def get_edit_url(self, doc=None):
        if not doc:
            return
        kintaro_collection = ''
        kintaro_document = doc.base
        doc_pod_path = self._normalize(doc.collection.pod_path)
        for binding in self.config.bind:
            if self._normalize(binding.collection) == doc_pod_path:
                kintaro_collection = binding.kintaro_collection
        if kintaro_collection:
            return KINTARO_EDIT_PATH_FORMAT.format(
                host=self.config.host,
                project=self.config.project,
                repo=self.config.repo,
                collection=kintaro_collection,
                document=kintaro_document)

    def can_inject(self, doc=None, collection=None):
        if not self.injected:
            return False
        if doc:
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    return True
        return False

    def inject(self, doc=None, collection=None):
        if doc:
            document_id = doc.base
            doc_pod_path = self._normalize(doc.collection.pod_path)
            for binding in self.config.bind:
                if self._normalize(binding.collection) == doc_pod_path:
                    entry = self.download_entry(
                        document_id=document_id,
                        collection_id=binding.kintaro_collection,
                        repo_id=self.config.repo,
                        project_id=self.config.project)
                    fields, _, _ = self._parse_entry(doc, entry)
                    doc.inject(fields, body='')
                    return doc

    def run(self, *args, **kwargs):
        for binding in self.config.bind:
            collection_pod_path = binding.collection
            kintaro_collection = binding.kintaro_collection
            entries = self.download_entries(
                repo_id=self.config.repo,
                collection_id=kintaro_collection,
                project_id=self.config.project)
            self.bind_collection(entries, collection_pod_path)

def schema_name_to_partial(value, sep='-', directory='views/partials',
                           use_sub_directory=False, prefix='partial'):
    """Parse a kintaro schema name to determine if it is a partial."""
    if value.lower().startswith(prefix):
        basename = value[len(prefix):]
        basename = PARTIAL_CONVERSION.sub(r'{}\1'.format(sep), basename)[1:]
        if use_sub_directory:
            directory = '{}/{}'.format(directory, basename.lower())
        return '/{}/{}.html'.format(directory, basename.lower())
    return None

class KintaroExtension(Extension):
    """Add a filter for jinja2 that assists with kintaro transformations."""

    def __init__(self, environment):
        super(KintaroExtension, self).__init__(environment)
        environment.filters[
            'kintaro.schema_name_to_partial'] = schema_name_to_partial
<|endoftext|>


<commit_before>from kivy.app import App
from kivy.uix.screenmanager import Screen, ScreenManager

class Model(object):
    def __init__(self, name):
        self.name = name
        self.presenters = []

    # return data for id here
    def get(self, id):
        raise Exception("not implemented")

    # set data for id here
    def _set(self, id, data):
        raise Exception("not implemented")

    def set(self, id, data):
        self._set(id, data)
        for p in self.presenters:
            p.modelEvent(self, id)

class DictModel(Model):
    def __init__(self, name):
        super(DictModel, self).__init__(name)
        self.data = {}

    def get(self, id):
        if id in self.data.keys():
            return self.data[id]
        else:
            return None

    def _set(self, id, data):
        self.data[id] = data

# e.g. for HttpModel you can just subclass DictModel and overload get, set s.t. you call
# super on get, if no hit, then fallback to http and call set of DictModel with result

class View(Screen):
    def __init__(self, presenter, **kwargs):
        super(View, self).__init__(**kwargs)
        self.presenter = presenter

    # update view based on new data here
    def _update(self, data):
        pass

    def update(self, data):
        self._update(data)
        self.canvas.ask_update()

    def event(self, e):
        self.presenter.userEvent(e)

# a View is just a small wrapper around kivy screens; no need for lots of functionality here.

class Presenter(object):
    def __init__(self, ctrl, viewClass, models):
        self.bus = ctrl.bus
        self.view = viewClass(self, name=self._name())
        ctrl.sm.add_widget(self.view)
        self.models = {}
        for model in models:
            self.models[model.name] = model
            model.presenters.append(self)
            self.modelEvent(model)

    # provide name here
    def _name(self):
        raise Exception("not implemented")

    def emit(self, event):
        self.bus.emit(event)

    # generic event from app controller or other presenter
    def receive(self, e):
        pass

    # associated view notifies us of user event, update model appropriately
    def userEvent(self, e):
        pass

    # model notfies us of update, refresh the view
    def modelEvent(self, model, e=None):
        pass

class AppController(object):
    def __init__(self):
        class EventBus(object):
            def __init__(self):
                self.listeners = []

            def register(self, obj):
                self.listeners.append(obj)

            def emit(self, event):
                for listener in self.listeners:
                    listener.receive(event)
        self.bus = EventBus()
        self.bus.register(self)
        self.sm = ScreenManager()
        self.presenters = {}
        sm = self.sm

        class KivyMVPApp(App):
            def build(self):
                return sm
            def on_pause(self):
                pass
            def on_resume(self):
                pass

        self.app = KivyMVPApp()

    def go(self, firstView):
        self.sm.current = firstView
        self.app.run()

    # listen to events from presenters here, e.g. switch triggers
    def receive(self, e):
        pass

    def add(self, pres):
        if pres._name() in self.presenters:
            raise Exception("presenter with name %s exists" % name)
        self.presenters[pres._name()] = pres
        self.bus.register(pres)

if __name__ == '__main__':
    from kivy.graphics import Color, Rectangle
    from kivy.uix.button import Button
    from kivy.uix.floatlayout import FloatLayout
    from kivy.uix.label import Label

    class TestAppController(AppController):
        def receive(self, e):
            if e == "switch":
                for p in self.presenters:
                    if self.sm.current != p:
                        self.sm.current = p
                        break

    ctrl = TestAppController()

    model = DictModel("aSingleNumber")
    model.set(0, 0)

    # This is a very basic example. Of course we should not duplicate code
    # for such a small difference in functionality. It is just to outline
    # how the framework is intended to be used.
    class BlackPresenter(Presenter):
        def _name(self):
            return "black"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "add":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x+1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    class WhitePresenter(Presenter):
        def _name(self):
            return "white"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "subtract":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x-1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    class ColorLayout(FloatLayout):
        def __init__(self, color, **kwargs):
            super(ColorLayout, self).__init__(**kwargs)
            with self.canvas.before:
                Color(color[0], color[1], color[2], color[3])
                self.rect = Rectangle(size=self.size, pos=self.pos)
            self.bind(size=self._update_rect, pos=self._update_rect)

        def _update_rect(self, instance, value):
            self.rect.pos = instance.pos
            self.rect.size = instance.size

    class BlackView(View):
        def __init__(self, presenter, **kwargs):
            super(BlackView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((0,0,0,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                     color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='add', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("add"))
                f.add_widget(b)
                b = Button(text='to white', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    class WhiteView(View):
        def __init__(self, presenter, **kwargs):
            super(WhiteView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((1,1,1,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                    color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='subtract', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("subtract"))
                f.add_widget(b)
                b = Button(text='to black', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    black_pres = BlackPresenter(ctrl, BlackView, [model])
    white_pres = WhitePresenter(ctrl, WhiteView, [model])

    ctrl.add(white_pres)
    ctrl.add(black_pres)

    ctrl.go('black')
<commit_msg>better explanations<commit_after>from kivy.app import App
from kivy.uix.screenmanager import Screen, ScreenManager

class Model(object):
    def __init__(self, name):
        self.name = name
        self.presenters = []

    # return data for id here
    def get(self, id):
        raise Exception("not implemented")

    # set data for id here
    def _set(self, id, data):
        raise Exception("not implemented")

    def set(self, id, data):
        self._set(id, data)
        for p in self.presenters:
            p.modelEvent(self, id)

class DictModel(Model):
    def __init__(self, name):
        super(DictModel, self).__init__(name)
        self.data = {}

    def get(self, id):
        if id in self.data.keys():
            return self.data[id]
        else:
            return None

    def _set(self, id, data):
        self.data[id] = data

# E.g. for HttpModel you can just subclass DictModel and overload get, set s.t. you call
# super on get, if no hit, then fallback to http and call set of DictModel with result

# More complex models will have new actions. They may also be unrelated to the actual UI
# and trigger remote actions in some backend.

class View(Screen):
    def __init__(self, presenter, **kwargs):
        super(View, self).__init__(**kwargs)
        self.presenter = presenter

    # update view based on new data here
    def _update(self, data):
        pass

    def update(self, data):
        self._update(data)
        self.canvas.ask_update()

    def event(self, e):
        self.presenter.userEvent(e)

# A View is just a small wrapper around kivy screens; no need for lots of functionality here.

class Presenter(object):
    def __init__(self, ctrl, viewClass, models):
        self.bus = ctrl.bus
        self.view = viewClass(self, name=self._name())
        ctrl.sm.add_widget(self.view)
        self.models = {}
        for model in models:
            self.models[model.name] = model
            model.presenters.append(self)
            self.modelEvent(model)

    # provide name here
    def _name(self):
        raise Exception("not implemented")

    def emit(self, event):
        self.bus.emit(event)

    # generic event from app controller or other presenter
    def receive(self, e):
        pass

    # associated view notifies us of user event, update model appropriately
    def userEvent(self, e):
        pass

    # model notfies us of update, refresh the view
    def modelEvent(self, model, e=None):
        pass

class AppController(object):
    def __init__(self):
        class EventBus(object):
            def __init__(self):
                self.listeners = []

            def register(self, obj):
                self.listeners.append(obj)

            def emit(self, event):
                for listener in self.listeners:
                    listener.receive(event)
        self.bus = EventBus()
        self.bus.register(self)
        self.sm = ScreenManager()
        self.presenters = {}
        sm = self.sm

        # TODO: Let's see what we have to expose here.
        class KivyMVPApp(App):
            def build(self):
                return sm
            def on_pause(self):
                pass
            def on_resume(self):
                pass

        self.app = KivyMVPApp()

    def go(self, firstView):
        self.sm.current = firstView
        self.app.run()

    # listen to events from presenters here, e.g. switch triggers
    def receive(self, e):
        pass

    def add(self, pres):
        if pres._name() in self.presenters:
            raise Exception("presenter with name %s exists" % name)
        self.presenters[pres._name()] = pres
        self.bus.register(pres)

if __name__ == '__main__':
    from kivy.graphics import Color, Rectangle
    from kivy.uix.button import Button
    from kivy.uix.floatlayout import FloatLayout
    from kivy.uix.label import Label

    # Our app controller simply listens for "switch" events and switches between
    # the two presenters, if it receives one.
    class TestAppController(AppController):
        def receive(self, e):
            if e == "switch":
                for p in self.presenters:
                    if self.sm.current != p:
                        self.sm.current = p
                        break

    ctrl = TestAppController()

    # Our model is a simple dictionary containing a single integer at key 0.
    model = DictModel("aSingleNumber")
    model.set(0, 0)

    # This is a very basic example. Of course we should not duplicate code
    # for such a small difference in functionality. It is just to outline
    # how the framework is intended to be used.

    # The black presenter listens for two user events.
    # If it receives "done" it signals "switch" to the app controller's event bus.
    # (Note: all presenters and the app controller are registered at the event bus
    #  and can response to events if required)
    # It it receives an "add" event it retrieves the current number from the model,
    # increments by one and puts it back into the model.
    # On receiving any event from the model it simply retrieves the currently stored
    # number and instructs the view to update based on it.
    class BlackPresenter(Presenter):
        def _name(self):
            return "black"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "add":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x+1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    # The white presenter listens for two user events.
    # If it receives "done" it signals "switch" to the app controller's event bus.
    # (Note: all presenters and the app controller are registered at the event bus
    #  and can response to events if required)
    # It it receives an "subtract" event it retrieves the current number from the model,
    # decrements by one and puts it back into the model.
    # On receiving any event from the model it simply retrieves the currently stored
    # number and instructs the view to update based on it.
    class WhitePresenter(Presenter):
        def _name(self):
            return "white"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "subtract":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x-1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    # This is just a simple layout with a background color such that we can easily
    # distinguish our two views.
    class ColorLayout(FloatLayout):
        def __init__(self, color, **kwargs):
            super(ColorLayout, self).__init__(**kwargs)
            with self.canvas.before:
                Color(color[0], color[1], color[2], color[3])
                self.rect = Rectangle(size=self.size, pos=self.pos)
            self.bind(size=self._update_rect, pos=self._update_rect)

        def _update_rect(self, instance, value):
            self.rect.pos = instance.pos
            self.rect.size = instance.size

    # The black view has a button "add" and a button "to_white" on a black background.
    # Pressing "add" triggers emits the event "add" and pressing "to white" triggers "done".
    # When it receives an update event it updates the label text with the new data.
    # Note that all kivy events should just trigger self.event with the appropriate data to
    # integrate into the MVP workflow.
    class BlackView(View):
        def __init__(self, presenter, **kwargs):
            super(BlackView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((0,0,0,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                     color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='add', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("add"))
                f.add_widget(b)
                b = Button(text='to white', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    # The white view has a button "add" and a button "to_black" on a white background.
    # Pressing "add" triggers emits the event "add" and pressing "to black" triggers "done".
    # When it receives an update event it updates the label text with the new data.
    class WhiteView(View):
        def __init__(self, presenter, **kwargs):
            super(WhiteView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((1,1,1,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                    color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='subtract', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("subtract"))
                f.add_widget(b)
                b = Button(text='to black', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    black_pres = BlackPresenter(ctrl, BlackView, [model])
    white_pres = WhitePresenter(ctrl, WhiteView, [model])

    ctrl.add(white_pres)
    ctrl.add(black_pres)

    # Start black.
    ctrl.go('black')
<|endoftext|>