<commit_before>__version__ = '0.1'
__author__ = 'Decklin Foster'
__email__ = 'decklin@red-bean.com'
__url__ = 'http://www.red-bean.com/~decklin/software/mnemosyne/'

import os
import mailbox
import time
import stat
import em
import shutil
from rsthtml import publish_content

def magic_attr(obj, rep):
    class Magic(type(obj)):
        def __repr__(self): return rep
    return Magic(obj)

def clean(s, maxwords=None):
    if s:
        words = s.strip().lower().split()
        if maxwords: words = words[:maxwords]
        words = [''.join([c for c in w if c.isalnum()]) for w in words]
        return '-'.join(words)

namespaces = {}
def uniq(ns, k, tag):
    ns = namespaces.setdefault(ns, {})
    ns.setdefault(k, {})

    def qual(s, n):
        if n == 0: return s
        else: return '%s-%d' % (s, n)

    if tag not in ns[k].keys():
        ns[k][tag] = qual(k, len(ns[k].keys()))

    return ns[k][tag]

class BaseEntry:
    def __init__(self, m):
        def fixdate(d):
            # For some bizarre reason, getdate doesn't set wday/yday/isdst...
            return time.localtime(time.mktime(d))
        def getstamp(maildirpath):
            stamp, id, host = os.path.split(maildirpath)[1].split('.')
            return int(stamp)

        self.m = m
        self.date = fixdate(self.m.getdate('Date'))
        self.mtime = time.localtime(getstamp(self.m.fp.name))

    def __cmp__(self, other):
        return cmp(time.mktime(self.date), time.mktime(other.date))

    def get_content(self):
        s = self.m.fp.read()
        try: s = s[:s.rindex('-- \n')]
        except ValueError: pass

        return magic_attr(publish_content(s), s[:100])

    def get_subject(self):
        try:
            subject = self.m['Subject']
            cleaned = clean(subject, 3)
        except KeyError:
            subject = ''
            cleaned = 'entry'

        u = uniq(self.date[0:3], cleaned, time.mktime(self.date))
        return magic_attr(subject, u)

    def get_id(self):
        try:
            id = self.m['Message-Id']
            lhs, host = id[1:-1].split('@')
            date = time.strftime('%Y-%m-%d', self.date)
            return magic_attr(id, 'tag:%s,%s:%s' % (host, date, lhs))
        except KeyError:
            return None

    def get_author(self):
        author = self.m.getaddr('From')[0]
        return magic_attr(author, clean(author))

    def get_email(self):
        email = self.m.getaddr('From')[1]
        cleaned = email.replace('@', ' at ').replace('.', ' dot ')
        return magic_attr(email, cleaned)

    def get_tags(self):
        try:
            tags = [t.strip() for t in self.m['X-Tags'].split(',')]
            return [magic_attr(t, clean(t)) for t in tags]
        except KeyError:
            return []

    def get_year(self):
        return magic_attr(self.date[0], time.strftime('%Y', self.date))

    def get_month(self):
        return magic_attr(self.date[1], time.strftime('%m', self.date))

    def get_day(self):
        return magic_attr(self.date[2], time.strftime('%d', self.date))

class Muse:

    def __init__(self, config, force):
        DEF_BASE_DIR = os.path.join(os.environ['HOME'], 'Mnemosyne')
        DEF_IGNORE = ('.svn', 'CVS')

        self.force = force
        self.config = {
            'entry_dir': os.path.join(DEF_BASE_DIR, 'entries'),
            'layout_dir': os.path.join(DEF_BASE_DIR, 'layout'),
            'style_dir': os.path.join(DEF_BASE_DIR, 'style'),
            'output_dir': os.path.join(DEF_BASE_DIR, 'htdocs'),
            'ignore': DEF_IGNORE,
            'locals': {
                '__version__': __version__,
                '__author__': __author__,
                '__email__': __email__,
                '__url__': __url__,
                },
            }

        exec file(config) in self.config

        class NoMixin: pass
        Mixin = self.config.get('EntryMixin', NoMixin)

        class Entry(Mixin, BaseEntry):
            def __getattr__(self, a):
                for c in (Mixin, BaseEntry):
                    try:
                        method = getattr(c, 'get_'+a)
                        self.__dict__[a] = method(self)
                        return self.__dict__[a]
                    except AttributeError:
                        pass
                else:
                    return getattr(BaseEntry, a)

        box = mailbox.Maildir(self.config['entry_dir'])
        self.entries = [Entry(msg) for msg in box]
        self.entries.sort()

        self.where = []

    def sing(self, entries=None, spath=None, dpath=None, what=None):
        if not entries: entries = self.entries
        if not spath: spath = self.config['layout_dir']
        if not dpath: dpath = self.config['output_dir']

        if what:
            source, dest = what
            spath = os.path.join(spath, source)
            dpath = os.path.join(dpath, dest)
            if source not in self.config['ignore']:
                if os.path.isfile(spath):
                    if os.stat(spath).st_mode & stat.S_IXUSR:
                        self.sing_file(entries, spath, dpath)
                    else:
                        shutil.copyfile(spath, dpath)
                elif os.path.isdir(spath):
                    self.sing(entries, spath, dpath)
        else:
            if not os.path.isdir(dpath): os.makedirs(dpath)
            for f in os.listdir(spath):
                if f.startswith('__'):
                    self.sing_instances(entries, spath, dpath, f)
                else:
                    self.where.append(f)
                    self.sing(entries, spath, dpath, (f, f))
                    self.where.pop()

    def sing_instances(self, entries, spath, dpath, what):
        magic = what[:what.rfind('__')+2]
        name = magic[2:-2]

        def cheapiter(x):
            try:
                assert not isinstance((x), (str, unicode))
                for e in x: return iter(x)
                else: return iter(())
            except (AssertionError, TypeError):
                if x: return iter((x,))
                else: return iter(())

        inst = {}
        for e in entries:
            mv = getattr(e, name, None)
            for m in cheapiter(mv):
                inst.setdefault(repr(m), []).append(e)

        for k, entries in inst.iteritems():
            self.where.append(k)
            self.sing(entries, spath, dpath, (what, what.replace(magic, k)))
            self.where.pop()

    def expand(self, style, locals):
        style = os.path.join(self.config['style_dir'], '%s.empy' % style)
        return em.expand(file(style).read(), locals)

    def sing_file(self, entries, spath, dpath):
        if not self.force:
            if os.path.exists(dpath):
                smtime = max([e.mtime for e in entries])
                dmtime = time.localtime(os.stat(dpath).st_mtime)
                if dmtime > smtime:
                    return

        def write(data):
            file(dpath, 'w').write(data)
            print 'Wrote %s' % dpath

        locals = self.config['locals'].copy()
        locals['muse'] = self
        locals['write'] = write
        locals['entries'] = entries

        exec file(spath) in locals
<commit_msg>What a mess. To ensure proper ordering of the uniq'd subject, it must be evaled up front, in sort order, not as the pages would have it since that is probably reversed.<commit_after>__version__ = '0.1'
__author__ = 'Decklin Foster'
__email__ = 'decklin@red-bean.com'
__url__ = 'http://www.red-bean.com/~decklin/software/mnemosyne/'

import os
import mailbox
import time
import stat
import em
import shutil
from rsthtml import publish_content

def magic_attr(obj, rep):
    class Magic(type(obj)):
        def __repr__(self): return rep
    return Magic(obj)

def clean(s, maxwords=None):
    if s:
        words = s.strip().lower().split()
        if maxwords: words = words[:maxwords]
        words = [''.join([c for c in w if c.isalnum()]) for w in words]
        return '-'.join(words)

namespaces = {}
def uniq(ns, k, tag):
    ns = namespaces.setdefault(ns, {})
    ns.setdefault(k, {})

    def qual(s, n):
        if n == 0: return s
        else: return '%s-%d' % (s, n)

    if tag not in ns[k].keys():
        ns[k][tag] = qual(k, len(ns[k].keys()))

    return ns[k][tag]

class BaseEntry:
    def __init__(self, m):
        def fixdate(d):
            # For some bizarre reason, getdate doesn't set wday/yday/isdst...
            return time.localtime(time.mktime(d))
        def getstamp(maildirpath):
            stamp, id, host = os.path.split(maildirpath)[1].split('.')
            return int(stamp)

        self.m = m
        self.date = fixdate(self.m.getdate('Date'))
        self.mtime = time.localtime(getstamp(self.m.fp.name))

    def __cmp__(self, other):
        return cmp(time.mktime(self.date), time.mktime(other.date))

    def get_content(self):
        s = self.m.fp.read()
        try: s = s[:s.rindex('-- \n')]
        except ValueError: pass

        return magic_attr(publish_content(s), s[:100])

    def make_subject(self):
        try:
            subject = self.m['Subject']
            cleaned = clean(subject, 3)
        except KeyError:
            subject = ''
            cleaned = 'entry'

        u = uniq(self.date[0:3], cleaned, time.mktime(self.date))
        return magic_attr(subject, u)

    def get_id(self):
        try:
            id = self.m['Message-Id']
            lhs, host = id[1:-1].split('@')
            date = time.strftime('%Y-%m-%d', self.date)
            return magic_attr(id, 'tag:%s,%s:%s' % (host, date, lhs))
        except KeyError:
            return None

    def get_author(self):
        author = self.m.getaddr('From')[0]
        return magic_attr(author, clean(author))

    def get_email(self):
        email = self.m.getaddr('From')[1]
        cleaned = email.replace('@', ' at ').replace('.', ' dot ')
        return magic_attr(email, cleaned)

    def get_tags(self):
        try:
            tags = [t.strip() for t in self.m['X-Tags'].split(',')]
            return [magic_attr(t, clean(t)) for t in tags]
        except KeyError:
            return []

    def get_year(self):
        return magic_attr(self.date[0], time.strftime('%Y', self.date))

    def get_month(self):
        return magic_attr(self.date[1], time.strftime('%m', self.date))

    def get_day(self):
        return magic_attr(self.date[2], time.strftime('%d', self.date))

class Muse:

    def __init__(self, config, force):
        DEF_BASE_DIR = os.path.join(os.environ['HOME'], 'Mnemosyne')
        DEF_IGNORE = ('.svn', 'CVS')

        self.where = []
        self.force = force
        self.config = {
            'entry_dir': os.path.join(DEF_BASE_DIR, 'entries'),
            'layout_dir': os.path.join(DEF_BASE_DIR, 'layout'),
            'style_dir': os.path.join(DEF_BASE_DIR, 'style'),
            'output_dir': os.path.join(DEF_BASE_DIR, 'htdocs'),
            'ignore': DEF_IGNORE,
            'locals': {
                '__version__': __version__,
                '__author__': __author__,
                '__email__': __email__,
                '__url__': __url__,
                },
            }

        exec file(config) in self.config

        class NoMixin: pass
        Mixin = self.config.get('EntryMixin', NoMixin)

        # get_* is evaluated on demand
        class Entry(Mixin, BaseEntry):
            def __getattr__(self, a):
                for c in (Mixin, BaseEntry):
                    try:
                        method = getattr(c, 'get_'+a)
                        self.__dict__[a] = method(self)
                        return self.__dict__[a]
                    except AttributeError:
                        pass
                else:
                    return getattr(BaseEntry, a)

        # have to do this after we define Entry but before we iter it
        box = mailbox.Maildir(self.config['entry_dir'])
        self.entries = [Entry(msg) for msg in box]
        self.entries.sort()

        # make_* is evaluated now
        for c in (Mixin, BaseEntry):
            for n, m in c.__dict__.iteritems():
                if n.startswith('make_'):
                    for e in self.entries:
                        setattr(e, n[5:], m(e))

    def sing(self, entries=None, spath=None, dpath=None, what=None):
        if not entries: entries = self.entries
        if not spath: spath = self.config['layout_dir']
        if not dpath: dpath = self.config['output_dir']

        if what:
            source, dest = what
            spath = os.path.join(spath, source)
            dpath = os.path.join(dpath, dest)
            if source not in self.config['ignore']:
                if os.path.isfile(spath):
                    if os.stat(spath).st_mode & stat.S_IXUSR:
                        self.sing_file(entries, spath, dpath)
                    else:
                        shutil.copyfile(spath, dpath)
                elif os.path.isdir(spath):
                    self.sing(entries, spath, dpath)
        else:
            if not os.path.isdir(dpath): os.makedirs(dpath)
            for f in os.listdir(spath):
                if f.startswith('__'):
                    self.sing_instances(entries, spath, dpath, f)
                else:
                    self.where.append(f)
                    self.sing(entries, spath, dpath, (f, f))
                    self.where.pop()

    def sing_instances(self, entries, spath, dpath, what):
        magic = what[:what.rfind('__')+2]
        name = magic[2:-2]

        def cheapiter(x):
            try:
                assert not isinstance((x), (str, unicode))
                for e in x: return iter(x)
                else: return iter(())
            except (AssertionError, TypeError):
                if x: return iter((x,))
                else: return iter(())

        inst = {}
        for e in entries:
            mv = getattr(e, name, None)
            for m in cheapiter(mv):
                inst.setdefault(repr(m), []).append(e)

        for k, entries in inst.iteritems():
            self.where.append(k)
            self.sing(entries, spath, dpath, (what, what.replace(magic, k)))
            self.where.pop()

    def expand(self, style, locals):
        style = os.path.join(self.config['style_dir'], '%s.empy' % style)
        return em.expand(file(style).read(), locals)

    def sing_file(self, entries, spath, dpath):
        if not self.force:
            if os.path.exists(dpath):
                smtime = max([e.mtime for e in entries])
                dmtime = time.localtime(os.stat(dpath).st_mtime)
                if dmtime > smtime:
                    return

        def write(data):
            file(dpath, 'w').write(data)
            print 'Wrote %s' % dpath

        locals = self.config['locals'].copy()
        locals['muse'] = self
        locals['write'] = write
        locals['entries'] = entries

        exec file(spath) in locals
<|endoftext|>


<commit_before>import yaml

class RoleBuilder():

    # Global list of roles
    ROLE_LIST = ['clc',
                 'user-facing',
                 'walrus',
                 'midonet-api',
                 'cluster-controller',
                 'storage-controller',
                 'node-controller',
                 'midolman',
                 'mon-bootstrap',
                 'ceph-mons',
                 'ceph-osds',
                 'riak-head',
                 'riak-node',
                 'haproxy',
                 'nginx',
                 'all']

    def __init__(self, environment_file='environment.yml'):
        self.environment_file = environment_file
        self.env_dict = self.get_all_attributes()
        self.roles = self.get_roles()
        self.all_hosts = self.roles['all']

    def read_environment(self):
        with open(self.environment_file) as env_file:
            return yaml.load(env_file.read())

    def get_all_attributes(self):
        env_dicts = self.read_environment()
        return env_dicts['default_attributes']

    def get_euca_attributes(self):
        try:
            return self.env_dict['eucalyptus']
        except:
            return None

    def get_riak_attributes(self):
        try:
            return self.env_dict['riakcs_cluster']
        except:
            return None

    def get_ceph_attributes(self):
        try:
            return self.env_dict['ceph']
        except:
            return None

    def _initialize_roles(self):
        roles = {}
        for role in self.ROLE_LIST:
            roles[role] = set()
        return roles

    def get_euca_hosts(self):
        roles = self.get_roles()

        # Create set of Eucalytpus only componnents
        euca_components = ['user-facing', 'cluster-controller',
                           'storage-controller', 'node-controller']
        if roles['walrus']:
            euca_components.append('walrus')

        all_hosts = roles['clc']
        for component in euca_components:
            all_hosts.update(roles[component])
        return all_hosts

    def get_roles(self):
        roles = self._initialize_roles()
        euca_attributes = self.get_euca_attributes()
        ceph_attributes = self.get_ceph_attributes()
        riak_attributes = self.get_riak_attributes()

        roles['all'] = set([])

        if riak_attributes:
            riak_topology = riak_attributes['topology']
            if riak_topology['head']:
                roles['riak-head'] = set([riak_topology['head']['ipaddr']])
                roles['all'].add(riak_topology['head']['ipaddr'])
            else:
                raise Exception("No head node found for RiakCS cluster!")

            if riak_topology.get('nodes'):
                for n in riak_topology['nodes']:
                    roles['riak-node'].add(n)
                    roles['all'].add(n)
            if riak_topology.get('load_balancer'):
                riak_lb = None
                if self.env_dict.get('nginx'):
                    riak_lb = 'nginx'
                    raise Exception("Nginx: Not implemented yet.")
                elif self.env_dict.get('haproxy'):
                    riak_lb = 'haproxy'
                else:
                    raise Exception("No Load-Balancer found for RiakCS cluster.")
                roles[riak_lb] = set([riak_topology['load_balancer']])
                roles['all'].add(riak_topology['load_balancer'])

        if ceph_attributes:
            ceph_topology = ceph_attributes['topology']
            if ceph_topology['mon_bootstrap']:
                roles['mon-bootstrap'] = set([ceph_topology['mon_bootstrap']['ipaddr']])
                roles['all'].add(ceph_topology['mon_bootstrap']['ipaddr'])
            else:
                raise Exception("No Monitor found for bootstraping!")

            if ceph_topology.get('mons'):
                monset = set()
                for mon in ceph_topology['mons']:
                    monset.add(mon['ipaddr'])
                    roles['all'].add(mon['ipaddr'])
                roles['ceph-mons'] = monset

            if ceph_topology['osds']:
                osdset = set()
                for osd in ceph_topology['osds']:
                    osdset.add(osd['ipaddr'])
                    roles['all'].add(osd['ipaddr'])
                roles['ceph-osds'] = osdset
            else:
                raise Exception("No OSD Found!")

        if euca_attributes:
            topology = euca_attributes['topology']

            # Add CLC
            roles['clc'] = set([topology['clc-1']])
            roles['all'].add(topology['clc-1'])

            # Add UFS
            roles['user-facing'] = set(topology['user-facing'])
            for ufs in topology['user-facing']:
                roles['all'].add(ufs)

            # Add Walrus
            if 'walrus' in topology:
                roles['walrus'] = set([topology['walrus']])
                roles['all'].add(topology['walrus'])
            else:
                # No walrus defined assuming RiakCS
                roles['walrus'] = set()

            # Add cluster level components
            for name in topology['clusters']:
                roles['cluster'] = {}
                if 'cc-1' in topology['clusters'][name]:
                    cc = topology['clusters'][name]['cc-1']
                    roles['cluster-controller'].add(cc)
                else:
                    raise IndexError("Unable to find CC in topology for cluster " + name)

                if 'sc-1' in topology['clusters'][name]:
                    sc = topology['clusters'][name]['sc-1']
                    roles['storage-controller'].add(sc)
                else:
                    raise IndexError("Unable to find SC in topology for cluster " + name)

                roles['cluster'][name] = set([cc, sc])
                if 'nodes' in topology['clusters'][name]:
                    nodes = topology['clusters'][name]['nodes'].split()
                else:
                    raise IndexError("Unable to find nodes in topology for cluster " + name)
                for node in nodes:
                    roles['node-controller'].add(node)
                    roles['cluster'][name].add(node)
                roles['all'].update(roles['cluster'][name])

            # Add Midokura roles
            midokura_attributes = self.env_dict.get('midokura', None)
            if midokura_attributes and euca_attributes['network']['mode'] == 'VPCMIDO':
                try:
                    mido = euca_attributes['network']['config-json']['Mido']
                    mido_gw_hostname = mido.get('EucanetdHost', None)
                    midolman_host_mapping = midokura_attributes.get('midolman-host-mapping', None)
                    if midolman_host_mapping:
                        mido_api_ip = midolman_host_mapping.get(mido_gw_hostname, None)
                        # Add the host IP for the midonet gw
                        roles['midonet-api'].add(mido_api_ip)
                        # Add hosts from the midonet host mapping, and all nodes
                        for hostname, host_ip in midolman_host_mapping.iteritems():
                            roles['midolman'].add(host_ip)
                        for node in roles['node-controller']:
                            roles['midolman'].add(node)
                except KeyError:
                    roles['midolman'] = roles['node-controller']
                    roles['midonet-api'] = roles['clc']
        return roles
<commit_msg>Better error message when midonet-api is not in midolman hostmapping<commit_after>import yaml

class RoleBuilder():

    # Global list of roles
    ROLE_LIST = ['clc',
                 'user-facing',
                 'walrus',
                 'midonet-api',
                 'cluster-controller',
                 'storage-controller',
                 'node-controller',
                 'midolman',
                 'mon-bootstrap',
                 'ceph-mons',
                 'ceph-osds',
                 'riak-head',
                 'riak-node',
                 'haproxy',
                 'nginx',
                 'all']

    def __init__(self, environment_file='environment.yml'):
        self.environment_file = environment_file
        self.env_dict = self.get_all_attributes()
        self.roles = self.get_roles()
        self.all_hosts = self.roles['all']

    def read_environment(self):
        with open(self.environment_file) as env_file:
            return yaml.load(env_file.read())

    def get_all_attributes(self):
        env_dicts = self.read_environment()
        return env_dicts['default_attributes']

    def get_euca_attributes(self):
        try:
            return self.env_dict['eucalyptus']
        except:
            return None

    def get_riak_attributes(self):
        try:
            return self.env_dict['riakcs_cluster']
        except:
            return None

    def get_ceph_attributes(self):
        try:
            return self.env_dict['ceph']
        except:
            return None

    def _initialize_roles(self):
        roles = {}
        for role in self.ROLE_LIST:
            roles[role] = set()
        return roles

    def get_euca_hosts(self):
        roles = self.get_roles()

        # Create set of Eucalytpus only componnents
        euca_components = ['user-facing', 'cluster-controller',
                           'storage-controller', 'node-controller']
        if roles['walrus']:
            euca_components.append('walrus')

        all_hosts = roles['clc']
        for component in euca_components:
            all_hosts.update(roles[component])
        return all_hosts

    def get_roles(self):
        roles = self._initialize_roles()
        euca_attributes = self.get_euca_attributes()
        ceph_attributes = self.get_ceph_attributes()
        riak_attributes = self.get_riak_attributes()

        roles['all'] = set([])

        if riak_attributes:
            riak_topology = riak_attributes['topology']
            if riak_topology['head']:
                roles['riak-head'] = set([riak_topology['head']['ipaddr']])
                roles['all'].add(riak_topology['head']['ipaddr'])
            else:
                raise Exception("No head node found for RiakCS cluster!")

            if riak_topology.get('nodes'):
                for n in riak_topology['nodes']:
                    roles['riak-node'].add(n)
                    roles['all'].add(n)
            if riak_topology.get('load_balancer'):
                riak_lb = None
                if self.env_dict.get('nginx'):
                    riak_lb = 'nginx'
                    raise Exception("Nginx: Not implemented yet.")
                elif self.env_dict.get('haproxy'):
                    riak_lb = 'haproxy'
                else:
                    raise Exception("No Load-Balancer found for RiakCS cluster.")
                roles[riak_lb] = set([riak_topology['load_balancer']])
                roles['all'].add(riak_topology['load_balancer'])

        if ceph_attributes:
            ceph_topology = ceph_attributes['topology']
            if ceph_topology['mon_bootstrap']:
                roles['mon-bootstrap'] = set([ceph_topology['mon_bootstrap']['ipaddr']])
                roles['all'].add(ceph_topology['mon_bootstrap']['ipaddr'])
            else:
                raise Exception("No Monitor found for bootstraping!")

            if ceph_topology.get('mons'):
                monset = set()
                for mon in ceph_topology['mons']:
                    monset.add(mon['ipaddr'])
                    roles['all'].add(mon['ipaddr'])
                roles['ceph-mons'] = monset

            if ceph_topology['osds']:
                osdset = set()
                for osd in ceph_topology['osds']:
                    osdset.add(osd['ipaddr'])
                    roles['all'].add(osd['ipaddr'])
                roles['ceph-osds'] = osdset
            else:
                raise Exception("No OSD Found!")

        if euca_attributes:
            topology = euca_attributes['topology']

            # Add CLC
            roles['clc'] = set([topology['clc-1']])
            roles['all'].add(topology['clc-1'])

            # Add UFS
            roles['user-facing'] = set(topology['user-facing'])
            for ufs in topology['user-facing']:
                roles['all'].add(ufs)

            # Add Walrus
            if 'walrus' in topology:
                roles['walrus'] = set([topology['walrus']])
                roles['all'].add(topology['walrus'])
            else:
                # No walrus defined assuming RiakCS
                roles['walrus'] = set()

            # Add cluster level components
            for name in topology['clusters']:
                roles['cluster'] = {}
                if 'cc-1' in topology['clusters'][name]:
                    cc = topology['clusters'][name]['cc-1']
                    roles['cluster-controller'].add(cc)
                else:
                    raise IndexError("Unable to find CC in topology for cluster " + name)

                if 'sc-1' in topology['clusters'][name]:
                    sc = topology['clusters'][name]['sc-1']
                    roles['storage-controller'].add(sc)
                else:
                    raise IndexError("Unable to find SC in topology for cluster " + name)

                roles['cluster'][name] = set([cc, sc])
                if 'nodes' in topology['clusters'][name]:
                    nodes = topology['clusters'][name]['nodes'].split()
                else:
                    raise IndexError("Unable to find nodes in topology for cluster " + name)
                for node in nodes:
                    roles['node-controller'].add(node)
                    roles['cluster'][name].add(node)
                roles['all'].update(roles['cluster'][name])

            # Add Midokura roles
            midokura_attributes = self.env_dict.get('midokura', None)
            if midokura_attributes and euca_attributes['network']['mode'] == 'VPCMIDO':
                try:
                    mido = euca_attributes['network']['config-json']['Mido']
                    mido_gw_hostname = mido.get('EucanetdHost', None)
                    midolman_host_mapping = midokura_attributes.get('midolman-host-mapping', None)
                    if midolman_host_mapping:
                        mido_api_ip = midolman_host_mapping.get(mido_gw_hostname, None)
                        if not mido_api_ip:
                            raise Exception('Unable to find midonet-api ({0}) host in midolman-host-mapping'.format(mido_gw_hostname))
                        # Add the host IP for the midonet gw
                        roles['midonet-api'].add(mido_api_ip)
                        # Add hosts from the midonet host mapping, and all nodes
                        for hostname, host_ip in midolman_host_mapping.iteritems():
                            roles['midolman'].add(host_ip)
                        for node in roles['node-controller']:
                            roles['midolman'].add(node)
                except KeyError:
                    roles['midolman'] = roles['node-controller']
                    roles['midonet-api'] = roles['clc']
        return roles
<|endoftext|>


<commit_before>
def import_question(posts, namespaces, id, title, body, tags, last_activity_date, last_updated_date, score, answer_count, has_accepted_answer):
    namespaces_for_post = {}
    for name, n in namespaces.items():
        namespace_tags = n.get_tags()
        if not(namespace_tags) or any(map(lambda x: x in tags, namespace_tags)):
            ids = n.get_ids(title, body, tags)
            if len(ids) > 0:
                ids = map(lambda x: x.lower(), ids)
                namespaces_for_post[name] = ids

    if len(namespaces_for_post):
        post = posts.find_one({"question_id": int(id)})
        previously_existed = False
        update = True
        if post:
            previously_existed = True
            # Only update title, score etc. if this is the latest data
            update = post["last_activity"] < last_activity_date
        else:
            post = {
                "question_id": int(id),
                "url": "http://stackoverflow.com/questions/%s" % id
            }

        post["namespaces"] = namespaces_for_post
        if update:
            post["title"] = title
            post["score"] = int(score)
            post["answers"] = int(answer_count)
            post["accepted_answer"] = has_accepted_answer
            post["last_activity"] = last_activity_date
            post["last_updated"] = last_updated_date

        if previously_existed:
            posts.update({"question_id": int(id)}, post)
        else:
            posts.insert(post)

        update_text = "Fully updated" if update else "Partially updated"
        print "%s %s question from %s (%s)" % (update_text if previously_existed else "Inserted", ", ".join(namespaces_for_post.keys()), str(last_activity_date), id)
<commit_msg>Rewrite inserting function. It can be a lot simpler now we know we're always dealing with up-to-date data.<commit_after>
def import_question(posts, namespaces, id, title, body, tags, last_activity_date, last_updated_date, score, answer_count, has_accepted_answer):
    namespaces_for_post = {}
    for name, n in namespaces.items():
        namespace_tags = n.get_tags()
        if not(namespace_tags) or any(map(lambda x: x in tags, namespace_tags)):
            ids = n.get_ids(title, body, tags)
            if len(ids) > 0:
                ids = map(lambda x: x.lower(), ids)
                namespaces_for_post[name] = ids

    if len(namespaces_for_post):
        post = {
            "question_id": id,
            "url": "http://stackoverflow.com/questions/%s" % id,
            "namespaces": namespaces_for_post,
            "title": title,
            "score": int(score),
            "accepted_answer": has_accepted_answer,
            "last_activity": last_activity_date,
            "last_updated": last_updated_date
        }

        posts.update({"question_id": id}, post, True);

        print "Processed %s question from %s (%s)" % (", ".join(namespaces_for_post.keys()), str(last_activity_date), id)
<|endoftext|>


<commit_before>import json
import logging
from copy import copy

from channels import Group
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.db import IntegrityError
from rest_framework import serializers

from lily.accounts.models import Account
from lily.calls.models import CallRecord, CallParticipant, CallTransfer
from lily.contacts.models import Contact
from lily.users.models import LilyUser

logger = logging.getLogger(__name__)

def create_or_get(model_cls, lookup, data):
    """
    This function assumes that you have a unique field in the model class.
    The unique field will raise an IntegrityError if there is an existing record already.
    """
    try:
        return model_cls.objects.create(**data)
    except IntegrityError:
        logger.exception('create_or_get IntegrityError')
        return model_cls.objects.get(**lookup)

class CallNotificationSerializer(serializers.Serializer):
    call_id = serializers.CharField()
    timestamp = serializers.DateTimeField()
    status = serializers.ChoiceField(choices=('ringing', 'in-progress', 'ended', 'transfer', ))
    direction = serializers.ChoiceField(choices=('inbound', 'outbound', ))

    # For ringing, in-progress and ended events.
    caller = serializers.DictField(required=False)
    destination = serializers.DictField(required=False)

    # For transfer events.
    party1 = serializers.DictField(required=False)
    party2 = serializers.DictField(required=False)
    redirector = serializers.DictField(required=False)
    merged_id = serializers.CharField(required=False)

    def to_representation(self, instance):
        # Because the CallRecord model doesn't have the same fields DRF will complain when returning the object.
        # We don't need to return an object after creation because VG doesn't care.
        return {}

    def create(self, validated_data):
        destination = validated_data.get('destination', {})

        if destination and not destination.get('number', '').startswith('+'):
            # We don't save phone calls to internal numbers because:
            # - They aren't visible anywhere.
            # - They are temporary phonecalls when a user transfers the call.
            return CallRecord()

        save_func = getattr(self, 'save_{}'.format(validated_data['status'].replace('-', '_')))

        return save_func(validated_data)

    def save_participant(self, data, metadata=False):
        name = data.get('name', '') or ''
        number = data.get('number', '') or ''
        internal_number = data.get('account_number', '') or ''

        # First try to autocomplete the name using a user.
        if internal_number and not name:
            user = LilyUser.objects.filter(internal_number=internal_number).first()
            if user:
                name = user.full_name

        meta = {
            'destination': 'create',
            'icon': static('app/images/notification_icons/add-account.png'),
            'params': {
                'name': name,
                'number': number,
            },
        }

        # Second try to autocomplete the name using a contact.
        if number and not name:
            contact = Contact.objects.filter(phone_numbers__number=number).first()

            if contact:
                name = contact.full_name
                meta = {
                    'destination': 'contact',
                    'icon': static('app/images/notification_icons/contact.png'),
                    'params': {
                        'name': name,
                        'number': number,
                        'id': contact.id,
                    },
                }

        # Third try to autocomplete the name using an account.
        if number and not name:
            account = Account.objects.filter(phone_numbers__number=number).first()

            if account:
                name = account.name
                meta = {
                    'destination': 'account',
                    'icon': static('app/images/notification_icons/account.png'),
                    'params': {
                        'name': name,
                        'number': number,
                        'id': account.id,
                    },
                }

        participant = CallParticipant.objects.get_or_create(
            name=name,
            number=number,
            internal_number=internal_number
        )[0]

        if metadata:
            return participant, meta

        return participant

    def save_ringing(self, data):
        caller, caller_meta = self.save_participant(data['caller'], metadata=True)
        destination = data['destination']

        data.update({
            'start': data.pop('timestamp'),
            'caller': caller,
            'destination': None,  # During ringing we don't want to store destination yet, only when it's picked up.
            'status': CallRecord.RINGING,
            'direction': CallRecord.INBOUND if data['direction'] == 'inbound' else CallRecord.OUTBOUND,
        })
        cr = create_or_get(CallRecord, lookup={'call_id': data['call_id']}, data=data)

        user_list = LilyUser.objects.filter(internal_number=destination['account_number'])
        for user in user_list:
            # Sends the data as a notification event to the users.
            Group('user-%s' % user.id).send({
                'text': json.dumps({
                    'event': 'notification',
                    'data': caller_meta
                }),
            })

        return cr

    def save_in_progress(self, data):
        try:
            cr = CallRecord.objects.get(call_id=data['call_id'])
        except CallRecord.DoesNotExist:
            logger.exception('Unexpected in-progress notification for call {}'.format(data['call_id']))
            return CallRecord()

        updated_data = {
            'destination': self.save_participant(data['destination']),
            'status': CallRecord.IN_PROGRESS,
        }

        for attr, value in updated_data.items():
            setattr(cr, attr, value)
        cr.save()

        return cr

    def save_transfer(self, data):
        crs = CallRecord.objects.filter(
            call_id=data['call_id']
        ) or CallRecord.objects.filter(
            call_id=data['merged_id']
        )

        if not crs:
            return CallRecord()

        CallTransfer.objects.create(**{
            'timestamp': data['timestamp'],
            'call': crs[0],
            'destination': self.save_participant(data['party2']),
        })

        return crs[0]

    def save_ended(self, data):
        try:
            cr = CallRecord.objects.get(call_id=data['call_id'])
        except CallRecord.DoesNotExist:
            # This can happen if the destination is unavailable. Then you get the ended notification right away.
            # Use the save ringing to create the cr and immediately overwrite the data afterwards.
            cr = self.save_ringing(copy(data))

        updated_data = {
            'end': data.pop('timestamp'),
            'status': CallRecord.ENDED,
        }

        for attr, value in updated_data.items():
            setattr(cr, attr, value)
        cr.save()

        return cr
<commit_msg>Refactor log statement in create_or_get<commit_after>import json
import logging
from copy import copy

from channels import Group
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.db import IntegrityError
from rest_framework import serializers

from lily.accounts.models import Account
from lily.calls.models import CallRecord, CallParticipant, CallTransfer
from lily.contacts.models import Contact
from lily.users.models import LilyUser

logger = logging.getLogger(__name__)

def create_or_get(model_cls, lookup, data):
    """
    This function assumes that you have a unique field in the model class.
    The unique field will raise an IntegrityError if there is an existing record already.
    """
    try:
        return model_cls.objects.create(**data)
    except IntegrityError as e:
        logger.warning('create_or_get IntegrityError: {}'.format(''.join(e.message.splitlines())))
        return model_cls.objects.get(**lookup)

class CallNotificationSerializer(serializers.Serializer):
    call_id = serializers.CharField()
    timestamp = serializers.DateTimeField()
    status = serializers.ChoiceField(choices=('ringing', 'in-progress', 'ended', 'transfer', ))
    direction = serializers.ChoiceField(choices=('inbound', 'outbound', ))

    # For ringing, in-progress and ended events.
    caller = serializers.DictField(required=False)
    destination = serializers.DictField(required=False)

    # For transfer events.
    party1 = serializers.DictField(required=False)
    party2 = serializers.DictField(required=False)
    redirector = serializers.DictField(required=False)
    merged_id = serializers.CharField(required=False)

    def to_representation(self, instance):
        # Because the CallRecord model doesn't have the same fields DRF will complain when returning the object.
        # We don't need to return an object after creation because VG doesn't care.
        return {}

    def create(self, validated_data):
        destination = validated_data.get('destination', {})

        if destination and not destination.get('number', '').startswith('+'):
            # We don't save phone calls to internal numbers because:
            # - They aren't visible anywhere.
            # - They are temporary phonecalls when a user transfers the call.
            return CallRecord()

        save_func = getattr(self, 'save_{}'.format(validated_data['status'].replace('-', '_')))

        return save_func(validated_data)

    def save_participant(self, data, metadata=False):
        name = data.get('name', '') or ''
        number = data.get('number', '') or ''
        internal_number = data.get('account_number', '') or ''

        # First try to autocomplete the name using a user.
        if internal_number and not name:
            user = LilyUser.objects.filter(internal_number=internal_number).first()
            if user:
                name = user.full_name

        meta = {
            'destination': 'create',
            'icon': static('app/images/notification_icons/add-account.png'),
            'params': {
                'name': name,
                'number': number,
            },
        }

        # Second try to autocomplete the name using a contact.
        if number and not name:
            contact = Contact.objects.filter(phone_numbers__number=number).first()

            if contact:
                name = contact.full_name
                meta = {
                    'destination': 'contact',
                    'icon': static('app/images/notification_icons/contact.png'),
                    'params': {
                        'name': name,
                        'number': number,
                        'id': contact.id,
                    },
                }

        # Third try to autocomplete the name using an account.
        if number and not name:
            account = Account.objects.filter(phone_numbers__number=number).first()

            if account:
                name = account.name
                meta = {
                    'destination': 'account',
                    'icon': static('app/images/notification_icons/account.png'),
                    'params': {
                        'name': name,
                        'number': number,
                        'id': account.id,
                    },
                }

        participant = CallParticipant.objects.get_or_create(
            name=name,
            number=number,
            internal_number=internal_number
        )[0]

        if metadata:
            return participant, meta

        return participant

    def save_ringing(self, data):
        caller, caller_meta = self.save_participant(data['caller'], metadata=True)
        destination = data['destination']

        data.update({
            'start': data.pop('timestamp'),
            'caller': caller,
            'destination': None,  # During ringing we don't want to store destination yet, only when it's picked up.
            'status': CallRecord.RINGING,
            'direction': CallRecord.INBOUND if data['direction'] == 'inbound' else CallRecord.OUTBOUND,
        })
        cr = create_or_get(CallRecord, lookup={'call_id': data['call_id']}, data=data)

        user_list = LilyUser.objects.filter(internal_number=destination['account_number'])
        for user in user_list:
            # Sends the data as a notification event to the users.
            Group('user-%s' % user.id).send({
                'text': json.dumps({
                    'event': 'notification',
                    'data': caller_meta
                }),
            })

        return cr

    def save_in_progress(self, data):
        try:
            cr = CallRecord.objects.get(call_id=data['call_id'])
        except CallRecord.DoesNotExist:
            logger.exception('Unexpected in-progress notification for call {}'.format(data['call_id']))
            return CallRecord()

        updated_data = {
            'destination': self.save_participant(data['destination']),
            'status': CallRecord.IN_PROGRESS,
        }

        for attr, value in updated_data.items():
            setattr(cr, attr, value)
        cr.save()

        return cr

    def save_transfer(self, data):
        crs = CallRecord.objects.filter(
            call_id=data['call_id']
        ) or CallRecord.objects.filter(
            call_id=data['merged_id']
        )

        if not crs:
            return CallRecord()

        CallTransfer.objects.create(**{
            'timestamp': data['timestamp'],
            'call': crs[0],
            'destination': self.save_participant(data['party2']),
        })

        return crs[0]

    def save_ended(self, data):
        try:
            cr = CallRecord.objects.get(call_id=data['call_id'])
        except CallRecord.DoesNotExist:
            # This can happen if the destination is unavailable. Then you get the ended notification right away.
            # Use the save ringing to create the cr and immediately overwrite the data afterwards.
            cr = self.save_ringing(copy(data))

        updated_data = {
            'end': data.pop('timestamp'),
            'status': CallRecord.ENDED,
        }

        for attr, value in updated_data.items():
            setattr(cr, attr, value)
        cr.save()

        return cr
<|endoftext|>


<commit_before>"""Functions for builtin CherryPy tools."""

import re

import cherrypy
from cherrypy.lib import http as _http

#                     Conditional HTTP request support                     #

def validate_etags(autotags=False):
    """Validate the current ETag against If-Match, If-None-Match headers.

    If autotags is True, an ETag response-header value will be provided
    from an MD5 hash of the response body (unless some other code has
    already provided an ETag header). If False, the ETag will not be
    automatic, and if no other code has provided an ETag value, then no
    checks will be performed against If-Match or If-None-Match headers.
    """
    response = cherrypy.response

    # Guard against being run twice.
    if hasattr(response, "ETag"):
        return

    etag = response.headers.get('ETag')

    if (not etag) and autotags:
        import md5
        etag = '"%s"' % md5.new(response.collapse_body()).hexdigest()
        response.headers['ETag'] = etag

    if etag:
        response.ETag = etag

        status, reason, msg = _http.valid_status(response.status)

        request = cherrypy.request

        conditions = request.headers.elements('If-Match') or []
        conditions = [str(x) for x in conditions]
        if conditions and not (conditions == ["*"] or etag in conditions):
            if status >= 200 and status < 299:
                raise cherrypy.HTTPError(412)

        conditions = request.headers.elements('If-None-Match') or []
        conditions = [str(x) for x in conditions]
        if conditions == ["*"] or etag in conditions:
            if status >= 200 and status < 299:
                if request.method in ("GET", "HEAD"):
                    raise cherrypy.HTTPRedirect([], 304)
                else:
                    raise cherrypy.HTTPError(412)

def validate_since():
    """Validate the current Last-Modified against If-Modified-Since headers.

    If no code has set the Last-Modified response header, then no validation
    will be performed.
    """
    response = cherrypy.response
    lastmod = response.headers.get('Last-Modified')
    if lastmod:
        status, reason, msg = _http.valid_status(response.status)

        request = cherrypy.request

        since = request.headers.get('If-Unmodified-Since')
        if since and since != lastmod:
            if (status >= 200 and status < 299) or status == 412:
                raise cherrypy.HTTPError(412)

        since = request.headers.get('If-Modified-Since')
        if since and since == lastmod:
            if (status >= 200 and status < 299) or status == 304:
                if request.method in ("GET", "HEAD"):
                    raise cherrypy.HTTPRedirect([], 304)
                else:
                    raise cherrypy.HTTPError(412)

#                                Tool code                                #

def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For',
          scheme='X-Forwarded-Proto'):
    """Change the base URL (scheme://host[:port][/path]).

    For running a CP server behind Apache, lighttpd, or other HTTP server.

    If you want the new request.base to include path info (not just the host),
    you must explicitly set base to the full base path, and ALSO set 'local'
    to '', so that the X-Forwarded-Host request header (which never includes
    path info) does not override it.

    cherrypy.request.remote.ip (the IP address of the client) will be
    rewritten if the header specified by the 'remote' arg is valid.
    By default, 'remote' is set to 'X-Forwarded-For'. If you do not
    want to rewrite remote.ip, set the 'remote' arg to an empty string.
    """

    request = cherrypy.request

    if scheme:
        scheme = request.headers.get(scheme, None)
    if not scheme:
        scheme = request.base[:request.base.find("://")]

    if local:
        base = request.headers.get(local, base)
    if not base:
        port = cherrypy.request.local.port
        if port == 80:
            base = 'localhost'
        else:
            base = 'localhost:%s' % port

    if base.find("://") == -1:
        # add http:// or https:// if needed
        base = scheme + "://" + base

    request.base = base

    if remote:
        xff = request.headers.get(remote)
        if xff:
            if remote == 'X-Forwarded-For':
                # See http://bob.pythonmac.org/archives/2005/09/23/apache-x-forwarded-for-caveat/
                xff = xff.split(',')[-1].strip()
            request.remote.ip = xff

def ignore_headers(headers=('Range',)):
    """Delete request headers whose field names are included in 'headers'.

    This is a useful tool for working behind certain HTTP servers;
    for example, Apache duplicates the work that CP does for 'Range'
    headers, and will doubly-truncate the response.
    """
    request = cherrypy.request
    for name in headers:
        if name in request.headers:
            del request.headers[name]

def response_headers(headers=None):
    """Set headers on the response."""
    for name, value in (headers or []):
        cherrypy.response.headers[name] = value
response_headers.failsafe = True

def referer(pattern, accept=True, accept_missing=False, error=403,
            message='Forbidden Referer header.'):
    """Raise HTTPError if Referer header does not pass our test.

    pattern: a regular expression pattern to test against the Referer.
    accept: if True, the Referer must match the pattern; if False,
        the Referer must NOT match the pattern.
    accept_missing: if True, permit requests with no Referer header.
    error: the HTTP error code to return to the client on failure.
    message: a string to include in the response body on failure.
    """
    try:
        match = bool(re.match(pattern, cherrypy.request.headers['Referer']))
        if accept == match:
            return
    except KeyError:
        if accept_missing:
            return

    raise cherrypy.HTTPError(error, message)

class SessionAuth(object):
    """Assert that the user is logged in."""

    session_key = "username"

    def check_username_and_password(self, username, password):
        pass

    def anonymous(self):
        """Provide a temporary user name for anonymous users."""
        pass

    def on_login(self, username):
        pass

    def on_logout(self, username):
        pass

    def on_check(self, username):
        pass

    def login_screen(self, from_page='..', username='', error_msg=''):
        return """
Message: %(error_msg)s
<form method="post" action="do_login">
    Login: <input type="text" name="username" value="%(username)s" size="10" />

    Password: <input type="password" name="password" size="10" />

    <input type="hidden" name="from_page" value="%(from_page)s" />

    <input type="submit" />
</form>
""" % {'from_page': from_page, 'username': username,
                     'error_msg': error_msg}

    def do_login(self, username, password, from_page='..'):
        """Login. May raise redirect, or return True if request handled."""
        error_msg = self.check_username_and_password(username, password)
        if error_msg:
            body = self.login_screen(from_page, username, error_msg)
            cherrypy.response.body = body
            if cherrypy.response.headers.has_key("Content-Length"):
                # Delete Content-Length header so finalize() recalcs it.
                del cherrypy.response.headers["Content-Length"]
            return True
        else:
            cherrypy.session[self.session_key] = username
            self.on_login(username)
            raise cherrypy.HTTPRedirect(from_page or "/")

    def do_logout(self, from_page='..'):
        """Logout. May raise redirect, or return True if request handled."""
        sess = cherrypy.session
        username = sess.get(self.session_key)
        sess[self.session_key] = None
        if username:
            self.on_logout(username)
        raise cherrypy.HTTPRedirect(from_page)

    def do_check(self):
        """Assert username. May raise redirect, or return True if request handled."""
        sess = cherrypy.session
        request = cherrypy.request

        username = sess.get(self.session_key)
        if not username:
            sess[self.session_key] = username = self.anonymous()
        if not username:
            cherrypy.response.body = self.login_screen(cherrypy.url(qs=request.query_string))
            if cherrypy.response.headers.has_key("Content-Length"):
                # Delete Content-Length header so finalize() recalcs it.
                del cherrypy.response.headers["Content-Length"]
            return True

        self.on_check(username)

    def run(self):
        request = cherrypy.request
        path = request.path_info
        if path.endswith('login_screen'):
            return self.login_screen()
        elif path.endswith('do_login'):
            return self.do_login(**request.params)
        elif path.endswith('do_logout'):
            return self.do_logout(**request.params)
        else:
            return self.do_check()

def session_auth(**kwargs):
    sa = SessionAuth()
    for k, v in kwargs.iteritems():
        setattr(sa, k, v)
    return sa.run()

def virtual_host(use_x_forwarded_host=True, **domains):
    """Redirect internally based on the Host header.

    Useful when running multiple sites within one CP server.

    From http://groups.google.com/group/cherrypy-users/browse_thread/thread/f393540fe278e54d:

    For various reasons I need several domains to point to different parts of a
    single website structure as well as to their own "homepage"   EG

    http://www.mydom1.com  ->  root
    http://www.mydom2.com  ->  root/mydom2/
    http://www.mydom3.com  ->  root/mydom3/
    http://www.mydom4.com  ->  under construction page

    but also to have  http://www.mydom1.com/mydom2/  etc to be valid pages in
    their own right.
    """
    request = cherrypy.request

    # Guard against running twice.
    if hasattr(request, "virtual_prefix"):
        return

    domain = request.headers.get('Host', '')
    if use_x_forwarded_host:
        domain = request.headers.get("X-Forwarded-Host", domain)

    request.virtual_prefix = prefix = domains.get(domain, "")
    if prefix:
        raise cherrypy.InternalRedirect(_http.urljoin(prefix, request.path_info))

def log_traceback():
    """Write the last error's traceback to the cherrypy error log."""
    from cherrypy import _cperror
    cherrypy.log(_cperror.format_exc(), "HTTP")

def log_request_headers():
    """Write request headers to the cherrypy error log."""
    h = ["  %s: %s" % (k, v) for k, v in cherrypy.request.header_list]
    cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP")

def redirect(url='', internal=True):
    """Raise InternalRedirect or HTTPRedirect to the given url."""
    if internal:
        raise cherrypy.InternalRedirect(url)
    else:
        raise cherrypy.HTTPRedirect(url)

def trailing_slash(missing=True, extra=False):
    """Redirect if path_info has (missing|extra) trailing slash."""
    request = cherrypy.request
    pi = request.path_info

    if request.is_index is True:
        if missing:
            if pi[-1:] != '/':
                new_url = cherrypy.url(pi + '/', request.query_string)
                raise cherrypy.HTTPRedirect(new_url)
    elif request.is_index is False:
        if extra:
            # If pi == '/', don't redirect to ''!
            if pi[-1:] == '/' and pi != '/':
                new_url = cherrypy.url(pi[:-1], request.query_string)
                raise cherrypy.HTTPRedirect(new_url)

<commit_msg>Probable fix for #583 (session_auth login_screen).<commit_after>"""Functions for builtin CherryPy tools."""

import re

import cherrypy
from cherrypy.lib import http as _http

#                     Conditional HTTP request support                     #

def validate_etags(autotags=False):
    """Validate the current ETag against If-Match, If-None-Match headers.

    If autotags is True, an ETag response-header value will be provided
    from an MD5 hash of the response body (unless some other code has
    already provided an ETag header). If False, the ETag will not be
    automatic, and if no other code has provided an ETag value, then no
    checks will be performed against If-Match or If-None-Match headers.
    """
    response = cherrypy.response

    # Guard against being run twice.
    if hasattr(response, "ETag"):
        return

    etag = response.headers.get('ETag')

    if (not etag) and autotags:
        import md5
        etag = '"%s"' % md5.new(response.collapse_body()).hexdigest()
        response.headers['ETag'] = etag

    if etag:
        response.ETag = etag

        status, reason, msg = _http.valid_status(response.status)

        request = cherrypy.request

        conditions = request.headers.elements('If-Match') or []
        conditions = [str(x) for x in conditions]
        if conditions and not (conditions == ["*"] or etag in conditions):
            if status >= 200 and status < 299:
                raise cherrypy.HTTPError(412)

        conditions = request.headers.elements('If-None-Match') or []
        conditions = [str(x) for x in conditions]
        if conditions == ["*"] or etag in conditions:
            if status >= 200 and status < 299:
                if request.method in ("GET", "HEAD"):
                    raise cherrypy.HTTPRedirect([], 304)
                else:
                    raise cherrypy.HTTPError(412)

def validate_since():
    """Validate the current Last-Modified against If-Modified-Since headers.

    If no code has set the Last-Modified response header, then no validation
    will be performed.
    """
    response = cherrypy.response
    lastmod = response.headers.get('Last-Modified')
    if lastmod:
        status, reason, msg = _http.valid_status(response.status)

        request = cherrypy.request

        since = request.headers.get('If-Unmodified-Since')
        if since and since != lastmod:
            if (status >= 200 and status < 299) or status == 412:
                raise cherrypy.HTTPError(412)

        since = request.headers.get('If-Modified-Since')
        if since and since == lastmod:
            if (status >= 200 and status < 299) or status == 304:
                if request.method in ("GET", "HEAD"):
                    raise cherrypy.HTTPRedirect([], 304)
                else:
                    raise cherrypy.HTTPError(412)

#                                Tool code                                #

def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For',
          scheme='X-Forwarded-Proto'):
    """Change the base URL (scheme://host[:port][/path]).

    For running a CP server behind Apache, lighttpd, or other HTTP server.

    If you want the new request.base to include path info (not just the host),
    you must explicitly set base to the full base path, and ALSO set 'local'
    to '', so that the X-Forwarded-Host request header (which never includes
    path info) does not override it.

    cherrypy.request.remote.ip (the IP address of the client) will be
    rewritten if the header specified by the 'remote' arg is valid.
    By default, 'remote' is set to 'X-Forwarded-For'. If you do not
    want to rewrite remote.ip, set the 'remote' arg to an empty string.
    """

    request = cherrypy.request

    if scheme:
        scheme = request.headers.get(scheme, None)
    if not scheme:
        scheme = request.base[:request.base.find("://")]

    if local:
        base = request.headers.get(local, base)
    if not base:
        port = cherrypy.request.local.port
        if port == 80:
            base = 'localhost'
        else:
            base = 'localhost:%s' % port

    if base.find("://") == -1:
        # add http:// or https:// if needed
        base = scheme + "://" + base

    request.base = base

    if remote:
        xff = request.headers.get(remote)
        if xff:
            if remote == 'X-Forwarded-For':
                # See http://bob.pythonmac.org/archives/2005/09/23/apache-x-forwarded-for-caveat/
                xff = xff.split(',')[-1].strip()
            request.remote.ip = xff

def ignore_headers(headers=('Range',)):
    """Delete request headers whose field names are included in 'headers'.

    This is a useful tool for working behind certain HTTP servers;
    for example, Apache duplicates the work that CP does for 'Range'
    headers, and will doubly-truncate the response.
    """
    request = cherrypy.request
    for name in headers:
        if name in request.headers:
            del request.headers[name]

def response_headers(headers=None):
    """Set headers on the response."""
    for name, value in (headers or []):
        cherrypy.response.headers[name] = value
response_headers.failsafe = True

def referer(pattern, accept=True, accept_missing=False, error=403,
            message='Forbidden Referer header.'):
    """Raise HTTPError if Referer header does not pass our test.

    pattern: a regular expression pattern to test against the Referer.
    accept: if True, the Referer must match the pattern; if False,
        the Referer must NOT match the pattern.
    accept_missing: if True, permit requests with no Referer header.
    error: the HTTP error code to return to the client on failure.
    message: a string to include in the response body on failure.
    """
    try:
        match = bool(re.match(pattern, cherrypy.request.headers['Referer']))
        if accept == match:
            return
    except KeyError:
        if accept_missing:
            return

    raise cherrypy.HTTPError(error, message)

class SessionAuth(object):
    """Assert that the user is logged in."""

    session_key = "username"

    def check_username_and_password(self, username, password):
        pass

    def anonymous(self):
        """Provide a temporary user name for anonymous users."""
        pass

    def on_login(self, username):
        pass

    def on_logout(self, username):
        pass

    def on_check(self, username):
        pass

    def login_screen(self, from_page='..', username='', error_msg=''):
        return """
Message: %(error_msg)s
<form method="post" action="do_login">
    Login: <input type="text" name="username" value="%(username)s" size="10" />

    Password: <input type="password" name="password" size="10" />

    <input type="hidden" name="from_page" value="%(from_page)s" />

    <input type="submit" />
</form>
""" % {'from_page': from_page, 'username': username,
                     'error_msg': error_msg}

    def do_login(self, username, password, from_page='..'):
        """Login. May raise redirect, or return True if request handled."""
        error_msg = self.check_username_and_password(username, password)
        if error_msg:
            body = self.login_screen(from_page, username, error_msg)
            cherrypy.response.body = body
            if cherrypy.response.headers.has_key("Content-Length"):
                # Delete Content-Length header so finalize() recalcs it.
                del cherrypy.response.headers["Content-Length"]
            return True
        else:
            cherrypy.session[self.session_key] = username
            self.on_login(username)
            raise cherrypy.HTTPRedirect(from_page or "/")

    def do_logout(self, from_page='..'):
        """Logout. May raise redirect, or return True if request handled."""
        sess = cherrypy.session
        username = sess.get(self.session_key)
        sess[self.session_key] = None
        if username:
            self.on_logout(username)
        raise cherrypy.HTTPRedirect(from_page)

    def do_check(self):
        """Assert username. May raise redirect, or return True if request handled."""
        sess = cherrypy.session
        request = cherrypy.request

        username = sess.get(self.session_key)
        if not username:
            sess[self.session_key] = username = self.anonymous()
        if not username:
            cherrypy.response.body = self.login_screen(cherrypy.url(qs=request.query_string))
            if cherrypy.response.headers.has_key("Content-Length"):
                # Delete Content-Length header so finalize() recalcs it.
                del cherrypy.response.headers["Content-Length"]
            return True

        self.on_check(username)

    def run(self):
        request = cherrypy.request
        path = request.path_info
        if path.endswith('login_screen'):
            return self.login_screen(**request.params)
        elif path.endswith('do_login'):
            return self.do_login(**request.params)
        elif path.endswith('do_logout'):
            return self.do_logout(**request.params)
        else:
            return self.do_check()

def session_auth(**kwargs):
    sa = SessionAuth()
    for k, v in kwargs.iteritems():
        setattr(sa, k, v)
    return sa.run()

def virtual_host(use_x_forwarded_host=True, **domains):
    """Redirect internally based on the Host header.

    Useful when running multiple sites within one CP server.

    From http://groups.google.com/group/cherrypy-users/browse_thread/thread/f393540fe278e54d:

    For various reasons I need several domains to point to different parts of a
    single website structure as well as to their own "homepage"   EG

    http://www.mydom1.com  ->  root
    http://www.mydom2.com  ->  root/mydom2/
    http://www.mydom3.com  ->  root/mydom3/
    http://www.mydom4.com  ->  under construction page

    but also to have  http://www.mydom1.com/mydom2/  etc to be valid pages in
    their own right.
    """
    request = cherrypy.request

    # Guard against running twice.
    if hasattr(request, "virtual_prefix"):
        return

    domain = request.headers.get('Host', '')
    if use_x_forwarded_host:
        domain = request.headers.get("X-Forwarded-Host", domain)

    request.virtual_prefix = prefix = domains.get(domain, "")
    if prefix:
        raise cherrypy.InternalRedirect(_http.urljoin(prefix, request.path_info))

def log_traceback():
    """Write the last error's traceback to the cherrypy error log."""
    from cherrypy import _cperror
    cherrypy.log(_cperror.format_exc(), "HTTP")

def log_request_headers():
    """Write request headers to the cherrypy error log."""
    h = ["  %s: %s" % (k, v) for k, v in cherrypy.request.header_list]
    cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP")

def redirect(url='', internal=True):
    """Raise InternalRedirect or HTTPRedirect to the given url."""
    if internal:
        raise cherrypy.InternalRedirect(url)
    else:
        raise cherrypy.HTTPRedirect(url)

def trailing_slash(missing=True, extra=False):
    """Redirect if path_info has (missing|extra) trailing slash."""
    request = cherrypy.request
    pi = request.path_info

    if request.is_index is True:
        if missing:
            if pi[-1:] != '/':
                new_url = cherrypy.url(pi + '/', request.query_string)
                raise cherrypy.HTTPRedirect(new_url)
    elif request.is_index is False:
        if extra:
            # If pi == '/', don't redirect to ''!
            if pi[-1:] == '/' and pi != '/':
                new_url = cherrypy.url(pi[:-1], request.query_string)
                raise cherrypy.HTTPRedirect(new_url)

<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Creates a heatmap of vocabulary used in different sessions.

Use with e.g. "find ~/Documents/Projects/Tangrams/Data/Ready/ -iname "utts.tsv" -exec ./vocabulary_heatmap.py {} +"
"""

__author__ = "Todd Shore <errantlinguist+github@gmail.com>"
__copyright__ = "Copyright 2017 Todd Shore"
__license__ = "Apache License, Version 2.0"

import argparse
import os
import sys
import typing
from collections import Counter
from enum import Enum, unique
from typing import Iterable

import pandas as pd

import utterances

@unique
class SessionVocabularyCountDataColumn(Enum):
	DYAD_ID = "DYAD"
	TOKEN_TYPE = "TOKEN_TYPE"
	TOKEN_COUNT = "TOKEN_COUNT"

__SESSION_WORD_COUNT_DF_COLS = (
	SessionVocabularyCountDataColumn.DYAD_ID.value, SessionVocabularyCountDataColumn.TOKEN_TYPE.value,
	SessionVocabularyCountDataColumn.TOKEN_COUNT.value)
# noinspection PyTypeChecker
assert len(__SESSION_WORD_COUNT_DF_COLS) == len(SessionVocabularyCountDataColumn)

def count_tokens(token_seqs: Iterable[Iterable[str]]) -> typing.Counter[str]:
	result = Counter()
	for token_seq in token_seqs:
		result.update(token_seq)
	return result

def create_session_word_count_df(utts: pd.DataFrame) -> pd.DataFrame:
	session_utts = utts.groupby(SessionVocabularyCountDataColumn.DYAD_ID.value)
	session_token_counts = session_utts[utterances.UtteranceTabularDataColumn.TOKEN_SEQ.value].agg(count_tokens)
	session_word_count_rows = []
	for session, token_counts in session_token_counts.iteritems():
		for token_type, count in token_counts.items():
			session_word_count_rows.append((session, token_type, count))
	return pd.DataFrame(data=session_word_count_rows, columns=__SESSION_WORD_COUNT_DF_COLS)

def __create_argparser() -> argparse.ArgumentParser:
	result = argparse.ArgumentParser(
		description="Visualizes usage of vocabulary across sessions as a heatmap.")
	result.add_argument("inpaths", metavar="INPATH", nargs='+',
						help="The files to process.")

	return result

def __dyad_id(infile: str) -> str:
	session_dir = os.path.dirname(infile)
	return os.path.split(session_dir)[1]

def __read_utts_file(inpath: str, utt_reader: utterances.UtteranceTabularDataReader) -> pd.DataFrame:
	dyad_id = __dyad_id(inpath)
	print("Reading utt file at \"{}\" as dyad \"{}\"".format(inpath, dyad_id), file=sys.stderr)
	result = utt_reader(inpath)
	result[SessionVocabularyCountDataColumn.DYAD_ID.value] = dyad_id
	return result

def __main(args):
	inpaths = args.inpaths
	print("Will read {} file(s).".format(len(inpaths)), file=sys.stderr)
	utt_reader = utterances.UtteranceTabularDataReader()
	utts = pd.concat((__read_utts_file(inpath, utt_reader) for inpath in inpaths))
	print("Read {} unique utterance(s) from {} file(s) with {} column(s).".format(utts.shape[0], len(inpaths),
																				  utts.shape[1]), file=sys.stderr)
	session_word_count_df = create_session_word_count_df(utts)
	session_word_count_df.sort_values(
		[SessionVocabularyCountDataColumn.DYAD_ID.value, SessionVocabularyCountDataColumn.TOKEN_TYPE.value],
		inplace=True)
	print(session_word_count_df)

if __name__ == "__main__":
	__main(__create_argparser().parse_args())
<commit_msg>re-implemented "create_session_word_count_df(DataFrame)" so that it works with other versions of pandas<commit_after>#!/usr/bin/env python3

"""
Creates a heatmap of vocabulary used in different sessions.

Use with e.g. "find ~/Documents/Projects/Tangrams/Data/Ready/ -iname "utts.tsv" -exec ./vocabulary_heatmap.py {} +"
"""

__author__ = "Todd Shore <errantlinguist+github@gmail.com>"
__copyright__ = "Copyright 2017 Todd Shore"
__license__ = "Apache License, Version 2.0"

import argparse
import os
import sys
from collections import Counter
from enum import Enum, unique

import pandas as pd

import utterances

@unique
class SessionVocabularyCountDataColumn(Enum):
	DYAD_ID = "DYAD"
	TOKEN_TYPE = "TOKEN_TYPE"
	TOKEN_COUNT = "TOKEN_COUNT"

__SESSION_WORD_COUNT_DF_COLS = (
	SessionVocabularyCountDataColumn.DYAD_ID.value, SessionVocabularyCountDataColumn.TOKEN_TYPE.value,
	SessionVocabularyCountDataColumn.TOKEN_COUNT.value)
# noinspection PyTypeChecker
assert len(__SESSION_WORD_COUNT_DF_COLS) == len(SessionVocabularyCountDataColumn)

def create_session_word_count_df(utts: pd.DataFrame) -> pd.DataFrame:
	session_utts = utts.groupby(SessionVocabularyCountDataColumn.DYAD_ID.value)

	rows = []
	for session, utts in session_utts:
		token_type_counts = Counter()
		for row in utts.itertuples(index=False):
			# noinspection PyProtectedMember
			row_as_dict = row._asdict()
			token_seq = row_as_dict[utterances.UtteranceTabularDataColumn.TOKEN_SEQ.value]
			token_type_counts.update(token_seq)

		session_rows = ((session, token_type, count) for token_type, count in token_type_counts.items())
		rows.extend(session_rows)

	return pd.DataFrame(data=rows, columns=__SESSION_WORD_COUNT_DF_COLS)

def __create_argparser() -> argparse.ArgumentParser:
	result = argparse.ArgumentParser(
		description="Visualizes usage of vocabulary across sessions as a heatmap.")
	result.add_argument("inpaths", metavar="INPATH", nargs='+',
						help="The files to process.")

	return result

def __dyad_id(infile: str) -> str:
	session_dir = os.path.dirname(infile)
	return os.path.split(session_dir)[1]

def __read_utts_file(inpath: str, utt_reader: utterances.UtteranceTabularDataReader) -> pd.DataFrame:
	dyad_id = __dyad_id(inpath)
	print("Reading utt file at \"{}\" as dyad \"{}\"".format(inpath, dyad_id), file=sys.stderr)
	result = utt_reader(inpath)
	result[SessionVocabularyCountDataColumn.DYAD_ID.value] = dyad_id
	return result

def __main(args):
	inpaths = args.inpaths
	print("Will read {} file(s).".format(len(inpaths)), file=sys.stderr)
	utt_reader = utterances.UtteranceTabularDataReader()
	utts = pd.concat((__read_utts_file(inpath, utt_reader) for inpath in inpaths))
	print("Read {} unique utterance(s) from {} file(s) with {} column(s).".format(utts.shape[0], len(inpaths),
																				  utts.shape[1]), file=sys.stderr)
	session_word_count_df = create_session_word_count_df(utts)
	session_word_count_df.sort_values(
		[SessionVocabularyCountDataColumn.DYAD_ID.value, SessionVocabularyCountDataColumn.TOKEN_TYPE.value],
		inplace=True)
	print(session_word_count_df)

if __name__ == "__main__":
	__main(__create_argparser().parse_args())
<|endoftext|>


<commit_before>#! /usr/bin/python

import os, sys, subprocess, re, traceback, smtplib, shlex

my_folder = os.path.dirname(os.path.abspath(__file__))
hash_pat = '[a-fA-F0-9]{40}'
commit_pat = re.compile(r'^\s*commit\s+(%s)\s*$' % hash_pat, re.MULTILINE)
status_pat = re.compile(r'\s*(%s)\s+([a-zA-Z]+)' % hash_pat)
to_addr = 'daniel.hardman@gmail.com'
from_addr = 'seada@builditbreakit.org'
status_file = os.path.join(my_folder, 'integrate-status.txt')

def get_latest_commit_id():
    answer = subprocess.check_output(shlex.split('git log -1'))
    m = commit_pat.search(answer)
    return m.group(1)

def report(subject, msg):
    smtpserver = smtplib.SMTP('localhost', 25)
    header = 'To: %s\nFrom: "SEADA bibifi vm" <%s>\nSubject: %s\n' % (to_addr, from_addr, subject)
    msg = header + '\n' + msg
    #print(msg)
    smtpserver.sendmail('seada@builditbreakit.org', to_addr, msg)
    smtpserver.close()

def get_integrate_status():
    if os.path.isfile(status_file):
        with open(status_file, 'r') as f:
            txt = f.read()
        m = status_pat.search(txt)
        if m:
            return (m.group(1), m.group(2))
    return ('no commit id', 'unknown')

def set_integrate_status(commit_id, status):
    with open(status_file, 'w') as f:
        f.write('%s %s\n' % (commit_id, status))

def read_f(fpath):
    with open(fpath, 'r') as f:
        return f.read()
    os.remove(fpath)

def integrate():
    os.chdir(my_folder)
    new_status = 'error'
    new_commit_id = 'unknown'
    try:
        try:
            old_commmit_id, old_status = get_integrate_status()
            current_commit_id = get_latest_commit_id()
            subprocess.check_call(shlex.split('git pull'))
            new_commit_id = get_latest_commit_id()
            if current_commit_id == new_commit_id and old_commmit_id == new_commit_id:
                #print('No commits since last integrate.')
                return
            else:
                if current_commit_id != new_commit_id:
                    # We might have a new integrate script; run it instead.
                    os.system('./integrate.py')
                    return

            my_stdout = open('.make.stdout', 'w')
            my_stderr = open('.make.stderr', 'w')
            child = subprocess.Popen(shlex.split('make -C build'), stdout=my_stdout, stderr=my_stderr)
            child.communicate()
            my_stdout.close()
            my_stderr.close()
            my_stdout = read_f('.make.stdout')
            my_stderr = read_f('.make.stderr')

            if child.returncode == 0:
                new_status = 'OK'
                if old_status == 'OK':
                    #print('Build is still clean.')
                    return
                subject = 'build is fixed (commit %s...)' % (new_commit_id[:10])
                msg = my_stdout
            else:
                new_status = 'fail'
                subject = 'build is still broken (commit %s...)' % (new_commit_id[:10])
                if not my_stdout:
                    my_stdout = '(no stdout)'
                if not my_stderr:
                    my_stderr = '(no stderr)'
                msg = 'make returned %d:\n\n%s\n%s' % (child.returncode, my_stdout, my_stderr)
            report(subject, msg)
        except:
            new_status = 'error'
            new_commit_id = 'unknown'
            msg = traceback.format_exc()
            report('unable to integrate', msg)
    finally:
        os.chdir(my_folder)
        set_integrate_status(new_commit_id, new_status)

if __name__ == '__main__':
    integrate()<commit_msg>fix bug in integrate script<commit_after>#! /usr/bin/python

import os, sys, subprocess, re, traceback, smtplib, shlex

my_folder = os.path.dirname(os.path.abspath(__file__))
hash_pat = '[a-fA-F0-9]{40}'
commit_pat = re.compile(r'^\s*commit\s+(%s)\s*$' % hash_pat, re.MULTILINE)
status_pat = re.compile(r'\s*(%s)\s+([a-zA-Z]+)' % hash_pat)
to_addr = 'daniel.hardman@gmail.com'
from_addr = 'seada@builditbreakit.org'
status_file = os.path.join(my_folder, 'integrate-status.txt')

def get_latest_commit_id():
    answer = subprocess.check_output(shlex.split('git log -1'))
    m = commit_pat.search(answer)
    return m.group(1)

def report(subject, msg):
    smtpserver = smtplib.SMTP('localhost', 25)
    header = 'To: %s\nFrom: "SEADA bibifi vm" <%s>\nSubject: %s\n' % (to_addr, from_addr, subject)
    msg = header + '\n' + msg
    #print(msg)
    smtpserver.sendmail('seada@builditbreakit.org', to_addr, msg)
    smtpserver.close()

def get_integrate_status():
    if os.path.isfile(status_file):
        with open(status_file, 'r') as f:
            txt = f.read()
        m = status_pat.search(txt)
        if m:
            return (m.group(1), m.group(2))
    return ('no commit id', 'unknown')

def set_integrate_status(commit_id, status):
    with open(status_file, 'w') as f:
        f.write('%s %s\n' % (commit_id, status))

def read_f(fpath):
    with open(fpath, 'r') as f:
        return f.read()
    os.remove(fpath)

def integrate():
    os.chdir(my_folder)
    new_status = 'error'
    new_commit_id = 'unknown'
    try:
        try:
            old_commit_id, old_status = get_integrate_status()
            current_commit_id = get_latest_commit_id()
            subprocess.check_call(shlex.split('git pull'))
            new_commit_id = get_latest_commit_id()
            if current_commit_id == new_commit_id and old_commit_id == new_commit_id:
                #print('No commits since last integrate.')
                return
            else:
                if current_commit_id != new_commit_id:
                    # We might have a new integrate script; run it instead.
                    os.system('./integrate.py')
                    return

            my_stdout = open('.make.stdout', 'w')
            my_stderr = open('.make.stderr', 'w')
            child = subprocess.Popen(shlex.split('make -C build'), stdout=my_stdout, stderr=my_stderr)
            child.communicate()
            my_stdout.close()
            my_stderr.close()
            my_stdout = read_f('.make.stdout')
            my_stderr = read_f('.make.stderr')

            if child.returncode == 0:
                new_status = 'OK'
                if old_status == 'OK':
                    #print('Build is still clean.')
                    return
                subject = 'build is fixed (commit %s...)' % (new_commit_id[:10])
                msg = my_stdout
            else:
                new_status = 'fail'
                subject = 'build is still broken (commit %s...)' % (new_commit_id[:10])
                if not my_stdout:
                    my_stdout = '(no stdout)'
                if not my_stderr:
                    my_stderr = '(no stderr)'
                msg = 'make returned %d:\n\n%s\n%s' % (child.returncode, my_stdout, my_stderr)
            report(subject, msg)
        except:
            new_status = 'error'
            new_commit_id = 'unknown'
            msg = traceback.format_exc()
            report('unable to integrate', msg)
    finally:
        os.chdir(my_folder)
        set_integrate_status(new_commit_id, new_status)

if __name__ == '__main__':
    integrate()<|endoftext|>