<commit_before>#! /usr/bin/python
import argparse
import os
import sys
from ConfigParser import SafeConfigParser
from collections import OrderedDict
from subprocess import call
import textwrap

class Command(object):

    def init(self, args):
        self.args = args
        self.loc = self.expand_path('~/.vcd')
        self.cfg = self.load_config()

    def expand_path(self, path):
        path = os.path.expanduser(path)
        path = os.path.expandvars(path)
        path = os.path.abspath(path)
        return path

    def load_config(self):
        """Returns a dictionary representation of the current config file."""
        cp = SafeConfigParser()
        cp.read(self.loc)
        cfg = OrderedDict()

        for section in ['venvs', 'cmd_map']:
            cfg[section] = OrderedDict()
            if cp.has_section(section):
                for k in cp.options(section):
                    cfg[section][k] = cp.get(section, k)
        return cfg

    def write_cfg(self):
        """Writes the current self.cfg dictionary to the default config file
        location as a config file, overwriting anything already in the config.
        """
        cp = SafeConfigParser()
        for section in ['venvs', 'cmd_map']:
            cp.add_section(section)
            for k, v in self.cfg[section].iteritems():
                cp.set(section, k, value=v)
        fp = open(self.loc, "w")
        cp.write(fp)
        fp.close()

class register_venv(Command):
    def __init__(self, args):
        self.init(args)
        loc = self.expand_path(self.args.location)
        self.cfg['venvs'].update({self.args.name: loc})
        self.write_cfg()

class register_cmd(Command):
    def __init__(self, args):
        self.init(args)
        if self.args.venv_alias not in self.cfg['venvs'].keys():
            sys.exit(
                "Could not register command: no virtualenv has been registered"
                " with the name {venv}".format(venv=self.args.venv_alias))
        cmd_str = self.args.venv_alias
        if self.args.command:
            cmd_str = "{venv},{cmd}".format(
                venv=self.args.venv_alias, cmd=self.args.command)
        self.cfg['cmd_map'].update({self.args.command_alias: cmd_str})
        self.write_cfg()

class list_resources(Command):
    def __init__(self, args):
        self.init(args)
        resource = args.resource
        resource_name = 'cmd_map' if resource == 'cmds' else resource
        resources = self.cfg.get(resource_name, {})
        k_justify = len(max(resources.keys(), key=len))
        v_justify = len(max(resources.values(), key=len))

        print "\n{0:^{1}} @ {2:^{3}}".format(
            resource_name, k_justify, "location", v_justify)
        print "-"*(k_justify+v_justify+3)
        for k, v in resources.iteritems():
            print "{0:<{1}} @ {2:<{3}}".format(k, k_justify, v, v_justify)

class exec_cmd(Command):
    def __init__(self, args):
        self.init(args)
        if not args.command:
            sys.exit("No command provided")
        cmd = args.command[0]
        remainder = " ".join(args.command[1:])
        info = self.cfg['cmd_map'].get(cmd).split(',')
        venv_loc = os.path.expanduser(self.cfg['venvs'].get(info[0]))
        if len(info) > 1:
            cmd = info[1]
        cmd_str = ". {venv_loc}/bin/activate; {cmd} {rmdr}".format(
            venv_loc=venv_loc, cmd=cmd, rmdr=remainder)
        call(
            cmd_str, stdout=sys.stdout, stderr=sys.stderr, shell=True)

def vcd_run():
    """ This is a seperate entry point to avoid colisions between vcd
    options and possible command aliases entered by the user"""
    parser = argparse.ArgumentParser()

    # Read the config and set the current commands as argaprser options
    cmd = Command()
    cmd.init(list())
    cmds = cmd.cfg['cmd_map'].keys()
    cmds.sort()
    padded_cmds = " ".join([x.ljust(len(max(cmds, key=len))) for x in cmds])
    parser.add_argument(
        'command', type=str, choices=cmds, nargs=argparse.REMAINDER,
        metavar="\n" + textwrap.fill(padded_cmds))
    parser.set_defaults(func=exec_cmd)

    # parse args and trigger actions
    args = parser.parse_args()
    args.func(args)

def vcd_config():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(metavar="add, list")

    # Registration subparsers
    register_parser = subparsers.add_parser(
        "add", usage="\n  Add new venvs and commands to vcd")
    register_subparsers = register_parser.add_subparsers(metavar="venv, cmd")

    # register venv <name> <location>
    register_venv_sparser = register_subparsers.add_parser("venv")
    register_venv_sparser.add_argument(
        'venv_alias', type=str, help="An alias used to address the virtualenv")
    register_venv_sparser.add_argument(
        'path_to_venv_dir', type=str,
        help="Location of the virtualenv top level directory")
    register_venv_sparser.set_defaults(func=register_venv)

    # register command <name> <location>
    register_venv_sparser = register_subparsers.add_parser("cmd")
    register_venv_sparser.add_argument(
        'venv_alias', type=str, help="An alias used to address the virtualenv")
    register_venv_sparser.add_argument(
        'command_alias', type=str,
        help="An alias used to trigger the registered command.  The actual "
        "command can be used here.")
    register_venv_sparser.add_argument(
        'command', nargs='?', default='', type=str,
        help="Optional.  If omitted, it is assumed that the command-alias is "
        "the as the actual command")
    register_venv_sparser.set_defaults(func=register_cmd)

    # Listing subparsers
    list_parser = subparsers.add_parser(
        "list", usage="\n  List registered venvs and commands")
    list_parser.add_argument(
        'resource', type=str, choices=['venvs', 'cmds'], metavar="venvs, cmds")
    list_parser.set_defaults(func=list_resources)

    args = parser.parse_args()
    args.func(args)
<commit_msg>Running unregistered commands results in a more helpful message now<commit_after>#! /usr/bin/python
import argparse
import os
import sys
from ConfigParser import SafeConfigParser
from collections import OrderedDict
from subprocess import call
import textwrap

class Command(object):

    def init(self, args):
        self.args = args
        self.loc = self.expand_path('~/.vcd')
        self.cfg = self.load_config()

    def expand_path(self, path):
        path = os.path.expanduser(path)
        path = os.path.expandvars(path)
        path = os.path.abspath(path)
        return path

    def load_config(self):
        """Returns a dictionary representation of the current config file."""
        cp = SafeConfigParser()
        cp.read(self.loc)
        cfg = OrderedDict()

        for section in ['venvs', 'cmd_map']:
            cfg[section] = OrderedDict()
            if cp.has_section(section):
                for k in cp.options(section):
                    cfg[section][k] = cp.get(section, k)
        return cfg

    def write_cfg(self):
        """Writes the current self.cfg dictionary to the default config file
        location as a config file, overwriting anything already in the config.
        """
        cp = SafeConfigParser()
        for section in ['venvs', 'cmd_map']:
            cp.add_section(section)
            for k, v in self.cfg[section].iteritems():
                cp.set(section, k, value=v)
        fp = open(self.loc, "w")
        cp.write(fp)
        fp.close()

class register_venv(Command):
    def __init__(self, args):
        self.init(args)
        loc = self.expand_path(self.args.location)
        self.cfg['venvs'].update({self.args.name: loc})
        self.write_cfg()

class register_cmd(Command):
    def __init__(self, args):
        self.init(args)
        if self.args.venv_alias not in self.cfg['venvs'].keys():
            sys.exit(
                "Could not register command: no virtualenv has been registered"
                " with the name {venv}".format(venv=self.args.venv_alias))
        cmd_str = self.args.venv_alias
        if self.args.command:
            cmd_str = "{venv},{cmd}".format(
                venv=self.args.venv_alias, cmd=self.args.command)
        self.cfg['cmd_map'].update({self.args.command_alias: cmd_str})
        self.write_cfg()

class list_resources(Command):
    def __init__(self, args):
        self.init(args)
        resource = args.resource
        resource_name = 'cmd_map' if resource == 'cmds' else resource
        resources = self.cfg.get(resource_name, {})
        k_justify = len(max(resources.keys(), key=len))
        v_justify = len(max(resources.values(), key=len))

        print "\n{0:^{1}} @ {2:^{3}}".format(
            resource_name, k_justify, "location", v_justify)
        print "-"*(k_justify+v_justify+3)
        for k, v in resources.iteritems():
            print "{0:<{1}} @ {2:<{3}}".format(k, k_justify, v, v_justify)

class exec_cmd(Command):
    def __init__(self, args):
        self.init(args)
        if not args.command:
            sys.exit("No command provided")
        if args.command not in self.cfg.get('cmds', []):
            sys.exit(
                "'{}' is not a registered command".format(args.command[0]))
        cmd = args.command[0]
        remainder = " ".join(args.command[1:])
        info = self.cfg['cmd_map'].get(cmd).split(',')
        venv_loc = os.path.expanduser(self.cfg['venvs'].get(info[0]))
        if len(info) > 1:
            cmd = info[1]
        cmd_str = ". {venv_loc}/bin/activate; {cmd} {rmdr}".format(
            venv_loc=venv_loc, cmd=cmd, rmdr=remainder)
        call(
            cmd_str, stdout=sys.stdout, stderr=sys.stderr, shell=True)

def vcd_run():
    """ This is a seperate entry point to avoid colisions between vcd
    options and possible command aliases entered by the user"""
    parser = argparse.ArgumentParser()

    # Read the config and set the current commands as argaprser options
    cmd = Command()
    cmd.init(list())
    cmds = cmd.cfg['cmd_map'].keys()
    cmds.sort()
    padded_cmds = " ".join([x.ljust(len(max(cmds, key=len))) for x in cmds])
    parser.add_argument(
        'command', type=str, choices=cmds, nargs=argparse.REMAINDER,
        metavar="\n" + textwrap.fill(padded_cmds))
    parser.set_defaults(func=exec_cmd)

    # parse args and trigger actions
    args = parser.parse_args()
    args.func(args)

def vcd_config():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(metavar="add, list")

    # Registration subparsers
    register_parser = subparsers.add_parser(
        "add", usage="\n  Add new venvs and commands to vcd")
    register_subparsers = register_parser.add_subparsers(metavar="venv, cmd")

    # register venv <name> <location>
    register_venv_sparser = register_subparsers.add_parser("venv")
    register_venv_sparser.add_argument(
        'venv_alias', type=str, help="An alias used to address the virtualenv")
    register_venv_sparser.add_argument(
        'path_to_venv_dir', type=str,
        help="Location of the virtualenv top level directory")
    register_venv_sparser.set_defaults(func=register_venv)

    # register command <name> <location>
    register_venv_sparser = register_subparsers.add_parser("cmd")
    register_venv_sparser.add_argument(
        'venv_alias', type=str, help="An alias used to address the virtualenv")
    register_venv_sparser.add_argument(
        'command_alias', type=str,
        help="An alias used to trigger the registered command.  The actual "
        "command can be used here.")
    register_venv_sparser.add_argument(
        'command', nargs='?', default='', type=str,
        help="Optional.  If omitted, it is assumed that the command-alias is "
        "the as the actual command")
    register_venv_sparser.set_defaults(func=register_cmd)

    # Listing subparsers
    list_parser = subparsers.add_parser(
        "list", usage="\n  List registered venvs and commands")
    list_parser.add_argument(
        'resource', type=str, choices=['venvs', 'cmds'], metavar="venvs, cmds")
    list_parser.set_defaults(func=list_resources)

    args = parser.parse_args()
    args.func(args)
<|endoftext|>


<commit_before>from django.urls import reverse
from django.utils.html import format_html, format_html_join
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from wagtail.core import hooks
from wagtail.admin.menu import MenuItem
from wagtail.core.whitelist import attribute_rule, check_url

@hooks.register('register_settings_menu_item')
def register_django_admin_menu_item():
    return MenuItem(_('Django Admin'), urlresolvers.reverse('admin:index'),
                    classnames='icon icon-cogs', order=700)

@hooks.register('construct_whitelister_element_rules')
def whitelister_element_rules():
    # Whitelist custom elements to the hallo.js editor
    return {
        'a': attribute_rule({'href': check_url, 'target': True}),
        'blockquote': attribute_rule({'class': True})
    }

@hooks.register('insert_editor_js')
def editor_js():
    # Add extra JS files to the admin
    js_files = [
        'js/hallo-custom.js',
    ]
    js_includes = format_html_join(
        '\n', '<script src="{0}{1}"></script>',
        ((settings.STATIC_URL, filename) for filename in js_files)
    )

    return js_includes + format_html(
        """
        <script>
            registerHalloPlugin('blockquotebutton');
            registerHalloPlugin('blockquotebuttonwithclass');
        </script>
        """
    )

@hooks.register('insert_editor_css')
def editor_css():
    # Add extra CSS files to the admin like font-awesome
    css_files = [
        'libs/font-awesome/css/font-awesome.min.css'
    ]

    css_includes = format_html_join(
        '\n', '<link rel="stylesheet" href="{0}{1}">',
        ((settings.STATIC_URL, filename) for filename in css_files)
    )

    return css_includes
<commit_msg>wagtail reverse url<commit_after>from django.urls import reverse
from django.utils.html import format_html, format_html_join
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from wagtail.core import hooks
from wagtail.admin.menu import MenuItem
from wagtail.core.whitelist import attribute_rule, check_url

@hooks.register('register_settings_menu_item')
def register_django_admin_menu_item():
    return MenuItem(_('Django Admin'), reverse('admin:index'),
                    classnames='icon icon-cogs', order=700)

@hooks.register('construct_whitelister_element_rules')
def whitelister_element_rules():
    # Whitelist custom elements to the hallo.js editor
    return {
        'a': attribute_rule({'href': check_url, 'target': True}),
        'blockquote': attribute_rule({'class': True})
    }

@hooks.register('insert_editor_js')
def editor_js():
    # Add extra JS files to the admin
    js_files = [
        'js/hallo-custom.js',
    ]
    js_includes = format_html_join(
        '\n', '<script src="{0}{1}"></script>',
        ((settings.STATIC_URL, filename) for filename in js_files)
    )

    return js_includes + format_html(
        """
        <script>
            registerHalloPlugin('blockquotebutton');
            registerHalloPlugin('blockquotebuttonwithclass');
        </script>
        """
    )

@hooks.register('insert_editor_css')
def editor_css():
    # Add extra CSS files to the admin like font-awesome
    css_files = [
        'libs/font-awesome/css/font-awesome.min.css'
    ]

    css_includes = format_html_join(
        '\n', '<link rel="stylesheet" href="{0}{1}">',
        ((settings.STATIC_URL, filename) for filename in css_files)
    )

    return css_includes
<|endoftext|>


<commit_before>from django.utils.safestring import mark_safe
import markdown

# Convert a markdown string into HTML5, and prevent Django from escaping it
def mdToHTML(value):
	return mark_safe(markdown.markdown(value, output_format="html5"))

# Get a summary of a post
def summarize(fullBody):
	firstNewline = fullBody.find("\n")

	if firstNewline > 0:
		return str(fullBody)[:firstNewline]

	return fullBody
<commit_msg>Fix UTF-8 Bug In summarize Function<commit_after>from django.utils.safestring import mark_safe
import markdown

# Convert a markdown string into HTML5, and prevent Django from escaping it
def mdToHTML(value):
	return mark_safe(markdown.markdown(value, output_format="html5"))

# Get a summary of a post
def summarize(fullBody):
	firstNewline = fullBody.find("\n")

	if firstNewline > 0:
		return unicode(fullBody)[:firstNewline]

	return fullBody
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2018-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import sys
import logging
from openquake.baselib import sap, datastore
from openquake.commonlib import logs
from openquake.calculators.extract import WebExtractor
from openquake.engine import engine
from openquake.server import dbserver

@sap.script
def importcalc(calc_id):
    """
    Import a remote calculation into the local database. server, username
    and password must be specified in an openquake.cfg file.
    NB: calc_id can be a local pathname to a datastore not already
    present in the database: in that case it is imported in the db.
    """
    dbserver.ensure_on()
    try:
        calc_id = int(calc_id)
    except ValueError:  # assume calc_id is a pathname
        calc_id, datadir = datastore.extract_calc_id_datadir(calc_id)
        status = 'complete'
        remote = False
    else:
        remote = True
    job = logs.dbcmd('get_job', calc_id)
    if job is not None:
        sys.exit('There is already a job #%d in the local db' % calc_id)
    if remote:
        datadir = datastore.get_datadir()
        webex = WebExtractor(calc_id)
        status = webex.status['status']
        hc_id = webex.oqparam.hazard_calculation_id
        if hc_id:
            sys.exit('The job has a parent (#%d) and cannot be '
                     'downloaded' % hc_id)
        webex.dump('%s/calc_%d.hdf5' % (datadir, calc_id))
        webex.close()
    with datastore.read(calc_id) as dstore:
        engine.expose_outputs(dstore, status=status)
    logging.info('Imported calculation %d successfully', calc_id)

importcalc.arg('calc_id', 'calculation ID or pathname')
<commit_msg>Improved importcalc when importing local datastores [skip hazardlib]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2018-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import sys
import logging
from openquake.baselib import sap, datastore
from openquake.commonlib import logs
from openquake.calculators.extract import WebExtractor
from openquake.engine import engine
from openquake.server import dbserver

@sap.script
def importcalc(calc_id):
    """
    Import a remote calculation into the local database. server, username
    and password must be specified in an openquake.cfg file.
    NB: calc_id can be a local pathname to a datastore not already
    present in the database: in that case it is imported in the db.
    """
    dbserver.ensure_on()
    try:
        calc_id = int(calc_id)
    except ValueError:  # assume calc_id is a pathname
        status = 'complete'
        remote = False
    else:
        remote = True
        job = logs.dbcmd('get_job', calc_id)
        if job is not None:
            sys.exit('There is already a job #%d in the local db' % calc_id)
    if remote:
        datadir = datastore.get_datadir()
        webex = WebExtractor(calc_id)
        status = webex.status['status']
        hc_id = webex.oqparam.hazard_calculation_id
        if hc_id:
            sys.exit('The job has a parent (#%d) and cannot be '
                     'downloaded' % hc_id)
        webex.dump('%s/calc_%d.hdf5' % (datadir, calc_id))
        webex.close()
    with datastore.read(calc_id) as dstore:
        engine.expose_outputs(dstore, status=status)
    logging.info('Imported calculation %d successfully', calc_id)

importcalc.arg('calc_id', 'calculation ID or pathname')
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2015, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import ast
import pydoc
from openquake.baselib.python3compat import pickle
import collections

import numpy
try:
    import h5py
except ImportError:
    # there is no need of h5py in the workers
    class mock_h5py(object):
        def __getattr__(self, name):
            raise ImportError('Could not import h5py.%s' % name)
    h5py = mock_h5py()

from openquake.baselib.hdf5 import Hdf5Dataset
from openquake.baselib.general import CallableDict
from openquake.commonlib.writers import write_csv

# a dictionary of views datastore -> array
view = CallableDict()

DATADIR = os.environ.get('OQ_DATADIR', os.path.expanduser('~/oqdata'))

def get_nbytes(dset):
    """
    If the dataset has an attribute 'nbytes', return it. Otherwise get the size
    of the underlying array. Returns None if the dataset is actually a group.
    """
    if 'nbytes' in dset.attrs:
        # look if the dataset has an attribute nbytes
        return dset.attrs['nbytes']
    elif hasattr(dset, 'value'):
        # else extract nbytes from the underlying array
        return dset.size * numpy.zeros(1, dset.dtype).nbytes

class ByteCounter(object):
    """
    A visitor used to measure the dimensions of a HDF5 dataset or group.
    Use it as ByteCounter.get_nbytes(dset_or_group).
    """
    @classmethod
    def get_nbytes(cls, dset):
        nbytes = get_nbytes(dset)
        if nbytes is not None:
            return nbytes
        # else dip in the tree
        self = cls()
        dset.visititems(self)
        return self.nbytes

    def __init__(self, nbytes=0):
        self.nbytes = nbytes

    def __call__(self, name, dset_or_group):
        nbytes = get_nbytes(dset_or_group)
        if nbytes:
            self.nbytes += nbytes

def get_calc_ids(datadir=DATADIR):
    """
    Extract the available calculation IDs from the datadir, in order.
    """
    if not os.path.exists(datadir):
        return []
    calc_ids = []
    for f in os.listdir(DATADIR):
        mo = re.match(r'calc_(\d+)\.hdf5', f)
        if mo:
            calc_ids.append(int(mo.group(1)))
    return sorted(calc_ids)

def get_last_calc_id(datadir):
    """
    Extract the latest calculation ID from the given directory.
    If none is found, return 0.
    """
    calcs = get_calc_ids(datadir)
    if not calcs:
        return 0
    return calcs[-1]

class DataStore(collections.MutableMapping):
    """
    DataStore class to store the inputs/outputs of a calculation on the
    filesystem.

    Here is a minimal example of usage:

    >>> ds = DataStore()
    >>> ds['example'] = 'hello world'
    >>> ds.items()
    [(u'example', 'hello world')]
    >>> ds.clear()

    When reading the items, the DataStore will return a generator. The
    items will be ordered lexicographically according to their name.

    There is a serialization protocol to store objects in the datastore.
    An object is serializable if it has a method `__toh5__` returning
    an array and a dictionary, and a method `__fromh5__` taking an array
    and a dictionary and populating the object.
    For an example of use see :class:`openquake.hazardlib.site.SiteCollection`.
    """
    def __init__(self, calc_id=None, datadir=DATADIR, parent=(),
                 export_dir='.', params=(), mode=None):
        if not os.path.exists(datadir):
            os.makedirs(datadir)
        if calc_id is None:  # use a new datastore
            self.calc_id = get_last_calc_id(datadir) + 1
        elif calc_id < 0:  # use an old datastore
            calc_ids = get_calc_ids(datadir)
            try:
                self.calc_id = calc_ids[calc_id]
            except IndexError:
                raise IndexError('There are %d old calculations, cannot '
                                 'retrieve the %s' % (len(calc_ids), calc_id))
        else:  # use the given datastore
            self.calc_id = calc_id
        self.parent = parent  # parent datastore (if any)
        self.datadir = datadir
        self.calc_dir = os.path.join(datadir, 'calc_%s' % self.calc_id)
        self.export_dir = export_dir
        self.hdf5path = self.calc_dir + '.hdf5'
        mode = mode or 'r+' if os.path.exists(self.hdf5path) else 'w'
        self.hdf5 = h5py.File(self.hdf5path, mode, libver='latest')
        self.attrs = self.hdf5.attrs
        for name, value in params:
            self.attrs[name] = value
        if not parent and 'hazard_calculation_id' in self.attrs:
            parent_id = ast.literal_eval(self.attrs['hazard_calculation_id'])
            if parent_id:
                self.parent = self.__class__(parent_id)

    def set_parent(self, parent):
        """
        Give a parent to a datastore and update its .attrs with the parent
        attributes, which are assumed to be literal strings.
        """
        self.parent = parent
        # merge parent attrs into child attrs
        for name, value in self.parent.attrs.items():
            if name not in self.attrs:  # add missing parameter
                self.attrs[name] = value

    def set_nbytes(self, key):
        """
        Set the `nbytes` attribute on the HDF5 object identified by `key`.
        """
        obj = self.hdf5[key]
        obj.attrs['nbytes'] = ByteCounter.get_nbytes(obj)

    def set_attrs(self, key, **kw):
        """
        Set the HDF5 attributes of the given key
        """
        for k, v in kw.items():
            self.hdf5[key].attrs[k] = v

    def get_attr(self, key, name, default=None):
        """
        :param key: dataset path
        :param name: name of the attribute
        :param default: value to return if the attribute is missing
        """
        obj = self.hdf5[key]
        try:
            return obj.attrs[name]
        except KeyError:
            if default is None:
                raise
            return default

    def create_dset(self, key, dtype, size=None, compression=None):
        """
        Create a one-dimensional HDF5 dataset.

        :param key: name of the dataset
        :param dtype: dtype of the dataset (usually composite)
        :param size: size of the dataset (if None, the dataset is extendable)
        """
        return Hdf5Dataset.create(self.hdf5, key, dtype, size, compression)

    def export_path(self, relname, export_dir=None):
        """
        Return the path of the exported file by adding the export_dir in
        front, the calculation ID at the end.

        :param relname: relative file name
        :param export_dir: export directory (if None use .export_dir)
        """
        assert not os.path.dirname(relname), relname
        name, ext = relname.rsplit('.', 1)
        newname = '%s_%s.%s' % (name, self.calc_id, ext)
        if export_dir is None:
            export_dir = self.export_dir
        return os.path.join(export_dir, newname)

    def export_csv(self, key):
        """
        Generic csv exporter
        """
        return write_csv(self.export_path(key, 'csv'), self[key])

    def reopen(self):
        """Reopen a closed datastore"""
        parent = () if self.parent is () else self.parent.reopen()
        return self.__class__(self.calc_id, self.datadir, parent,
                              self.export_dir)

    def close(self):
        """Close the underlying hdf5 file"""
        if self.parent:
            self.parent.close()
        if self.hdf5:  # is open
            self.hdf5.close()

    def clear(self):
        """Remove the datastore from the file system"""
        self.close()
        os.remove(self.hdf5path)

    def getsize(self, key=None):
        """
        Return the size in byte of the output associated to the given key.
        If no key is given, returns the total size of all files.
        """
        if key is None:
            return os.path.getsize(self.hdf5path)
        return ByteCounter.get_nbytes(self.hdf5[key])

    def get(self, key, default):
        """
        :returns: the value associated to the datastore key, or the default
        """
        try:
            return self[key]
        except KeyError:
            return default

    def __getitem__(self, key):
        try:
            val = self.hdf5[key]
        except KeyError:
            if self.parent:
                try:
                    val = self.parent.hdf5[key]
                except KeyError:
                    raise KeyError(
                        'No %r found in %s' % (key, [self, self.parent]))
            else:
                raise KeyError('No %r found in %s' % (key, self))
        try:
            shape = val.shape
        except AttributeError:  # val is a group
            return val
        if '__pyclass__' in val.attrs:  # serialized object
            value, attrs = val.value, dict(val.attrs)
            cls = pydoc.locate(attrs.pop('__pyclass__'))
            val = cls.__new__(cls)
            val.__fromh5__(value, attrs)
        if not shape:
            val = pickle.loads(val.value)
        return val

    def __setitem__(self, key, value):
        attrs = {}
        if hasattr(value, '__toh5__'):
            val, attrs = value.__toh5__()
            attrs['__pyclass__'] = '.'.join([value.__class__.__module__,
                                             value.__class__.__name__])
        elif (not isinstance(value, numpy.ndarray) or
                value.dtype is numpy.dtype(object)):
            val = numpy.array(pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
        else:
            val = value
        if key in self.hdf5:
            # there is a bug in the current version of HDF5 for composite
            # arrays: is impossible to save twice the same key; so we remove
            # the key first, then it is possible to save it again
            del self[key]
        try:
            self.hdf5[key] = val
        except RuntimeError as exc:
            raise RuntimeError('Could not save %s: %s in %s' %
                               (key, exc, self.hdf5path))
        # save attributes if any
        for k, v in attrs.items():
            self.hdf5[key].attrs[k] = v

    def __delitem__(self, key):
        if (h5py.version.version <= '2.0.1' and not
                hasattr(self.hdf5[key], 'shape')):
            # avoid bug when deleting groups that produces a segmentation fault
            return
        del self.hdf5[key]

    def __iter__(self):
        if not self.hdf5:
            raise RuntimeError('%s is closed' % self)
        for path in sorted(self.hdf5):
            yield path

    def __contains__(self, key):
        return key in self.hdf5

    def __len__(self):
        return sum(1 for f in self)

    def __repr__(self):
        return '<%s %d>' % (self.__class__.__name__, self.calc_id)

class Fake(dict):
    """
    A fake datastore as a dict subclass, useful in tests and such
    """
    def __init__(self, attrs=None, **kwargs):
        self.attrs = {k: repr(v) for k, v in attrs.items()} if attrs else {}
        self.update(kwargs)

def persistent_attribute(key):
    """
    Persistent attributes are persisted to the datastore and cached.
    Modifications to mutable objects are not automagically persisted.
    If you have a huge object that does not fit in memory use the datastore
    directory (for instance, open a HDF5 file to create an empty array, then
    populate it). Notice that you can use any dict-like data structure in
    place of the datastore, provided you can set attributes on it.
    Here is an example:

    >>> class Datastore(dict):
    ...     "A fake datastore"

    >>> class Store(object):
    ...     a = persistent_attribute('a')
    ...     def __init__(self, a):
    ...         self.datastore = Datastore()
    ...         self.a = a  # this assegnation will store the attribute

    >>> store = Store([1])
    >>> store.a  # this retrieves the attribute
    [1]
    >>> store.a.append(2)
    >>> store.a = store.a  # remember to store the modified attribute!

    :param key: the name of the attribute to be made persistent
    :returns: a property to be added to a class with a .datastore attribute
    """
    privatekey = '_' + key

    def getter(self):
        # Try to get the value from the privatekey attribute (i.e. from
        # the cache of the datastore); if not possible, get the value
        # from the datastore and set the cache; if not possible, get the
        # value from the parent and set the cache. If the value cannot
        # be retrieved, raise an AttributeError.
        try:
            return getattr(self.datastore, privatekey)
        except AttributeError:
            value = self.datastore[key]
            setattr(self.datastore, privatekey, value)
            return value

    def setter(self, value):
        # Update the datastore and the private key
        self.datastore[key] = value
        setattr(self.datastore, privatekey, value)

    return property(getter, setter)
<commit_msg>Added an utility datastore.read<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2015, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import ast
import pydoc
from openquake.baselib.python3compat import pickle
import collections

import numpy
try:
    import h5py
except ImportError:
    # there is no need of h5py in the workers
    class mock_h5py(object):
        def __getattr__(self, name):
            raise ImportError('Could not import h5py.%s' % name)
    h5py = mock_h5py()

from openquake.baselib.hdf5 import Hdf5Dataset
from openquake.baselib.general import CallableDict
from openquake.commonlib.writers import write_csv

# a dictionary of views datastore -> array
view = CallableDict()

DATADIR = os.environ.get('OQ_DATADIR', os.path.expanduser('~/oqdata'))

def get_nbytes(dset):
    """
    If the dataset has an attribute 'nbytes', return it. Otherwise get the size
    of the underlying array. Returns None if the dataset is actually a group.
    """
    if 'nbytes' in dset.attrs:
        # look if the dataset has an attribute nbytes
        return dset.attrs['nbytes']
    elif hasattr(dset, 'value'):
        # else extract nbytes from the underlying array
        return dset.size * numpy.zeros(1, dset.dtype).nbytes

class ByteCounter(object):
    """
    A visitor used to measure the dimensions of a HDF5 dataset or group.
    Use it as ByteCounter.get_nbytes(dset_or_group).
    """
    @classmethod
    def get_nbytes(cls, dset):
        nbytes = get_nbytes(dset)
        if nbytes is not None:
            return nbytes
        # else dip in the tree
        self = cls()
        dset.visititems(self)
        return self.nbytes

    def __init__(self, nbytes=0):
        self.nbytes = nbytes

    def __call__(self, name, dset_or_group):
        nbytes = get_nbytes(dset_or_group)
        if nbytes:
            self.nbytes += nbytes

def get_calc_ids(datadir=DATADIR):
    """
    Extract the available calculation IDs from the datadir, in order.
    """
    if not os.path.exists(datadir):
        return []
    calc_ids = []
    for f in os.listdir(DATADIR):
        mo = re.match(r'calc_(\d+)\.hdf5', f)
        if mo:
            calc_ids.append(int(mo.group(1)))
    return sorted(calc_ids)

def get_last_calc_id(datadir):
    """
    Extract the latest calculation ID from the given directory.
    If none is found, return 0.
    """
    calcs = get_calc_ids(datadir)
    if not calcs:
        return 0
    return calcs[-1]

def read(calc_id, mode='r'):
    """
    Read the datastore identified by the calculation ID, if it exists
    and it is accessible.
    """
    if calc_id < 0:  # use an old datastore
        calc_id = get_calc_ids(DATADIR)[calc_id]
    fname = os.path.join(DATADIR, 'calc_%s.hdf5' % calc_id)
    assert os.path.exists(fname), fname
    assert os.access(fname, os.R_OK if mode == 'r' else os.W_OK), fname
    return DataStore(calc_id, mode=mode)

class DataStore(collections.MutableMapping):
    """
    DataStore class to store the inputs/outputs of a calculation on the
    filesystem.

    Here is a minimal example of usage:

    >>> ds = DataStore()
    >>> ds['example'] = 'hello world'
    >>> ds.items()
    [(u'example', 'hello world')]
    >>> ds.clear()

    When reading the items, the DataStore will return a generator. The
    items will be ordered lexicographically according to their name.

    There is a serialization protocol to store objects in the datastore.
    An object is serializable if it has a method `__toh5__` returning
    an array and a dictionary, and a method `__fromh5__` taking an array
    and a dictionary and populating the object.
    For an example of use see :class:`openquake.hazardlib.site.SiteCollection`.
    """
    def __init__(self, calc_id=None, datadir=DATADIR, parent=(),
                 export_dir='.', params=(), mode=None):
        if not os.path.exists(datadir):
            os.makedirs(datadir)
        if calc_id is None:  # use a new datastore
            self.calc_id = get_last_calc_id(datadir) + 1
        elif calc_id < 0:  # use an old datastore
            calc_ids = get_calc_ids(datadir)
            try:
                self.calc_id = calc_ids[calc_id]
            except IndexError:
                raise IndexError('There are %d old calculations, cannot '
                                 'retrieve the %s' % (len(calc_ids), calc_id))
        else:  # use the given datastore
            self.calc_id = calc_id
        self.parent = parent  # parent datastore (if any)
        self.datadir = datadir
        self.calc_dir = os.path.join(datadir, 'calc_%s' % self.calc_id)
        self.export_dir = export_dir
        self.hdf5path = self.calc_dir + '.hdf5'
        mode = mode or 'r+' if os.path.exists(self.hdf5path) else 'w'
        self.hdf5 = h5py.File(self.hdf5path, mode, libver='latest')
        self.attrs = self.hdf5.attrs
        for name, value in params:
            self.attrs[name] = value
        if not parent and 'hazard_calculation_id' in self.attrs:
            parent_id = ast.literal_eval(self.attrs['hazard_calculation_id'])
            if parent_id:
                self.parent = self.__class__(parent_id)

    def set_parent(self, parent):
        """
        Give a parent to a datastore and update its .attrs with the parent
        attributes, which are assumed to be literal strings.
        """
        self.parent = parent
        # merge parent attrs into child attrs
        for name, value in self.parent.attrs.items():
            if name not in self.attrs:  # add missing parameter
                self.attrs[name] = value

    def set_nbytes(self, key):
        """
        Set the `nbytes` attribute on the HDF5 object identified by `key`.
        """
        obj = self.hdf5[key]
        obj.attrs['nbytes'] = ByteCounter.get_nbytes(obj)

    def set_attrs(self, key, **kw):
        """
        Set the HDF5 attributes of the given key
        """
        for k, v in kw.items():
            self.hdf5[key].attrs[k] = v

    def get_attr(self, key, name, default=None):
        """
        :param key: dataset path
        :param name: name of the attribute
        :param default: value to return if the attribute is missing
        """
        obj = self.hdf5[key]
        try:
            return obj.attrs[name]
        except KeyError:
            if default is None:
                raise
            return default

    def create_dset(self, key, dtype, size=None, compression=None):
        """
        Create a one-dimensional HDF5 dataset.

        :param key: name of the dataset
        :param dtype: dtype of the dataset (usually composite)
        :param size: size of the dataset (if None, the dataset is extendable)
        """
        return Hdf5Dataset.create(self.hdf5, key, dtype, size, compression)

    def export_path(self, relname, export_dir=None):
        """
        Return the path of the exported file by adding the export_dir in
        front, the calculation ID at the end.

        :param relname: relative file name
        :param export_dir: export directory (if None use .export_dir)
        """
        assert not os.path.dirname(relname), relname
        name, ext = relname.rsplit('.', 1)
        newname = '%s_%s.%s' % (name, self.calc_id, ext)
        if export_dir is None:
            export_dir = self.export_dir
        return os.path.join(export_dir, newname)

    def export_csv(self, key):
        """
        Generic csv exporter
        """
        return write_csv(self.export_path(key, 'csv'), self[key])

    def reopen(self):
        """Reopen a closed datastore"""
        parent = () if self.parent is () else self.parent.reopen()
        return self.__class__(self.calc_id, self.datadir, parent,
                              self.export_dir)

    def close(self):
        """Close the underlying hdf5 file"""
        if self.parent:
            self.parent.close()
        if self.hdf5:  # is open
            self.hdf5.close()

    def clear(self):
        """Remove the datastore from the file system"""
        self.close()
        os.remove(self.hdf5path)

    def getsize(self, key=None):
        """
        Return the size in byte of the output associated to the given key.
        If no key is given, returns the total size of all files.
        """
        if key is None:
            return os.path.getsize(self.hdf5path)
        return ByteCounter.get_nbytes(self.hdf5[key])

    def get(self, key, default):
        """
        :returns: the value associated to the datastore key, or the default
        """
        try:
            return self[key]
        except KeyError:
            return default

    def __getitem__(self, key):
        try:
            val = self.hdf5[key]
        except KeyError:
            if self.parent:
                try:
                    val = self.parent.hdf5[key]
                except KeyError:
                    raise KeyError(
                        'No %r found in %s' % (key, [self, self.parent]))
            else:
                raise KeyError('No %r found in %s' % (key, self))
        try:
            shape = val.shape
        except AttributeError:  # val is a group
            return val
        if '__pyclass__' in val.attrs:  # serialized object
            value, attrs = val.value, dict(val.attrs)
            cls = pydoc.locate(attrs.pop('__pyclass__'))
            val = cls.__new__(cls)
            val.__fromh5__(value, attrs)
        if not shape:
            val = pickle.loads(val.value)
        return val

    def __setitem__(self, key, value):
        attrs = {}
        if hasattr(value, '__toh5__'):
            val, attrs = value.__toh5__()
            attrs['__pyclass__'] = '.'.join([value.__class__.__module__,
                                             value.__class__.__name__])
        elif (not isinstance(value, numpy.ndarray) or
                value.dtype is numpy.dtype(object)):
            val = numpy.array(pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
        else:
            val = value
        if key in self.hdf5:
            # there is a bug in the current version of HDF5 for composite
            # arrays: is impossible to save twice the same key; so we remove
            # the key first, then it is possible to save it again
            del self[key]
        try:
            self.hdf5[key] = val
        except RuntimeError as exc:
            raise RuntimeError('Could not save %s: %s in %s' %
                               (key, exc, self.hdf5path))
        # save attributes if any
        for k, v in attrs.items():
            self.hdf5[key].attrs[k] = v

    def __delitem__(self, key):
        if (h5py.version.version <= '2.0.1' and not
                hasattr(self.hdf5[key], 'shape')):
            # avoid bug when deleting groups that produces a segmentation fault
            return
        del self.hdf5[key]

    def __iter__(self):
        if not self.hdf5:
            raise RuntimeError('%s is closed' % self)
        for path in sorted(self.hdf5):
            yield path

    def __contains__(self, key):
        return key in self.hdf5

    def __len__(self):
        return sum(1 for f in self)

    def __repr__(self):
        return '<%s %d>' % (self.__class__.__name__, self.calc_id)

class Fake(dict):
    """
    A fake datastore as a dict subclass, useful in tests and such
    """
    def __init__(self, attrs=None, **kwargs):
        self.attrs = {k: repr(v) for k, v in attrs.items()} if attrs else {}
        self.update(kwargs)

def persistent_attribute(key):
    """
    Persistent attributes are persisted to the datastore and cached.
    Modifications to mutable objects are not automagically persisted.
    If you have a huge object that does not fit in memory use the datastore
    directory (for instance, open a HDF5 file to create an empty array, then
    populate it). Notice that you can use any dict-like data structure in
    place of the datastore, provided you can set attributes on it.
    Here is an example:

    >>> class Datastore(dict):
    ...     "A fake datastore"

    >>> class Store(object):
    ...     a = persistent_attribute('a')
    ...     def __init__(self, a):
    ...         self.datastore = Datastore()
    ...         self.a = a  # this assegnation will store the attribute

    >>> store = Store([1])
    >>> store.a  # this retrieves the attribute
    [1]
    >>> store.a.append(2)
    >>> store.a = store.a  # remember to store the modified attribute!

    :param key: the name of the attribute to be made persistent
    :returns: a property to be added to a class with a .datastore attribute
    """
    privatekey = '_' + key

    def getter(self):
        # Try to get the value from the privatekey attribute (i.e. from
        # the cache of the datastore); if not possible, get the value
        # from the datastore and set the cache; if not possible, get the
        # value from the parent and set the cache. If the value cannot
        # be retrieved, raise an AttributeError.
        try:
            return getattr(self.datastore, privatekey)
        except AttributeError:
            value = self.datastore[key]
            setattr(self.datastore, privatekey, value)
            return value

    def setter(self, value):
        # Update the datastore and the private key
        self.datastore[key] = value
        setattr(self.datastore, privatekey, value)

    return property(getter, setter)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2020 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This application demonstrates how to receive messages with the
Pub/Sub Lite API. For more information, see the root level README.md and the
documentation at https://cloud.google.com/pubsub/lite/docs/subscribing.
"""

import argparse

from google.cloud.pubsublite.types import MessageMetadata
from google.pubsub_v1 import PubsubMessage

def receive_messages(
    project_number, cloud_region, zone_id, subscription_id, timeout, regional
):
    # [START pubsublite_quickstart_subscriber]
    from concurrent.futures._base import TimeoutError
    from google.cloud.pubsublite.cloudpubsub import SubscriberClient
    from google.cloud.pubsublite.types import (
        CloudRegion,
        CloudZone,
        FlowControlSettings,
        SubscriptionPath,
    )

    # TODO(developer):
    # project_number = 1122334455
    # cloud_region = "us-central1"
    # zone_id = "a"
    # subscription_id = "your-subscription-id"
    # timeout = 90
    # regional = True

    if regional:
        location = CloudRegion(cloud_region)
    else:
        location = CloudZone(CloudRegion(cloud_region), zone_id)

    subscription_path = SubscriptionPath(project_number, location, subscription_id)
    # Configure when to pause the message stream for more incoming messages based on the
    # maximum size or number of messages that a single-partition subscriber has received,
    # whichever condition is met first.
    per_partition_flow_control_settings = FlowControlSettings(
        # 1,000 outstanding messages. Must be >0.
        messages_outstanding=1000,
        # 10 MiB. Must be greater than the allowed size of the largest message (1 MiB).
        bytes_outstanding=10 * 1024 * 1024,
    )

    def callback(message: PubsubMessage):
        message_data = message.data.decode("utf-8")
        metadata = MessageMetadata.decode(message.message_id)
        print(
            f"Received {message_data} of ordering key {message.ordering_key} with id {metadata}."
        )
        message.ack()

    # SubscriberClient() must be used in a `with` block or have __enter__() called before use.
    with SubscriberClient() as subscriber_client:

        streaming_pull_future = subscriber_client.subscribe(
            subscription_path,
            callback=callback,
            per_partition_flow_control_settings=per_partition_flow_control_settings,
        )

        print(f"Listening for messages on {str(subscription_path)}...")

        try:
            streaming_pull_future.result(timeout=timeout)
        except TimeoutError or KeyboardInterrupt:
            streaming_pull_future.cancel()
            assert streaming_pull_future.done()
    # [END pubsublite_quickstart_subscriber]

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument("project_number", help="Your Google Cloud Project Number")
    parser.add_argument("cloud_region", help="Your Cloud Region, e.g. 'us-central1'")
    parser.add_argument("zone_id", help="Your Zone ID, e.g. 'a'")
    parser.add_argument("subscription_id", help="Your subscription ID")
    parser.add_argument(
        "timeout", nargs="?", type=int, help="Timeout in second",
    )
    parser.add_argument("regional", "True if using a regional resource else zonal")

    args = parser.parse_args()

    receive_messages(
        args.project_number,
        args.cloud_region,
        args.zone_id,
        args.subscription_id,
        args.timeout,
        args.regional,
    )
<commit_msg>samples: move imports inside region tags (#318)<commit_after>#!/usr/bin/env python

# Copyright 2020 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This application demonstrates how to receive messages with the
Pub/Sub Lite API. For more information, see the root level README.md and the
documentation at https://cloud.google.com/pubsub/lite/docs/subscribing.
"""

import argparse

def receive_messages(
    project_number, cloud_region, zone_id, subscription_id, timeout, regional
):
    # [START pubsublite_quickstart_subscriber]
    from concurrent.futures._base import TimeoutError
    from google.pubsub_v1 import PubsubMessage
    from google.cloud.pubsublite.cloudpubsub import SubscriberClient
    from google.cloud.pubsublite.types import (
        CloudRegion,
        CloudZone,
        FlowControlSettings,
        MessageMetadata,
        SubscriptionPath,
    )

    # TODO(developer):
    # project_number = 1122334455
    # cloud_region = "us-central1"
    # zone_id = "a"
    # subscription_id = "your-subscription-id"
    # timeout = 90
    # regional = True

    if regional:
        location = CloudRegion(cloud_region)
    else:
        location = CloudZone(CloudRegion(cloud_region), zone_id)

    subscription_path = SubscriptionPath(project_number, location, subscription_id)
    # Configure when to pause the message stream for more incoming messages based on the
    # maximum size or number of messages that a single-partition subscriber has received,
    # whichever condition is met first.
    per_partition_flow_control_settings = FlowControlSettings(
        # 1,000 outstanding messages. Must be >0.
        messages_outstanding=1000,
        # 10 MiB. Must be greater than the allowed size of the largest message (1 MiB).
        bytes_outstanding=10 * 1024 * 1024,
    )

    def callback(message: PubsubMessage):
        message_data = message.data.decode("utf-8")
        metadata = MessageMetadata.decode(message.message_id)
        print(
            f"Received {message_data} of ordering key {message.ordering_key} with id {metadata}."
        )
        message.ack()

    # SubscriberClient() must be used in a `with` block or have __enter__() called before use.
    with SubscriberClient() as subscriber_client:

        streaming_pull_future = subscriber_client.subscribe(
            subscription_path,
            callback=callback,
            per_partition_flow_control_settings=per_partition_flow_control_settings,
        )

        print(f"Listening for messages on {str(subscription_path)}...")

        try:
            streaming_pull_future.result(timeout=timeout)
        except TimeoutError or KeyboardInterrupt:
            streaming_pull_future.cancel()
            assert streaming_pull_future.done()
    # [END pubsublite_quickstart_subscriber]

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument("project_number", help="Your Google Cloud Project Number")
    parser.add_argument("cloud_region", help="Your Cloud Region, e.g. 'us-central1'")
    parser.add_argument("zone_id", help="Your Zone ID, e.g. 'a'")
    parser.add_argument("subscription_id", help="Your subscription ID")
    parser.add_argument(
        "timeout", nargs="?", type=int, help="Timeout in second",
    )
    parser.add_argument("regional", "True if using a regional resource else zonal")

    args = parser.parse_args()

    receive_messages(
        args.project_number,
        args.cloud_region,
        args.zone_id,
        args.subscription_id,
        args.timeout,
        args.regional,
    )
<|endoftext|>


<commit_before>""" Contains AccessibleTime, class that represents the period of time when a course/task is accessible """
from datetime import datetime

class AccessibleTime(object):

    """ represents the period of time when a course/task is accessible """

    def __init__(self, val=None):
        """
            Parse a string/a boolean to get the correct time period.
            Correct values for val:
            True (task always open)
            False (task always closed)
            2014-07-16 11:24:00 (task is open from 2014-07-16 at 11:24:00)
            2014-07-16 (task is open from 2014-07-16)
            / 2014-07-16 11:24:00 (task is only open before the 2014-07-16 at 11:24:00)
            / 2014-07-16 (task is only open before the 2014-07-16)
            2014-07-16 11:24:00 / 2014-07-20 11:24:00 (task is open from 2014-07-16 11:24:00 and will be closed the 2014-07-20 at 11:24:00)
            2014-07-16 / 2014-07-20 11:24:00 (...)
            2014-07-16 11:24:00 / 2014-07-20 (...)
            2014-07-16 / 2014-07-20 (...)
        """
        if val is None or val == "":
            self.val = [None, None]
        elif val == False:
            self.val = [0, 0]
        else:  # str
            values = val.split("/")
            if len(values) == 1:
                self.val = [self._parse_date(values[0].strip()), None]
            else:
                self.val = [self._parse_date(values[0].strip()), self._parse_date(values[1].strip())]

    @classmethod
    def _parse_date(cls, date):
        """ Parse a valid date """
        if date == "":
            return None

        print date
        for format_type in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d", "%d/%m/%Y %H:%M:%S", "%d/%m/%Y %H:%M", "%d/%m/%Y %H", "%d/%m/%Y"]:
            try:
                print datetime.strptime(date, format_type)
                return datetime.strptime(date, format_type)
            except ValueError:
                pass
        raise Exception("Unknow format for " + date)

    def is_open(self, when=None):
        """ Returns True if the course/task is still open """
        if when is None:
            when = datetime.now()

        first, second = self.val
        if first is not None and second is not None and first >= second:
            return False
        if first is not None and first > when:
            return False
        if second is not None and second < when:
            return False
        return True
<commit_msg>Woops<commit_after>""" Contains AccessibleTime, class that represents the period of time when a course/task is accessible """
from datetime import datetime

class AccessibleTime(object):

    """ represents the period of time when a course/task is accessible """

    def __init__(self, val=None):
        """
            Parse a string/a boolean to get the correct time period.
            Correct values for val:
            True (task always open)
            False (task always closed)
            2014-07-16 11:24:00 (task is open from 2014-07-16 at 11:24:00)
            2014-07-16 (task is open from 2014-07-16)
            / 2014-07-16 11:24:00 (task is only open before the 2014-07-16 at 11:24:00)
            / 2014-07-16 (task is only open before the 2014-07-16)
            2014-07-16 11:24:00 / 2014-07-20 11:24:00 (task is open from 2014-07-16 11:24:00 and will be closed the 2014-07-20 at 11:24:00)
            2014-07-16 / 2014-07-20 11:24:00 (...)
            2014-07-16 11:24:00 / 2014-07-20 (...)
            2014-07-16 / 2014-07-20 (...)
        """
        if val is None or val == "":
            self.val = [None, None]
        elif val == False:
            self.val = [0, 0]
        else:  # str
            values = val.split("/")
            if len(values) == 1:
                self.val = [self._parse_date(values[0].strip()), None]
            else:
                self.val = [self._parse_date(values[0].strip()), self._parse_date(values[1].strip())]

    @classmethod
    def _parse_date(cls, date):
        """ Parse a valid date """
        if date == "":
            return None

        for format_type in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d", "%d/%m/%Y %H:%M:%S", "%d/%m/%Y %H:%M", "%d/%m/%Y %H", "%d/%m/%Y"]:
            try:
                return datetime.strptime(date, format_type)
            except ValueError:
                pass
        raise Exception("Unknow format for " + date)

    def is_open(self, when=None):
        """ Returns True if the course/task is still open """
        if when is None:
            when = datetime.now()

        first, second = self.val
        if first is not None and second is not None and first >= second:
            return False
        if first is not None and first > when:
            return False
        if second is not None and second < when:
            return False
        return True
<|endoftext|>


<commit_before>
def burrows_wheeler(text):
    """Returns the burrows wheeler transform of <text>.

    The text is assumed to not contain the character $"""

    text += "$"
    all_permutations = []
    for i in range(len(text)):
        all_permutations.append(text[i:] + text[:i])

    all_permutations.sort()
    return "".join([w[-1] for w in all_permutations])
<commit_msg>Return suffix array indices from burrows wheeler<commit_after>
def burrows_wheeler(text):
    """Calculates the burrows wheeler transform of <text>.

    returns the burrows wheeler string and the suffix array indices
    The text is assumed to not contain the character $"""

    text += "$"
    all_permutations = []
    for i in range(len(text)):
        all_permutations.append((text[i:] + text[:i],i))

    all_permutations.sort()
    bw_l = [] # burrows wheeler as list
    sa_i = [] # suffix array indices

    for w,j in all_permutations:
        bw_l.append(w[-1])
        sa_i.append(j)

    return "".join(bw_l), sa_i
<|endoftext|>


<commit_before><commit_msg>typo<commit_after><|endoftext|>


<commit_before>import six
import numpy as np
import scipy.integrate as integrate

class ContinuousFactor(object):
    """
    Base class for factors representing various multivariate
    representations.
    """
    def __init__(self, variables, pdf):
        """
        Parameters
        ----------
        variables: list or array-like
            The variables for wich the distribution is defined.

        pdf: function
            The probability density function of the distribution.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        # Two variable drichlet ditribution with alpha = (1,2)
        >>> def drichlet_pdf(x, y):
        ...     return (np.power(x, 1)*np.power(y, 2))/beta(x, y)
        >>> from pgmpy.factors import ContinuousFactor
        >>> drichlet_factor = ContinuousFactor(['x', 'y'], drichlet_pdf)
        >>> drichlet_factor.scope()
        ['x', 'y']
        >>> drichlet_factor.assignemnt(5,6)
        226800.0
        """
        if not isinstance(variables, list):
            raise TypeError("variables: Expected type list or array-like,\
                             got type {var_type}".format(var_type=type(variables)))

        self.variables = variables
        self.pdf = pdf

    def scope(self):
        """
        Returns the scope of the factor.

        Returns
        -------
        list: List of variable names in the scope of the factor.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> normal_pdf = lambda x: multivariate_normal(x, [0, 0], [[1, 0], [0, 1]])
        >>> phi = ContinuousFactor(['x1', 'x2'], normal_pdf)
        >>> phi.scope()
        ['x1', 'x2']
        """
        return self.variables

    def assignment(self, *args):
        """
        Returns a list of pdf assignments for the corresponding values.

        Parameters
        ----------
        values: A list of arrays of dimension 1 x n
            List of values whose assignment is to be computed.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> normal_pdf = lambda x: multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> phi = ContinuousFactor(['x1', 'x2'], normal_pdf)
        >>> phi.assignment([1,2])
        0.013064233284684921
        """
        return self.pdf(*args)

    def copy(self):
        """
        Return a copy of the distribution.

        Returns
        -------
        ContinuousFactor object: copy of the distribution

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        # Two variable drichlet ditribution with alpha = (1,2)
        >>> def drichlet_pdf(x, y):
        ...     return (np.power(x, 1)*np.power(y, 2))/beta(x, y)
        >>> from pgmpy.factors import ContinuousFactor
        >>> drichlet_factor = ContinuousFactor(['x', 'y'], drichlet_pdf)
        >>> drichlet_factor.variables
        ['x', 'y']
        >>> copy_factor = drichlet_factor.copy()
        copy_factor.variables

        """
        return ContinuousFactor(self.scope(), self.pdf)

    def reduce(self, values, inplace=True):
        """
        Reduces the factor to the context of the given variable values.

        Parameters
        ----------
        values: list, array-like
            A list of tuples of the form (variable_name, variable_value).

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new ContinuosFactor object.

        Returns
        -------
        ContinuousFactor or None: if inplace=True (default) returns None
                                  if inplace=False returns a new `Factor` instance.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        >>> def custom_pdf(x, y, z):
        ...     return z*(np.power(x, 1)*np.power(y, 2))/beta(x, y)
        >>> from pgmpy.factors import ContinuousFactor
        >>> custom_factor = ContinuousFactor(['x', 'y', 'z'], custom_pdf)
        >>> custom_factor.variables
        ['x', 'y', 'z']
        >>> custom_factor.assignment(1, 2, 3)
        24.0

        >>> custom_factor.reduce([('y', 2)])
        >>> custom_factor.variables
        ['x', 'z']
        >>> custom_factor.assignment(1, 3)
        24.0
        """
        if not isinstance(values, list):
            raise TypeError("values: Expected type list or array-like,\
                             got type {var_type}".format(var_type=type(values)))

        phi = self if inplace else self.copy()

        var_to_remove = [var for var, value in values]
        var_to_keep = [var for var in self.variables if var not in var_to_remove]

        reduced_var_index = [(self.variables.index(var), value) for var, value in values]
        pdf = self.pdf

        def reduced_pdf(*args, **kwargs):
            reduced_args = list(args)
            reduced_kwargs = kwargs.copy()

            if reduced_args:
                for index, value in reduced_var_index:
                    reduced_args.insert(index, value)
            if reduced_kwargs:
                for var, value in values:
                    reduced_kwargs[var] = value
            if reduced_args and reduced_kwargs:
                reduced_args = [arg for arg in reduced_args if arg not in reduced_kwargs.values()]

            return pdf(*reduced_args, **reduced_kwargs)

        phi.variables = var_to_keep
        phi.pdf = reduced_pdf

        if not inplace:
            return phi

    def marginalize(self, variables, inplace=True):
        """
        Maximizes the factor with respect to the given variables.

        Parameters
        ----------
        variables: list, array-like
            List of variables with respect to which factor is to be maximized.

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new ContinuousFactor instance.

        Returns
        -------
        Factor or None: if inplace=True (default) returns None
                        if inplace=False returns a new ContinuousFactor instance.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> std_normal_pdf = lambda x: multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> std_normal = ContinuousFactor(['x1', 'x2'], std_normal_pdf)
        >>> std_normal.scope()
        ['x1', 'x2']
        >>> std_normal.assignment([1, 1])
        0.058549831524319168
        >>> std_normal.marginalize(['x2'])
        >>> std_normal.scope()
        ['x1']
        >>> std_normal.assignment([1])

        """
        if not isinstance(variables, list):
            raise TypeError("variables: Expected type list or array-like,\
                             got type {var_type}".format(var_type=type(variables)))

        phi = self if inplace else self.copy()

        for var in variables:
            if var not in phi.variables:
                raise ValueError("{var} not in scope.".format(var=var))

        all_var = [var for var in self.variables]
        var_to_keep = [var for var in self.variables if var not in variables]
        reordered_var_index = [all_var.index(var) for var in variables + var_to_keep]
        pdf = phi.pdf

        # The arguments need to be reordered because integrate.nquad integrates the first n-arguments
        # of the function passed.

        def reordered_pdf(*args):
            # ordered_args restores the original order as it was in self.variables
            ordered_args = [args[reordered_var_index.index(index_id)] for index_id in range(len(all_var))]
            return pdf(*ordered_args)

        def marginalized_pdf(*args):
            return integrate.nquad(reordered_pdf, [[-np.inf, np.inf] for i in range(len(variables))], args=args)[0]

        phi.pdf = marginalized_pdf
        phi.variables = var_to_keep

        if not inplace:
            return phi

    def discretize(self, method, *args, **kwargs):
        """
        Discretizes the continuous distribution into discrete
        probability masses using various methods.

        Returns
        -------
        An n-D array or a Factor object according to the discretiztion
        method used.

        Parameters
        ----------
        method : A Discretizer Class from pgmpy.discretize

        *args, **kwargs:
            The parameters to be given to the Discretizer Class.
        """
        return method(self, *args, **kwargs).get_discrete_values()
<commit_msg>Added ContinuousFactor.normalize<commit_after>from __future__ import division

import six
import numpy as np
import scipy.integrate as integrate

class ContinuousFactor(object):
    """
    Base class for factors representing various multivariate
    representations.
    """
    def __init__(self, variables, pdf):
        """
        Parameters
        ----------
        variables: list or array-like
            The variables for wich the distribution is defined.

        pdf: function
            The probability density function of the distribution.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        # Two variable drichlet ditribution with alpha = (1,2)
        >>> def drichlet_pdf(x, y):
        ...     return (np.power(x, 1)*np.power(y, 2))/beta(x, y)
        >>> from pgmpy.factors import ContinuousFactor
        >>> drichlet_factor = ContinuousFactor(['x', 'y'], drichlet_pdf)
        >>> drichlet_factor.scope()
        ['x', 'y']
        >>> drichlet_factor.assignemnt(5,6)
        226800.0
        """
        if not isinstance(variables, list):
            raise TypeError("variables: Expected type list or array-like,\
                             got type {var_type}".format(var_type=type(variables)))

        self.variables = variables
        self.pdf = pdf

    def scope(self):
        """
        Returns the scope of the factor.

        Returns
        -------
        list: List of variable names in the scope of the factor.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> normal_pdf = lambda x: multivariate_normal(x, [0, 0], [[1, 0], [0, 1]])
        >>> phi = ContinuousFactor(['x1', 'x2'], normal_pdf)
        >>> phi.scope()
        ['x1', 'x2']
        """
        return self.variables

    def assignment(self, *args):
        """
        Returns a list of pdf assignments for the corresponding values.

        Parameters
        ----------
        values: A list of arrays of dimension 1 x n
            List of values whose assignment is to be computed.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> normal_pdf = lambda x: multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> phi = ContinuousFactor(['x1', 'x2'], normal_pdf)
        >>> phi.assignment([1,2])
        0.013064233284684921
        """
        return self.pdf(*args)

    def copy(self):
        """
        Return a copy of the distribution.

        Returns
        -------
        ContinuousFactor object: copy of the distribution

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        # Two variable drichlet ditribution with alpha = (1,2)
        >>> def drichlet_pdf(x, y):
        ...     return (np.power(x, 1)*np.power(y, 2))/beta(x, y)
        >>> from pgmpy.factors import ContinuousFactor
        >>> drichlet_factor = ContinuousFactor(['x', 'y'], drichlet_pdf)
        >>> drichlet_factor.variables
        ['x', 'y']
        >>> copy_factor = drichlet_factor.copy()
        copy_factor.variables

        """
        return ContinuousFactor(self.scope(), self.pdf)

    def reduce(self, values, inplace=True):
        """
        Reduces the factor to the context of the given variable values.

        Parameters
        ----------
        values: list, array-like
            A list of tuples of the form (variable_name, variable_value).

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new ContinuosFactor object.

        Returns
        -------
        ContinuousFactor or None: if inplace=True (default) returns None
                                  if inplace=False returns a new `Factor` instance.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        >>> def custom_pdf(x, y, z):
        ...     return z*(np.power(x, 1)*np.power(y, 2))/beta(x, y)
        >>> from pgmpy.factors import ContinuousFactor
        >>> custom_factor = ContinuousFactor(['x', 'y', 'z'], custom_pdf)
        >>> custom_factor.variables
        ['x', 'y', 'z']
        >>> custom_factor.assignment(1, 2, 3)
        24.0

        >>> custom_factor.reduce([('y', 2)])
        >>> custom_factor.variables
        ['x', 'z']
        >>> custom_factor.assignment(1, 3)
        24.0
        """
        if not isinstance(values, list):
            raise TypeError("values: Expected type list or array-like,\
                             got type {var_type}".format(var_type=type(values)))

        phi = self if inplace else self.copy()

        var_to_remove = [var for var, value in values]
        var_to_keep = [var for var in self.variables if var not in var_to_remove]

        reduced_var_index = [(self.variables.index(var), value) for var, value in values]
        pdf = self.pdf

        def reduced_pdf(*args, **kwargs):
            reduced_args = list(args)
            reduced_kwargs = kwargs.copy()

            if reduced_args:
                for index, value in reduced_var_index:
                    reduced_args.insert(index, value)
            if reduced_kwargs:
                for var, value in values:
                    reduced_kwargs[var] = value
            if reduced_args and reduced_kwargs:
                reduced_args = [arg for arg in reduced_args if arg not in reduced_kwargs.values()]

            return pdf(*reduced_args, **reduced_kwargs)

        phi.variables = var_to_keep
        phi.pdf = reduced_pdf

        if not inplace:
            return phi

    def marginalize(self, variables, inplace=True):
        """
        Maximizes the factor with respect to the given variables.

        Parameters
        ----------
        variables: list, array-like
            List of variables with respect to which factor is to be maximized.

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new ContinuousFactor instance.

        Returns
        -------
        Factor or None: if inplace=True (default) returns None
                        if inplace=False returns a new ContinuousFactor instance.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> std_normal_pdf = lambda x: multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> std_normal = ContinuousFactor(['x1', 'x2'], std_normal_pdf)
        >>> std_normal.scope()
        ['x1', 'x2']
        >>> std_normal.assignment([1, 1])
        0.058549831524319168
        >>> std_normal.marginalize(['x2'])
        >>> std_normal.scope()
        ['x1']
        >>> std_normal.assignment([1])

        """
        if not isinstance(variables, list):
            raise TypeError("variables: Expected type list or array-like,\
                             got type {var_type}".format(var_type=type(variables)))

        phi = self if inplace else self.copy()

        for var in variables:
            if var not in phi.variables:
                raise ValueError("{var} not in scope.".format(var=var))

        all_var = [var for var in self.variables]
        var_to_keep = [var for var in self.variables if var not in variables]
        reordered_var_index = [all_var.index(var) for var in variables + var_to_keep]
        pdf = phi.pdf

        # The arguments need to be reordered because integrate.nquad integrates the first n-arguments
        # of the function passed.

        def reordered_pdf(*args):
            # ordered_args restores the original order as it was in self.variables
            ordered_args = [args[reordered_var_index.index(index_id)] for index_id in range(len(all_var))]
            return pdf(*ordered_args)

        def marginalized_pdf(*args):
            return integrate.nquad(reordered_pdf, [[-np.inf, np.inf] for i in range(len(variables))], args=args)[0]

        phi.pdf = marginalized_pdf
        phi.variables = var_to_keep

        if not inplace:
            return phi

    def normalize(self, inplace=True):
        """
        Normalizes the pdf of the continuous factor so that it integrates to
        1 over all the variables.

        Parameters
        ----------
        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new factor.

        Returns
        -------
        ContinuousFactor or None:
             if inplace=True (default) returns None
             if inplace=False returns a new ContinuousFactor instance.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> std_normal_pdf = lambda x: 2 * multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> std_normal = ContinuousFactor(['x1', 'x2'], std_normal_pdf)
        >>> std_normal.assignment(1, 1)
        0.117099663049
        >>> std_normal.normalize()
        >>> std_normal.assignment(1, 1)
        0.0585498315243

        """
        phi = self if inplace else self.copy()
        pdf = self.pdf

        pdf_mod = integrate.nquad(pdf, [[-np.inf, np.inf] for var in self.variables])[0]

        phi.pdf = lambda *args: pdf(*args) / pdf_mod

        if not inplace:
            return phi

    def discretize(self, method, *args, **kwargs):
        """
        Discretizes the continuous distribution into discrete
        probability masses using various methods.

        Returns
        -------
        An n-D array or a Factor object according to the discretiztion
        method used.

        Parameters
        ----------
        method : A Discretizer Class from pgmpy.discretize

        *args, **kwargs:
            The parameters to be given to the Discretizer Class.
        """
        return method(self, *args, **kwargs).get_discrete_values()
<|endoftext|>


<commit_before>from __future__ import absolute_import

from widgy.site import WidgySite

class DemoWidgySite(WidgySite):
    def valid_parent_of(self, parent, child_class, child=None):
        from widgy.contrib.page_builder.models import Accordion, Section
        from demo.demo_widgets.models import TwoContentLayout

        if isinstance(parent, Accordion) and isinstance(parent.get_root(), TwoContentLayout) and issubclass(child_class, Section) and len(parent.children) >= 2:
            return False
        return super(DemoWidgySite, self).valid_parent_of(parent, child_class, child)

widgy_site = DemoWidgySite()
<commit_msg>only 2 sections per accordion validation for real<commit_after>from __future__ import absolute_import

from widgy.site import WidgySite

class DemoWidgySite(WidgySite):
    def valid_parent_of(self, parent, child_class, child=None):
        from widgy.contrib.page_builder.models import Accordion, Section
        from demo.demo_widgets.models import TwoContentLayout

        if isinstance(parent, Accordion) and isinstance(parent.get_root(), TwoContentLayout) and issubclass(child_class, Section) and len(parent.children) >= 2:
            if not child or child not in parent.children:
                return False
        return super(DemoWidgySite, self).valid_parent_of(parent, child_class, child)

widgy_site = DemoWidgySite()
<|endoftext|>