<commit_before>import arrow
import datetime
import re
from settings import (DEFAULT_THEME_DATE_FORMAT, DEFAULT_THEME_TIMEZONE)
from superdesk import get_resource_service
import logging

logger = logging.getLogger('superdesk')

DEFAULT_IFRAME_WIDTH = "350"
DEFAULT_IFRAME_HEIGHT = "350"

def moment_date_filter_container(theme):
    def moment_date_filter(date, format=None):
        """
        Jinja2 filter for moment.js compatible dates.
        :param date:
        :param format:
        :return: str
        """
        settings = get_resource_service('themes').get_default_settings(theme)
        parsed = arrow.get(date)
        if not format:
            format = settings.get('datetimeFormat', DEFAULT_THEME_DATE_FORMAT)
        # Workaround for "x" unsupported format
        if format == 'x':
            return parsed.timestamp
        # TODO: implement momentjs `Localized formats`
        elif re.search('l+', format.lower()):
            format = DEFAULT_THEME_DATE_FORMAT

        formated = parsed.to(DEFAULT_THEME_TIMEZONE).format(DEFAULT_THEME_DATE_FORMAT)
        try:
            formated = parsed.to(DEFAULT_THEME_TIMEZONE).format(format, locale=settings.get('language', 'en'))
        except Exception:
            logger.info("language not supported")

        return formated

    return moment_date_filter

def regaddten(obj):
    print(obj)
    val = int(obj.group(1))
    return str(val + 10)

def addten(date):

    if isinstance(date, datetime.datetime):
        try:
            date = date.replace(year=date.year + 10)
        except ValueError:
            # Leap day in a leap year, move date to February 28th
            date = date.replace(year=date.year + 10, day=28)

    if isinstance(date, str):
        return re.sub(r'(\d{4})', regaddten, date)

    return date

def ampify(html):
    if re.search('iframe', html, re.IGNORECASE):
        src = re.search(r'src\s*=\s*"(?P<src>[^\"]+)"', html)
        width = re.search(r'width\s*=\s*"(?P<width>[^\"]+)"', html)
        height = re.search(r'height\s*=\s*"(?P<height>[^\"]+)"', html)

        width = width.group('width') if width else DEFAULT_IFRAME_WIDTH
        if "%" in width:
            width = DEFAULT_IFRAME_WIDTH

        # adding also fallback for height in case is wrongly provided in %
        height = height.group('height') if height else DEFAULT_IFRAME_HEIGHT
        if "%" in height:
            height = DEFAULT_IFRAME_HEIGHT

        return '''
<amp-iframe
    width={width}
    height={height}
    layout="responsive"
    frameborder="0"
    sandbox="allow-scripts allow-same-origin allow-popups"
    src="{src}">

Loading...

</amp-iframe>
'''.format(
            width=width,
            height=height,
            src=src.group('src') if src else '')
    return html
<commit_msg>Add support for relative date format<commit_after>import arrow
import datetime
import re
from settings import (DEFAULT_THEME_DATE_FORMAT, DEFAULT_THEME_TIMEZONE)
from superdesk import get_resource_service
import logging

logger = logging.getLogger('superdesk')

DEFAULT_IFRAME_WIDTH = "350"
DEFAULT_IFRAME_HEIGHT = "350"

def moment_date_filter_container(theme):
    def moment_date_filter(date, format=None):
        """
        Jinja2 filter for moment.js compatible dates.
        :param date:
        :param format:
        :return: str
        """
        settings = get_resource_service('themes').get_default_settings(theme)
        locale = settings.get('language', 'en')
        parsed = arrow.get(date)
        if not format:
            format = settings.get('datetimeFormat', DEFAULT_THEME_DATE_FORMAT)
        # Workaround for "x" unsupported format
        if format == 'x':
            return parsed.timestamp
        # TODO: implement momentjs `Localized formats`
        elif re.search('l+', format.lower()):
            format = DEFAULT_THEME_DATE_FORMAT

        if format == 'ago':
            formated = parsed.humanize()
        else:
            formated = parsed.to(DEFAULT_THEME_TIMEZONE).format(DEFAULT_THEME_DATE_FORMAT)

        try:
            if format == 'ago':
                formated = parsed.humanize(locale=locale)
            else:
                formated = parsed.to(DEFAULT_THEME_TIMEZONE).format(format, locale=locale)
        except Exception:
            logger.info("language not supported")

        return formated

    return moment_date_filter

def regaddten(obj):
    print(obj)
    val = int(obj.group(1))
    return str(val + 10)

def addten(date):

    if isinstance(date, datetime.datetime):
        try:
            date = date.replace(year=date.year + 10)
        except ValueError:
            # Leap day in a leap year, move date to February 28th
            date = date.replace(year=date.year + 10, day=28)

    if isinstance(date, str):
        return re.sub(r'(\d{4})', regaddten, date)

    return date

def ampify(html):
    if re.search('iframe', html, re.IGNORECASE):
        src = re.search(r'src\s*=\s*"(?P<src>[^\"]+)"', html)
        width = re.search(r'width\s*=\s*"(?P<width>[^\"]+)"', html)
        height = re.search(r'height\s*=\s*"(?P<height>[^\"]+)"', html)

        width = width.group('width') if width else DEFAULT_IFRAME_WIDTH
        if "%" in width:
            width = DEFAULT_IFRAME_WIDTH

        # adding also fallback for height in case is wrongly provided in %
        height = height.group('height') if height else DEFAULT_IFRAME_HEIGHT
        if "%" in height:
            height = DEFAULT_IFRAME_HEIGHT

        return '''
<amp-iframe
    width={width}
    height={height}
    layout="responsive"
    frameborder="0"
    sandbox="allow-scripts allow-same-origin allow-popups"
    src="{src}">

Loading...

</amp-iframe>
'''.format(
            width=width,
            height=height,
            src=src.group('src') if src else '')
    return html
<|endoftext|>


<commit_before>import ddapp
import os
import sys
import argparse

class DRCArgParser(object):

    def __init__(self):
        self._args = None
        self._parser = None

    def getArgs(self):
        if self._args is None:
            self.parseArgs()
        return self._args

    def getParser(self):
        if self._parser is None:
            self._parser = argparse.ArgumentParser()
            self.addDefaultArgs(self._parser)
        return self._parser

    def parseArgs(self):
        parser = self.getParser()
        self._args = parser.parse_args()

    def getDefaultBotConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(), 'software/config/drc_robot_02_mit.cfg')

    def addDefaultArgs(self, parser):

        parser.add_argument('-c', '--config_file', type=str, help='Robot cfg file',
                            default=self.getDefaultBotConfigFile())

        parser.add_argument('data_files', type=str, nargs='*',
                            help='data files to load at startup')

_argParser = None
def getGlobalArgParser():
    global _argParser
    if not _argParser:
        _argParser = DRCArgParser()
    return _argParser

def args():
    return getGlobalArgParser().getArgs()
<commit_msg>convert tuple to list because argparse doesn't like tuples<commit_after>import ddapp
import os
import sys
import argparse

class DRCArgParser(object):

    def __init__(self):
        self._args = None
        self._parser = None

    def getArgs(self):
        if self._args is None:
            self.parseArgs()
        return self._args

    def getParser(self):
        if self._parser is None:
            self._parser = argparse.ArgumentParser()
            self.addDefaultArgs(self._parser)
        return self._parser

    def parseArgs(self):
        parser = self.getParser()
        sys.argv = [str(v) for v in sys.argv]
        self._args = parser.parse_args()

    def getDefaultBotConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(), 'software/config/drc_robot_02_mit.cfg')

    def addDefaultArgs(self, parser):

        parser.add_argument('-c', '--config_file', type=str, help='Robot cfg file',
                            default=self.getDefaultBotConfigFile())

        parser.add_argument('data_files', type=str, nargs='*',
                            help='data files to load at startup')

_argParser = None
def getGlobalArgParser():
    global _argParser
    if not _argParser:
        _argParser = DRCArgParser()
    return _argParser

def args():
    return getGlobalArgParser().getArgs()
<|endoftext|>


<commit_before>"""
A 'universal' WHOIS proxy server.
"""

import logging
import logging.config
import os.path
import re
import socket
import sys

from uwhoisd import net, utils

USAGE = "Usage: %s <config>"

PORT = socket.getservbyname('whois', 'tcp')

CONFIG = """
[uwhoisd]
iface=0.0.0.0
port=4343
registry_whois=false
suffix=whois-servers.net

[overrides]

[prefixes]

[recursion_patterns]
"""

logger = logging.getLogger('uwhoisd')

class UWhois(object):
    """
    Universal WHOIS proxy.
    """

    __slots__ = (
        'conservative',
        'overrides',
        'prefixes',
        'recursion_patterns',
        'registry_whois',
        'suffix',
    )

    def __init__(self):
        super(UWhois, self).__init__()
        self.suffix = None
        self.overrides = {}
        self.prefixes = {}
        self.recursion_patterns = {}
        self.registry_whois = False
        self.conservative = ()

    def _get_dict(self, parser, section):
        """
        Pull a dictionary out of the config safely.
        """
        if parser.has_section(section):
            values = dict(
                (key, utils.decode_value(value))
                for key, value in parser.items(section))
        else:
            values = {}
        setattr(self, section, values)

    def read_config(self, parser):
        """
        Read the configuration for this object from a config file.
        """
        self.registry_whois = utils.to_bool(
            parser.get('uwhoisd', 'registry_whois'))
        self.suffix = parser.get('uwhoisd', 'suffix')
        self.conservative = [
            zone
            for zone in parser.get('uwhoisd', 'conservative').split("\n")
            if zone != '']

        for section in ('overrides', 'prefixes'):
            self._get_dict(parser, section)

        for zone, pattern in parser.items('recursion_patterns'):
            self.recursion_patterns[zone] = re.compile(
                utils.decode_value(pattern),
                re.I)

    def get_whois_server(self, zone):
        """
        Get the WHOIS server for the given zone.
        """
        if zone in self.overrides:
            server = self.overrides[zone]
        else:
            server = zone + '.' + self.suffix
        if ':' in server:
            server, port = server.split(':', 1)
            port = int(port)
        else:
            port = PORT
        return server, port

    def get_registrar_whois_server(self, zone, response):
        """
        Extract the registrar's WHOIS server from the registry response.
        """
        matches = self.recursion_patterns[zone].search(response)
        return None if matches is None else matches.group('server')

    def get_prefix(self, zone):
        """
        Gets the prefix required when querying the servers for the given zone.
        """
        return self.prefixes[zone] if zone in self.prefixes else ''

    def _thin_query(self, server_index, response, port, query):
        server = self.get_registrar_whois_server(server_index, response)
        if server is not None:
            if not self.registry_whois:
                response = ""
            with net.WhoisClient(server, port) as client:
                logger.info(
                    "Recursive query to %s about %s",
                    server, query)
                response += client.whois(query)

    def whois(self, query):
        """
        Query the appropriate WHOIS server.
        """
        # Figure out the zone whose WHOIS server we're meant to be querying.
        for zone in self.conservative:
            if query.endswith('.' + zone):
                break
        else:
            if query.split('.')[-1].isdigit():
                zone = query.split('.')[0]
            else:
                _, zone = utils.split_fqdn(query)

        # Query the registry's WHOIS server.
        server, port = self.get_whois_server(zone)
        with net.WhoisClient(server, port) as client:
            logger.info("Querying %s about %s", server, query)
            prefix = self.get_prefix(zone)
            if len(prefix) == 0:
                prefix = self.get_prefix(server)
            response = client.whois(prefix + query)

        # Thin registry? Query the registrar's WHOIS server.
        if zone in self.recursion_patterns:
            self._thin_query(zone, response, port, query)
        elif server in self.recursion_patterns:
            self._thin_query(server, response, port, query)
        return response

def main():
    """
    Execute the daemon.
    """
    if len(sys.argv) != 2:
        print >> sys.stderr, USAGE % os.path.basename(sys.argv[0])
        return 1

    logging.config.fileConfig(sys.argv[1])

    try:
        logger.info("Reading config file at '%s'", sys.argv[1])
        parser = utils.make_config_parser(CONFIG, sys.argv[1])

        iface = parser.get('uwhoisd', 'iface')
        port = parser.getint('uwhoisd', 'port')
        logger.info("Listen on %s:%d", iface, port)

        uwhois = UWhois()
        uwhois.read_config(parser)
        cache = utils.to_bool(parser.get('cache', 'enable'))
        redis_cache = utils.to_bool(parser.get('redis_cache', 'enable'))

        if cache:
            logger.info("Caching activated")
            cache = utils.Cache(
                max_size=parser.getint('cache', 'max_size'),
                max_age=parser.getint('cache', 'max_age'))

            def whois(query):
                """Caching wrapper around UWhois."""
                cache.evict_expired()
                if query in cache:
                    logger.info("Cache hit for %s", query)
                    response = cache[query]
                else:
                    response = uwhois.whois(query)
                    cache[query] = response
                return response
        elif redis_cache:
            logger.info("Redis caching activated")
            import redis
            redis_host = parser.get('redis_cache', 'host')
            redis_port = parser.getint('redis_cache', 'port')
            redis_database = parser.getint('redis_cache', 'db')
            redis_expire = parser.getint('redis_cache', 'expire')
            redis_cache = redis.StrictRedis(redis_host, redis_port,
                                            redis_database)

            def whois(query):
                """Redis caching wrapper around UWhois."""
                response = redis_cache.get(query)
                if response is None:
                    response = uwhois.whois(query)
                    redis_cache.setex(query, redis_expire, response)
                return response
        else:
            logger.info("Caching deactivated")
            whois = uwhois.whois
    except Exception, ex:  # pylint: disable-msg=W0703
        print >> sys.stderr, "Could not parse config file: %s" % str(ex)
        return 1

    net.start_service(iface, port, whois)
    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Fix thin queries.<commit_after>"""
A 'universal' WHOIS proxy server.
"""

import logging
import logging.config
import os.path
import re
import socket
import sys

from uwhoisd import net, utils

USAGE = "Usage: %s <config>"

PORT = socket.getservbyname('whois', 'tcp')

CONFIG = """
[uwhoisd]
iface=0.0.0.0
port=4343
registry_whois=false
suffix=whois-servers.net

[overrides]

[prefixes]

[recursion_patterns]
"""

logger = logging.getLogger('uwhoisd')

class UWhois(object):
    """
    Universal WHOIS proxy.
    """

    __slots__ = (
        'conservative',
        'overrides',
        'prefixes',
        'recursion_patterns',
        'registry_whois',
        'suffix',
    )

    def __init__(self):
        super(UWhois, self).__init__()
        self.suffix = None
        self.overrides = {}
        self.prefixes = {}
        self.recursion_patterns = {}
        self.registry_whois = False
        self.conservative = ()

    def _get_dict(self, parser, section):
        """
        Pull a dictionary out of the config safely.
        """
        if parser.has_section(section):
            values = dict(
                (key, utils.decode_value(value))
                for key, value in parser.items(section))
        else:
            values = {}
        setattr(self, section, values)

    def read_config(self, parser):
        """
        Read the configuration for this object from a config file.
        """
        self.registry_whois = utils.to_bool(
            parser.get('uwhoisd', 'registry_whois'))
        self.suffix = parser.get('uwhoisd', 'suffix')
        self.conservative = [
            zone
            for zone in parser.get('uwhoisd', 'conservative').split("\n")
            if zone != '']

        for section in ('overrides', 'prefixes'):
            self._get_dict(parser, section)

        for zone, pattern in parser.items('recursion_patterns'):
            self.recursion_patterns[zone] = re.compile(
                utils.decode_value(pattern),
                re.I)

    def get_whois_server(self, zone):
        """
        Get the WHOIS server for the given zone.
        """
        if zone in self.overrides:
            server = self.overrides[zone]
        else:
            server = zone + '.' + self.suffix
        if ':' in server:
            server, port = server.split(':', 1)
            port = int(port)
        else:
            port = PORT
        return server, port

    def get_registrar_whois_server(self, zone, response):
        """
        Extract the registrar's WHOIS server from the registry response.
        """
        matches = self.recursion_patterns[zone].search(response)
        return None if matches is None else matches.group('server')

    def get_prefix(self, zone):
        """
        Gets the prefix required when querying the servers for the given zone.
        """
        return self.prefixes[zone] if zone in self.prefixes else ''

    def _thin_query(self, server_index, response, port, query):
        server = self.get_registrar_whois_server(server_index, response)
        if server is not None:
            if not self.registry_whois:
                response = ""
            with net.WhoisClient(server, port) as client:
                logger.info(
                    "Recursive query to %s about %s",
                    server, query)
                response += client.whois(query)
        return response

    def whois(self, query):
        """
        Query the appropriate WHOIS server.
        """
        # Figure out the zone whose WHOIS server we're meant to be querying.
        for zone in self.conservative:
            if query.endswith('.' + zone):
                break
        else:
            if query.split('.')[-1].isdigit():
                zone = query.split('.')[0]
            else:
                _, zone = utils.split_fqdn(query)

        # Query the registry's WHOIS server.
        server, port = self.get_whois_server(zone)
        with net.WhoisClient(server, port) as client:
            logger.info("Querying %s about %s", server, query)
            prefix = self.get_prefix(zone)
            if len(prefix) == 0:
                prefix = self.get_prefix(server)
            response = client.whois(prefix + query)

        # Thin registry? Query the registrar's WHOIS server.
        if zone in self.recursion_patterns:
            response = self._thin_query(zone, response, port, query)
        elif server in self.recursion_patterns:
            response = self._thin_query(server, response, port, query)
        return response

def main():
    """
    Execute the daemon.
    """
    if len(sys.argv) != 2:
        print >> sys.stderr, USAGE % os.path.basename(sys.argv[0])
        return 1

    logging.config.fileConfig(sys.argv[1])

    try:
        logger.info("Reading config file at '%s'", sys.argv[1])
        parser = utils.make_config_parser(CONFIG, sys.argv[1])

        iface = parser.get('uwhoisd', 'iface')
        port = parser.getint('uwhoisd', 'port')
        logger.info("Listen on %s:%d", iface, port)

        uwhois = UWhois()
        uwhois.read_config(parser)
        cache = utils.to_bool(parser.get('cache', 'enable'))
        redis_cache = utils.to_bool(parser.get('redis_cache', 'enable'))

        if cache:
            logger.info("Caching activated")
            cache = utils.Cache(
                max_size=parser.getint('cache', 'max_size'),
                max_age=parser.getint('cache', 'max_age'))

            def whois(query):
                """Caching wrapper around UWhois."""
                cache.evict_expired()
                if query in cache:
                    logger.info("Cache hit for %s", query)
                    response = cache[query]
                else:
                    response = uwhois.whois(query)
                    cache[query] = response
                return response
        elif redis_cache:
            logger.info("Redis caching activated")
            import redis
            redis_host = parser.get('redis_cache', 'host')
            redis_port = parser.getint('redis_cache', 'port')
            redis_database = parser.getint('redis_cache', 'db')
            redis_expire = parser.getint('redis_cache', 'expire')
            redis_cache = redis.StrictRedis(redis_host, redis_port,
                                            redis_database)

            def whois(query):
                """Redis caching wrapper around UWhois."""
                response = redis_cache.get(query)
                if response is None:
                    response = uwhois.whois(query)
                    redis_cache.setex(query, redis_expire, response)
                return response
        else:
            logger.info("Caching deactivated")
            whois = uwhois.whois
    except Exception, ex:  # pylint: disable-msg=W0703
        print >> sys.stderr, "Could not parse config file: %s" % str(ex)
        return 1

    net.start_service(iface, port, whois)
    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>from data_structures.stack import Stack

class Node(object):
    def __init__(self, key=None, val=None):
        self.count = 0
        if key is not None:
            self.count = 1
        self.elems = [(key, val), (None, None), (None, None), ]
        self.parent = None
        self.left = None
        self.mid = None
        self.right = None

    def __repr__(self):
        """For printing out the heap and its nodes
        It's here to save me typing during debugging"""
        result = "["
        for i in range(3):
            result += '%s, ' % str(self.elems[i])
        return '%s]' % result

    def add_to_node(self, key, val):
        for i in range(3):
            if self.elems[i][0] is None or self.elems[i][0] > key:
                self.elems.pop()
                self.elems.insert(i, (key, val))
                self.count += 1
                break

    def del_from_node(self, idx):
        self.elems.pop(idx)
        self.count -= 1
        self.elems.append((None, None))

    def has(self, key):
        for i in range(3):
            if self.elems[i][0] == key:
                return True
        return False

    def split_node(self):
        return (
            Node(self.elems[0][0], self.elems[0][1]),
            self.elems[1],
            Node(self.elems[2][0], self.elems[2][1])
            )

class BTree(object):
    def __init__(self):
        self.root = Node()

    def search(self, node, key):
        if node.has(key):
            if node.elems[0][0] == key:
                return node.elems[0][1]
            return node.elems[1][1]
        elif node.left is not None and key < node.elems[0][0]:
            return self.search(node.left, key)
        elif node.mid is not None and key < node.elems[1][0]:
            return self.search(node.mid, key)
        elif node.right is not None:
            return self.search(node.right, key)
        raise MissingError

    def insert(self, key, val):
        pass

    def delete(self, key):
        pass

class MissingError(BaseException):
    pass
<commit_msg>Rudimentary insert, no overflow handling, no stack implementation<commit_after>from data_structures.stack import Stack

class Node(object):
    def __init__(self, key=None, val=None):
        self.count = 0
        if key is not None:
            self.count = 1
        self.elems = [(key, val), (None, None), (None, None), ]
        self.parent = None
        self.left = None
        self.mid = None
        self.right = None

    def __repr__(self):
        """For printing out the heap and its nodes
        It's here to save me typing during debugging"""
        result = "["
        for i in range(3):
            result += '%s, ' % str(self.elems[i])
        return '%s]' % result

    def add_to_node(self, key, val):
        for i in range(3):
            if self.elems[i][0] is None or self.elems[i][0] > key:
                self.elems.pop()
                self.elems.insert(i, (key, val))
                self.count += 1
                break

    def del_from_node(self, idx):
        self.elems.pop(idx)
        self.count -= 1
        self.elems.append((None, None))

    def has(self, key):
        for i in range(3):
            if self.elems[i][0] == key:
                return True
        return False

    def split_node(self):
        return (
            Node(self.elems[0][0], self.elems[0][1]),
            self.elems[1],
            Node(self.elems[2][0], self.elems[2][1])
            )

class BTree(object):
    def __init__(self):
        self.root = Node()

    def search(self, node, key):
        if node.has(key):
            if node.elems[0][0] == key:
                return node.elems[0][1]
            return node.elems[1][1]
        elif node.left is not None and key < node.elems[0][0]:
            return self.search(node.left, key)
        elif node.mid is not None and key < node.elems[1][0]:
            return self.search(node.mid, key)
        elif node.right is not None:
            return self.search(node.right, key)
        raise MissingError

    def insert(self, node, key, val):
        if node.has(key):
            return
        elif node.left is not None and key < node.elems[0][0]:
            return self.insert(node.left, key, val)
        elif node.mid is not None and key < node.elems[1][0]:
            return self.insert(node.mid, key, val)
        elif node.right is not None:
            return self.insert(node.right, key, val)
        node.add_to_node(key, val)

    def delete(self, key):
        pass

class MissingError(BaseException):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# Licensed to Crate (https://crate.io) under one or more contributor
# license agreements.  See the NOTICE file distributed with this work for
# additional information regarding copyright ownership.  Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

from crate.theme.rtd.conf import *

project = u'CrateDB: Admin UI'
html_title = project

url_path = 'docs/crate/admin-u/en/latest/'

# For sitemap extension
site_url = 'https://crate.io/%s' % url_path

html_theme_options.update({
    'canonical_url_path': url_path, # For rel="canonical" links
})
<commit_msg>Fix broken URL<commit_after># -*- coding: utf-8; -*-
#
# Licensed to Crate (https://crate.io) under one or more contributor
# license agreements.  See the NOTICE file distributed with this work for
# additional information regarding copyright ownership.  Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

from crate.theme.rtd.conf import *

project = u'CrateDB: Admin UI'
html_title = project

url_path = 'docs/crate/admin-ui/en/latest/'

# For sitemap extension
site_url = 'https://crate.io/%s' % url_path

html_theme_options.update({
    'canonical_url_path': url_path, # For rel="canonical" links
})
<|endoftext|>


<commit_before>"""
Slack is a micro-framework for scraping the web using selenium.
Written by Wykleph in April of 2016.

Create models in the `Models.py` file.

Once you have defined your models, they are accessed using the
`Models.ModelName` syntax.
"""

from time import sleep
# Database models used to interact with databases.
from Project import Models
# The environment variable loader.  These variables can be set in the .env file.
from Config.Environment import env, env_driver
# Controllers are kept in the SiteAutomations folder.
from SiteAutomations.Examples import GoogleExample, BingExample
from Helpers.Contexts import quitting
# Pull in the command factory for the second example.
from Helpers.Commands import CommandFactory, Kwargs
from selenium.webdriver.support.wait import WebDriverWait

# This is where selenium starts up.
# This could be written as:
#
#   browser = env("BROWSER")
#   web_driver = env_driver(browser)
#   with quitting(web_driver()) as driver:
#       pass
#
with quitting(env_driver(env("BROWSER"))()) as driver:
    # Write your selenium code here.

    # Here is an example of a `SiteAutomation` taken from the
    # `GoogleExample.py` file.
    wait = WebDriverWait(driver, 30)
    # Pass the web driver to the site automation along with anything
    # else it might need to do its job. This could include an
    # instance of WebDriverWait, and even the collection of
    # Models.
    google_search = GoogleExample.GoogleSearch(driver, wait, Models)
    bing_search = BingExample.BingSearch(driver, wait, Models)
    google_search.do_search('google wiki')
    sleep(1)
    bing_search.do_search('bing wiki')
    sleep(1)

# Optionally, you can use the command manager to do the searches.
# This will make each controller use it's own personal WebDriver.
# Define some controllers to pass to CommandFactory.
# Note that the controllers being used are subclasses of IndependentController.
# These classes don't handle any threading, but they are set up so that the
# command factory will be able to create commands with them.
controllers = {
    'google': GoogleExample.ThreadedGoogleSearch(Models),
    'bing': BingExample.ThreadedBingSearch(Models)  # Check out the example files for more info on threading.
}
# We use the CommandFactory instead of the ThreadedCommandFactory
# so that the each controller has it's own WebDriver instance
# and each request is made in the main thread.
cmd_factory = CommandFactory(controllers, logging=False)

# Register arguments to pass to each controller.  They are
# matched by the key in the controllers dictionary.
# Command packs are passed as *args.  If you need any
# **kwargs, just instantiate a Kwargs object with the
# dictionary containing the **kwargs and make sure
# the method you are calling with the command pack
# is decorated with @has_kwargs.
search_command = {
    'google': ('star wars', Kwargs({'some_kwarg': 'NEW KWARG VALUE!'})),  # You can override keyword arguments as well!
    'bing': ('star wars',)
}

# Create the command.  Pass a function as the first parameter and
# the command pack as the second parameter.  A Command instance
# is returned when the command is created.  These Command
# objects are used to start the work!
cmd = cmd_factory.create_command(lambda controller, *search_term: controller.do_search(*search_term), search_command)

# Start the command.  Each search will be executed one after the
# other.
cmd.start()
print 'finished first search'
sleep(5)

# Close the WebDrivers down.
cmd_factory.shutdown()
<commit_msg>Added whitespace.<commit_after>"""
Slack is a micro-framework for scraping the web using selenium.
Written by Wykleph in April of 2016.

Create models in the `Models.py` file.

Once you have defined your models, they are accessed using the
`Models.ModelName` syntax.
"""

from time import sleep
# Database models used to interact with databases.
from Project import Models
# The environment variable loader.  These variables can be set in the .env file.
from Config.Environment import env, env_driver
# Controllers are kept in the SiteAutomations folder.
from SiteAutomations.Examples import GoogleExample, BingExample
from Helpers.Contexts import quitting
# Pull in the command factory for the second example.
from Helpers.Commands import CommandFactory, Kwargs
from selenium.webdriver.support.wait import WebDriverWait

# This is where selenium starts up.
# This could be written as:
#
#   browser = env("BROWSER")
#   web_driver = env_driver(browser)
#   with quitting(web_driver()) as driver:
#       pass
#
with quitting(env_driver(env("BROWSER"))()) as driver:
    # Write your selenium code here.

    # Here is an example of a `SiteAutomation` taken from the
    # `GoogleExample.py` file.
    wait = WebDriverWait(driver, 30)

    # Pass the web driver to the site automation along with anything
    # else it might need to do its job. This could include an
    # instance of WebDriverWait, and even the collection of
    # Models.
    google_search = GoogleExample.GoogleSearch(driver, wait, Models)
    bing_search = BingExample.BingSearch(driver, wait, Models)
    google_search.do_search('google wiki')
    sleep(1)
    bing_search.do_search('bing wiki')
    sleep(1)

# Optionally, you can use the command manager to do the searches.
# This will make each controller use it's own personal WebDriver.
# Define some controllers to pass to CommandFactory.
# Note that the controllers being used are subclasses of IndependentController.
# These classes don't handle any threading, but they are set up so that the
# command factory will be able to create commands with them.
controllers = {
    'google': GoogleExample.ThreadedGoogleSearch(Models),
    'bing': BingExample.ThreadedBingSearch(Models)  # Check out the example files for more info on threading.
}

# We use the CommandFactory instead of the ThreadedCommandFactory
# so that the each controller has it's own WebDriver instance
# and each request is made in the main thread.
cmd_factory = CommandFactory(controllers, logging=False)

# Register arguments to pass to each controller.  They are
# matched by the key in the controllers dictionary.
# Command packs are passed as *args.  If you need any
# **kwargs, just instantiate a Kwargs object with the
# dictionary containing the **kwargs and make sure
# the method you are calling with the command pack
# is decorated with @has_kwargs.
search_command = {
    'google': ('star wars', Kwargs({'some_kwarg': 'NEW KWARG VALUE!'})),  # You can override keyword arguments as well!
    'bing': ('star wars',)
}

# Create the command.  Pass a function as the first parameter and
# the command pack as the second parameter.  A Command instance
# is returned when the command is created.  These Command
# objects are used to start the work!
cmd = cmd_factory.create_command(lambda controller, *search_term: controller.do_search(*search_term), search_command)

# Start the command.  Each search will be executed one after the
# other.
cmd.start()
print 'finished first search'
sleep(5)

# Close the WebDrivers down.
cmd_factory.shutdown()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import mavri
import requests
from bs4 import BeautifulSoup

xx = mavri.login('tr.wikipedia', 'Mavrikant')
trwiki = 'https://tr.wikipedia.org'

nextpage = '/w/index.php?title=%C3%96zel:BekleyenDe%C4%9Fi%C5%9Fiklikler&limit=100&namespace=&level=-1&size=0'
while nextpage != 'DONE':
    soup = BeautifulSoup(requests.get(trwiki + nextpage).text, 'html.parser')
    try:
        nextpage = soup.findAll("a", {"class": "mw-nextlink"})[0].get('href')
    except:
        nextpage = 'DONE'

    for line in soup.find("div", {"id": "mw-content-text"}).ul.find_all('li'):
        title = line.find_all('a')[2].get('title')
        link = line.find_all('a')[2].get('href')
        FARK = requests.get(trwiki + link, cookies=xx.cookies).text

        if FARK.find('
(Fark yok)
') != -1:
            diff = FARK.split('<input id="mw-fr-input-oldid" type="hidden" value="')[1].split('" name="oldid" />')[0]
            RAPOR = '\n* ' + title + ' -  [[Special:Diff/' + str(diff) + ']]\n** '
            RAPOR += mavri.review_diff('tr.wikipedia', diff, xx).text
            mavri.appendtext_on_page('tr.wikipedia', 'Kullanıcı:Mavrikant/Log/FarkYok', RAPOR,
                                     '+Bot ile oto onaylanmış değişilik', xx)
<commit_msg>dzn. sorunsuz çalışıyor<commit_after># -*- coding: utf-8 -*-
import mavri
import requests
from bs4 import BeautifulSoup

xx = mavri.login('tr.wikipedia', 'Mavrikant')
trwiki = 'https://tr.wikipedia.org'

nextpage = '/w/index.php?title=%C3%96zel:BekleyenDe%C4%9Fi%C5%9Fiklikler&limit=100&namespace=&level=-1&size=0'
while nextpage != 'DONE':
    soup = BeautifulSoup(requests.get(trwiki + nextpage).text, 'html.parser')
    try:
        nextpage = soup.findAll("a", {"class": "mw-nextlink"})[0].get('href')
    except:
        nextpage = 'DONE'

    for line in soup.find("div", {"id": "mw-content-text"}).ul.find_all('li'):
        title = line.find_all('a')[2].get('title')
        link = line.find_all('a')[2].get('href')
        FARK = requests.get(trwiki + link, cookies=xx.cookies).text

        if FARK.find('
(Fark yok)
') != -1:
            diff = FARK.split('<input id="mw-fr-input-oldid" type="hidden" value="')[1].split('" name="oldid" />')[0]
            RAPOR = '\n* [[' + title + ']] -  [[Special:Diff/' + str(diff) + ']]'
            mavri.review_diff('tr.wikipedia', diff, xx)
            mavri.appendtext_on_page('tr.wikipedia', 'Kullanıcı:Mavrikant/Log/FarkYok', RAPOR,
                                     title + ' - Bot ile oto onaylanmış değişilik', xx)
<|endoftext|>


<commit_before>import discord
from discord.ext import commands

class Fox:
    """My custom cog that does stuff!"""

    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def fox(self):
        """This does stuff!"""

        #Your code will go here
        await self.bot.say("I can do stuff!")

    @fox.commands()
    async def punch(self, user : discord.Member):
        """I will puch anyone! >.<"""
        #Your code will go here
        await self.bot.say("ONE PUNCH! And " + user.mention + " is out! ლ(ಠ益ಠლ)")

def setup(bot):
    bot.add_cog(FoxMain(bot))
<commit_msg>Subcommand attempts<commit_after>import discord
from discord.ext import commands

class Fox:
    """My custom cog that does stuff!"""

    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def fox(self, ctx):
        """This does stuff!"""

        #Your code will go here
        if ctx.invoked_subcommand is None:
            await self.bot.send_cmd_help(ctx)
        #await self.bot.say("I can do stuff!")

    @fox.command(pass_context=True)
    async def punch(self, user : discord.Member):
        """I will puch anyone! >.<"""
        #Your code will go here
        await self.bot.say("ONE PUNCH! And " + user.mention + " is out! ლ(ಠ益ಠლ)")

def setup(bot):
    bot.add_cog(FoxMain(bot))
<|endoftext|>


<commit_before>from django.conf import settings

def bitgroup_cache_key(slug):
    return "%s:%s" % (
        getattr(settings, 'PAGEBIT_CACHE_PREFIX', 'pagebits'),
        slug
    )
<commit_msg>Fake commit to push to github<commit_after>from django.conf import settings

def bitgroup_cache_key(slug):
    return "%s:%s" % (
        getattr(settings, 'PAGEBIT_CACHE_PREFIX', 'pagebits'),
        slug
    )

<|endoftext|>


<commit_before># Copyright (c) 2014, Djaodjin Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from django.conf.urls import patterns, url

from pages.api.edition import PageElementDetail
from pages.api.upload_media import FileUploadView, MediaDestroyAPIView, ImageListAPIView
from pages.api.upload_template import TemplateUploadView, UploadedTemplateListAPIView
# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()

urlpatterns = patterns('',
    url(r'^delete-image/(?P<pk>\d+)/',
        MediaDestroyAPIView.as_view(), name='image_element'),
    url(r'^list/uploaded-images/',
        ImageListAPIView.as_view(), name='upload_image_element'),
    url(r'^editables/upload-image/',
        FileUploadView.as_view(), name='upload_image_element'),
    url(r'^editables/(?P<slug>[\w-]+)/',
        PageElementDetail.as_view(), name='edit_page_element'),
    url(r'^templates/upload-templates/',
        TemplateUploadView.as_view(), name='upload_template'),
    url(r'^uploaded-templates/list/', #pylint: disable=line-too-long
        UploadedTemplateListAPIView.as_view(),
        name='get_uploadedtemplate_list'),
)
<commit_msg>pylint code<commit_after># Copyright (c) 2014, Djaodjin Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from django.conf.urls import patterns, url

from pages.api.edition import PageElementDetail
from pages.api.upload_media import (
    FileUploadView,
    MediaDestroyAPIView,
    ImageListAPIView)

from pages.api.upload_template import (
    TemplateUploadView,
    UploadedTemplateListAPIView)
# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()

urlpatterns = patterns('',
    url(r'^delete-image/(?P<pk>\d+)/',
        MediaDestroyAPIView.as_view(), name='image_element'),
    url(r'^list/uploaded-images/',
        ImageListAPIView.as_view(), name='upload_image_element'),
    url(r'^editables/upload-image/',
        FileUploadView.as_view(), name='upload_image_element'),
    url(r'^editables/(?P<slug>[\w-]+)/',
        PageElementDetail.as_view(), name='edit_page_element'),
    url(r'^templates/upload-templates/',
        TemplateUploadView.as_view(), name='upload_template'),
    url(r'^uploaded-templates/list/', #pylint: disable=line-too-long
        UploadedTemplateListAPIView.as_view(),
        name='get_uploadedtemplate_list'),
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Views for Pagetags Django CMS plugin.

.. seealso::
    http://docs.djangoproject.com/en/1.4/ref/class-based-views/
"""
from django.http import JsonResponse

from tagging.models import Tag

def list_tags(request):
    q = request.GET.get('q')
    tags = []
    if q:
        tags = (Tag.objects
                .filter(name__istartswith=q)
                .values_list('name', flat=True))
    response = [ {'id':tag, 'label':tag, 'value':tag}  for tag in tags ]
    return JsonResponse(response)
<commit_msg>JsonResponse requires safe flag to false when response is a list<commit_after># -*- coding: utf-8 -*-
"""
Views for Pagetags Django CMS plugin.

.. seealso::
    http://docs.djangoproject.com/en/1.4/ref/class-based-views/
"""
from django.http import JsonResponse

from tagging.models import Tag

def list_tags(request):
    q = request.GET.get('q')
    tags = []
    if q:
        tags = (Tag.objects
                .filter(name__istartswith=q)
                .values_list('name', flat=True))
    response = [ {'id':tag, 'label':tag, 'value':tag}  for tag in tags ]
    return JsonResponse(response, safe=False)
<|endoftext|>


<commit_before>from enum import Enum
from typing import (
    TYPE_CHECKING,
    KT,
    VT,
    Any,
    Union,
    Type,
    TypeVar,
    get_type_hints,
    no_type_check,
    )
try:
    from typing import OrderedDict
except ImportError:
    # Python < 3.7.2 compatibility; see
    # https://github.com/python/cpython/commit/68b56d0
    import collections
    from typing import _alias
    OrderedDict = _alias(collections.OrderedDict, (KT, VT))

import pydantic
from pydantic import DictError, Extra, ValidationError
from pydantic.class_validators import make_generic_validator

class Resource(str, Enum):
    """Enumeration of SDMX REST API endpoints.

    ====================== ================================================
    :class:`Enum` member   :mod:`pandasdmx.model` class
    ====================== ================================================
    ``categoryscheme``     :class:`CategoryScheme \
                                   <pandasdmx.model.CategoryScheme>`
    ``codelist``           :class:`Codelist \
                                   <pandasdmx.model.Codelist>`
    ``conceptscheme``      :class:`ConceptScheme \
                                   <pandasdmx.model.ConceptScheme>`
    ``data``               :class:`DataSet \
                                   <pandasdmx.model.DataSet>`
    ``dataflow``           :class:`DataflowDefinition \
                                   <pandasdmx.model.DataflowDefinition>`
    ``datastructure``      :class:`DataStructureDefinition \
                                   <pandasdmx.model.DataStructureDefinition>`
    ``provisionagreement`` :class:`ProvisionAgreement \
                                   <pandasdmx.model.ProvisionAgreement>`
    ====================== ================================================
    """

    # agencyscheme = 'agencyscheme'
    # attachementconstraint = 'attachementconstraint'
    # categorisation = 'categorisation'
    categoryscheme = 'categoryscheme'
    codelist = 'codelist'
    conceptscheme = 'conceptscheme'
    # contentconstraint = 'contentconstraint'
    data = 'data'
    # dataconsumerscheme = 'dataconsumerscheme'
    dataflow = 'dataflow'
    # dataproviderscheme = 'dataproviderscheme'
    datastructure = 'datastructure'
    # hierarchicalcodelist = 'hierarchicalcodelist'
    # metadata = 'metadata'
    # metadataflow = 'metadataflow'
    # metadatastructure = 'metadatastructure'
    # organisationscheme = 'organisationscheme'
    # organisationunitscheme = 'organisationunitscheme'
    # process = 'process'
    provisionagreement = 'provisionagreement'
    # reportingtaxonomy = 'reportingtaxonomy'
    # schema = 'schema'
    # structure = 'structure'
    # structureset = 'structureset'

    @classmethod
    def from_obj(cls, obj):
        """Return an enumeration value based on the class of *obj*."""
        clsname = {
            'DataStructureDefinition': 'datastructure',
            }.get(obj.__class__.__name__, obj.__class__.__name__)
        return cls[clsname.lower()]

    @classmethod
    def describe(cls):
        return '{' + ' '.join(v.name for v in cls._member_map_.values()) + '}'

if TYPE_CHECKING:
    Model = TypeVar('Model', bound='BaseModel')

class BaseModel(pydantic.BaseModel):
    class Config:
        validate_assignment = 'limited'
        validate_assignment_exclude = []

    # Workaround for https://github.com/samuelcolvin/pydantic/issues/521:
    # - When cls.attr is typed as BaseModel (or a subclass), then
    #   a.attr is b.attr is always False, even when set to the same reference
    # - Same as pydantic.BaseModel.validate, but without copy().
    # - Issue marked as wontfix by pydantic maintainer.
    @classmethod
    def validate(cls: Type['Model'], value: Any) -> 'Model':
        if isinstance(value, dict):
            return cls(**value)
        elif isinstance(value, cls):
            return value
        elif cls.__config__.orm_mode:
            return cls.from_orm(value)
        else:
            try:
                value_as_dict = dict(value)
            except (TypeError, ValueError) as e:
                raise DictError() from e
            return cls(**value_as_dict)

    # Workaround for https://github.com/samuelcolvin/pydantic/issues/524:
    @no_type_check
    def __setattr__(self, name, value):
        if (self.__config__.extra is not Extra.allow and name not in
                self.__fields__):
            raise ValueError(f'"{self.__class__.__name__}" object has no field'
                             '"{name}"')
        elif not self.__config__.allow_mutation:
            raise TypeError(f'"{self.__class__.__name__}" is immutable and '
                            'does not support item assignment')
        elif (self.__config__.validate_assignment and name not in
              self.__config__.validate_assignment_exclude):
            if self.__config__.validate_assignment == 'limited':
                kw = {'include': {}}
            else:
                kw = {'exclude': {name}}
            known_field = self.__fields__.get(name, None)
            if known_field:
                value, error_ = known_field.validate(value, self.dict(**kw),
                                                     loc=name)
                if error_:
                    raise ValidationError([error_], type(self))
        self.__dict__[name] = value
        self.__fields_set__.add(name)

def get_class_hint(obj, attr):
    klass = get_type_hints(obj.__class__)[attr].__args__[0]
    if getattr(klass, '__origin__', None) is Union:
        klass = klass.__args__[0]
    return klass

class DictLike(OrderedDict[KT, VT]):
    """Container with features of a dict & list, plus attribute access."""
    def __getitem__(self, key):
        try:
            return super().__getitem__(key)
        except KeyError:
            if isinstance(key, int):
                return list(self.values())[key]
            elif isinstance(key, str) and key.startswith('__'):
                raise AttributeError
            else:
                raise

    def __setitem__(self, key, value):
        key = self._apply_validators('key', key)
        value = self._apply_validators('value', value)
        super().__setitem__(key, value)

    # Access items as attributes
    def __getattr__(self, name):
        try:
            return self.__getitem__(name)
        except KeyError as e:
            raise AttributeError(*e.args) from None

    def validate(cls, value, field):
        if not isinstance(value, (dict, DictLike)):
            raise ValueError(value)

        result = DictLike()
        result.__fields = {'key': field.key_field, 'value': field}
        result.update(value)
        return result

    def _apply_validators(self, which, value):
        try:
            field = self.__fields[which]
        except AttributeError:
            return value
        result, error = field._apply_validators(
            value, validators=field.validators, values={}, loc=(), cls=None)
        if error:
            raise ValidationError([error], self.__class__)
        else:
            return result

def summarize_dictlike(dl, maxwidth=72):
    """Return a string summary of the DictLike contents."""
    value_cls = dl[0].__class__.__name__
    count = len(dl)
    keys = ' '.join(dl.keys())
    result = f'{value_cls} ({count}): {keys}'

    if len(result) > maxwidth:
        # Truncate the list of keys
        result = result[:maxwidth - 3] + '...'

    return result

def validate_dictlike(*fields):
    def decorator(cls):
        v = make_generic_validator(DictLike.validate)
        for field in fields:
            cls.__fields__[field].whole_post_validators = [v]
        return cls

    return decorator
<commit_msg>Update util.validate_dictlike<commit_after>from enum import Enum
from typing import (
    TYPE_CHECKING,
    KT,
    VT,
    Any,
    Union,
    Type,
    TypeVar,
    get_type_hints,
    no_type_check,
    )
try:
    from typing import OrderedDict
except ImportError:
    # Python < 3.7.2 compatibility; see
    # https://github.com/python/cpython/commit/68b56d0
    import collections
    from typing import _alias
    OrderedDict = _alias(collections.OrderedDict, (KT, VT))

import pydantic
from pydantic import DictError, Extra, ValidationError
from pydantic.class_validators import make_generic_validator

class Resource(str, Enum):
    """Enumeration of SDMX REST API endpoints.

    ====================== ================================================
    :class:`Enum` member   :mod:`pandasdmx.model` class
    ====================== ================================================
    ``categoryscheme``     :class:`CategoryScheme \
                                   <pandasdmx.model.CategoryScheme>`
    ``codelist``           :class:`Codelist \
                                   <pandasdmx.model.Codelist>`
    ``conceptscheme``      :class:`ConceptScheme \
                                   <pandasdmx.model.ConceptScheme>`
    ``data``               :class:`DataSet \
                                   <pandasdmx.model.DataSet>`
    ``dataflow``           :class:`DataflowDefinition \
                                   <pandasdmx.model.DataflowDefinition>`
    ``datastructure``      :class:`DataStructureDefinition \
                                   <pandasdmx.model.DataStructureDefinition>`
    ``provisionagreement`` :class:`ProvisionAgreement \
                                   <pandasdmx.model.ProvisionAgreement>`
    ====================== ================================================
    """

    # agencyscheme = 'agencyscheme'
    # attachementconstraint = 'attachementconstraint'
    # categorisation = 'categorisation'
    categoryscheme = 'categoryscheme'
    codelist = 'codelist'
    conceptscheme = 'conceptscheme'
    # contentconstraint = 'contentconstraint'
    data = 'data'
    # dataconsumerscheme = 'dataconsumerscheme'
    dataflow = 'dataflow'
    # dataproviderscheme = 'dataproviderscheme'
    datastructure = 'datastructure'
    # hierarchicalcodelist = 'hierarchicalcodelist'
    # metadata = 'metadata'
    # metadataflow = 'metadataflow'
    # metadatastructure = 'metadatastructure'
    # organisationscheme = 'organisationscheme'
    # organisationunitscheme = 'organisationunitscheme'
    # process = 'process'
    provisionagreement = 'provisionagreement'
    # reportingtaxonomy = 'reportingtaxonomy'
    # schema = 'schema'
    # structure = 'structure'
    # structureset = 'structureset'

    @classmethod
    def from_obj(cls, obj):
        """Return an enumeration value based on the class of *obj*."""
        clsname = {
            'DataStructureDefinition': 'datastructure',
            }.get(obj.__class__.__name__, obj.__class__.__name__)
        return cls[clsname.lower()]

    @classmethod
    def describe(cls):
        return '{' + ' '.join(v.name for v in cls._member_map_.values()) + '}'

if TYPE_CHECKING:
    Model = TypeVar('Model', bound='BaseModel')

class BaseModel(pydantic.BaseModel):
    class Config:
        validate_assignment = 'limited'
        validate_assignment_exclude = []

    # Workaround for https://github.com/samuelcolvin/pydantic/issues/521:
    # - When cls.attr is typed as BaseModel (or a subclass), then
    #   a.attr is b.attr is always False, even when set to the same reference
    # - Same as pydantic.BaseModel.validate, but without copy().
    # - Issue marked as wontfix by pydantic maintainer.
    @classmethod
    def validate(cls: Type['Model'], value: Any) -> 'Model':
        if isinstance(value, dict):
            return cls(**value)
        elif isinstance(value, cls):
            return value
        elif cls.__config__.orm_mode:
            return cls.from_orm(value)
        else:
            try:
                value_as_dict = dict(value)
            except (TypeError, ValueError) as e:
                raise DictError() from e
            return cls(**value_as_dict)

    # Workaround for https://github.com/samuelcolvin/pydantic/issues/524:
    @no_type_check
    def __setattr__(self, name, value):
        if (self.__config__.extra is not Extra.allow and name not in
                self.__fields__):
            raise ValueError(f'"{self.__class__.__name__}" object has no field'
                             '"{name}"')
        elif not self.__config__.allow_mutation:
            raise TypeError(f'"{self.__class__.__name__}" is immutable and '
                            'does not support item assignment')
        elif (self.__config__.validate_assignment and name not in
              self.__config__.validate_assignment_exclude):
            if self.__config__.validate_assignment == 'limited':
                kw = {'include': {}}
            else:
                kw = {'exclude': {name}}
            known_field = self.__fields__.get(name, None)
            if known_field:
                value, error_ = known_field.validate(value, self.dict(**kw),
                                                     loc=name)
                if error_:
                    raise ValidationError([error_], type(self))
        self.__dict__[name] = value
        self.__fields_set__.add(name)

def get_class_hint(obj, attr):
    klass = get_type_hints(obj.__class__)[attr].__args__[0]
    if getattr(klass, '__origin__', None) is Union:
        klass = klass.__args__[0]
    return klass

class DictLike(OrderedDict[KT, VT]):
    """Container with features of a dict & list, plus attribute access."""
    def __getitem__(self, key):
        try:
            return super().__getitem__(key)
        except KeyError:
            if isinstance(key, int):
                return list(self.values())[key]
            elif isinstance(key, str) and key.startswith('__'):
                raise AttributeError
            else:
                raise

    def __setitem__(self, key, value):
        key = self._apply_validators('key', key)
        value = self._apply_validators('value', value)
        super().__setitem__(key, value)

    # Access items as attributes
    def __getattr__(self, name):
        try:
            return self.__getitem__(name)
        except KeyError as e:
            raise AttributeError(*e.args) from None

    def validate(cls, value, field):
        if not isinstance(value, (dict, DictLike)):
            raise ValueError(value)

        result = DictLike()
        result.__fields = {'key': field.key_field, 'value': field}
        result.update(value)
        return result

    def _apply_validators(self, which, value):
        try:
            field = self.__fields[which]
        except AttributeError:
            return value
        result, error = field._apply_validators(
            value, validators=field.validators, values={}, loc=(), cls=None)
        if error:
            raise ValidationError([error], self.__class__)
        else:
            return result

def summarize_dictlike(dl, maxwidth=72):
    """Return a string summary of the DictLike contents."""
    value_cls = dl[0].__class__.__name__
    count = len(dl)
    keys = ' '.join(dl.keys())
    result = f'{value_cls} ({count}): {keys}'

    if len(result) > maxwidth:
        # Truncate the list of keys
        result = result[:maxwidth - 3] + '...'

    return result

def validate_dictlike(*fields):
    def decorator(cls):
        v = make_generic_validator(DictLike.validate)
        for field in fields:
            cls.__fields__[field].post_validators = [v]
        return cls

    return decorator
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import io
import os
import sys
import json
import yaml
import fnmatch
import nbformat
import requests
import warnings
import entrypoints

from contextlib import contextmanager

from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential

from . import __version__
from .log import logger
from .utils import chdir
from .exceptions import (
    PapermillException,
    PapermillRateLimitException,
    missing_dependency_generator,
    missing_environment_variable_generator,
)

try:
    from .s3 import S3
except ImportError:
    S3 = missing_dependency_generator("boto3", "s3")
try:
    from .adl import ADL
except ImportError:
    ADL = missing_dependency_generator("azure.datalake.store", "azure")
except KeyError as exc:
    if exc.args[0] == "APPDATA":
        ADL = missing_environment_variable_generator("azure.datalake.store", "APPDATA")
    else:
        raise
try:
    from .abs import AzureBlobStore
except ImportError:
    AzureBlobStore = missing_dependency_generator("azure.storage.blob", "azure")
try:
    from gcsfs import GCSFileSystem
except ImportError:
    GCSFileSystem = missing_dependency_generator("gcsfs", "gcs")

# Handle newer and older gcsfs versions
try:
    try:
        from gcsfs.utils import HttpError as GCSHttpError
    except ImportError:
        from gcsfs.utils import HtmlError as GCSHttpError
except ImportError:
    # Fall back to a sane import if gcsfs is missing
    GCSHttpError = Exception

try:
    from gcsfs.utils import RateLimitException as GCSRateLimitException
except ImportError:
    # Fall back to GCSHttpError when using older library
    GCSRateLimitException = GCSHttpError

try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError

class PapermillIO(object):
    '''
    The holder which houses any io system registered with the system.
    This object is used in a singleton manner to save and load particular
    named Handler objects for reference externally.
    '''

    def __init__(self):
        self.reset()

    def read(self, path, extensions=['.ipynb', '.json']):
        if path == '-':
            return sys.stdin.read()

        if not fnmatch.fnmatch(os.path.basename(path), '*.*'):
            warnings.warn(
                "the file is not specified with any extension : " + os.path.basename(path)
            )
        elif not any(fnmatch.fnmatch(os.path.basename(path), '*' + ext) for ext in extensions):
            warnings.warn(
                "The specified input file ({}) does not end in one of {}".format(path, extensions)
            )
        # Handle https://github.com/nteract/papermill/issues/317
        notebook_metadata = self.get_handler(path).read(path)
        if isinstance(notebook_metadata, (bytes, bytearray)):
            return notebook_metadata.decode('utf-8')
        return notebook_metadata

    def write(self, buf, path, extensions=['.ipynb', '.json']):
        if path == '-':
            return sys.stdout.write(buf)

        # Usually no return object here
        if not fnmatch.fnmatch(os.path.basename(path), '*.*'):
            warnings.warn(
                "the file is not specified with any extension : " + os.path.basename(path)
            )
        elif not any(fnmatch.fnmatch(os.path.basename(path), '*' + ext) for ext in extensions):
            warnings.warn(
                "The specified input file ({}) does not end in one of {}".format(path, extensions)
            )
        return self.get_handler(path).write(buf, path)

    def listdir(self, path):
        return self.get_handler(path).listdir(path)

    def pretty_path(self, path):
        return self.get_handler(path).pretty_path(path)

    def reset(self):
        self._handlers = []

    def register(self, scheme, handler):
        # Keep these ordered as LIFO
        self._handlers.insert(0, (scheme, handler))

    def register_entry_points(self):
        # Load handlers provided by other packages
        for entrypoint in entrypoints.get_group_all("papermill.io"):
            self.register(entrypoint.name, entrypoint.load())

    def get_handler(self, path):
        local_handler = None
        for scheme, handler in self._handlers:
            if scheme == 'local':
                local_handler = handler

            if path.startswith(scheme):
                return handler

        if local_handler is None:
            raise PapermillException(
                "Could not find a registered schema handler for: {}".format(path)
            )

        return local_handler

class HttpHandler(object):
    @classmethod
    def read(cls, path):
        return requests.get(path, headers={'Accept': 'application/json'}).text

    @classmethod
    def listdir(cls, path):
        raise PapermillException('listdir is not supported by HttpHandler')

    @classmethod
    def write(cls, buf, path):
        result = requests.put(path, json=json.loads(buf))
        result.raise_for_status()

    @classmethod
    def pretty_path(cls, path):
        return path

class LocalHandler(object):
    def __init__(self):
        self._cwd = None

    def read(self, path):
        try:
            with chdir(self._cwd):
                with io.open(path, 'r', encoding="utf-8") as f:
                    return f.read()
        except IOError as e:
            try:
                # Check if path could be a notebook passed in as a
                # string
                json.loads(path)
                return path
            except ValueError:
                # Propagate the IOError
                raise e

    def listdir(self, path):
        with chdir(self._cwd):
            return [os.path.join(path, fn) for fn in os.listdir(path)]

    def write(self, buf, path):
        with chdir(self._cwd):
            dirname = os.path.dirname(path)
            if dirname and not os.path.exists(dirname):
                raise FileNotFoundError("output folder {} doesn't exist.".format(dirname))
            with io.open(path, 'w', encoding="utf-8") as f:
                f.write(buf)

    def pretty_path(self, path):
        return path

    def cwd(self, new_path):
        '''Sets the cwd during reads and writes'''
        old_cwd = self._cwd
        self._cwd = new_path
        return old_cwd

class S3Handler(object):
    @classmethod
    def read(cls, path):
        return "\n".join(S3().read(path))

    @classmethod
    def listdir(cls, path):
        return S3().listdir(path)

    @classmethod
    def write(cls, buf, path):
        return S3().cp_string(buf, path)

    @classmethod
    def pretty_path(cls, path):
        return path

class ADLHandler(object):
    def __init__(self):
        self._client = None

    def _get_client(self):
        if self._client is None:
            self._client = ADL()
        return self._client

    def read(self, path):
        lines = self._get_client().read(path)
        return "\n".join(lines)

    def listdir(self, path):
        return self._get_client().listdir(path)

    def write(self, buf, path):
        return self._get_client().write(buf, path)

    def pretty_path(self, path):
        return path

class ABSHandler(object):
    def __init__(self):
        self._client = None

    def _get_client(self):
        if self._client is None:
            self._client = AzureBlobStore()
        return self._client

    def read(self, path):
        lines = self._get_client().read(path)
        return "\n".join(lines)

    def listdir(self, path):
        return self._get_client().listdir(path)

    def write(self, buf, path):
        return self._get_client().write(buf, path)

    def pretty_path(self, path):
        return path

class GCSHandler(object):
    RATE_LIMIT_RETRIES = 3
    RETRY_DELAY = 1
    RETRY_MULTIPLIER = 1
    RETRY_MAX_DELAY = 4

    def __init__(self):
        self._client = None

    def _get_client(self):
        if self._client is None:
            self._client = GCSFileSystem()
        return self._client

    def read(self, path):
        with self._get_client().open(path) as f:
            return f.read()

    def listdir(self, path):
        return self._get_client().ls(path)

    def write(self, buf, path):
        # Wrapped so we can mock retry options during testing
        @retry(
            retry=retry_if_exception_type(PapermillRateLimitException),
            stop=stop_after_attempt(self.RATE_LIMIT_RETRIES),
            wait=wait_exponential(
                multiplier=self.RETRY_MULTIPLIER, min=self.RETRY_DELAY, max=self.RETRY_MAX_DELAY
            ),
            reraise=True,
        )
        def retry_write():
            try:
                with self._get_client().open(path, 'w') as f:
                    return f.write(buf)
            except (GCSHttpError, GCSRateLimitException) as e:
                try:
                    # If code is assigned but unknown, optimistically retry
                    if e.code is None or e.code == 429:
                        raise PapermillRateLimitException(e.message)
                except AttributeError:
                    try:
                        message = e.message
                    except AttributeError:
                        message = "Generic exception {} raised, retrying".format(type(e))
                    raise PapermillRateLimitException(message)
                # Reraise the original exception without retries
                raise

        return retry_write()

    def pretty_path(self, path):
        return path

# Hack to make YAML loader not auto-convert datetimes
# https://stackoverflow.com/a/52312810
NoDatesSafeLoader = yaml.SafeLoader
NoDatesSafeLoader.yaml_implicit_resolvers = {
    k: [r for r in v if r[0] != 'tag:yaml.org,2002:timestamp'] for
    k, v in NoDatesSafeLoader.yaml_implicit_resolvers.items()
}

# Instantiate a PapermillIO instance and register Handlers.
papermill_io = PapermillIO()
papermill_io.register("local", LocalHandler())
papermill_io.register("s3://", S3Handler)
papermill_io.register("adl://", ADLHandler)
papermill_io.register("abs://", ABSHandler())
papermill_io.register("http://", HttpHandler)
papermill_io.register("https://", HttpHandler)
papermill_io.register("gs://", GCSHandler())
papermill_io.register_entry_points()

def read_yaml_file(path):
    """Reads a YAML file from the location specified at 'path'."""
    return yaml.load(papermill_io.read(path, ['.json', '.yaml', '.yml']), Loader=NoDatesSafeLoader)

def write_ipynb(nb, path):
    """Saves a notebook object to the specified path.
    Args:
        nb_node (nbformat.NotebookNode): Notebook object to save.
        notebook_path (str): Path to save the notebook object to.
    """
    papermill_io.write(nbformat.writes(nb), path)

def load_notebook_node(notebook_path):
    """Returns a notebook object with papermill metadata loaded from the specified path.

    Args:
        notebook_path (str): Path to the notebook file.

    Returns:
        nbformat.NotebookNode

    """
    nb = nbformat.reads(papermill_io.read(notebook_path), as_version=4)

    if not hasattr(nb.metadata, 'papermill'):
        nb.metadata['papermill'] = {
            'parameters': dict(),
            'environment_variables': dict(),
            'version': __version__,
        }

    for cell in nb.cells:
        if not hasattr(cell.metadata, 'tags'):
            cell.metadata['tags'] = []  # Create tags attr if one doesn't exist.

        if not hasattr(cell.metadata, 'papermill'):
            cell.metadata['papermill'] = dict()
    return nb

def list_notebook_files(path):
    """Returns a list of all the notebook files in a directory."""
    return [p for p in papermill_io.listdir(path) if p.endswith('.ipynb')]

def get_pretty_path(path):
    return papermill_io.pretty_path(path)

@contextmanager
def local_file_io_cwd(path=None):
    try:
        local_handler = papermill_io.get_handler("local")
    except PapermillException:
        logger.warning("No local file handler detected")
    else:
        try:
            old_cwd = local_handler.cwd(path or os.getcwd())
        except AttributeError:
            logger.warning("Local file handler does not support cwd assignment")
        else:
            try:
                yield
            finally:
                local_handler.cwd(old_cwd)
<commit_msg>fix ADLHandler registration in papermill_io<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import io
import os
import sys
import json
import yaml
import fnmatch
import nbformat
import requests
import warnings
import entrypoints

from contextlib import contextmanager

from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential

from . import __version__
from .log import logger
from .utils import chdir
from .exceptions import (
    PapermillException,
    PapermillRateLimitException,
    missing_dependency_generator,
    missing_environment_variable_generator,
)

try:
    from .s3 import S3
except ImportError:
    S3 = missing_dependency_generator("boto3", "s3")
try:
    from .adl import ADL
except ImportError:
    ADL = missing_dependency_generator("azure.datalake.store", "azure")
except KeyError as exc:
    if exc.args[0] == "APPDATA":
        ADL = missing_environment_variable_generator("azure.datalake.store", "APPDATA")
    else:
        raise
try:
    from .abs import AzureBlobStore
except ImportError:
    AzureBlobStore = missing_dependency_generator("azure.storage.blob", "azure")
try:
    from gcsfs import GCSFileSystem
except ImportError:
    GCSFileSystem = missing_dependency_generator("gcsfs", "gcs")

# Handle newer and older gcsfs versions
try:
    try:
        from gcsfs.utils import HttpError as GCSHttpError
    except ImportError:
        from gcsfs.utils import HtmlError as GCSHttpError
except ImportError:
    # Fall back to a sane import if gcsfs is missing
    GCSHttpError = Exception

try:
    from gcsfs.utils import RateLimitException as GCSRateLimitException
except ImportError:
    # Fall back to GCSHttpError when using older library
    GCSRateLimitException = GCSHttpError

try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError

class PapermillIO(object):
    '''
    The holder which houses any io system registered with the system.
    This object is used in a singleton manner to save and load particular
    named Handler objects for reference externally.
    '''

    def __init__(self):
        self.reset()

    def read(self, path, extensions=['.ipynb', '.json']):
        if path == '-':
            return sys.stdin.read()

        if not fnmatch.fnmatch(os.path.basename(path), '*.*'):
            warnings.warn(
                "the file is not specified with any extension : " + os.path.basename(path)
            )
        elif not any(fnmatch.fnmatch(os.path.basename(path), '*' + ext) for ext in extensions):
            warnings.warn(
                "The specified input file ({}) does not end in one of {}".format(path, extensions)
            )
        # Handle https://github.com/nteract/papermill/issues/317
        notebook_metadata = self.get_handler(path).read(path)
        if isinstance(notebook_metadata, (bytes, bytearray)):
            return notebook_metadata.decode('utf-8')
        return notebook_metadata

    def write(self, buf, path, extensions=['.ipynb', '.json']):
        if path == '-':
            return sys.stdout.write(buf)

        # Usually no return object here
        if not fnmatch.fnmatch(os.path.basename(path), '*.*'):
            warnings.warn(
                "the file is not specified with any extension : " + os.path.basename(path)
            )
        elif not any(fnmatch.fnmatch(os.path.basename(path), '*' + ext) for ext in extensions):
            warnings.warn(
                "The specified input file ({}) does not end in one of {}".format(path, extensions)
            )
        return self.get_handler(path).write(buf, path)

    def listdir(self, path):
        return self.get_handler(path).listdir(path)

    def pretty_path(self, path):
        return self.get_handler(path).pretty_path(path)

    def reset(self):
        self._handlers = []

    def register(self, scheme, handler):
        # Keep these ordered as LIFO
        self._handlers.insert(0, (scheme, handler))

    def register_entry_points(self):
        # Load handlers provided by other packages
        for entrypoint in entrypoints.get_group_all("papermill.io"):
            self.register(entrypoint.name, entrypoint.load())

    def get_handler(self, path):
        local_handler = None
        for scheme, handler in self._handlers:
            if scheme == 'local':
                local_handler = handler

            if path.startswith(scheme):
                return handler

        if local_handler is None:
            raise PapermillException(
                "Could not find a registered schema handler for: {}".format(path)
            )

        return local_handler

class HttpHandler(object):
    @classmethod
    def read(cls, path):
        return requests.get(path, headers={'Accept': 'application/json'}).text

    @classmethod
    def listdir(cls, path):
        raise PapermillException('listdir is not supported by HttpHandler')

    @classmethod
    def write(cls, buf, path):
        result = requests.put(path, json=json.loads(buf))
        result.raise_for_status()

    @classmethod
    def pretty_path(cls, path):
        return path

class LocalHandler(object):
    def __init__(self):
        self._cwd = None

    def read(self, path):
        try:
            with chdir(self._cwd):
                with io.open(path, 'r', encoding="utf-8") as f:
                    return f.read()
        except IOError as e:
            try:
                # Check if path could be a notebook passed in as a
                # string
                json.loads(path)
                return path
            except ValueError:
                # Propagate the IOError
                raise e

    def listdir(self, path):
        with chdir(self._cwd):
            return [os.path.join(path, fn) for fn in os.listdir(path)]

    def write(self, buf, path):
        with chdir(self._cwd):
            dirname = os.path.dirname(path)
            if dirname and not os.path.exists(dirname):
                raise FileNotFoundError("output folder {} doesn't exist.".format(dirname))
            with io.open(path, 'w', encoding="utf-8") as f:
                f.write(buf)

    def pretty_path(self, path):
        return path

    def cwd(self, new_path):
        '''Sets the cwd during reads and writes'''
        old_cwd = self._cwd
        self._cwd = new_path
        return old_cwd

class S3Handler(object):
    @classmethod
    def read(cls, path):
        return "\n".join(S3().read(path))

    @classmethod
    def listdir(cls, path):
        return S3().listdir(path)

    @classmethod
    def write(cls, buf, path):
        return S3().cp_string(buf, path)

    @classmethod
    def pretty_path(cls, path):
        return path

class ADLHandler(object):
    def __init__(self):
        self._client = None

    def _get_client(self):
        if self._client is None:
            self._client = ADL()
        return self._client

    def read(self, path):
        lines = self._get_client().read(path)
        return "\n".join(lines)

    def listdir(self, path):
        return self._get_client().listdir(path)

    def write(self, buf, path):
        return self._get_client().write(buf, path)

    def pretty_path(self, path):
        return path

class ABSHandler(object):
    def __init__(self):
        self._client = None

    def _get_client(self):
        if self._client is None:
            self._client = AzureBlobStore()
        return self._client

    def read(self, path):
        lines = self._get_client().read(path)
        return "\n".join(lines)

    def listdir(self, path):
        return self._get_client().listdir(path)

    def write(self, buf, path):
        return self._get_client().write(buf, path)

    def pretty_path(self, path):
        return path

class GCSHandler(object):
    RATE_LIMIT_RETRIES = 3
    RETRY_DELAY = 1
    RETRY_MULTIPLIER = 1
    RETRY_MAX_DELAY = 4

    def __init__(self):
        self._client = None

    def _get_client(self):
        if self._client is None:
            self._client = GCSFileSystem()
        return self._client

    def read(self, path):
        with self._get_client().open(path) as f:
            return f.read()

    def listdir(self, path):
        return self._get_client().ls(path)

    def write(self, buf, path):
        # Wrapped so we can mock retry options during testing
        @retry(
            retry=retry_if_exception_type(PapermillRateLimitException),
            stop=stop_after_attempt(self.RATE_LIMIT_RETRIES),
            wait=wait_exponential(
                multiplier=self.RETRY_MULTIPLIER, min=self.RETRY_DELAY, max=self.RETRY_MAX_DELAY
            ),
            reraise=True,
        )
        def retry_write():
            try:
                with self._get_client().open(path, 'w') as f:
                    return f.write(buf)
            except (GCSHttpError, GCSRateLimitException) as e:
                try:
                    # If code is assigned but unknown, optimistically retry
                    if e.code is None or e.code == 429:
                        raise PapermillRateLimitException(e.message)
                except AttributeError:
                    try:
                        message = e.message
                    except AttributeError:
                        message = "Generic exception {} raised, retrying".format(type(e))
                    raise PapermillRateLimitException(message)
                # Reraise the original exception without retries
                raise

        return retry_write()

    def pretty_path(self, path):
        return path

# Hack to make YAML loader not auto-convert datetimes
# https://stackoverflow.com/a/52312810
NoDatesSafeLoader = yaml.SafeLoader
NoDatesSafeLoader.yaml_implicit_resolvers = {
    k: [r for r in v if r[0] != 'tag:yaml.org,2002:timestamp'] for
    k, v in NoDatesSafeLoader.yaml_implicit_resolvers.items()
}

# Instantiate a PapermillIO instance and register Handlers.
papermill_io = PapermillIO()
papermill_io.register("local", LocalHandler())
papermill_io.register("s3://", S3Handler)
papermill_io.register("adl://", ADLHandler())
papermill_io.register("abs://", ABSHandler())
papermill_io.register("http://", HttpHandler)
papermill_io.register("https://", HttpHandler)
papermill_io.register("gs://", GCSHandler())
papermill_io.register_entry_points()

def read_yaml_file(path):
    """Reads a YAML file from the location specified at 'path'."""
    return yaml.load(papermill_io.read(path, ['.json', '.yaml', '.yml']), Loader=NoDatesSafeLoader)

def write_ipynb(nb, path):
    """Saves a notebook object to the specified path.
    Args:
        nb_node (nbformat.NotebookNode): Notebook object to save.
        notebook_path (str): Path to save the notebook object to.
    """
    papermill_io.write(nbformat.writes(nb), path)

def load_notebook_node(notebook_path):
    """Returns a notebook object with papermill metadata loaded from the specified path.

    Args:
        notebook_path (str): Path to the notebook file.

    Returns:
        nbformat.NotebookNode

    """
    nb = nbformat.reads(papermill_io.read(notebook_path), as_version=4)

    if not hasattr(nb.metadata, 'papermill'):
        nb.metadata['papermill'] = {
            'parameters': dict(),
            'environment_variables': dict(),
            'version': __version__,
        }

    for cell in nb.cells:
        if not hasattr(cell.metadata, 'tags'):
            cell.metadata['tags'] = []  # Create tags attr if one doesn't exist.

        if not hasattr(cell.metadata, 'papermill'):
            cell.metadata['papermill'] = dict()
    return nb

def list_notebook_files(path):
    """Returns a list of all the notebook files in a directory."""
    return [p for p in papermill_io.listdir(path) if p.endswith('.ipynb')]

def get_pretty_path(path):
    return papermill_io.pretty_path(path)

@contextmanager
def local_file_io_cwd(path=None):
    try:
        local_handler = papermill_io.get_handler("local")
    except PapermillException:
        logger.warning("No local file handler detected")
    else:
        try:
            old_cwd = local_handler.cwd(path or os.getcwd())
        except AttributeError:
            logger.warning("Local file handler does not support cwd assignment")
        else:
            try:
                yield
            finally:
                local_handler.cwd(old_cwd)
<|endoftext|>


<commit_before>from parcels.codegenerator import KernelGenerator, LoopGenerator
from py import path
import math  # NOQA get flake8 to ignore unused import.
import numpy.ctypeslib as npct
from ctypes import c_int, c_float, c_double, c_void_p, byref
from ast import parse, FunctionDef, Module
import inspect
from copy import deepcopy

class Kernel(object):
    """Kernel object that encapsulates auto-generated code.

    :arg filename: Basename for kernel files to generate"""

    def __init__(self, grid, ptype, pyfunc=None, funcname=None,
                 py_ast=None, funcvars=None):
        self.grid = grid
        self.ptype = ptype

        if pyfunc is not None:
            self.funcname = pyfunc.__name__
            self.funcvars = list(pyfunc.__code__.co_varnames)
            # Parse the Python code into an AST
            self.py_ast = parse(inspect.getsource(pyfunc.__code__))
            self.py_ast = self.py_ast.body[0]
            self.pyfunc = pyfunc
        else:
            self.funcname = funcname
            self.py_ast = py_ast
            self.funcvars = funcvars
            # Compile and generate Python function from AST
            py_mod = Module(body=[self.py_ast])
            user_ctx = inspect.stack()[-1][0].f_globals
            exec(compile(py_mod, "<ast>", "exec"), user_ctx)
            self.pyfunc = user_ctx[self.funcname]

        self.name = "%s%s" % (ptype.name, funcname)

        self.src_file = str(path.local("%s.c" % self.name))
        self.lib_file = str(path.local("%s.so" % self.name))
        self.log_file = str(path.local("%s.log" % self.name))
        self._lib = None

        # Generate the kernel function and add the outer loop
        if self.ptype.uses_jit:
            kernelgen = KernelGenerator(grid, ptype)
            kernel_ccode = kernelgen.generate(deepcopy(self.py_ast),
                                              self.funcvars)
            self.field_args = kernelgen.field_args
            loopgen = LoopGenerator(grid, ptype)
            self.ccode = loopgen.generate(self.funcname,
                                          self.field_args,
                                          kernel_ccode)

    def compile(self, compiler):
        """ Writes kernel code to file and compiles it."""
        with open(self.src_file, 'w') as f:
            f.write(self.ccode)
        compiler.compile(self.src_file, self.lib_file, self.log_file)

    def load_lib(self):
        self._lib = npct.load_library(self.lib_file, '.')
        self._function = self._lib.particle_loop

    def execute(self, pset, timesteps, time, dt):
        if self.ptype.uses_jit:
            fargs = [byref(f.ctypes_struct) for f in self.field_args.values()]
            particle_data = pset._particle_data.ctypes.data_as(c_void_p)
            self._function(c_int(len(pset)), particle_data, c_int(timesteps),
                           c_double(time), c_float(dt), *fargs)
        else:
            for _ in range(timesteps):
                for p in pset.particles:
                    self.pyfunc(p, pset.grid, time, dt)
                time += dt

    def merge(self, kernel):
        funcname = self.funcname + kernel.funcname
        func_ast = FunctionDef(name=funcname, args=self.py_ast.args,
                               body=self.py_ast.body + kernel.py_ast.body,
                               decorator_list=[], lineno=1, col_offset=0)
        return Kernel(self.grid, self.ptype, pyfunc=None, funcname=funcname,
                      py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars)

    def __add__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.grid, self.ptype, pyfunc=kernel)
        return self.merge(kernel)

    def __radd__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.grid, self.ptype, pyfunc=kernel)
        return kernel.merge(self)
<commit_msg>Kernel: Make sure we remove cyclic references when accessing call stack<commit_after>from parcels.codegenerator import KernelGenerator, LoopGenerator
from py import path
import math  # NOQA get flake8 to ignore unused import.
import numpy.ctypeslib as npct
from ctypes import c_int, c_float, c_double, c_void_p, byref
from ast import parse, FunctionDef, Module
import inspect
from copy import deepcopy

class Kernel(object):
    """Kernel object that encapsulates auto-generated code.

    :arg filename: Basename for kernel files to generate"""

    def __init__(self, grid, ptype, pyfunc=None, funcname=None,
                 py_ast=None, funcvars=None):
        self.grid = grid
        self.ptype = ptype

        if pyfunc is not None:
            self.funcname = pyfunc.__name__
            self.funcvars = list(pyfunc.__code__.co_varnames)
            # Parse the Python code into an AST
            self.py_ast = parse(inspect.getsource(pyfunc.__code__))
            self.py_ast = self.py_ast.body[0]
            self.pyfunc = pyfunc
        else:
            self.funcname = funcname
            self.py_ast = py_ast
            self.funcvars = funcvars
            # Extract user context by inspecting the call stack
            stack = inspect.stack()
            try:
                user_ctx = stack[-1][0].f_globals
            except:
                print("Warning: Could not access user context when merging kernels")
                user_ctx = globals()
            finally:
                del stack  # Remove cyclic references
            # Compile and generate Python function from AST
            py_mod = Module(body=[self.py_ast])
            exec(compile(py_mod, "<ast>", "exec"), user_ctx)
            self.pyfunc = user_ctx[self.funcname]

        self.name = "%s%s" % (ptype.name, funcname)

        self.src_file = str(path.local("%s.c" % self.name))
        self.lib_file = str(path.local("%s.so" % self.name))
        self.log_file = str(path.local("%s.log" % self.name))
        self._lib = None

        # Generate the kernel function and add the outer loop
        if self.ptype.uses_jit:
            kernelgen = KernelGenerator(grid, ptype)
            kernel_ccode = kernelgen.generate(deepcopy(self.py_ast),
                                              self.funcvars)
            self.field_args = kernelgen.field_args
            loopgen = LoopGenerator(grid, ptype)
            self.ccode = loopgen.generate(self.funcname,
                                          self.field_args,
                                          kernel_ccode)

    def compile(self, compiler):
        """ Writes kernel code to file and compiles it."""
        with open(self.src_file, 'w') as f:
            f.write(self.ccode)
        compiler.compile(self.src_file, self.lib_file, self.log_file)

    def load_lib(self):
        self._lib = npct.load_library(self.lib_file, '.')
        self._function = self._lib.particle_loop

    def execute(self, pset, timesteps, time, dt):
        if self.ptype.uses_jit:
            fargs = [byref(f.ctypes_struct) for f in self.field_args.values()]
            particle_data = pset._particle_data.ctypes.data_as(c_void_p)
            self._function(c_int(len(pset)), particle_data, c_int(timesteps),
                           c_double(time), c_float(dt), *fargs)
        else:
            for _ in range(timesteps):
                for p in pset.particles:
                    self.pyfunc(p, pset.grid, time, dt)
                time += dt

    def merge(self, kernel):
        funcname = self.funcname + kernel.funcname
        func_ast = FunctionDef(name=funcname, args=self.py_ast.args,
                               body=self.py_ast.body + kernel.py_ast.body,
                               decorator_list=[], lineno=1, col_offset=0)
        return Kernel(self.grid, self.ptype, pyfunc=None, funcname=funcname,
                      py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars)

    def __add__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.grid, self.ptype, pyfunc=kernel)
        return self.merge(kernel)

    def __radd__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.grid, self.ptype, pyfunc=kernel)
        return kernel.merge(self)
<|endoftext|>