<commit_before>#!/usr/bin/env python
"""runner [-v|-q] [-c config] taskdir"""
import os
import time
import subprocess
import copy
import itertools
import sys
from ConfigParser import RawConfigParser

import logging
log = logging.getLogger(__name__)

def run_task(t, env, max_time):
    start = time.time()
    proc = subprocess.Popen(t, stdin=open(os.devnull, 'r'), env=env)
    while True:
        if proc.poll() is not None:
            break

        if max_time == 0:
            # if we've set to run forever, we can sleep for a lot longer
            # than 1 second.
            time.sleep(20)
        elif time.time() - start > max_time:
            # Try killing it
            log.warn("exceeded max_time; killing")
            proc.terminate()
            return "RETRY"
        else:
            time.sleep(1)

    rv = proc.wait()
    if rv == 0:
        return "OK"
    elif rv == 2:
        return "HALT"
    else:
        return "RETRY"

class Config(object):
    sleep_time = 1
    max_tries = 5
    max_time = 600
    halt_task = "halt.sh"
    filename = None
    options = None

    def load_config(self, filename):
        self.filename = filename
        self.options = RawConfigParser()
        # The default optionxform converts option names to lower case. We want
        # to preserve case, so change the transform function to just return the
        # str value
        self.options.optionxform = str
        if not self.options.read([filename]):
            log.warn("Couldn't load %s", filename)
            self.options = None
            return
        if self.options.has_option('runner', 'include_dir'):
            # reload the object including files in config.d
            config_dir = self.options.get('runner', 'include_dir')
            configs = [os.path.join(config_dir, c) for c in list_directory(config_dir)]
            if not self.options.read([filename] + configs):
                log.warn("Couldn't load %s", config_dir)
                return

        if self.options.has_option('runner', 'sleep_time'):
            self.sleep_time = self.options.getint('runner', 'sleep_time')
        if self.options.has_option('runner', 'max_tries'):
            self.max_tries = self.options.getint('runner', 'max_tries')
        if self.options.has_option('runner', 'max_time'):
            self.max_time = self.options.getint('runner', 'max_time')
        if self.options.has_option('runner', 'halt_task'):
            self.halt_task = self.options.get('runner', 'halt_task')

    def get(self, section, option):
        if self.options and self.options.has_option(section, option):
            return self.options.get(section, option)
        return None

    def get_env(self):
        retval = {}
        if self.options and self.options.has_section('env'):
            for option, value in self.options.items('env'):
                retval[str(option)] = str(value)
        if self.filename:
            retval['RUNNER_CONFIG_CMD'] = '{python} {runner} -c {configfile}'.format(
                python=sys.executable,
                runner=os.path.abspath(sys.argv[0]),
                configfile=os.path.abspath(self.filename),
            )
        return retval

    def get_task_config(self, taskname):
        """Returns a dict of the config options for [taskname]
        or an empty dict otherwise
        """
        if self.options is not None and self.options.has_section(taskname):
            return dict(self.options.items(taskname))
        return {}

def list_directory(dirname):
    # List the files in the directory, and sort them
    files = os.listdir(dirname)
    # Filter out files with leading .
    return [f for f in files if f[0] != '.']

def get_task_name(taskfile):
    """
    >>> get_task_name('3-buildbot.py')
    'buildbot'
    >>> get_task_name('buildbot.py')
    'buildbot'
    >>> get_task_name('buildbot')
    'buildbot'
    """
    task_no_prefix = ''.join(taskfile.split('-')[1:])
    taskname = task_no_prefix if task_no_prefix != '' else taskfile

    task_no_suffix = ''.join(taskname.split('.')[0:-1])
    taskname = task_no_suffix if task_no_suffix != '' else taskname

    return taskname

class CycleError(Exception):
    pass

class DependencyDoesNotExistError(Exception):
    pass

class TaskGraph(object):
    def __init__(self, taskconfigs):
        self._nodes = {}
        for task in taskconfigs:
            self._nodes[task.name] = task

        self._refresh()

        missing = self._missing_tasks()
        if missing:
            raise DependencyDoesNotExistError("The following are depended on "
                                              "but do not exist: "
                                              + ", ".join(missing))

    def _refresh(self):
        """Refreshes the pointers between graph nodes"""
        for node in self._nodes.values():
            for sd in node.stated_dependencies:
                if sd in self._nodes and sd not in node.dependencies:
                    node.dependencies.add(self._nodes[sd])

    def _missing_tasks(self):
        """List of missing tasks (ie. unfulfilled dependencies)"""
        lst = [node._missing_dependencies() for node in self._nodes.values()]
        return set(itertools.chain.from_iterable(lst))

    def sequential_ordering(self):
        """Topological sort, ignores parallelisation possibilities
        Algorithm is Kahn (1962),
        http://en.wikipedia.org/wiki/Topological_sorting#Algorithms
        """

        to_ret = []
        graph = copy.deepcopy(self._nodes.values())
        no_inc_edges = self._start_nodes(graph)

        while no_inc_edges:
            n = no_inc_edges.pop()
            to_ret.append(n.name)
            for m in self._nodes_with_edges_from(graph, n):
                self._remove_edge(graph, n, m)
                if not set(self._nodes_with_edges_to(graph, m)) - set([n]):
                    no_inc_edges.add(m)

        if self._has_edges(graph):
            # we've got a cycle in our graph!
            raise CycleError("Graph of task dependencies has cycles")

        to_ret.reverse()  # because we point TO our dependents
        return to_ret

    @classmethod
    def _start_nodes(cls, graph):
        """Returns the nodes in the graph with no dependencies"""
        return {n for n in graph
                if not cls._nodes_with_edges_to(graph, n)}

    @staticmethod
    def _has_edges(graph):
        """Returns True if there are any edges remaining in the graph,
        False otherwise"""

        for node in graph:
            if node.dependencies:
                return True
        return False

    @staticmethod
    def _nodes_with_edges_to(graph, n):
        """Returns the list of task nodes which depend on n.

        More generally this means it returns the list of nodes which have
        directed edges pointed to n.
        """
        return {m for m in graph if m is not n and n in m.dependencies}

    @staticmethod
    def _nodes_with_edges_from(graph, n):
        """Returns the list of task nodes which n depends on

        More generally this means it returns the list of nodes which n points
        to
        """
        return copy.copy(n.dependencies)

    @staticmethod
    def _remove_edge(graph, n, m):
        """Remove the edge from n to m"""
        if m not in n.dependencies:
            return

        n.dependencies.remove(m)

    def __str__(self):
        return ", ".join(map(str, self._nodes.values()))

class TaskConfig(object):
    def __init__(self, name, dependencies):
        self.name = name
        self.stated_dependencies = set(dependencies) - set([name])
        self.dependencies = set()

    @classmethod
    def fromtuple(cls, pair):
        return cls(pair[0], pair[1])

    @classmethod
    def fromdict(cls, mapping):
        return cls(mapping['name'], mapping['dependencies'])

    def _missing_dependencies(self):
        return self.stated_dependencies - {d.name for d in self.dependencies}

    def __str__(self):
        return "({}, {}, {})".format(self.name,
                                     self.stated_dependencies,
                                     self.dependencies)

def process_taskdir(config, dirname):
    tasks = list_directory(dirname)
    # Filter out the halting task
    if config.halt_task in tasks:
        tasks.remove(config.halt_task)

    # Get a list of a TaskConfig objects mapping task to their dependencies
    taskconfigs = []
    for t in tasks:
        deps = config.get(get_task_name(t), 'depends_on')
        if deps is not None:
            taskconfigs.append(TaskConfig(t, map(str.strip, deps.split(','))))
        else:
            taskconfigs.append(TaskConfig(t, []))

    tg = TaskGraph(taskconfigs)  # construct the dependency graph
    task_list = tg.sequential_ordering()  # get a topologically sorted order

    log.debug("tasks: %s", task_list)

    env = os.environ.copy()
    new_env = config.get_env()
    log.debug("Updating env with %s", new_env)
    env.update(new_env)

    default_config = {
        "max_time": config.max_time,
        "max_tries": config.max_tries,
        "sleep_time": config.sleep_time
    }

    for try_num in range(1, config.max_tries + 1):
        for t in task_list:
            # Get the portion of a task's config that can override default_config
            task_config = config.get_task_config(get_task_name(t))
            task_config = {k: int(v) for k, v in task_config.items() if k in default_config}

            # do the override
            for k, v in default_config.items():
                if k not in task_config:
                    task_config[k] = v

            log.debug("%s: starting (max time %is)", t, task_config['max_time'])
            r = run_task(os.path.join(dirname, t), env, max_time=task_config['max_time'])
            log.debug("%s: %s", t, r)

            if r == "OK":
                continue
            elif r == "RETRY":
                # No point in sleeping if we're on our last try
                if try_num == task_config['max_tries']:
                    log.warn("maximum attempts reached")
                    # TODO: halt here too?
                    return False
                # Sleep and try again
                log.debug("sleeping for %i", task_config['sleep_time'])
                time.sleep(task_config['sleep_time'])
                break
            elif r == "HALT":
                # stop/halt/reboot?
                log.info("halting")
                run_task(os.path.join(dirname, config.halt_task), env, max_time=task_config['max_time'])
                return False
        else:
            log.debug("all tasks completed!")
            return True

def make_argument_parser():
    import argparse
    parser = argparse.ArgumentParser(__doc__)
    parser.set_defaults(
        loglevel=logging.INFO,
    )
    parser.add_argument("-q", "--quiet", dest="loglevel", action="store_const", const=logging.WARN, help="quiet")
    parser.add_argument("-v", "--verbose", dest="loglevel", action="store_const", const=logging.DEBUG, help="verbose")
    parser.add_argument("-c", "--config", dest="config_file")
    parser.add_argument("-g", "--get", dest="get", help="get configuration value")
    parser.add_argument("-n", "--times", dest="times", type=int, help="run this many times (default is forever)")
    parser.add_argument("taskdir", help="task directory", nargs="?")

    return parser

def runner(config, taskdir, times):
    """Runs tasks in the taskdir up to `times` number of times

    times can be None to run forever
    """
    t = 0
    while True:
        t += 1
        if times and t > times:
            break
        log.info("iteration %i", t)
        if not process_taskdir(config, taskdir):
            exit(1)

def main():
    parser = make_argument_parser()
    args = parser.parse_args()
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=args.loglevel)

    config = Config()
    if args.config_file:
        config.load_config(args.config_file)

    if args.get:
        log.debug("getting %s", args.get)
        section, option = args.get.split(".", 1)
        v = config.get(section, option)
        if v is not None:
            print v
        exit(0)
    elif not args.taskdir:
        parser.error("taskdir required")

    if not os.path.exists(args.taskdir):
        log.error("%s doesn't exist", args.taskdir)
        exit(1)

    runner(config, args.taskdir, args.times)

if __name__ == '__main__':
    main()
<commit_msg>Bug 1055794 - Allows explicit setting of an interpreter for running tasks<commit_after>#!/usr/bin/env python
"""runner [-v|-q] [-c config] taskdir"""
import os
import time
import shlex
import subprocess
import copy
import itertools
import sys
from ConfigParser import RawConfigParser

import logging
log = logging.getLogger(__name__)

def run_task(t, env, max_time):
    start = time.time()
    proc = subprocess.Popen(t, stdin=open(os.devnull, 'r'), env=env)
    while True:
        if proc.poll() is not None:
            break

        if max_time == 0:
            # if we've set to run forever, we can sleep for a lot longer
            # than 1 second.
            time.sleep(20)
        elif time.time() - start > max_time:
            # Try killing it
            log.warn("exceeded max_time; killing")
            proc.terminate()
            return "RETRY"
        else:
            time.sleep(1)

    rv = proc.wait()
    if rv == 0:
        return "OK"
    elif rv == 2:
        return "HALT"
    else:
        return "RETRY"

class Config(object):
    sleep_time = 1
    max_tries = 5
    max_time = 600
    halt_task = "halt.sh"
    task_interpreter = None
    filename = None
    options = None

    def load_config(self, filename):
        self.filename = filename
        self.options = RawConfigParser()
        # The default optionxform converts option names to lower case. We want
        # to preserve case, so change the transform function to just return the
        # str value
        self.options.optionxform = str
        if not self.options.read([filename]):
            log.warn("Couldn't load %s", filename)
            self.options = None
            return
        if self.options.has_option('runner', 'include_dir'):
            # reload the object including files in config.d
            config_dir = self.options.get('runner', 'include_dir')
            configs = [os.path.join(config_dir, c) for c in list_directory(config_dir)]
            if not self.options.read([filename] + configs):
                log.warn("Couldn't load %s", config_dir)
                return

        if self.options.has_option('runner', 'sleep_time'):
            self.sleep_time = self.options.getint('runner', 'sleep_time')
        if self.options.has_option('runner', 'max_tries'):
            self.max_tries = self.options.getint('runner', 'max_tries')
        if self.options.has_option('runner', 'max_time'):
            self.max_time = self.options.getint('runner', 'max_time')
        if self.options.has_option('runner', 'halt_task'):
            self.halt_task = self.options.get('runner', 'halt_task')
        if self.options.has_option('runner', 'task_interpreter'):
            self.task_interpreter = self.options.get('runner', 'task_interpreter')

    def get(self, section, option):
        if self.options and self.options.has_option(section, option):
            return self.options.get(section, option)
        return None

    def get_env(self):
        retval = {}
        if self.options and self.options.has_section('env'):
            for option, value in self.options.items('env'):
                retval[str(option)] = str(value)
        if self.filename:
            retval['RUNNER_CONFIG_CMD'] = '{python} {runner} -c {configfile}'.format(
                python=sys.executable,
                runner=os.path.abspath(sys.argv[0]),
                configfile=os.path.abspath(self.filename),
            )
        return retval

    def get_task_config(self, taskname):
        """Returns a dict of the config options for [taskname]
        or an empty dict otherwise
        """
        if self.options is not None and self.options.has_section(taskname):
            return dict(self.options.items(taskname))
        return {}

def list_directory(dirname):
    # List the files in the directory, and sort them
    files = os.listdir(dirname)
    # Filter out files with leading .
    return [f for f in files if f[0] != '.']

def get_task_name(taskfile):
    """
    >>> get_task_name('3-buildbot.py')
    'buildbot'
    >>> get_task_name('buildbot.py')
    'buildbot'
    >>> get_task_name('buildbot')
    'buildbot'
    """
    task_no_prefix = ''.join(taskfile.split('-')[1:])
    taskname = task_no_prefix if task_no_prefix != '' else taskfile

    task_no_suffix = ''.join(taskname.split('.')[0:-1])
    taskname = task_no_suffix if task_no_suffix != '' else taskname

    return taskname

class CycleError(Exception):
    pass

class DependencyDoesNotExistError(Exception):
    pass

class TaskGraph(object):
    def __init__(self, taskconfigs):
        self._nodes = {}
        for task in taskconfigs:
            self._nodes[task.name] = task

        self._refresh()

        missing = self._missing_tasks()
        if missing:
            raise DependencyDoesNotExistError("The following are depended on "
                                              "but do not exist: "
                                              + ", ".join(missing))

    def _refresh(self):
        """Refreshes the pointers between graph nodes"""
        for node in self._nodes.values():
            for sd in node.stated_dependencies:
                if sd in self._nodes and sd not in node.dependencies:
                    node.dependencies.add(self._nodes[sd])

    def _missing_tasks(self):
        """List of missing tasks (ie. unfulfilled dependencies)"""
        lst = [node._missing_dependencies() for node in self._nodes.values()]
        return set(itertools.chain.from_iterable(lst))

    def sequential_ordering(self):
        """Topological sort, ignores parallelisation possibilities
        Algorithm is Kahn (1962),
        http://en.wikipedia.org/wiki/Topological_sorting#Algorithms
        """

        to_ret = []
        graph = copy.deepcopy(self._nodes.values())
        no_inc_edges = self._start_nodes(graph)

        while no_inc_edges:
            n = no_inc_edges.pop()
            to_ret.append(n.name)
            for m in self._nodes_with_edges_from(graph, n):
                self._remove_edge(graph, n, m)
                if not set(self._nodes_with_edges_to(graph, m)) - set([n]):
                    no_inc_edges.add(m)

        if self._has_edges(graph):
            # we've got a cycle in our graph!
            raise CycleError("Graph of task dependencies has cycles")

        to_ret.reverse()  # because we point TO our dependents
        return to_ret

    @classmethod
    def _start_nodes(cls, graph):
        """Returns the nodes in the graph with no dependencies"""
        return {n for n in graph
                if not cls._nodes_with_edges_to(graph, n)}

    @staticmethod
    def _has_edges(graph):
        """Returns True if there are any edges remaining in the graph,
        False otherwise"""

        for node in graph:
            if node.dependencies:
                return True
        return False

    @staticmethod
    def _nodes_with_edges_to(graph, n):
        """Returns the list of task nodes which depend on n.

        More generally this means it returns the list of nodes which have
        directed edges pointed to n.
        """
        return {m for m in graph if m is not n and n in m.dependencies}

    @staticmethod
    def _nodes_with_edges_from(graph, n):
        """Returns the list of task nodes which n depends on

        More generally this means it returns the list of nodes which n points
        to
        """
        return copy.copy(n.dependencies)

    @staticmethod
    def _remove_edge(graph, n, m):
        """Remove the edge from n to m"""
        if m not in n.dependencies:
            return

        n.dependencies.remove(m)

    def __str__(self):
        return ", ".join(map(str, self._nodes.values()))

class TaskConfig(object):
    def __init__(self, name, dependencies):
        self.name = name
        self.stated_dependencies = set(dependencies) - set([name])
        self.dependencies = set()

    @classmethod
    def fromtuple(cls, pair):
        return cls(pair[0], pair[1])

    @classmethod
    def fromdict(cls, mapping):
        return cls(mapping['name'], mapping['dependencies'])

    def _missing_dependencies(self):
        return self.stated_dependencies - {d.name for d in self.dependencies}

    def __str__(self):
        return "({}, {}, {})".format(self.name,
                                     self.stated_dependencies,
                                     self.dependencies)

def process_taskdir(config, dirname):
    tasks = list_directory(dirname)
    # Filter out the halting task
    if config.halt_task in tasks:
        tasks.remove(config.halt_task)

    # Get a list of a TaskConfig objects mapping task to their dependencies
    taskconfigs = []
    for t in tasks:
        deps = config.get(get_task_name(t), 'depends_on')
        if deps is not None:
            taskconfigs.append(TaskConfig(t, map(str.strip, deps.split(','))))
        else:
            taskconfigs.append(TaskConfig(t, []))

    tg = TaskGraph(taskconfigs)  # construct the dependency graph
    task_list = tg.sequential_ordering()  # get a topologically sorted order

    log.debug("tasks: %s", task_list)

    env = os.environ.copy()
    new_env = config.get_env()
    log.debug("Updating env with %s", new_env)
    env.update(new_env)

    default_config = {
        "max_time": int(config.max_time),
        "max_tries": int(config.max_tries),
        "sleep_time": int(config.sleep_time),
        "task_interpreter": config.task_interpreter,
    }

    for try_num in range(1, config.max_tries + 1):
        for t in task_list:
            # Get the portion of a task's config that can override default_config
            task_config = config.get_task_config(get_task_name(t))
            task_config = {k: v for k, v in task_config.items() if k in default_config}

            # do the override
            for k, v in default_config.items():
                if k not in task_config:
                    task_config[k] = v

            log.debug("%s: starting (max time %is)", t, task_config['max_time'])
            task_cmd = os.path.join(dirname, t)
            if task_config['task_interpreter'] is not None:
                log.debug("%s: running with interpreter (%s)", t, task_config['task_interpreter'])
                # using shlex affords the ability to pass arguments to the
                # interpreter as well (i.e. bash -c)
                task_cmd = shlex.split("{} '{}'".format(task_config['task_interpreter'], task_cmd))
            r = run_task(task_cmd, env, max_time=task_config['max_time'])
            log.debug("%s: %s", t, r)

            if r == "OK":
                continue
            elif r == "RETRY":
                # No point in sleeping if we're on our last try
                if try_num == task_config['max_tries']:
                    log.warn("maximum attempts reached")
                    # TODO: halt here too?
                    return False
                # Sleep and try again
                log.debug("sleeping for %i", task_config['sleep_time'])
                time.sleep(task_config['sleep_time'])
                break
            elif r == "HALT":
                # stop/halt/reboot?
                log.info("halting")
                halt_cmd = os.path.join(dirname, config.halt_task)
                if config.task_interpreter is not None:
                    # if a global task interpreter was set, it should apply
                    # here as well
                    halt_cmd = shlex.split("{} '{}'".format(config.task_interpreter, halt_cmd))
                run_task(halt_cmd, env, max_time=task_config['max_time'])
                return False
        else:
            log.debug("all tasks completed!")
            return True

def make_argument_parser():
    import argparse
    parser = argparse.ArgumentParser(__doc__)
    parser.set_defaults(
        loglevel=logging.INFO,
    )
    parser.add_argument("-q", "--quiet", dest="loglevel", action="store_const", const=logging.WARN, help="quiet")
    parser.add_argument("-v", "--verbose", dest="loglevel", action="store_const", const=logging.DEBUG, help="verbose")
    parser.add_argument("-c", "--config", dest="config_file")
    parser.add_argument("-g", "--get", dest="get", help="get configuration value")
    parser.add_argument("-n", "--times", dest="times", type=int, help="run this many times (default is forever)")
    parser.add_argument("taskdir", help="task directory", nargs="?")

    return parser

def runner(config, taskdir, times):
    """Runs tasks in the taskdir up to `times` number of times

    times can be None to run forever
    """
    t = 0
    while True:
        t += 1
        if times and t > times:
            break
        log.info("iteration %i", t)
        if not process_taskdir(config, taskdir):
            exit(1)

def main():
    parser = make_argument_parser()
    args = parser.parse_args()
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=args.loglevel)

    config = Config()
    if args.config_file:
        config.load_config(args.config_file)

    if args.get:
        log.debug("getting %s", args.get)
        section, option = args.get.split(".", 1)
        v = config.get(section, option)
        if v is not None:
            print v
        exit(0)
    elif not args.taskdir:
        parser.error("taskdir required")

    if not os.path.exists(args.taskdir):
        log.error("%s doesn't exist", args.taskdir)
        exit(1)

    runner(config, args.taskdir, args.times)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from re import search, IGNORECASE

"Some baseline features for testing the classifier."

def make_searcher(pattern, field='article_snippet', flags=IGNORECASE):
	"""Returns a feature template function that defines the feature
	   'has_pattern_' + pattern, which is present whenever the datum
	   contains that pattern.  Default by case-insensitive.
	"""
	def result(datum):
		if search(pattern, datum.__dict__[field], flags):
			return [field + '_has_pattern_' + pattern]
		else:
			return []
	return result

def words_in(field):
	"""Makes a feature for each word in the given field."""
	def result(datum):
		wordset = set()
		def mkfeature(string): return field + '_has_word_' + string
		for word in datum.__dict__[field].split():
			wordset.add(word.lower())
		return map(mkfeature, wordset) # TODO: The lambda is a stub.
	return result

feature_templates = map(make_searcher, [
	# These seem likely to show up in articles about disease:
	'estimated',
	'exposure',
	'percent',
	'population',
	'virus',
	'disease',
	'outbreak',
	'report',
	'prevention',
	'control',
	'disease',
	'confirmed',
	'diagnosed',
	'diagnosis',
	'associated',
	'emergency',
	'season', # TODO: Maybe exclude 'hurricane'.
	'dea(d|th)',
	'authorities',
	# 'sick', 		# Doesn't distinguish.
	# '\sill\s', 		# Doesn't distinguish.
	# 'issued.*advisory', 	# Too sparse.

	# Things I'm trying out because of their counts:
	'president',
	'<img', # Has a picture.  Good for 'unrelated'.

	# These seem like they would show up in non-outbreak or non-disease articles:
	'rival',
	'sport',
	'champ',
	'protest',
	'hurricane',
	'economy',
	'wedding', # Good but sparse.
	'research',
	'prosecut'] # as in 'prosecution' and 'prosecuted'.
	) + [
	# TODO: URL features.
	make_searcher('(epi|pan)demic', 'title'),
	make_searcher('H\dN\d', field='article_snippet', flags=0), # names of flu subtypes, like H1N1
	words_in('title'),
	# TODO: Maybe strip away non-word symbols like in "(ap)" and "prosecutors:".
	# TODO: Maybe get the word STEMS in the title, not the words.
	# TODO: Replace all numbers with a single digit or something.

	]
<commit_msg>Added named_entities feature and reorganized.<commit_after>from re import search, IGNORECASE
import nltk

"Features for the classifier."

def make_searcher(pattern, field='article_snippet', flags=IGNORECASE):
	"""Returns a feature template function that defines the feature
	   'has_pattern_' + pattern, which is present whenever the datum
	   contains that pattern.  Defaults to case-insensitive.
	"""
	def result(datum):
		if search(pattern, datum.__dict__[field], flags):
			return [field + '_has_pattern_' + pattern]
		else:
			return []
	return result

def words_in(field):
	"""Makes a feature for each word in the given field."""
	def result(datum):
		wordset = set()
		def mkfeature(string): return field + '_has_word_' + string
		for word in datum.__dict__[field].split():
			wordset.add(word.lower())
		return map(mkfeature, wordset)
	return result

# Since NER is slow, only do it for the first cutoff_for_ner characters of the
# article_snippet:
cutoff_for_ner = 1000
# Again, since NER is slow, only do it for the first
# per_sentence_cutoff_for_ner characters of each sentence:
per_sentence_cutoff_for_ner = 10

def named_entities(datum):
	result = []
	for sent in nltk.sent_tokenize(datum.article_snippet[:cutoff_for_ner]):
		for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent[:per_sentence_cutoff_for_ner]))):
			if hasattr(chunk, 'node'):
				result.append('article_snippet_has_entity_' + chunk.node + '_' +  '_'.join(c[0].lower() for c in chunk.leaves()))
	return result

matcher_features = map(make_searcher, [
	# These seem likely to show up in articles about disease:
	'estimated',
	'exposure',
	'percent',
	'population',
	'virus',
	'disease',
	'outbreak',
	'report',
	'prevention',
	'control',
	'disease',
	'confirmed',
	'diagnosed',
	'diagnosis',
	'associated',
	'emergency',
	'season', # TODO: Maybe exclude 'hurricane'.
	'dea(d|th)',
	'authorities',
	# 'sick', 		# Doesn't distinguish.
	# '\sill\s', 		# Doesn't distinguish.
	# 'issued.*advisory', 	# Too sparse.

	# Things I'm trying out because of their counts:
	'president',
	'<img', # Has a picture.  Good for 'unrelated'.

	# These seem like they would show up in non-outbreak or non-disease articles:
	'rival',
	'sport',
	'champ',
	'protest',
	'hurricane',
	'economy',
	'wedding', # Good but sparse.
	'linux',
	'research',
	'prosecut'] # as in 'prosecution' and 'prosecuted'.
	)

other_features = [
	# TODO: URL features.
	named_entities,
	make_searcher('(epi|pan)demic', 'title'),
	make_searcher('H\dN\d', field='article_snippet', flags=0), # names of flu subtypes, like H1N1
	words_in('title'),
	# TODO: Maybe strip away non-word symbols like in "(ap)" and "prosecutors:".
	# TODO: Maybe get the word STEMS in the title, not the words.
	# TODO: Replace all numbers with a single digit or something.

	]

feature_templates = matcher_features + other_features
<|endoftext|>


<commit_before>import typing

from django.contrib.auth.models import User

from accounts.db_facade import fetch_accounts_for_user
from projects.permission_models import ProjectAgentRole
from projects.project_models import Project

class FilterOption(typing.NamedTuple):
    key: str
    label: str
    description: str

FILTER_OPTIONS = (
    FilterOption('created', 'Created by me', 'Projects that were created by you.'),
    FilterOption('account', 'Account projects', 'Projects owned by Accounts to which you belong.'),
    FilterOption('shared', 'Shared with me', 'Projects to which you or one of your Teams have access.'),
    FilterOption('public', 'Public', 'Projects that are publicly visible.')
)

FILTER_LOOKUP = {f.key: f for f in FILTER_OPTIONS}

class ProjectFetchResult(typing.NamedTuple):
    filter: FilterOption
    projects: typing.Iterable[Project]
    available_results: int
    remaining_results: typing.Optional[int]

def get_projects(user: User, filter_key: typing.Optional[str],
                 limit: typing.Optional[int] = None) -> ProjectFetchResult:
    if not user.is_authenticated:
        projects = Project.objects.filter(public=True)
        filter_key = 'public'
    else:
        if filter_key == 'account':
            accounts = fetch_accounts_for_user(user)
            projects = Project.objects.filter(account__in=accounts)
        elif filter_key == 'shared':
            project_ids = ProjectAgentRole.filter_with_user_teams(user=user).exclude(
                project__creator=user).values_list('project', flat=True)
            projects = Project.objects.filter(pk__in=project_ids)
            # all_projects = map(lambda par: par.project, roles)
            # projects = filter(lambda proj: proj.creator != user, all_projects)
        elif filter_key == 'public':
            projects = Project.objects.filter(public=True)
        else:
            filter_key = 'created'
            projects = Project.objects.filter(creator=user)

    available_results = projects.count()

    if limit:
        remaining_results = available_results - limit
        projects = projects[:limit]
    else:
        remaining_results = None

    return ProjectFetchResult(FILTER_LOOKUP[filter_key], projects, available_results, remaining_results)
<commit_msg>fix: Fixed display of negative projects available<commit_after>import typing

from django.contrib.auth.models import User

from accounts.db_facade import fetch_accounts_for_user
from projects.permission_models import ProjectAgentRole
from projects.project_models import Project

class FilterOption(typing.NamedTuple):
    key: str
    label: str
    description: str

FILTER_OPTIONS = (
    FilterOption('created', 'Created by me', 'Projects that were created by you.'),
    FilterOption('account', 'Account projects', 'Projects owned by Accounts to which you belong.'),
    FilterOption('shared', 'Shared with me', 'Projects to which you or one of your Teams have access.'),
    FilterOption('public', 'Public', 'Projects that are publicly visible.')
)

FILTER_LOOKUP = {f.key: f for f in FILTER_OPTIONS}

class ProjectFetchResult(typing.NamedTuple):
    filter: FilterOption
    projects: typing.Iterable[Project]
    available_results: int
    remaining_results: typing.Optional[int]

def get_projects(user: User, filter_key: typing.Optional[str],
                 limit: typing.Optional[int] = None) -> ProjectFetchResult:
    if not user.is_authenticated:
        projects = Project.objects.filter(public=True)
        filter_key = 'public'
    else:
        if filter_key == 'account':
            accounts = fetch_accounts_for_user(user)
            projects = Project.objects.filter(account__in=accounts)
        elif filter_key == 'shared':
            project_ids = ProjectAgentRole.filter_with_user_teams(user=user).exclude(
                project__creator=user).values_list('project', flat=True)
            projects = Project.objects.filter(pk__in=project_ids)
            # all_projects = map(lambda par: par.project, roles)
            # projects = filter(lambda proj: proj.creator != user, all_projects)
        elif filter_key == 'public':
            projects = Project.objects.filter(public=True)
        else:
            filter_key = 'created'
            projects = Project.objects.filter(creator=user)

    available_results = projects.count()

    if limit:
        remaining_results = max(available_results - limit, 0)
        projects = projects[:limit]
    else:
        remaining_results = None

    return ProjectFetchResult(FILTER_LOOKUP[filter_key], projects, available_results, remaining_results)
<|endoftext|>


<commit_before>"""
Some code to manage the Amazon Certificate Service.
"""
import logging

import boto3
import botocore
from .resource_helper import throttled_call

logger = logging.getLogger(__name__)

CERT_SUMMARY_LIST_KEY = 'CertificateSummaryList'
CERT_ARN_KEY = 'CertificateArn'
CERT_ALT_NAMES_KEY = 'SubjectAlternativeNames'
DOMAIN_NAME_KEY = 'DomainName'
CERT_KEY = 'Certificate'
CERT_ISSUED_DATE_KEY = 'IssuedAt'

class DiscoACM(object):
    """
    A class to manage the Amazon Certificate Service

    """
    WILDCARD_PREFIX = "*."

    def __init__(self, connection=None):
        self._acm = connection

    def _in_domain(self, domain, dns_name):
        """
        Returns whether the host is in the ACM certificate domain
        Only supports top level domain wildcard matching
        e.g. *.blah.com will match, but not *.*.blah.com
        It would be good to use a standard library here if becomes available.
        """
        if not (domain and dns_name):
            return False

        # sanity check left-most label
        name, subdomain = dns_name.split('.', 1)
        if not name or name == '*':
            logger.error('Left-most label "%s" of "%s" is invalid', name, dns_name)
            return False

        # exact match
        if dns_name == domain:
            return True

        # handle wildcard cert domains
        if domain.startswith(self.WILDCARD_PREFIX):
            domain = domain[len(self.WILDCARD_PREFIX):]
        return subdomain == domain

    @property
    def acm(self):
        """
        Lazily creates ACM connection

        Return None if service does not exist in current region
        """
        if not self._acm:
            try:
                self._acm = boto3.client('acm')
            except Exception:
                logger.warning("ACM service does not exist in current region")
                return None
        return self._acm

    def get_certificate_arn(self, dns_name):
        """Returns a Certificate ARN from the Amazon Certificate Service given the DNS name"""
        if not self.acm:
            return None

        try:
            cert_summary = throttled_call(self.acm.list_certificates)[CERT_SUMMARY_LIST_KEY]

            cert_matches = []
            for cert in cert_summary:
                response = throttled_call(self.acm.describe_certificate, CertificateArn=cert[CERT_ARN_KEY])
                cert = response[CERT_KEY]
                # search for the dns_name in the cert's alternative domain names (includes the main one)
                for alt_name in cert[CERT_ALT_NAMES_KEY]:
                    if self._in_domain(alt_name, dns_name):
                        # a tuple of the matching cert domain name and the actual cert
                        cert_matches.append((alt_name, cert))

            # sort the matches by name, longest first
            cert_matches.sort(key=lambda cert: (len(cert[0]), cert[1][CERT_ISSUED_DATE_KEY]), reverse=True)

            if not cert_matches:
                logger.warning("No ACM certificates returned for %s", dns_name)
                return None

            # pick the cert for the longest matched domain name
            return cert_matches[0][1][CERT_ARN_KEY]
        except (botocore.exceptions.EndpointConnectionError,
                botocore.vendored.requests.exceptions.ConnectionError):
            # some versions of botocore(1.3.26) will try to connect to acm even if outside us-east-1
            logger.exception("Unable to get ACM certificate")
            return None
<commit_msg>Fix comment<commit_after>"""
Some code to manage the Amazon Certificate Service.
"""
import logging

import boto3
import botocore
from .resource_helper import throttled_call

logger = logging.getLogger(__name__)

CERT_SUMMARY_LIST_KEY = 'CertificateSummaryList'
CERT_ARN_KEY = 'CertificateArn'
CERT_ALT_NAMES_KEY = 'SubjectAlternativeNames'
DOMAIN_NAME_KEY = 'DomainName'
CERT_KEY = 'Certificate'
CERT_ISSUED_DATE_KEY = 'IssuedAt'

class DiscoACM(object):
    """
    A class to manage the Amazon Certificate Service

    """
    WILDCARD_PREFIX = "*."

    def __init__(self, connection=None):
        self._acm = connection

    def _in_domain(self, domain, dns_name):
        """
        Returns whether the host is in the ACM certificate domain
        Only supports top level domain wildcard matching
        e.g. *.blah.com will match, but not *.*.blah.com
        It would be good to use a standard library here if becomes available.
        """
        if not (domain and dns_name):
            return False

        # sanity check left-most label
        name, subdomain = dns_name.split('.', 1)
        if not name or name == '*':
            logger.error('Left-most label "%s" of "%s" is invalid', name, dns_name)
            return False

        # exact match
        if dns_name == domain:
            return True

        # handle wildcard cert domains
        if domain.startswith(self.WILDCARD_PREFIX):
            domain = domain[len(self.WILDCARD_PREFIX):]
        return subdomain == domain

    @property
    def acm(self):
        """
        Lazily creates ACM connection

        Return None if service does not exist in current region
        """
        if not self._acm:
            try:
                self._acm = boto3.client('acm')
            except Exception:
                logger.warning("ACM service does not exist in current region")
                return None
        return self._acm

    def get_certificate_arn(self, dns_name):
        """Returns a Certificate ARN from the Amazon Certificate Service given the DNS name"""
        if not self.acm:
            return None

        try:
            cert_summary = throttled_call(self.acm.list_certificates)[CERT_SUMMARY_LIST_KEY]

            cert_matches = []
            for cert in cert_summary:
                response = throttled_call(self.acm.describe_certificate, CertificateArn=cert[CERT_ARN_KEY])
                cert = response[CERT_KEY]
                # search for the dns_name in the cert's alternative domain names (includes the main one)
                for alt_name in cert[CERT_ALT_NAMES_KEY]:
                    if self._in_domain(alt_name, dns_name):
                        # a tuple of the matching cert domain name and the actual cert
                        cert_matches.append((alt_name, cert))

            # sort the matches by name, longest first, then by issued date, newest first
            cert_matches.sort(key=lambda cert: (len(cert[0]), cert[1][CERT_ISSUED_DATE_KEY]), reverse=True)

            if not cert_matches:
                logger.warning("No ACM certificates returned for %s", dns_name)
                return None

            # pick the cert for the longest matched domain name
            return cert_matches[0][1][CERT_ARN_KEY]
        except (botocore.exceptions.EndpointConnectionError,
                botocore.vendored.requests.exceptions.ConnectionError):
            # some versions of botocore(1.3.26) will try to connect to acm even if outside us-east-1
            logger.exception("Unable to get ACM certificate")
            return None
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2015-present Rapptz

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

from __future__ import annotations

from typing import Any, Callable, Deque, Dict, Optional, TYPE_CHECKING
from discord.enums import Enum
import time
import asyncio
from collections import deque

from ...abc import PrivateChannel
from .errors import MaxConcurrencyReached
from discord.app_commands import Cooldown as Cooldown

if TYPE_CHECKING:
    from typing_extensions import Self

    from ...message import Message

__all__ = (
    'BucketType',
    'Cooldown',
    'CooldownMapping',
    'DynamicCooldownMapping',
    'MaxConcurrency',
)

class BucketType(Enum):
    default = 0
    user = 1
    guild = 2
    channel = 3
    member = 4
    category = 5
    role = 6

    def get_key(self, msg: Message) -> Any:
        if self is BucketType.user:
            return msg.author.id
        elif self is BucketType.guild:
            return (msg.guild or msg.author).id
        elif self is BucketType.channel:
            return msg.channel.id
        elif self is BucketType.member:
            return ((msg.guild and msg.guild.id), msg.author.id)
        elif self is BucketType.category:
            return (msg.channel.category or msg.channel).id  # type: ignore
        elif self is BucketType.role:
            # we return the channel id of a private-channel as there are only roles in guilds
            # and that yields the same result as for a guild with only the @everyone role
            # NOTE: PrivateChannel doesn't actually have an id attribute but we assume we are
            # receiving a DMChannel or GroupChannel which inherit from PrivateChannel and do
            return (msg.channel if isinstance(msg.channel, PrivateChannel) else msg.author.top_role).id  # type: ignore

    def __call__(self, msg: Message) -> Any:
        return self.get_key(msg)

class CooldownMapping:
    def __init__(
        self,
        original: Optional[Cooldown],
        type: Callable[[Message], Any],
    ) -> None:
        if not callable(type):
            raise TypeError('Cooldown type must be a BucketType or callable')

        self._cache: Dict[Any, Cooldown] = {}
        self._cooldown: Optional[Cooldown] = original
        self._type: Callable[[Message], Any] = type

    def copy(self) -> CooldownMapping:
        ret = CooldownMapping(self._cooldown, self._type)
        ret._cache = self._cache.copy()
        return ret

    @property
    def valid(self) -> bool:
        return self._cooldown is not None

    @property
    def type(self) -> Callable[[Message], Any]:
        return self._type

    @classmethod
    def from_cooldown(cls, rate: float, per: float, type: Callable[[Message], Any]) -> Self:
        return cls(Cooldown(rate, per), type)

    def _bucket_key(self, msg: Message) -> Any:
        return self._type(msg)

    def _verify_cache_integrity(self, current: Optional[float] = None) -> None:
        # we want to delete all cache objects that haven't been used
        # in a cooldown window. e.g. if we have a  command that has a
        # cooldown of 60s and it has not been used in 60s then that key should be deleted
        current = current or time.time()
        dead_keys = [k for k, v in self._cache.items() if current > v._last + v.per]
        for k in dead_keys:
            del self._cache[k]

    def create_bucket(self, message: Message) -> Cooldown:
        return self._cooldown.copy()  # type: ignore

    def get_bucket(self, message: Message, current: Optional[float] = None) -> Cooldown:
        if self._type is BucketType.default:
            return self._cooldown  # type: ignore

        self._verify_cache_integrity(current)
        key = self._bucket_key(message)
        if key not in self._cache:
            bucket = self.create_bucket(message)
            if bucket is not None:
                self._cache[key] = bucket
        else:
            bucket = self._cache[key]

        return bucket

    def update_rate_limit(self, message: Message, current: Optional[float] = None) -> Optional[float]:
        bucket = self.get_bucket(message, current)
        return bucket.update_rate_limit(current)

class DynamicCooldownMapping(CooldownMapping):
    def __init__(
        self,
        factory: Callable[[Message], Cooldown],
        type: Callable[[Message], Any],
    ) -> None:
        super().__init__(None, type)
        self._factory: Callable[[Message], Cooldown] = factory

    def copy(self) -> DynamicCooldownMapping:
        ret = DynamicCooldownMapping(self._factory, self._type)
        ret._cache = self._cache.copy()
        return ret

    @property
    def valid(self) -> bool:
        return True

    def create_bucket(self, message: Message) -> Cooldown:
        return self._factory(message)

class _Semaphore:
    """This class is a version of a semaphore.

    If you're wondering why asyncio.Semaphore isn't being used,
    it's because it doesn't expose the internal value. This internal
    value is necessary because I need to support both `wait=True` and
    `wait=False`.

    An asyncio.Queue could have been used to do this as well -- but it is
    not as inefficient since internally that uses two queues and is a bit
    overkill for what is basically a counter.
    """

    __slots__ = ('value', 'loop', '_waiters')

    def __init__(self, number: int) -> None:
        self.value: int = number
        self.loop: asyncio.AbstractEventLoop = asyncio.get_running_loop()
        self._waiters: Deque[asyncio.Future] = deque()

    def __repr__(self) -> str:
        return f'<_Semaphore value={self.value} waiters={len(self._waiters)}>'

    def locked(self) -> bool:
        return self.value == 0

    def is_active(self) -> bool:
        return len(self._waiters) > 0

    def wake_up(self) -> None:
        while self._waiters:
            future = self._waiters.popleft()
            if not future.done():
                future.set_result(None)
                return

    async def acquire(self, *, wait: bool = False) -> bool:
        if not wait and self.value <= 0:
            # signal that we're not acquiring
            return False

        while self.value <= 0:
            future = self.loop.create_future()
            self._waiters.append(future)
            try:
                await future
            except:
                future.cancel()
                if self.value > 0 and not future.cancelled():
                    self.wake_up()
                raise

        self.value -= 1
        return True

    def release(self) -> None:
        self.value += 1
        self.wake_up()

class MaxConcurrency:
    __slots__ = ('number', 'per', 'wait', '_mapping')

    def __init__(self, number: int, *, per: BucketType, wait: bool) -> None:
        self._mapping: Dict[Any, _Semaphore] = {}
        self.per: BucketType = per
        self.number: int = number
        self.wait: bool = wait

        if number <= 0:
            raise ValueError('max_concurrency \'number\' cannot be less than 1')

        if not isinstance(per, BucketType):
            raise TypeError(f'max_concurrency \'per\' must be of type BucketType not {type(per)!r}')

    def copy(self) -> Self:
        return self.__class__(self.number, per=self.per, wait=self.wait)

    def __repr__(self) -> str:
        return f'<MaxConcurrency per={self.per!r} number={self.number} wait={self.wait}>'

    def get_key(self, message: Message) -> Any:
        return self.per.get_key(message)

    async def acquire(self, message: Message) -> None:
        key = self.get_key(message)

        try:
            sem = self._mapping[key]
        except KeyError:
            self._mapping[key] = sem = _Semaphore(self.number)

        acquired = await sem.acquire(wait=self.wait)
        if not acquired:
            raise MaxConcurrencyReached(self.number, self.per)

    async def release(self, message: Message) -> None:
        # Technically there's no reason for this function to be async
        # But it might be more useful in the future
        key = self.get_key(message)

        try:
            sem = self._mapping[key]
        except KeyError:
            # ...? peculiar
            return
        else:
            sem.release()

        if sem.value >= self.number and not sem.is_active():
            del self._mapping[key]
<commit_msg>[commands] Allow DynamicCooldownMapping factory to return None<commit_after>"""
The MIT License (MIT)

Copyright (c) 2015-present Rapptz

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

from __future__ import annotations

from typing import Any, Callable, Deque, Dict, Optional, TYPE_CHECKING
from discord.enums import Enum
import time
import asyncio
from collections import deque

from ...abc import PrivateChannel
from .errors import MaxConcurrencyReached
from discord.app_commands import Cooldown as Cooldown

if TYPE_CHECKING:
    from typing_extensions import Self

    from ...message import Message

__all__ = (
    'BucketType',
    'Cooldown',
    'CooldownMapping',
    'DynamicCooldownMapping',
    'MaxConcurrency',
)

class BucketType(Enum):
    default = 0
    user = 1
    guild = 2
    channel = 3
    member = 4
    category = 5
    role = 6

    def get_key(self, msg: Message) -> Any:
        if self is BucketType.user:
            return msg.author.id
        elif self is BucketType.guild:
            return (msg.guild or msg.author).id
        elif self is BucketType.channel:
            return msg.channel.id
        elif self is BucketType.member:
            return ((msg.guild and msg.guild.id), msg.author.id)
        elif self is BucketType.category:
            return (msg.channel.category or msg.channel).id  # type: ignore
        elif self is BucketType.role:
            # we return the channel id of a private-channel as there are only roles in guilds
            # and that yields the same result as for a guild with only the @everyone role
            # NOTE: PrivateChannel doesn't actually have an id attribute but we assume we are
            # receiving a DMChannel or GroupChannel which inherit from PrivateChannel and do
            return (msg.channel if isinstance(msg.channel, PrivateChannel) else msg.author.top_role).id  # type: ignore

    def __call__(self, msg: Message) -> Any:
        return self.get_key(msg)

class CooldownMapping:
    def __init__(
        self,
        original: Optional[Cooldown],
        type: Callable[[Message], Any],
    ) -> None:
        if not callable(type):
            raise TypeError('Cooldown type must be a BucketType or callable')

        self._cache: Dict[Any, Cooldown] = {}
        self._cooldown: Optional[Cooldown] = original
        self._type: Callable[[Message], Any] = type

    def copy(self) -> CooldownMapping:
        ret = CooldownMapping(self._cooldown, self._type)
        ret._cache = self._cache.copy()
        return ret

    @property
    def valid(self) -> bool:
        return self._cooldown is not None

    @property
    def type(self) -> Callable[[Message], Any]:
        return self._type

    @classmethod
    def from_cooldown(cls, rate: float, per: float, type: Callable[[Message], Any]) -> Self:
        return cls(Cooldown(rate, per), type)

    def _bucket_key(self, msg: Message) -> Any:
        return self._type(msg)

    def _verify_cache_integrity(self, current: Optional[float] = None) -> None:
        # we want to delete all cache objects that haven't been used
        # in a cooldown window. e.g. if we have a  command that has a
        # cooldown of 60s and it has not been used in 60s then that key should be deleted
        current = current or time.time()
        dead_keys = [k for k, v in self._cache.items() if current > v._last + v.per]
        for k in dead_keys:
            del self._cache[k]

    def create_bucket(self, message: Message) -> Cooldown:
        return self._cooldown.copy()  # type: ignore

    def get_bucket(self, message: Message, current: Optional[float] = None) -> Cooldown:
        if self._type is BucketType.default:
            return self._cooldown  # type: ignore

        self._verify_cache_integrity(current)
        key = self._bucket_key(message)
        if key not in self._cache:
            bucket = self.create_bucket(message)
            if bucket is not None:
                self._cache[key] = bucket
        else:
            bucket = self._cache[key]

        return bucket

    def update_rate_limit(self, message: Message, current: Optional[float] = None) -> Optional[float]:
        bucket = self.get_bucket(message, current)
        return bucket.update_rate_limit(current)

class DynamicCooldownMapping(CooldownMapping):
    def __init__(
        self,
        factory: Callable[[Message], Optional[Cooldown]],
        type: Callable[[Message], Any],
    ) -> None:
        super().__init__(None, type)
        self._factory: Callable[[Message], Optional[Cooldown]] = factory

    def copy(self) -> DynamicCooldownMapping:
        ret = DynamicCooldownMapping(self._factory, self._type)
        ret._cache = self._cache.copy()
        return ret

    @property
    def valid(self) -> bool:
        return True

    def create_bucket(self, message: Message) -> Optional[Cooldown]:
        return self._factory(message)

class _Semaphore:
    """This class is a version of a semaphore.

    If you're wondering why asyncio.Semaphore isn't being used,
    it's because it doesn't expose the internal value. This internal
    value is necessary because I need to support both `wait=True` and
    `wait=False`.

    An asyncio.Queue could have been used to do this as well -- but it is
    not as inefficient since internally that uses two queues and is a bit
    overkill for what is basically a counter.
    """

    __slots__ = ('value', 'loop', '_waiters')

    def __init__(self, number: int) -> None:
        self.value: int = number
        self.loop: asyncio.AbstractEventLoop = asyncio.get_running_loop()
        self._waiters: Deque[asyncio.Future] = deque()

    def __repr__(self) -> str:
        return f'<_Semaphore value={self.value} waiters={len(self._waiters)}>'

    def locked(self) -> bool:
        return self.value == 0

    def is_active(self) -> bool:
        return len(self._waiters) > 0

    def wake_up(self) -> None:
        while self._waiters:
            future = self._waiters.popleft()
            if not future.done():
                future.set_result(None)
                return

    async def acquire(self, *, wait: bool = False) -> bool:
        if not wait and self.value <= 0:
            # signal that we're not acquiring
            return False

        while self.value <= 0:
            future = self.loop.create_future()
            self._waiters.append(future)
            try:
                await future
            except:
                future.cancel()
                if self.value > 0 and not future.cancelled():
                    self.wake_up()
                raise

        self.value -= 1
        return True

    def release(self) -> None:
        self.value += 1
        self.wake_up()

class MaxConcurrency:
    __slots__ = ('number', 'per', 'wait', '_mapping')

    def __init__(self, number: int, *, per: BucketType, wait: bool) -> None:
        self._mapping: Dict[Any, _Semaphore] = {}
        self.per: BucketType = per
        self.number: int = number
        self.wait: bool = wait

        if number <= 0:
            raise ValueError('max_concurrency \'number\' cannot be less than 1')

        if not isinstance(per, BucketType):
            raise TypeError(f'max_concurrency \'per\' must be of type BucketType not {type(per)!r}')

    def copy(self) -> Self:
        return self.__class__(self.number, per=self.per, wait=self.wait)

    def __repr__(self) -> str:
        return f'<MaxConcurrency per={self.per!r} number={self.number} wait={self.wait}>'

    def get_key(self, message: Message) -> Any:
        return self.per.get_key(message)

    async def acquire(self, message: Message) -> None:
        key = self.get_key(message)

        try:
            sem = self._mapping[key]
        except KeyError:
            self._mapping[key] = sem = _Semaphore(self.number)

        acquired = await sem.acquire(wait=self.wait)
        if not acquired:
            raise MaxConcurrencyReached(self.number, self.per)

    async def release(self, message: Message) -> None:
        # Technically there's no reason for this function to be async
        # But it might be more useful in the future
        key = self.get_key(message)

        try:
            sem = self._mapping[key]
        except KeyError:
            # ...? peculiar
            return
        else:
            sem.release()

        if sem.value >= self.number and not sem.is_active():
            del self._mapping[key]
<|endoftext|>


<commit_before>"""
Syncs yoda etcd proxy nodes with route53
"""
import argparse
import time
import random
import os

from boto.route53.record import ResourceRecordSets
from urllib3.exceptions import ReadTimeoutError
import yoda
import etcd
import boto

from discover import logger
from discover.util import init_shutdown_handler

def etcd_client(parsed_args):
    return etcd.Client(host=parsed_args.etcd_host, port=parsed_args.etcd_port)

def yoda_client(etcd_cl):
    return yoda.Client(etcd_cl=etcd_cl)

def job_lock(etcd_cl, etcd_base):
    lock_key = '%s/route53/_lock' % etcd_base
    try:
        node = etcd_cl.write(lock_key, True, ttl=60, prevExist=False)
        return node.modifiedIndex
    except KeyError:
        return None

def job_unlock(etcd_cl, etcd_base, lock):
    if lock:
        lock_key = '%s/route53/_lock' % etcd_base
        try:
            etcd_cl.delete(lock_key, prevIndex=lock)
        except KeyError:
            logger.warn('Failed to unlock route53 job %d', lock)
            return False
    else:
        return False

class ApplyLock:
    def __init__(self, etcd_cl, etcd_base):
        self.etcd_cl = etcd_cl
        self.etcd_base = etcd_base
        self.lock = job_lock(etcd_cl, etcd_base)

    def __enter__(self):
        return self.lock

    def __exit__(self, type, value, traceback):
        job_unlock(self.etcd_cl, self.etcd_base, self.lock)

def set_sync_index(etcd_cl, etcd_base, sync_index):
    sync_key = '%s/route53/sync-index/' % etcd_base
    etcd_cl.write(sync_key, sync_index)

def get_sync_index(etcd_cl, etcd_base):
    sync_key = '%s/route53/sync-index/' % etcd_base
    try:
        return int(etcd_cl.read(sync_key, consistent=True).value)
    except KeyError:
        pass

def remove_sync_index(etcd_cl, etcd_base):
    sync_key = '%s/route53/sync-index/' % etcd_base
    try:
        etcd_cl.delete(sync_key)
    except KeyError:
        pass

def update_route53(node_name, value, parsed_args):
    conn = boto.connect_route53(
        aws_access_key_id=parsed_args.access_key_id,
        aws_secret_access_key=parsed_args.secret_access_key)
    records = conn.get_all_rrsets(parsed_args.zone_id,
                                  type=parsed_args.record_type,
                                  name=parsed_args.dns_record,
                                  identifier=node_name, maxitems=1)
    changes = ResourceRecordSets(conn, parsed_args.zone_id)
    change = changes.add_change(
        'UPSERT', name=parsed_args.dns_record,
        type=parsed_args.record_type, ttl=parsed_args.dns_ttl,
        identifier=node_name, weight=parsed_args.record_weight)
    change.add_value(value)
    if len(records) > 0 and records[0].name == parsed_args.dns_record + '.' \
            and records[0].identifier == node_name:
        if len(records[0].resource_records) > 0 and \
                records[0].resource_records[0] == value:
            logger.info('No change in Record %s. Skipping...',
                        parsed_args.dns_record)
        else:
            logger.info('Modifying record: %s:%s with identifier:%s',
                        parsed_args.dns_record, value, node_name)
            changes.commit()
    else:
        logger.info('Adding record: %s:%s with identifier:%s',
                    parsed_args.dns_record, value, node_name)
        changes.commit()

def delete_route53(node_name, parsed_args):
    conn = boto.connect_route53(
        aws_access_key_id=parsed_args.access_key_id,
        aws_secret_access_key=parsed_args.secret_access_key)
    records = conn.get_all_rrsets(parsed_args.zone_id,
                                  name=parsed_args.dns_record,
                                  identifier=node_name, maxitems=1)
    changes = ResourceRecordSets(conn, parsed_args.zone_id)
    change = changes.add_change(
        'DELETE', name=parsed_args.dns_record, identifier=node_name,
        type=parsed_args.record_type, ttl=records[0].ttl,
        weight=records[0].weight)
    change.add_value(records[0].resource_records[0])
    if len(records) > 0 and records[0].name == parsed_args.dns_record + '.' \
            and records[0].identifier == node_name:

        logger.info('Deleting record: %s with identifier %s',
                    parsed_args.dns_record, node_name)
        changes.commit()
    else:
        logger.info('Skip delete  for non existing record: %s with '
                    'identifier:%s', parsed_args.dns_record, node_name)

def route53_sync(parsed_args, poll=None):
    """
    Syncs yoda proxy nodes with route53 based on parsed arguments.
    :param parsed_args: Parsed arguments
    :param should_poll: Lambda or function that evaluates if polling should
        continue. Added for ease of unit testing
    :type should_poll: function
    :return: None
    """
    logger.info('Started sync for yoda proxy nodes with route53')
    etcd_cl = etcd_client(parsed_args)
    proxy_nodes_key = '%s/%s' % (parsed_args.etcd_base, 'proxy-nodes')
    etcd_args = {
        'key': proxy_nodes_key,
        'recursive': True,
        'wait': True,
        'timeout': 300
    }
    poll = poll or (lambda: True)
    while poll():
        sync_index = get_sync_index(etcd_cl, parsed_args.etcd_base)
        if sync_index:
            etcd_args['waitIndex'] = int(sync_index) + 1
        elif etcd_args.get('waitIndex'):
            del(etcd_args['waitIndex'])
        logger.info('Watching for changes for %s', proxy_nodes_key)
        try:
            result = etcd_cl.read(**etcd_args)
        except ReadTimeoutError:
            logger.info('Did not receive any changes. Will restart polling...')
            continue
        except etcd.EtcdException as etcd_error:
            etcd_msg = str(etcd_error).lower()
            if etcd_msg.startswith(
                    'the event in requested index is outdated and cleared'):
                logger.warn('Wait Index is stale. Removing the waitIndex')
                remove_sync_index(etcd_cl, parsed_args.etcd_base)
                # Adding a sleep to prevent high cpu during infinite looping.
                time.sleep(5)
                continue
            elif etcd_msg.startswith('unable to decode server response:'):
                # Let the job retry after 5s. This seems intermittent issue.
                time.sleep(5)
                continue
            else:
                raise

        with ApplyLock(etcd_cl, parsed_args.etcd_base) as lock:
            if lock:
                sync_index = get_sync_index(etcd_cl, parsed_args.etcd_base)
                if sync_index and sync_index >= result.modifiedIndex:
                    # Change is already processed by another node. Skip
                    continue
                else:
                    node_name = os.path.basename(result.key)
                    if result.action not in ('delete', 'expire'):
                        update_route53(node_name, result.value, parsed_args)
                    else:
                        delete_route53(node_name, parsed_args)
                    # Persist the new sync index so that other nodes can poll
                    # from sync_index+1
                    set_sync_index(etcd_cl, parsed_args.etcd_base,
                                   result.modifiedIndex)
                    etcd_args['waitIndex'] = result.modifiedIndex + 1
                    # Sleep for 5s before releasing lock. AWS API Limit
                    time.sleep(5)
            else:
                # Sleep for 5s before next poll.
                logger.info('Job locked by another node. '
                            'Will sleep and poll again')
                time.sleep(random.randint(1, 10))
                # Set default value of waitIndex to current modified index so
                # that the result is not lost in-case currently processing node
                # fails
                etcd_args['waitIndex'] = result.modifiedIndex

        if lock:
            # This node will not poll for next 11-15 seconds.
            # To give other nodes a fair chance
            time.sleep(random.randint(11, 15))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Syncs route53 nodes with etcd')

    parser.add_argument(
        '--etcd-host', metavar='<ETCD_HOST>',
        default=os.environ.get('ETCD_HOST', '172.17.42.1'),
        help='Docker URL (defaults to 172.17.42.1)')
    parser.add_argument(
        '--etcd-port', metavar='<ETCD_PORT>',
        default=os.environ.get('ETCD_PORT', '4001'), type=int,
        help='Docker URL (defaults to 4001)')
    parser.add_argument(
        '--etcd-base', metavar='<ETCD_BASE>',
        default=os.environ.get('ETCD_BASE', '/yoda'),
        help='Yoda base key (defaults to /yoda)')
    parser.add_argument(
        '--check-ports', metavar='<CHECK_PORTS>',
        default=os.environ.get('CHECK_PORTS', ''),
        help='Comma separated ports to be used for status check. '
             'Defaults to empty')
    parser.add_argument(
        '--access-key-id', metavar='<AWS_ACCESS_KEY_ID>',
        default=os.environ.get('AWS_ACCESS_KEY_ID'),
        help='AWS Access Key id for connecting to route53 API.')
    parser.add_argument(
        '--secret-access-key', metavar='<AWS_SECRET_ACCESS_KEY>',
        default=os.environ.get('AWS_SECRET_ACCESS_KEY'),
        help='AWS Secret Access Key for connecting to route53 API.')
    parser.add_argument(
        '--poll-interval', metavar='<POLL_INTERVAL">',
        help='Poll interval in seconds', type=int,
        default=os.environ.get('POLL_INTERVAL', '180'))
    parser.add_argument(
        '--dns-ttl', metavar='<ROUTE53_DNS_RECORD_TTL>', type=int,
        help='DNS Record TTL in seconds. Defaults to 60.',
        default=os.environ.get('ROUTE53_DNS_RECORD_TTL', '60'))
    parser.add_argument(
        '--record-weight', metavar='<ROUTE53_DNS_RECORD_WEIGHT>', type=int,
        help='Weight of the dns record for this node. Defaults to 1',
        default=os.environ.get('ROUTE53_DNS_RECORD_WEIGHT', '1'))
    parser.add_argument(
        '--record-type', metavar='<ROUTE53_DNS_RECORD_TYPE>',
        help='Type of dns record (CNAME, A). Defaults to CNAME',
        default=os.environ.get('ROUTE53_DNS_RECORD_TYPE', 'CNAME'))
    parser.add_argument(
        'zone_id', metavar='<ROUTE53_HOSTED_ZONE_ID>',
        help='Hosted zone id for route53.')
    parser.add_argument(
        'dns_record', metavar='<ROUTE53_DNS_RECORD>',
        help='DNS Record e.g. (mycluster.abc.com)')

    parsed_args = parser.parse_args()
    parsed_args.check_ports = parsed_args.check_ports.split(',')
    init_shutdown_handler()
    route53_sync(parsed_args)
<commit_msg>Fix typo in command help<commit_after>"""
Syncs yoda etcd proxy nodes with route53
"""
import argparse
import time
import random
import os

from boto.route53.record import ResourceRecordSets
from urllib3.exceptions import ReadTimeoutError
import yoda
import etcd
import boto

from discover import logger
from discover.util import init_shutdown_handler

def etcd_client(parsed_args):
    return etcd.Client(host=parsed_args.etcd_host, port=parsed_args.etcd_port)

def yoda_client(etcd_cl):
    return yoda.Client(etcd_cl=etcd_cl)

def job_lock(etcd_cl, etcd_base):
    lock_key = '%s/route53/_lock' % etcd_base
    try:
        node = etcd_cl.write(lock_key, True, ttl=60, prevExist=False)
        return node.modifiedIndex
    except KeyError:
        return None

def job_unlock(etcd_cl, etcd_base, lock):
    if lock:
        lock_key = '%s/route53/_lock' % etcd_base
        try:
            etcd_cl.delete(lock_key, prevIndex=lock)
        except KeyError:
            logger.warn('Failed to unlock route53 job %d', lock)
            return False
    else:
        return False

class ApplyLock:
    def __init__(self, etcd_cl, etcd_base):
        self.etcd_cl = etcd_cl
        self.etcd_base = etcd_base
        self.lock = job_lock(etcd_cl, etcd_base)

    def __enter__(self):
        return self.lock

    def __exit__(self, type, value, traceback):
        job_unlock(self.etcd_cl, self.etcd_base, self.lock)

def set_sync_index(etcd_cl, etcd_base, sync_index):
    sync_key = '%s/route53/sync-index/' % etcd_base
    etcd_cl.write(sync_key, sync_index)

def get_sync_index(etcd_cl, etcd_base):
    sync_key = '%s/route53/sync-index/' % etcd_base
    try:
        return int(etcd_cl.read(sync_key, consistent=True).value)
    except KeyError:
        pass

def remove_sync_index(etcd_cl, etcd_base):
    sync_key = '%s/route53/sync-index/' % etcd_base
    try:
        etcd_cl.delete(sync_key)
    except KeyError:
        pass

def update_route53(node_name, value, parsed_args):
    conn = boto.connect_route53(
        aws_access_key_id=parsed_args.access_key_id,
        aws_secret_access_key=parsed_args.secret_access_key)
    records = conn.get_all_rrsets(parsed_args.zone_id,
                                  type=parsed_args.record_type,
                                  name=parsed_args.dns_record,
                                  identifier=node_name, maxitems=1)
    changes = ResourceRecordSets(conn, parsed_args.zone_id)
    change = changes.add_change(
        'UPSERT', name=parsed_args.dns_record,
        type=parsed_args.record_type, ttl=parsed_args.dns_ttl,
        identifier=node_name, weight=parsed_args.record_weight)
    change.add_value(value)
    if len(records) > 0 and records[0].name == parsed_args.dns_record + '.' \
            and records[0].identifier == node_name:
        if len(records[0].resource_records) > 0 and \
                records[0].resource_records[0] == value:
            logger.info('No change in Record %s. Skipping...',
                        parsed_args.dns_record)
        else:
            logger.info('Modifying record: %s:%s with identifier:%s',
                        parsed_args.dns_record, value, node_name)
            changes.commit()
    else:
        logger.info('Adding record: %s:%s with identifier:%s',
                    parsed_args.dns_record, value, node_name)
        changes.commit()

def delete_route53(node_name, parsed_args):
    conn = boto.connect_route53(
        aws_access_key_id=parsed_args.access_key_id,
        aws_secret_access_key=parsed_args.secret_access_key)
    records = conn.get_all_rrsets(parsed_args.zone_id,
                                  name=parsed_args.dns_record,
                                  identifier=node_name, maxitems=1)
    changes = ResourceRecordSets(conn, parsed_args.zone_id)
    change = changes.add_change(
        'DELETE', name=parsed_args.dns_record, identifier=node_name,
        type=parsed_args.record_type, ttl=records[0].ttl,
        weight=records[0].weight)
    change.add_value(records[0].resource_records[0])
    if len(records) > 0 and records[0].name == parsed_args.dns_record + '.' \
            and records[0].identifier == node_name:

        logger.info('Deleting record: %s with identifier %s',
                    parsed_args.dns_record, node_name)
        changes.commit()
    else:
        logger.info('Skip delete  for non existing record: %s with '
                    'identifier:%s', parsed_args.dns_record, node_name)

def route53_sync(parsed_args, poll=None):
    """
    Syncs yoda proxy nodes with route53 based on parsed arguments.
    :param parsed_args: Parsed arguments
    :param should_poll: Lambda or function that evaluates if polling should
        continue. Added for ease of unit testing
    :type should_poll: function
    :return: None
    """
    logger.info('Started sync for yoda proxy nodes with route53')
    etcd_cl = etcd_client(parsed_args)
    proxy_nodes_key = '%s/%s' % (parsed_args.etcd_base, 'proxy-nodes')
    etcd_args = {
        'key': proxy_nodes_key,
        'recursive': True,
        'wait': True,
        'timeout': 300
    }
    poll = poll or (lambda: True)
    while poll():
        sync_index = get_sync_index(etcd_cl, parsed_args.etcd_base)
        if sync_index:
            etcd_args['waitIndex'] = int(sync_index) + 1
        elif etcd_args.get('waitIndex'):
            del(etcd_args['waitIndex'])
        logger.info('Watching for changes for %s', proxy_nodes_key)
        try:
            result = etcd_cl.read(**etcd_args)
        except ReadTimeoutError:
            logger.info('Did not receive any changes. Will restart polling...')
            continue
        except etcd.EtcdException as etcd_error:
            etcd_msg = str(etcd_error).lower()
            if etcd_msg.startswith(
                    'the event in requested index is outdated and cleared'):
                logger.warn('Wait Index is stale. Removing the waitIndex')
                remove_sync_index(etcd_cl, parsed_args.etcd_base)
                # Adding a sleep to prevent high cpu during infinite looping.
                time.sleep(5)
                continue
            elif etcd_msg.startswith('unable to decode server response:'):
                # Let the job retry after 5s. This seems intermittent issue.
                time.sleep(5)
                continue
            else:
                raise

        with ApplyLock(etcd_cl, parsed_args.etcd_base) as lock:
            if lock:
                sync_index = get_sync_index(etcd_cl, parsed_args.etcd_base)
                if sync_index and sync_index >= result.modifiedIndex:
                    # Change is already processed by another node. Skip
                    continue
                else:
                    node_name = os.path.basename(result.key)
                    if result.action not in ('delete', 'expire'):
                        update_route53(node_name, result.value, parsed_args)
                    else:
                        delete_route53(node_name, parsed_args)
                    # Persist the new sync index so that other nodes can poll
                    # from sync_index+1
                    set_sync_index(etcd_cl, parsed_args.etcd_base,
                                   result.modifiedIndex)
                    etcd_args['waitIndex'] = result.modifiedIndex + 1
                    # Sleep for 5s before releasing lock. AWS API Limit
                    time.sleep(5)
            else:
                # Sleep for 5s before next poll.
                logger.info('Job locked by another node. '
                            'Will sleep and poll again')
                time.sleep(random.randint(1, 10))
                # Set default value of waitIndex to current modified index so
                # that the result is not lost in-case currently processing node
                # fails
                etcd_args['waitIndex'] = result.modifiedIndex

        if lock:
            # This node will not poll for next 11-15 seconds.
            # To give other nodes a fair chance
            time.sleep(random.randint(11, 15))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Syncs route53 nodes with etcd')

    parser.add_argument(
        '--etcd-host', metavar='<ETCD_HOST>',
        default=os.environ.get('ETCD_HOST', '172.17.42.1'),
        help='Etcd Host (defaults to 172.17.42.1)')
    parser.add_argument(
        '--etcd-port', metavar='<ETCD_PORT>',
        default=os.environ.get('ETCD_PORT', '4001'), type=int,
        help='Etcd Port (defaults to 4001)')
    parser.add_argument(
        '--etcd-base', metavar='<ETCD_BASE>',
        default=os.environ.get('ETCD_BASE', '/yoda'),
        help='Yoda base key (defaults to /yoda)')
    parser.add_argument(
        '--check-ports', metavar='<CHECK_PORTS>',
        default=os.environ.get('CHECK_PORTS', ''),
        help='Comma separated ports to be used for status check. '
             'Defaults to empty')
    parser.add_argument(
        '--access-key-id', metavar='<AWS_ACCESS_KEY_ID>',
        default=os.environ.get('AWS_ACCESS_KEY_ID'),
        help='AWS Access Key id for connecting to route53 API.')
    parser.add_argument(
        '--secret-access-key', metavar='<AWS_SECRET_ACCESS_KEY>',
        default=os.environ.get('AWS_SECRET_ACCESS_KEY'),
        help='AWS Secret Access Key for connecting to route53 API.')
    parser.add_argument(
        '--poll-interval', metavar='<POLL_INTERVAL">',
        help='Poll interval in seconds', type=int,
        default=os.environ.get('POLL_INTERVAL', '180'))
    parser.add_argument(
        '--dns-ttl', metavar='<ROUTE53_DNS_RECORD_TTL>', type=int,
        help='DNS Record TTL in seconds. Defaults to 60.',
        default=os.environ.get('ROUTE53_DNS_RECORD_TTL', '60'))
    parser.add_argument(
        '--record-weight', metavar='<ROUTE53_DNS_RECORD_WEIGHT>', type=int,
        help='Weight of the dns record for this node. Defaults to 1',
        default=os.environ.get('ROUTE53_DNS_RECORD_WEIGHT', '1'))
    parser.add_argument(
        '--record-type', metavar='<ROUTE53_DNS_RECORD_TYPE>',
        help='Type of dns record (CNAME, A). Defaults to CNAME',
        default=os.environ.get('ROUTE53_DNS_RECORD_TYPE', 'CNAME'))
    parser.add_argument(
        'zone_id', metavar='<ROUTE53_HOSTED_ZONE_ID>',
        help='Hosted zone id for route53.')
    parser.add_argument(
        'dns_record', metavar='<ROUTE53_DNS_RECORD>',
        help='DNS Record e.g. (mycluster.abc.com)')

    parsed_args = parser.parse_args()
    parsed_args.check_ports = parsed_args.check_ports.split(',')
    init_shutdown_handler()
    route53_sync(parsed_args)
<|endoftext|>


<commit_before>from django.http import Http404
from django.shortcuts import get_object_or_404, render
from django.views.generic import DetailView

from django_jobvite import models as jobvite_models

import utils
from careers.careers.forms import PositionFilterForm
from careers.django_workable import models as workable_models

def home(request):
    return render(request, 'careers/home.html')

def listings(request):
    return render(request, 'careers/listings.html', {
        'positions': utils.get_all_positions(
            order_by=lambda x: '{0} {1}'.format(x.category.name, x.title)),
        'form': PositionFilterForm(request.GET or None),
    })

def position(request, job_id=None):
    # Cannot use __exact instead of __contains due to MySQL collation
    # which does not allow case sensitive matching.
    position = get_object_or_404(jobvite_models.Position, job_id__contains=job_id)
    positions = utils.get_all_positions(filters={'category__name': position.category.name},
                                        order_by=lambda x: x.title)

    # Add applicant source param for jobvite
    position.apply_url += '&s=PDN'

    return render(request, 'careers/position.html', {
        'position': position,
        'positions': positions,
    })

class WorkablePositionDetailView(DetailView):
    context_object_name = 'position'
    model = workable_models.Position
    template_name = 'careers/position.html'
    slug_field = 'shortcode'
    slug_url_kwarg = 'shortcode'

    def get_context_data(self, **kwargs):
        context = super(WorkablePositionDetailView, self).get_context_data(**kwargs)
        context['positions'] = utils.get_all_positions(
            filters={'category__name': context['position'].category.name},
            order_by=lambda x: x.title)
        return context
<commit_msg>Unicode fix.<commit_after>from django.http import Http404
from django.shortcuts import get_object_or_404, render
from django.views.generic import DetailView

from django_jobvite import models as jobvite_models

import utils
from careers.careers.forms import PositionFilterForm
from careers.django_workable import models as workable_models

def home(request):
    return render(request, 'careers/home.html')

def listings(request):
    return render(request, 'careers/listings.html', {
        'positions': utils.get_all_positions(
            order_by=lambda x: u'{0} {1}'.format(x.category.name, x.title)),
        'form': PositionFilterForm(request.GET or None),
    })

def position(request, job_id=None):
    # Cannot use __exact instead of __contains due to MySQL collation
    # which does not allow case sensitive matching.
    position = get_object_or_404(jobvite_models.Position, job_id__contains=job_id)
    positions = utils.get_all_positions(filters={'category__name': position.category.name},
                                        order_by=lambda x: x.title)

    # Add applicant source param for jobvite
    position.apply_url += '&s=PDN'

    return render(request, 'careers/position.html', {
        'position': position,
        'positions': positions,
    })

class WorkablePositionDetailView(DetailView):
    context_object_name = 'position'
    model = workable_models.Position
    template_name = 'careers/position.html'
    slug_field = 'shortcode'
    slug_url_kwarg = 'shortcode'

    def get_context_data(self, **kwargs):
        context = super(WorkablePositionDetailView, self).get_context_data(**kwargs)
        context['positions'] = utils.get_all_positions(
            filters={'category__name': context['position'].category.name},
            order_by=lambda x: x.title)
        return context
<|endoftext|>


<commit_before># Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE

import functools
import re

import frappe
from frappe import _

def load_address_and_contact(doc, key=None):
	"""Loads address list and contact list in `__onload`"""
	from frappe.contacts.doctype.address.address import get_address_display, get_condensed_address

	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Address"],
	]
	address_list = frappe.get_list("Address", filters=filters, fields=["*"])

	address_list = [a.update({"display": get_address_display(a)}) for a in address_list]

	address_list = sorted(
		address_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_address - b.is_primary_address))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("addr_list", address_list)

	contact_list = []
	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Contact"],
	]
	contact_list = frappe.get_list("Contact", filters=filters, fields=["*"])

	for contact in contact_list:
		contact["email_ids"] = frappe.get_all(
			"Contact Email",
			filters={"parenttype": "Contact", "parent": contact.name, "is_primary": 0},
			fields=["email_id"],
		)

		contact["phone_nos"] = frappe.get_all(
			"Contact Phone",
			filters={
				"parenttype": "Contact",
				"parent": contact.name,
				"is_primary_phone": 0,
				"is_primary_mobile_no": 0,
			},
			fields=["phone"],
		)

		if contact.address:
			address = frappe.get_doc("Address", contact.address)
			contact["address"] = get_condensed_address(address)

	contact_list = sorted(
		contact_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_contact - b.is_primary_contact))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("contact_list", contact_list)

def has_permission(doc, ptype, user):
	links = get_permitted_and_not_permitted_links(doc.doctype)
	if not links.get("not_permitted_links"):
		# optimization: don't determine permissions based on link fields
		return True

	# True if any one is True or all are empty
	names = []
	for df in links.get("permitted_links") + links.get("not_permitted_links"):
		doctype = df.options
		name = doc.get(df.fieldname)
		names.append(name)

		if name and frappe.has_permission(doctype, ptype, doc=name):
			return True

	if not any(names):
		return True
	return False

def get_permission_query_conditions_for_contact(user):
	return get_permission_query_conditions("Contact")

def get_permission_query_conditions_for_address(user):
	return get_permission_query_conditions("Address")

def get_permission_query_conditions(doctype):
	links = get_permitted_and_not_permitted_links(doctype)

	if not links.get("not_permitted_links"):
		# when everything is permitted, don't add additional condition
		return ""

	elif not links.get("permitted_links"):
		conditions = []

		# when everything is not permitted
		for df in links.get("not_permitted_links"):
			# like ifnull(customer, '')='' and ifnull(supplier, '')=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " and ".join(conditions) + " )"

	else:
		conditions = []

		for df in links.get("permitted_links"):
			# like ifnull(customer, '')!='' or ifnull(supplier, '')!=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " or ".join(conditions) + " )"

def get_permitted_and_not_permitted_links(doctype):
	permitted_links = []
	not_permitted_links = []

	meta = frappe.get_meta(doctype)
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	for df in meta.get_link_fields():
		if df.options not in ("Customer", "Supplier", "Company", "Sales Partner"):
			continue

		if df.options in allowed_doctypes:
			permitted_links.append(df)
		else:
			not_permitted_links.append(df)

	return {"permitted_links": permitted_links, "not_permitted_links": not_permitted_links}

def delete_contact_and_address(doctype, docname):
	for parenttype in ("Contact", "Address"):
		items = frappe.db.sql_list(
			"""select parent from `tabDynamic Link`
			where parenttype=%s and link_doctype=%s and link_name=%s""",
			(parenttype, doctype, docname),
		)

		for name in items:
			doc = frappe.get_doc(parenttype, name)
			if len(doc.links) == 1:
				doc.delete()

@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters):
	if not txt:
		txt = ""

	doctypes = frappe.db.get_all(
		"DocField", filters=filters, fields=["parent"], distinct=True, as_list=True
	)

	doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE))

	filters.update({"dt": ("not in", [d[0] for d in doctypes])})

	_doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True)

	_doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)])

	all_doctypes = [d[0] for d in doctypes + _doctypes]
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes)))
	valid_doctypes = [[doctype] for doctype in valid_doctypes]

	return valid_doctypes

def set_link_title(doc):
	if not doc.links:
		return
	for link in doc.links:
		if not link.link_title:
			linked_doc = frappe.get_doc(link.link_doctype, link.link_name)
			link.link_title = linked_doc.get_title() or link.link_name
<commit_msg>fix: change ordering to creation (#16863)<commit_after># Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE

import functools
import re

import frappe
from frappe import _

def load_address_and_contact(doc, key=None):
	"""Loads address list and contact list in `__onload`"""
	from frappe.contacts.doctype.address.address import get_address_display, get_condensed_address

	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Address"],
	]
	address_list = frappe.get_list("Address", filters=filters, fields=["*"], order_by="creation asc")

	address_list = [a.update({"display": get_address_display(a)}) for a in address_list]

	address_list = sorted(
		address_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_address - b.is_primary_address))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("addr_list", address_list)

	contact_list = []
	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Contact"],
	]
	contact_list = frappe.get_list("Contact", filters=filters, fields=["*"])

	for contact in contact_list:
		contact["email_ids"] = frappe.get_all(
			"Contact Email",
			filters={"parenttype": "Contact", "parent": contact.name, "is_primary": 0},
			fields=["email_id"],
		)

		contact["phone_nos"] = frappe.get_all(
			"Contact Phone",
			filters={
				"parenttype": "Contact",
				"parent": contact.name,
				"is_primary_phone": 0,
				"is_primary_mobile_no": 0,
			},
			fields=["phone"],
		)

		if contact.address:
			address = frappe.get_doc("Address", contact.address)
			contact["address"] = get_condensed_address(address)

	contact_list = sorted(
		contact_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_contact - b.is_primary_contact))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("contact_list", contact_list)

def has_permission(doc, ptype, user):
	links = get_permitted_and_not_permitted_links(doc.doctype)
	if not links.get("not_permitted_links"):
		# optimization: don't determine permissions based on link fields
		return True

	# True if any one is True or all are empty
	names = []
	for df in links.get("permitted_links") + links.get("not_permitted_links"):
		doctype = df.options
		name = doc.get(df.fieldname)
		names.append(name)

		if name and frappe.has_permission(doctype, ptype, doc=name):
			return True

	if not any(names):
		return True
	return False

def get_permission_query_conditions_for_contact(user):
	return get_permission_query_conditions("Contact")

def get_permission_query_conditions_for_address(user):
	return get_permission_query_conditions("Address")

def get_permission_query_conditions(doctype):
	links = get_permitted_and_not_permitted_links(doctype)

	if not links.get("not_permitted_links"):
		# when everything is permitted, don't add additional condition
		return ""

	elif not links.get("permitted_links"):
		conditions = []

		# when everything is not permitted
		for df in links.get("not_permitted_links"):
			# like ifnull(customer, '')='' and ifnull(supplier, '')=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " and ".join(conditions) + " )"

	else:
		conditions = []

		for df in links.get("permitted_links"):
			# like ifnull(customer, '')!='' or ifnull(supplier, '')!=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " or ".join(conditions) + " )"

def get_permitted_and_not_permitted_links(doctype):
	permitted_links = []
	not_permitted_links = []

	meta = frappe.get_meta(doctype)
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	for df in meta.get_link_fields():
		if df.options not in ("Customer", "Supplier", "Company", "Sales Partner"):
			continue

		if df.options in allowed_doctypes:
			permitted_links.append(df)
		else:
			not_permitted_links.append(df)

	return {"permitted_links": permitted_links, "not_permitted_links": not_permitted_links}

def delete_contact_and_address(doctype, docname):
	for parenttype in ("Contact", "Address"):
		items = frappe.db.sql_list(
			"""select parent from `tabDynamic Link`
			where parenttype=%s and link_doctype=%s and link_name=%s""",
			(parenttype, doctype, docname),
		)

		for name in items:
			doc = frappe.get_doc(parenttype, name)
			if len(doc.links) == 1:
				doc.delete()

@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters):
	if not txt:
		txt = ""

	doctypes = frappe.db.get_all(
		"DocField", filters=filters, fields=["parent"], distinct=True, as_list=True
	)

	doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE))

	filters.update({"dt": ("not in", [d[0] for d in doctypes])})

	_doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True)

	_doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)])

	all_doctypes = [d[0] for d in doctypes + _doctypes]
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes)))
	valid_doctypes = [[doctype] for doctype in valid_doctypes]

	return valid_doctypes

def set_link_title(doc):
	if not doc.links:
		return
	for link in doc.links:
		if not link.link_title:
			linked_doc = frappe.get_doc(link.link_doctype, link.link_name)
			link.link_title = linked_doc.get_title() or link.link_name
<|endoftext|>