<commit_before># coding=utf-8
# Copyright 2022 The Balloon Learning Environment Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Example running Acme's distributed QrDQN in JAX on the BLE."""

import functools
from typing import Any, Dict

from absl import app
from absl import flags
from acme import core
from acme import environment_loop
from acme import specs
from acme.jax import networks as networks_lib
from acme.jax.layouts import distributed_layout
from acme.utils import counting
from acme.utils import loggers
from acme.utils import lp_utils
from balloon_learning_environment import acme_utils
import launchpad as lp

FLAGS = flags.FLAGS
flags.DEFINE_string('base_dir', '~/acme',
                    'Directory where to store statistics/images.')
flags.DEFINE_integer('num_actors', 128, 'Number of actors.')
flags.DEFINE_integer('num_episodes', 10000, 'Number of episodes to train for.')
flags.DEFINE_integer('max_episode_length', 960,
                     'Maximum number of steps per episode. Assuming 3 days, '
                     'with each step lasting 3 minutes.')

def default_evaluator(
    environment_factory: distributed_layout.EnvironmentFactory,
    network_factory: distributed_layout.NetworkFactory,
    policy_factory: distributed_layout.PolicyFactory
) -> distributed_layout.EvaluatorFactory:
  """Returns a default evaluator process."""
  def evaluator(
      random_key: networks_lib.PRNGKey,
      variable_source: core.VariableSource,
      counter: counting.Counter,
      make_actor: distributed_layout.MakeActorFn,
  ):
    """The evaluation process."""

    # Create environment and evaluator networks
    dummy_seed = 1
    environment = environment_factory(dummy_seed)
    networks = network_factory(specs.make_environment_spec(environment))

    actor = make_actor(
        random_key, policy_factory(networks), variable_source)
    actor._per_episode_update = True  # pylint: disable=protected-access

    # Create logger and counter.
    counter = counting.Counter(counter, 'evaluator')
    logger = loggers.make_default_logger('evaluator')

    # Create the run loop and return it.
    return environment_loop.EnvironmentLoop(environment, actor, counter,
                                              logger)
  return evaluator

def get_program(params: Dict[str, Any]) -> lp.Program:
  """Constructs the program."""
  max_episode_length = FLAGS.max_episode_length
  env_factory = functools.partial(
      acme_utils.create_env, max_episode_length=max_episode_length)
  seed = params.pop('seed', 0)

  (rl_agent, config, dqn_network_fn, behavior_policy_fn, eval_policy_fn
   ) = acme_utils.create_dqn(params)

  agent = distributed_layout.DistributedLayout(
      seed=seed,
      environment_factory=lambda seed: env_factory(False),
      network_factory=dqn_network_fn,
      builder=rl_agent,
      policy_network=behavior_policy_fn,
      evaluator_factories=[
          default_evaluator(
              environment_factory=lambda seed: env_factory(True),
              network_factory=dqn_network_fn,
              policy_factory=eval_policy_fn),
      ],
      num_actors=FLAGS.num_actors,
      max_number_of_steps=FLAGS.num_episodes * FLAGS.max_episode_length,
      prefetch_size=config.prefetch_size,
      checkpointing_config=distributed_layout.CheckpointingConfig(
          directory=FLAGS.base_dir,
          add_uid=True,
      ),
  )
  return agent.build()

def main(_):
  program = get_program({
      'seed': 0,
      'num_sgd_steps_per_step': 2,
      'prefetch_size': 0
  })

  # Launch experiment.
  lp.launch(program, xm_resources=lp_utils.make_xm_docker_resources(program))

if __name__ == '__main__':
  app.run(main)
<commit_msg>Update of the environment factory type.<commit_after># coding=utf-8
# Copyright 2022 The Balloon Learning Environment Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Example running Acme's distributed QrDQN in JAX on the BLE."""

import functools
from typing import Any, Dict

from absl import app
from absl import flags
from acme import core
from acme import environment_loop
from acme import specs
from acme.jax import networks as networks_lib
from acme.jax import types
from acme.jax.layouts import distributed_layout
from acme.utils import counting
from acme.utils import loggers
from acme.utils import lp_utils
from balloon_learning_environment import acme_utils
import launchpad as lp

FLAGS = flags.FLAGS
flags.DEFINE_string('base_dir', '~/acme',
                    'Directory where to store statistics/images.')
flags.DEFINE_integer('num_actors', 128, 'Number of actors.')
flags.DEFINE_integer('num_episodes', 10000, 'Number of episodes to train for.')
flags.DEFINE_integer('max_episode_length', 960,
                     'Maximum number of steps per episode. Assuming 3 days, '
                     'with each step lasting 3 minutes.')

def default_evaluator(
    environment_factory: types.EnvironmentFactory,
    network_factory: distributed_layout.NetworkFactory,
    policy_factory: distributed_layout.PolicyFactory
) -> distributed_layout.EvaluatorFactory:
  """Returns a default evaluator process."""
  def evaluator(
      random_key: networks_lib.PRNGKey,
      variable_source: core.VariableSource,
      counter: counting.Counter,
      make_actor: distributed_layout.MakeActorFn,
  ):
    """The evaluation process."""

    # Create environment and evaluator networks
    dummy_seed = 1
    environment = environment_factory(dummy_seed)
    networks = network_factory(specs.make_environment_spec(environment))

    actor = make_actor(
        random_key, policy_factory(networks), variable_source)
    actor._per_episode_update = True  # pylint: disable=protected-access

    # Create logger and counter.
    counter = counting.Counter(counter, 'evaluator')
    logger = loggers.make_default_logger('evaluator')

    # Create the run loop and return it.
    return environment_loop.EnvironmentLoop(environment, actor, counter,
                                              logger)
  return evaluator

def get_program(params: Dict[str, Any]) -> lp.Program:
  """Constructs the program."""
  max_episode_length = FLAGS.max_episode_length
  env_factory = functools.partial(
      acme_utils.create_env, max_episode_length=max_episode_length)
  seed = params.pop('seed', 0)

  (rl_agent, config, dqn_network_fn, behavior_policy_fn, eval_policy_fn
   ) = acme_utils.create_dqn(params)

  agent = distributed_layout.DistributedLayout(
      seed=seed,
      environment_factory=lambda seed: env_factory(False),
      network_factory=dqn_network_fn,
      builder=rl_agent,
      policy_network=behavior_policy_fn,
      evaluator_factories=[
          default_evaluator(
              environment_factory=lambda seed: env_factory(True),
              network_factory=dqn_network_fn,
              policy_factory=eval_policy_fn),
      ],
      num_actors=FLAGS.num_actors,
      max_number_of_steps=FLAGS.num_episodes * FLAGS.max_episode_length,
      prefetch_size=config.prefetch_size,
      checkpointing_config=distributed_layout.CheckpointingConfig(
          directory=FLAGS.base_dir,
          add_uid=True,
      ),
  )
  return agent.build()

def main(_):
  program = get_program({
      'seed': 0,
      'num_sgd_steps_per_step': 2,
      'prefetch_size': 0
  })

  # Launch experiment.
  lp.launch(program, xm_resources=lp_utils.make_xm_docker_resources(program))

if __name__ == '__main__':
  app.run(main)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Basic Turbinia config."""

from __future__ import unicode_literals

import imp
import itertools
import logging
import os
import sys

log = logging.getLogger('turbinia')

# Look for config files with these names
CONFIGFILES = ['.turbiniarc', 'turbinia.conf', 'turbinia_config.py']
# Look in homedir first, then /etc/turbinia, and finally in the source
# config dir for config files
CONFIGPATH = [
    os.path.expanduser('~'),
    '/etc/turbinia',
    os.path.dirname(os.path.abspath(__file__))]
# Config vars that we expect to exist in the configuration
CONFIGVARS = [
    # Turbinia Config
    'TASK_MANAGER',
    'LOG_FILE',
    'LOCK_FILE',
    'OUTPUT_DIR',
    'SLEEP_TIME',
    'SINGLE_RUN',
    'MOUNT_DIR_PREFIX',
    'SHARED_FILESYSTEM',
    # TODO(aarontp): Move this to the recipe config when it's available.
    'DEBUG_TASKS',
    # GCE CONFIG
    'PROJECT',
    'ZONE',
    'TURBINIA_REGION',
    'BUCKET_NAME',
    'PSQ_TOPIC',
    'PUBSUB_TOPIC',
    'GCS_OUTPUT_PATH',
    'STATE_MANAGER',
    'INSTANCE_ID',
    # REDIS CONFIG
    'REDIS_HOST',
    'REDIS_PORT',
    'REDIS_DB',
    # Celery config
    'CELERY_BROKER',
    'CELERY_BACKEND',
    'KOMBU_BROKER',
    'KOMBU_CHANNEL',
    'KOMBU_DURABLE',]
# Environment variable to look for path data in
ENVCONFIGVAR = 'TURBINIA_CONFIG_PATH'

CONFIG = None

class TurbiniaConfigException(Exception):
  """Exception for Turbinia configuration."""
  pass

def LoadConfig():
  """Finds Turbinia config file and loads it."""
  # TODO(aarontp): Find way to not require global var here.  Maybe a singleton
  # pattern on the config class.
  # pylint: disable=global-statement
  global CONFIG
  if CONFIG:
    return CONFIG

  if ENVCONFIGVAR in os.environ:
    CONFIGPATH.extend(os.environ[ENVCONFIGVAR].split(':'))

  config_file = None
  # Load first file found
  for _dir, _file in itertools.product(CONFIGPATH, CONFIGFILES):
    if os.path.exists(os.path.join(_dir, _file)):
      config_file = os.path.join(_dir, _file)
      break

  if config_file is None:
    raise TurbiniaConfigException('No config files found')

  log.info('Loading config from {0:s}'.format(config_file))
  _config = imp.load_source('config', config_file)
  _config.configSource = config_file
  ValidateAndSetConfig(_config)
  CONFIG = _config
  return _config

def ValidateAndSetConfig(_config):
  """Makes sure that the config has the vars loaded and set in the module."""
  # TODO(aarontp): Allow for non-mandatory config options
  for var in CONFIGVARS:
    if not hasattr(_config, var):
      raise TurbiniaConfigException(
          'No config attribute {0:s}:{1:s}'.format(_config.configSource, var))
    if getattr(_config, var) is None:
      raise TurbiniaConfigException(
          'Config attribute {0:s}:{1:s} is not set'.format(
              _config.configSource, var))

    # Set the attribute in the current module
    setattr(sys.modules[__name__], var, getattr(_config, var))
<commit_msg>Make environment config var override default path (fixes #293) (#294)<commit_after># -*- coding: utf-8 -*-
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Basic Turbinia config."""

from __future__ import unicode_literals

import imp
import itertools
import logging
import os
import sys

log = logging.getLogger('turbinia')

# Look for config files with these names
CONFIGFILES = ['.turbiniarc', 'turbinia.conf', 'turbinia_config.py']
# Look in homedir first, then /etc/turbinia, and finally in the source
# config dir for config files
CONFIGPATH = [
    os.path.expanduser('~'),
    '/etc/turbinia',
    os.path.dirname(os.path.abspath(__file__))]
# Config vars that we expect to exist in the configuration
CONFIGVARS = [
    # Turbinia Config
    'TASK_MANAGER',
    'LOG_FILE',
    'LOCK_FILE',
    'OUTPUT_DIR',
    'SLEEP_TIME',
    'SINGLE_RUN',
    'MOUNT_DIR_PREFIX',
    'SHARED_FILESYSTEM',
    # TODO(aarontp): Move this to the recipe config when it's available.
    'DEBUG_TASKS',
    # GCE CONFIG
    'PROJECT',
    'ZONE',
    'TURBINIA_REGION',
    'BUCKET_NAME',
    'PSQ_TOPIC',
    'PUBSUB_TOPIC',
    'GCS_OUTPUT_PATH',
    'STATE_MANAGER',
    'INSTANCE_ID',
    # REDIS CONFIG
    'REDIS_HOST',
    'REDIS_PORT',
    'REDIS_DB',
    # Celery config
    'CELERY_BROKER',
    'CELERY_BACKEND',
    'KOMBU_BROKER',
    'KOMBU_CHANNEL',
    'KOMBU_DURABLE',]
# Environment variable to look for path data in
ENVCONFIGVAR = 'TURBINIA_CONFIG_PATH'

CONFIG = None

class TurbiniaConfigException(Exception):
  """Exception for Turbinia configuration."""
  pass

def LoadConfig():
  """Finds Turbinia config file and loads it."""
  # TODO(aarontp): Find way to not require global var here.  Maybe a singleton
  # pattern on the config class.
  # pylint: disable=global-statement
  global CONFIG
  if CONFIG:
    return CONFIG

  # If the environment variable is set, take precedence over the pre-defined
  # CONFIGPATHs.
  configpath = CONFIGPATH
  if ENVCONFIGVAR in os.environ:
    configpath = os.environ[ENVCONFIGVAR].split(':')

  config_file = None
  # Load first file found
  for _dir, _file in itertools.product(configpath, CONFIGFILES):
    if os.path.exists(os.path.join(_dir, _file)):
      config_file = os.path.join(_dir, _file)
      break

  if config_file is None:
    raise TurbiniaConfigException('No config files found')

  log.info('Loading config from {0:s}'.format(config_file))
  _config = imp.load_source('config', config_file)
  _config.configSource = config_file
  ValidateAndSetConfig(_config)
  CONFIG = _config
  return _config

def ValidateAndSetConfig(_config):
  """Makes sure that the config has the vars loaded and set in the module."""
  # TODO(aarontp): Allow for non-mandatory config options
  for var in CONFIGVARS:
    if not hasattr(_config, var):
      raise TurbiniaConfigException(
          'No config attribute {0:s}:{1:s}'.format(_config.configSource, var))
    if getattr(_config, var) is None:
      raise TurbiniaConfigException(
          'Config attribute {0:s}:{1:s} is not set'.format(
              _config.configSource, var))

    # Set the attribute in the current module
    setattr(sys.modules[__name__], var, getattr(_config, var))
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division

import collections
import re

import slumber
import slumber.exceptions
import xmlrpc2.client

_normalize_regex = re.compile(r"[^A-Za-z0-9.]+")

def get(d, attr, default=None):
    value = d.get(attr, default)

    if not value:
        value = default

    return value

class BaseProcessor(object):

    def __init__(self, index, warehouse, *args, **kwargs):
        super(BaseProcessor, self).__init__(*args, **kwargs)

        wargs, wkwargs = warehouse

        self.client = xmlrpc2.client.Client(index)
        self.warehouse = slumber.API(*wargs, **wkwargs)

    def process(self):
        raise NotImplementedError

    def get_releases(self, name, version=None):
        if version is None:
            versions = self.client.package_releases(name, True)

            if isinstance(versions, basestring):
                versions = [versions]
        else:
            versions = [version]

        for version in versions:
            item = self.client.release_data(name, version)
            url = self.client.release_urls(item["name"], item["version"])

            if isinstance(url, collections.Mapping):
                urls = [url]
            elif isinstance(url, collections.Iterable):
                urls = url
            else:
                raise RuntimeError("Do not understand the type returned by release_urls")

            item.update({
                "normalized": _normalize_regex.sub("-", item["name"]).lower(),
                "files": urls,
            })

            yield item

    def sync_release(self, release):
        # Get or Create Project
        try:
            project = self.warehouse.projects(release["normalized"]).get()
        except slumber.exceptions.HttpClientError as e:
            if not e.response.status_code == 404:
                raise

            data = self.to_warehouse_project(release)
            project = self.warehouse.projects.post(data)
        else:
            # @@@ Update project
            pass

        # Get or Create Version
        try:
            version = self.warehouse.projects(release["normalized"]).versions(release["version"]).get()
        except slumber.exceptions.HttpClientError as e:
            if not e.response.status_code == 404:
                raise

            data = self.to_warehouse_version(release, extra={"project": project["resource_uri"]})
            version = self.warehouse.projects(release["normalized"]).versions().post(data)
        else:
            # @@@ Update Version
            pass

        # @@@ Get/Create/Delete Files
        # @@@ Batch Update?

    def to_warehouse_project(self, release, extra=None):
        data = {"name": release["name"]}

        if extra is not None:
            data.update(extra)

        return data

    def to_warehouse_version(self, release, extra=None):
        data = {
            "version": release["version"],

            "summary": get(release, "summary", ""),
            "description": get(release, "description", ""),
            "license": get(release, "license", ""),

            "author": {
                "name": get(release, "author", ""),
                "email": get(release, "author_email", ""),
            },
            "maintainer": {
                "name": get(release, "maintainer", ""),
                "email": get(release, "maintainer_email", ""),
            },

            "classifiers": get(release, "classifiers"),
            "uris": {},

            "requires_python": get(release, "requires_python", ""),
        }

        if get(release, "download_url"):
            data["uris"]["Download"] = release["download_url"]

        if get(release, "home_page"):
            data["uris"]["Home page"] = release["home_page"]

        if get(release, "bugtrack_url"):
            data["uris"]["Bug tracker"] = release["bugtrack_url"]

        if get(release, "docs_url"):
            data["uris"]["Documentation"] = release["docs_url"]

        if get(release, "platform"):
            platforms = get(release, "platform")

            if isinstance(platforms, basestring):
                platforms = [platforms]

            data["platforms"] = platforms

        if extra is not None:
            data.update(extra)

        return data

    def to_warehouse_file(self, release, extra=None):
        pass

class BulkProcessor(BaseProcessor):

    def process(self):
        pass
<commit_msg>Count "UNKNOWN" the same as an empty value<commit_after>from __future__ import absolute_import
from __future__ import division

import collections
import re

import slumber
import slumber.exceptions
import xmlrpc2.client

_normalize_regex = re.compile(r"[^A-Za-z0-9.]+")

def get(d, attr, default=None):
    value = d.get(attr, default)

    if not value or value == "UNKNOWN":
        value = default

    return value

class BaseProcessor(object):

    def __init__(self, index, warehouse, *args, **kwargs):
        super(BaseProcessor, self).__init__(*args, **kwargs)

        wargs, wkwargs = warehouse

        self.client = xmlrpc2.client.Client(index)
        self.warehouse = slumber.API(*wargs, **wkwargs)

    def process(self):
        raise NotImplementedError

    def get_releases(self, name, version=None):
        if version is None:
            versions = self.client.package_releases(name, True)

            if isinstance(versions, basestring):
                versions = [versions]
        else:
            versions = [version]

        for version in versions:
            item = self.client.release_data(name, version)
            url = self.client.release_urls(item["name"], item["version"])

            if isinstance(url, collections.Mapping):
                urls = [url]
            elif isinstance(url, collections.Iterable):
                urls = url
            else:
                raise RuntimeError("Do not understand the type returned by release_urls")

            item.update({
                "normalized": _normalize_regex.sub("-", item["name"]).lower(),
                "files": urls,
            })

            yield item

    def sync_release(self, release):
        # Get or Create Project
        try:
            project = self.warehouse.projects(release["normalized"]).get()
        except slumber.exceptions.HttpClientError as e:
            if not e.response.status_code == 404:
                raise

            data = self.to_warehouse_project(release)
            project = self.warehouse.projects.post(data)
        else:
            # @@@ Update project
            pass

        # Get or Create Version
        try:
            version = self.warehouse.projects(release["normalized"]).versions(release["version"]).get()
        except slumber.exceptions.HttpClientError as e:
            if not e.response.status_code == 404:
                raise

            data = self.to_warehouse_version(release, extra={"project": project["resource_uri"]})
            version = self.warehouse.projects(release["normalized"]).versions().post(data)
        else:
            # @@@ Update Version
            pass

        # @@@ Get/Create/Delete Files
        # @@@ Batch Update?

    def to_warehouse_project(self, release, extra=None):
        data = {"name": release["name"]}

        if extra is not None:
            data.update(extra)

        return data

    def to_warehouse_version(self, release, extra=None):
        data = {
            "version": release["version"],

            "summary": get(release, "summary", ""),
            "description": get(release, "description", ""),
            "license": get(release, "license", ""),

            "author": {
                "name": get(release, "author", ""),
                "email": get(release, "author_email", ""),
            },
            "maintainer": {
                "name": get(release, "maintainer", ""),
                "email": get(release, "maintainer_email", ""),
            },

            "classifiers": get(release, "classifiers"),
            "uris": {},

            "requires_python": get(release, "requires_python", ""),
        }

        if get(release, "download_url"):
            data["uris"]["Download"] = release["download_url"]

        if get(release, "home_page"):
            data["uris"]["Home page"] = release["home_page"]

        if get(release, "bugtrack_url"):
            data["uris"]["Bug tracker"] = release["bugtrack_url"]

        if get(release, "docs_url"):
            data["uris"]["Documentation"] = release["docs_url"]

        if get(release, "platform"):
            platforms = get(release, "platform")

            if isinstance(platforms, basestring):
                platforms = [platforms]

            data["platforms"] = platforms

        if extra is not None:
            data.update(extra)

        return data

    def to_warehouse_file(self, release, extra=None):
        pass

class BulkProcessor(BaseProcessor):

    def process(self):
        pass
<|endoftext|>


<commit_before>import znc
import re

class inbox(znc.Module):
	triggers = set()
	lineStorage = "inboxLines"
	triggerStorage = "inboxTriggers"

	def OnLoad(self, args, msg):
		self.readTriggers()
		self.addTrigger(self.GetUser().GetNick())
		self.lines = self.read()
		self.commands = {
			'show': self.com_show,
			'add-trigger': self.com_addTrigger,
			'debug': self.com_debug
		}

		return znc.CONTINUE

	def OnChanMsg(self, nick, channel, msg):
		if self.regex.search(str(msg.s)) != None:
			self.lines.append("<%s> %s" % (nick, msg.s))
			self.write(self.lines)

		return znc.CONTINUE

	def OnModCommand(self, command):
		words = command.split(' ')
		keyword = words[0]
		if keyword in self.commands:
			self.commands[keyword](words[1:])

		return znc.CONTINUE

	def com_show(self, params):
		if len(params) == 0:
			self.PutModule("Specify how many lines you want to see, e.g.: show 5")
			return
		try:
			count = int(params[0])
			self.printout(self.lines[-count:])
		except ValueError:
			self.PutModule("Invalid syntax. %s does not appear to be a number" % params[0])

	def com_debug(self, params):
		self.PutModule("%d lines stored" % len(self.lines))
		self.PutModule(repr(self.lines))

	def com_addTrigger(self, params):
		if len(params) > 0 and len(params[0]) > 0:
			self.addTrigger(trigger)
		else:
			self.PutModule("Can't add an empty trigger")

	def write(self, lines):
		self.SetNV(self.lineStorage, "\n".join(lines))

	def read(self):
		stored = self.GetNV(self.lineStorage)
		return stored.split("\n") if stored != '' else []

	def printout(self, lines):
		if len(lines) == 0:
			self.PutModule("No lines found")
			return
		for line in lines:
			self.PutModule(line)

	def addTrigger(self, trigger):
		self.triggers.add(trigger)
		self.regex = re.compile(self.makeRegex(), flags=re.IGNORECASE)
		self.writeTriggers(self.triggers)
		self.PutModule("Current triggers: %s" % ", ".join(self.triggers))

	def writeTriggers(self, triggers):
		self.SetNV(self.triggerStorage, '\n'.join(triggers))

	def readTriggers(self):
		stored = self.GetNV(self.triggerStorage)
		if len(stored) > 0:
			triggers = stored.split('\n')
			for trigger in triggers:
				self.addTrigger(trigger)

	def makeRegex(self):
		escaped = map(lambda t: re.escape(t), self.triggers)
		return "(" + "|".join(escaped) + ")"
<commit_msg>Added command to remove triggers<commit_after>import znc
import re

class inbox(znc.Module):
	triggers = set()
	lineStorage = "inboxLines"
	triggerStorage = "inboxTriggers"

	def OnLoad(self, args, msg):
		self.readTriggers()
		self.addTrigger(self.GetUser().GetNick())
		self.lines = self.read()
		self.commands = {
			'show': self.com_show,
			'add-trigger': self.com_addTrigger,
			'remove-trigger': self.com_rmTrigger,
			'debug': self.com_debug
		}

		return znc.CONTINUE

	def OnChanMsg(self, nick, channel, msg):
		if self.regex.search(str(msg.s)) != None:
			self.lines.append("<%s> %s" % (nick, msg.s))
			self.write(self.lines)

		return znc.CONTINUE

	def OnModCommand(self, command):
		words = command.split(' ')
		keyword = words[0]
		if keyword in self.commands:
			self.commands[keyword](words[1:])

		return znc.CONTINUE

	def com_show(self, params):
		if len(params) == 0:
			self.PutModule("Specify how many lines you want to see, e.g.: show 5")
			return
		try:
			count = int(params[0])
			self.printout(self.lines[-count:])
		except ValueError:
			self.PutModule("Invalid syntax. %s does not appear to be a number" % params[0])

	def com_debug(self, params):
		self.PutModule("%d lines stored" % len(self.lines))
		self.PutModule(repr(self.lines))

	def com_addTrigger(self, params):
		if len(params) > 0 and len(params[0]) > 0:
			self.addTrigger(params[0])
		else:
			self.PutModule("Can't add an empty trigger")

	def com_rmTrigger(self, params):
		if len(params) > 0 and len(params[0]) > 0:
			if params[0] in self.triggers:
				self.triggers.remove(params[0])
				self.writeTriggers(self.triggers)
			else:
				self.PutModule("No such trigger")
		else:
			self.PutModule("Can't remove an empty trigger")

	def write(self, lines):
		self.SetNV(self.lineStorage, "\n".join(lines))

	def read(self):
		stored = self.GetNV(self.lineStorage)
		return stored.split("\n") if stored != '' else []

	def printout(self, lines):
		if len(lines) == 0:
			self.PutModule("No lines found")
			return
		for line in lines:
			self.PutModule(line)

	def addTrigger(self, trigger):
		self.triggers.add(trigger)
		self.regex = re.compile(self.makeRegex(), flags=re.IGNORECASE)
		self.writeTriggers(self.triggers)
		self.PutModule("Current triggers: %s" % ", ".join(self.triggers))

	def writeTriggers(self, triggers):
		self.SetNV(self.triggerStorage, '\n'.join(triggers))

	def readTriggers(self):
		stored = self.GetNV(self.triggerStorage)
		if len(stored) > 0:
			triggers = stored.split('\n')
			for trigger in triggers:
				self.addTrigger(trigger)

	def makeRegex(self):
		escaped = map(lambda t: re.escape(t), self.triggers)
		return "(" + "|".join(escaped) + ")"
<|endoftext|>


<commit_before>#! python3
# -*- coding: utf-8 -*-

# sookie is a recipe and food planning web application
# Copyright (C) 2015  Niko Wenselowski

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from flask import Flask, g, redirect, render_template, request, url_for
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.wtf import Form
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView
from wtforms import TextField, validators
from wtforms.ext.sqlalchemy.fields import QuerySelectField

app = Flask(__name__)
app.config.from_object("config")
app.config.from_envvar('SOOKIE_SETTINGS', silent=True)
db = SQLAlchemy(app)
admin = Admin(app, name='sookie', template_mode='bootstrap3')

class Category(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(50))

    def __init__(self, name=None):
        self.name = name

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<Category({!r})>'.format(self.name)

class PlannableItem(db.Model):
    __tablename__ = 'plannableitem'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), unique=True)
    type = db.Column(db.String(10))

    __mapper_args__ = {
        'polymorphic_on':type,
        'polymorphic_identity':'plannableitem',
        'with_polymorphic':'*'
    }

    def __init__(self, name=""):
        self.name = name

    def __repr__(self):
        return '<PlannableItem({!r})>'.format(self.name)

class Recipe(PlannableItem):
    __tablename__ = 'recipe'
    id = db.Column(db.Integer, db.ForeignKey('plannableitem.id'), primary_key=True)
    source = db.Column(db.String(120), unique=True)

    category_id = db.Column(db.Integer, db.ForeignKey('category.id'))
    category = db.relationship('Category',
                               backref=db.backref('recipe',
                                                  lazy='dynamic'))

    __mapper_args__ = {'polymorphic_identity': 'recipe'}

    def __init__(self, name="", source=None, category=None):
        self.name = name
        self.source = source
        self.category = category

    def __repr__(self):
        return '<Recipe({!r}, {!r})>'.format(self.name, self.source)

class RecipeForm(Form):
    name = TextField('Name', [validators.Required(),
                              validators.Length(min=4, max=80)])
    source = TextField('Source', [validators.Required(),
                                  validators.Length(min=6, max=35)])
    category = QuerySelectField(query_factory=lambda: Category.query.all())

def init_db():
    db.create_all()

    breakfast = Category('Frühstück')
    lunch = Category('Mittagessen')
    dinner = Category('Abendessen')

    try:
        db.session.add(breakfast)
        db.session.add(lunch)
        db.session.add(dinner)
        db.session.add(Recipe('Mandelfoo', 'VFF, S.123', breakfast))
        db.session.add(Recipe('Foobrot', 'VFF, S.124', lunch))
        db.session.add(Recipe('Banane', 'VFF, S.125', breakfast))
        db.session.commit()
    except Exception as error:
        print("Failed to load example data: {}".format(error))

@app.route('/recipe/<int:id>')
def show_recipe(id):
    recipe = Recipe.query.get_or_404(id)
    cat = recipe.category
    return render_template('recipe.html',
                           recipe=recipe,
                           category_link=url_for('show_category', id=cat.id),
                           category=cat.name)

@app.route('/')
def list_recipes():
    return render_template('start.html', categories=Category.query.all())

@app.route('/categories')
@app.route('/category/')
def list_categories():
    return render_template('categories.html', categories=Category.query.all())

@app.route('/category/<int:id>')
def show_category(id):
    category = Category.query.filter_by(id=id).first_or_404()
    recipes = sorted(Recipe.query.filter_by(category_id=id).all(),
                     key=lambda recipe: recipe.name.lower())
    return render_template('category_overview.html',
                           category=category,
                           recipes=recipes,
                           )

@app.route('/recipe/new', methods=('GET', 'POST'))
def submit_recipe():
    form = RecipeForm(request.form)

    if form.validate_on_submit():
        rec = Recipe(form.name.data, form.source.data, form.category.data)
        db.session.add(rec)
        db.session.commit()

        return redirect(url_for('show_recipe', id=rec.id))

    return render_template('new_recipe.html', form=form, errors=form.errors)

@app.errorhandler(404)
def error_occured(error):
    return render_template('404.html', error=error), 404

class CategoryModelView(ModelView):
    create_modal = True
    edit_modal = True
    form_excluded_columns = ['recipe']

admin.add_view(ModelView(Recipe, db.session))
admin.add_view(CategoryModelView(Category, db.session))

if __name__ == "__main__":
    init_db()

    app.run(debug=True)
<commit_msg>pep8.<commit_after>#! python3
# -*- coding: utf-8 -*-

# sookie is a recipe and food planning web application
# Copyright (C) 2015  Niko Wenselowski

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from flask import Flask, g, redirect, render_template, request, url_for
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.wtf import Form
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView
from wtforms import TextField, validators
from wtforms.ext.sqlalchemy.fields import QuerySelectField

app = Flask(__name__)
app.config.from_object("config")
app.config.from_envvar('SOOKIE_SETTINGS', silent=True)
db = SQLAlchemy(app)
admin = Admin(app, name='sookie', template_mode='bootstrap3')

class Category(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(50))

    def __init__(self, name=None):
        self.name = name

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<Category({!r})>'.format(self.name)

class PlannableItem(db.Model):
    __tablename__ = 'plannableitem'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), unique=True)
    type = db.Column(db.String(10))

    __mapper_args__ = {
        'polymorphic_on': type,
        'polymorphic_identity': 'plannableitem',
        'with_polymorphic': '*'
    }

    def __init__(self, name=""):
        self.name = name

    def __repr__(self):
        return '<PlannableItem({!r})>'.format(self.name)

class Recipe(PlannableItem):
    __tablename__ = 'recipe'
    id = db.Column(db.Integer, db.ForeignKey('plannableitem.id'), primary_key=True)
    source = db.Column(db.String(120), unique=True)

    category_id = db.Column(db.Integer, db.ForeignKey('category.id'))
    category = db.relationship('Category',
                               backref=db.backref('recipe',
                                                  lazy='dynamic'))

    __mapper_args__ = {'polymorphic_identity': 'recipe'}

    def __init__(self, name="", source=None, category=None):
        self.name = name
        self.source = source
        self.category = category

    def __repr__(self):
        return '<Recipe({!r}, {!r})>'.format(self.name, self.source)

class RecipeForm(Form):
    name = TextField('Name', [validators.Required(),
                              validators.Length(min=4, max=80)])
    source = TextField('Source', [validators.Required(),
                                  validators.Length(min=6, max=35)])
    category = QuerySelectField(query_factory=lambda: Category.query.all())

def init_db():
    db.create_all()

    breakfast = Category('Frühstück')
    lunch = Category('Mittagessen')
    dinner = Category('Abendessen')

    try:
        db.session.add(breakfast)
        db.session.add(lunch)
        db.session.add(dinner)
        db.session.add(Recipe('Mandelfoo', 'VFF, S.123', breakfast))
        db.session.add(Recipe('Foobrot', 'VFF, S.124', lunch))
        db.session.add(Recipe('Banane', 'VFF, S.125', breakfast))
        db.session.commit()
    except Exception as error:
        print("Failed to load example data: {}".format(error))

@app.route('/recipe/<int:id>')
def show_recipe(id):
    recipe = Recipe.query.get_or_404(id)
    cat = recipe.category
    return render_template('recipe.html',
                           recipe=recipe,
                           category_link=url_for('show_category', id=cat.id),
                           category=cat.name)

@app.route('/')
def list_recipes():
    return render_template('start.html', categories=Category.query.all())

@app.route('/categories')
@app.route('/category/')
def list_categories():
    return render_template('categories.html', categories=Category.query.all())

@app.route('/category/<int:id>')
def show_category(id):
    category = Category.query.filter_by(id=id).first_or_404()
    recipes = sorted(Recipe.query.filter_by(category_id=id).all(),
                     key=lambda recipe: recipe.name.lower())
    return render_template('category_overview.html',
                           category=category,
                           recipes=recipes,
                           )

@app.route('/recipe/new', methods=('GET', 'POST'))
def submit_recipe():
    form = RecipeForm(request.form)

    if form.validate_on_submit():
        rec = Recipe(form.name.data, form.source.data, form.category.data)
        db.session.add(rec)
        db.session.commit()

        return redirect(url_for('show_recipe', id=rec.id))

    return render_template('new_recipe.html', form=form, errors=form.errors)

@app.errorhandler(404)
def error_occured(error):
    return render_template('404.html', error=error), 404

class CategoryModelView(ModelView):
    create_modal = True
    edit_modal = True
    form_excluded_columns = ['recipe']

admin.add_view(ModelView(Recipe, db.session))
admin.add_view(CategoryModelView(Category, db.session))

if __name__ == "__main__":
    init_db()

    app.run(debug=True)
<|endoftext|>


<commit_before>from django.shortcuts import get_object_or_404, redirect, render, render_to_response
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.views.generic import TemplateView
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from django.db.models import Q
from django.core.urlresolvers import reverse, reverse_lazy

from farms.forms import FieldFormSet
from farms.models import Farm

farm_view_fields =  ('farmer',
                     'name',
                     'description',
                     'users',
                     'address_1',
                     'address_2',
                     'city',
                     'county',
                     'state',
                     'zipcode',
                 )

def farms_filter(user):
        return Farm.objects.filter( Q(farmer=user) |
                                    Q(users=user) ).distinct()

class FarmMixin:
    def get(self, request, *args, **kwargs):
        """
        Handles GET requests and instantiates blank versions of the form
        and its inline formsets.
        """
        self.object = self.get_object()

        form_class = self.get_form_class()
        form = self.get_form(form_class)
        field_form = FieldFormSet(prefix='field', instance=self.object)

        field_form_headers = map(lambda field: field.label,
                                 field_form[0])
        field_form_headers = map(lambda label: '' if label in ( 'Delete', 'Id', 'Farm' ) else label,
                                    field_form_headers)

        context = self.get_context_data(form=form,
                                        field_form=field_form,
                                        field_form_headers=field_form_headers,
                                    )
        return self.render_to_response(context)

    def post(self, request, *args, **kwargs):
        """
        Handles POST requests, instantiating a form instance and its inline
        formsets with the passed POST variables and then checking them for
        validity.
        """
        self.object = self.get_object()

        form_class = self.get_form_class()
        form = self.get_form(form_class)
        field_form = FieldFormSet(self.request.POST, prefix='field', instance=self.object)
        if (form.is_valid() and field_form.is_valid() ):
            return self.form_valid(form, field_form)
        else:
            return self.form_invalid(form, field_form)

    def form_valid(self, form, field_form):
        """
        Called if all forms are valid. Creates a Farm instance along with
        associated Fields and then redirects to a success page.
        """
        self.object = form.save()
        field_form.instance = self.object
        field_form.save()
        return redirect(self.get_success_url())

    def form_invalid(self, form, field_form):
        """
        Called if a form is invalid. Re-renders the context data with the
        data-filled forms and errors.
        """
        context = self.get_context_data(form=form,
                                        field_form=field_form)
        return self.render_to_response(context)

class FarmListView(TemplateView):
    template_name = "farms/farm_list.html"

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(FarmListView, self).dispatch(*args, **kwargs)

class FarmUpdateView(FarmMixin, UpdateView):
    template_name = "farms/farm_and_fields.html"
    model = Farm
    pk_field = 'pk'
    fields = farm_view_fields

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        """
        This function ensures that the user has the right to access
        this object by comparing the primary key of this object to the
        list of primary keys constructed from a query of objects where
        this user is either the farmer or the one of the users.
        """
        farm_list = farms_filter(self.request.user)
        user_pk = int(kwargs['pk'])
        pk_list = map( lambda x: int(x.pk), farm_list )
        if not user_pk in pk_list:
            return redirect( reverse('farm_list') )
        else:
            return super(FarmUpdateView, self).dispatch(*args, **kwargs)

    def get_success_url(self):
        path = reverse('farm_id', args=[self.object.pk])
        return path

class FarmCreateView(FarmMixin, CreateView):
    template_name = "farms/farm_and_fields.html"
    model = Farm
    pk_field = 'pk'
    fields = farm_view_fields

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(FarmCreateView, self).dispatch(*args, **kwargs)

    def get_success_url(self):
        path = self.request.path.replace('new', "%s" % self.object.pk)
        return path

    def get_initial(self):
        dict = super(FarmCreateView, self).get_initial()
        dict['farmer'] = self.request.user
        return dict

    def get_object(self, queryset=None):
        return None

class FarmDeleteView(DeleteView):
    template_name = "farms/farm_delete.html"
    model = Farm
    pk_field = 'pk'

    success_url = reverse_lazy('farm_list')

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        """
        This function ensures that the user has the right to access
        this object by comparing the primary key of this object to the
        list of primary keys constructed from a query of objects where
        this user is either the farmer or the one of the users.
        """
        farm_list = farms_filter(self.request.user)
        user_pk = int(kwargs['pk'])
        pk_list = map( lambda x: int(x.pk), farm_list )
        if not user_pk in pk_list:
            return redirect( reverse('farm_list') )
        else:
            return super(FarmUpdateView, self).dispatch(*args, **kwargs)

<commit_msg>Wrong superclass.<commit_after>from django.shortcuts import get_object_or_404, redirect, render, render_to_response
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.views.generic import TemplateView
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from django.db.models import Q
from django.core.urlresolvers import reverse, reverse_lazy

from farms.forms import FieldFormSet
from farms.models import Farm

farm_view_fields =  ('farmer',
                     'name',
                     'description',
                     'users',
                     'address_1',
                     'address_2',
                     'city',
                     'county',
                     'state',
                     'zipcode',
                 )

def farms_filter(user):
        return Farm.objects.filter( Q(farmer=user) |
                                    Q(users=user) ).distinct()

class FarmMixin:
    def get(self, request, *args, **kwargs):
        """
        Handles GET requests and instantiates blank versions of the form
        and its inline formsets.
        """
        self.object = self.get_object()

        form_class = self.get_form_class()
        form = self.get_form(form_class)
        field_form = FieldFormSet(prefix='field', instance=self.object)

        field_form_headers = map(lambda field: field.label,
                                 field_form[0])
        field_form_headers = map(lambda label: '' if label in ( 'Delete', 'Id', 'Farm' ) else label,
                                    field_form_headers)

        context = self.get_context_data(form=form,
                                        field_form=field_form,
                                        field_form_headers=field_form_headers,
                                    )
        return self.render_to_response(context)

    def post(self, request, *args, **kwargs):
        """
        Handles POST requests, instantiating a form instance and its inline
        formsets with the passed POST variables and then checking them for
        validity.
        """
        self.object = self.get_object()

        form_class = self.get_form_class()
        form = self.get_form(form_class)
        field_form = FieldFormSet(self.request.POST, prefix='field', instance=self.object)
        if (form.is_valid() and field_form.is_valid() ):
            return self.form_valid(form, field_form)
        else:
            return self.form_invalid(form, field_form)

    def form_valid(self, form, field_form):
        """
        Called if all forms are valid. Creates a Farm instance along with
        associated Fields and then redirects to a success page.
        """
        self.object = form.save()
        field_form.instance = self.object
        field_form.save()
        return redirect(self.get_success_url())

    def form_invalid(self, form, field_form):
        """
        Called if a form is invalid. Re-renders the context data with the
        data-filled forms and errors.
        """
        context = self.get_context_data(form=form,
                                        field_form=field_form)
        return self.render_to_response(context)

class FarmListView(TemplateView):
    template_name = "farms/farm_list.html"

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(FarmListView, self).dispatch(*args, **kwargs)

class FarmUpdateView(FarmMixin, UpdateView):
    template_name = "farms/farm_and_fields.html"
    model = Farm
    pk_field = 'pk'
    fields = farm_view_fields

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        """
        This function ensures that the user has the right to access
        this object by comparing the primary key of this object to the
        list of primary keys constructed from a query of objects where
        this user is either the farmer or the one of the users.
        """
        farm_list = farms_filter(self.request.user)
        user_pk = int(kwargs['pk'])
        pk_list = map( lambda x: int(x.pk), farm_list )
        if not user_pk in pk_list:
            return redirect( reverse('farm_list') )
        else:
            return super(FarmUpdateView, self).dispatch(*args, **kwargs)

    def get_success_url(self):
        path = reverse('farm_id', args=[self.object.pk])
        return path

class FarmCreateView(FarmMixin, CreateView):
    template_name = "farms/farm_and_fields.html"
    model = Farm
    pk_field = 'pk'
    fields = farm_view_fields

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(FarmCreateView, self).dispatch(*args, **kwargs)

    def get_success_url(self):
        path = self.request.path.replace('new', "%s" % self.object.pk)
        return path

    def get_initial(self):
        dict = super(FarmCreateView, self).get_initial()
        dict['farmer'] = self.request.user
        return dict

    def get_object(self, queryset=None):
        return None

class FarmDeleteView(DeleteView):
    template_name = "farms/farm_delete.html"
    model = Farm
    pk_field = 'pk'

    success_url = reverse_lazy('farm_list')

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        """
        This function ensures that the user has the right to access
        this object by comparing the primary key of this object to the
        list of primary keys constructed from a query of objects where
        this user is either the farmer or the one of the users.
        """
        farm_list = farms_filter(self.request.user)
        user_pk = int(kwargs['pk'])
        pk_list = map( lambda x: int(x.pk), farm_list )
        if not user_pk in pk_list:
            return redirect( reverse('farm_list') )
        else:
            return super(FarmDeleteView, self).dispatch(*args, **kwargs)

<|endoftext|>


<commit_before>from loom.format import load_encoder, load_decoder
from distributions.io.stream import open_compressed, json_load
import csv

class PreQL(object):
    def __init__(self, query_server, encoding, debug=False):
        self.query_server = query_server
        self.encoders = json_load(encoding)
        self.feature_names = [e['name'] for e in self.encoders]
        self.debug = debug

    def predict(self, rows_csv, count, result_out, id_offset=True):
        with open_compressed(rows_csv, 'rb') as fin:
            with open(result_out, 'w') as fout:
                reader = csv.reader(fin)
                writer = csv.writer(fout)
                feature_names = list(reader.next())
                writer.writerow(feature_names)
                name_to_pos = {name: i for i, name in enumerate(feature_names)}
                pos_to_decode = {}
                schema = []
                for encoder in self.encoders:
                    pos = name_to_pos.get(encoder['name'])
                    encode = load_encoder(encoder)
                    decode = load_decoder(encoder)
                    if pos is not None:
                        pos_to_decode[pos] = decode
                    schema.append((pos, encode))
                for row in reader:
                    conditioning_row = []
                    to_sample = []
                    if id_offset:
                        row_id = row.pop(0)
                    for pos, encode, in schema:
                        value = None if pos is None else row[pos].strip()
                        observed = bool(value)
                        to_sample.append((not observed))
                        if observed is False:
                            conditioning_row.append(None)
                        else:
                            conditioning_row.append(encode(value))
                    samples = self.query_server.sample(
                        to_sample,
                        conditioning_row,
                        count)
                    for sample in samples:
                        if id_offset:
                            out_row = [row_id]
                        else:
                            out_row = []
                        for name in feature_names:
                            pos = name_to_pos[name]
                            decode = pos_to_decode[pos]
                            val = sample[pos]
                            out_row.append(val)
                        writer.writerow(out_row)

    def cols_to_sample(self, cols):
        cols = set(cols)
        return [fname in cols for fname in self.feature_names]

    def normalize_mutual_information(self, mutual_info, joint_entropy):
        """
        Mutual information is normalized by joint entopy because:
            I(X; Y) = 0 if p(x, y) = p(x)p(y) (independence)
            and
            I(X; Y) = H(X) + H(Y) - H(X, Y)
            H(X, X) = H(X)
            => I(X; X) = H(X) = H(X, X)
        """
        return mutual_info / joint_entropy

    def relate(self, columns, result_out, sample_count=1000):
        """
        Compute pairwise related scores between all pairs of
        columns in columns.

        Related scores are defined to be:
            Related(X, Y) = I(X; Y) / H(X, Y)
        Where:
            I(X; Y) is the mutual information between X and Y:
                I(X; Y) = E[ log( p(x, y)) / ( p(x) p(y) ) ]; x, y ~ p(x, y)
            H(X) is the entropy of X:
                H(X) = E[ log( p(x) )]; x ~ p(x)
        Expectations are estimated via monte carlo with `sample_count` samples
        """
        with open(result_out, 'w') as f:
            writer = csv.writer(f)
            writer.writerow(self.feature_names)
            for target_column in set(columns):
                out_row = [target_column]
                to_sample1 = self.cols_to_sample([target_column])
                for to_relate in self.feature_names:
                    to_sample2 = self.cols_to_sample([to_relate])
                    mi = self.query_server.mutual_information(
                        to_sample1,
                        to_sample2,
                        sample_count=sample_count).mean
                    joined = [to_relate, target_column]
                    to_sample_both = self.cols_to_sample(joined)
                    joint_entropy = self.query_server.entropy(
                        to_sample_both,
                        sample_count=sample_count).mean
                    normalized_mi = self.normalize_mutual_information(
                        mi,
                        joint_entropy)
                    out_row.append(normalized_mi)
                writer.writerow(out_row)
<commit_msg>open compressed everywhere<commit_after>from loom.format import load_encoder, load_decoder
from distributions.io.stream import open_compressed, json_load
import csv

class PreQL(object):
    def __init__(self, query_server, encoding, debug=False):
        self.query_server = query_server
        self.encoders = json_load(encoding)
        self.feature_names = [e['name'] for e in self.encoders]
        self.debug = debug

    def predict(self, rows_csv, count, result_out, id_offset=True):
        with open_compressed(rows_csv, 'rb') as fin:
            with open_compressed(result_out, 'w') as fout:
                reader = csv.reader(fin)
                writer = csv.writer(fout)
                feature_names = list(reader.next())
                writer.writerow(feature_names)
                name_to_pos = {name: i for i, name in enumerate(feature_names)}
                pos_to_decode = {}
                schema = []
                for encoder in self.encoders:
                    pos = name_to_pos.get(encoder['name'])
                    encode = load_encoder(encoder)
                    decode = load_decoder(encoder)
                    if pos is not None:
                        pos_to_decode[pos] = decode
                    schema.append((pos, encode))
                for row in reader:
                    conditioning_row = []
                    to_sample = []
                    if id_offset:
                        row_id = row.pop(0)
                    for pos, encode, in schema:
                        value = None if pos is None else row[pos].strip()
                        observed = bool(value)
                        to_sample.append((not observed))
                        if observed is False:
                            conditioning_row.append(None)
                        else:
                            conditioning_row.append(encode(value))
                    samples = self.query_server.sample(
                        to_sample,
                        conditioning_row,
                        count)
                    for sample in samples:
                        if id_offset:
                            out_row = [row_id]
                        else:
                            out_row = []
                        for name in feature_names:
                            pos = name_to_pos[name]
                            decode = pos_to_decode[pos]
                            val = sample[pos]
                            out_row.append(val)
                        writer.writerow(out_row)

    def cols_to_sample(self, cols):
        cols = set(cols)
        return [fname in cols for fname in self.feature_names]

    def normalize_mutual_information(self, mutual_info, joint_entropy):
        """
        Mutual information is normalized by joint entopy because:
            I(X; Y) = 0 if p(x, y) = p(x)p(y) (independence)
            and
            I(X; Y) = H(X) + H(Y) - H(X, Y)
            H(X, X) = H(X)
            => I(X; X) = H(X) = H(X, X)
        """
        return mutual_info / joint_entropy

    def relate(self, columns, result_out, sample_count=1000):
        """
        Compute pairwise related scores between all pairs of
        columns in columns.

        Related scores are defined to be:
            Related(X, Y) = I(X; Y) / H(X, Y)
        Where:
            I(X; Y) is the mutual information between X and Y:
                I(X; Y) = E[ log( p(x, y)) / ( p(x) p(y) ) ]; x, y ~ p(x, y)
            H(X) is the entropy of X:
                H(X) = E[ log( p(x) )]; x ~ p(x)
        Expectations are estimated via monte carlo with `sample_count` samples
        """
        with open_compressed(result_out, 'w') as f:
            writer = csv.writer(f)
            writer.writerow(self.feature_names)
            for target_column in set(columns):
                out_row = [target_column]
                to_sample1 = self.cols_to_sample([target_column])
                for to_relate in self.feature_names:
                    to_sample2 = self.cols_to_sample([to_relate])
                    mi = self.query_server.mutual_information(
                        to_sample1,
                        to_sample2,
                        sample_count=sample_count).mean
                    joined = [to_relate, target_column]
                    to_sample_both = self.cols_to_sample(joined)
                    joint_entropy = self.query_server.entropy(
                        to_sample_both,
                        sample_count=sample_count).mean
                    normalized_mi = self.normalize_mutual_information(
                        mi,
                        joint_entropy)
                    out_row.append(normalized_mi)
                writer.writerow(out_row)
<|endoftext|>


<commit_before># coding=utf-8
import socket
import sys
import distutils.spawn
from string import Template
from subprocess import check_output, call, STDOUT, PIPE

dockerTemplate = '''
version: '2'
services:
    api:
        image: screwdrivercd/screwdriver:stable
        ports:
            - 9001:80
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock:rw
        environment:
            PORT: 80
            URI: http://${ip}:9001
            ECOSYSTEM_UI: http://${ip}:9000
            ECOSYSTEM_STORE: http://${ip}:9002
            DATASTORE_PLUGIN: sequelize
            DATASTORE_SEQUELIZE_DIALECT: sqlite
            EXECUTOR_PLUGIN: docker
            SECRET_WHITELIST: "[]"
            EXECUTOR_DOCKER_DOCKER: |
                {
                    "socketPath": "/var/run/docker.sock"
                }
            SECRET_OAUTH_CLIENT_ID: ${oauth_id}
            SECRET_OAUTH_CLIENT_SECRET: ${oauth_secret}
            SECRET_JWT_PRIVATE_KEY: |
${private_key}
            SECRET_JWT_PUBLIC_KEY: |
${public_key}
    ui:
        image: screwdrivercd/ui:stable
        ports:
            - 9000:80
        environment:
            ECOSYSTEM_API: http://${ip}:9001

    store:
        image: screwdrivercd/store:stable
        ports:
            - 9002:80
        environment:
            URI: http://${ip}:9002
            SECRET_JWT_PUBLIC_KEY: |
${public_key}
'''

# Get the external IP by poking at Google's DNS
def get_ip_address():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    return s.getsockname()[0]

# Left-Pad a set of lines (split by \n) with spaces
def pad_lines(lines, length):
    return '\n'.join(map((lambda row: ''.rjust(length, ' ') + row), lines.split('\n')))

# Generate a new JWT
def generate_jwt():
    junk = check_output(['openssl', 'genrsa', '-out', 'jwt.pem', '1024'], stderr=STDOUT)
    junk = check_output(['openssl', 'rsa', '-in', 'jwt.pem', '-pubout', '-out', 'jwt.pub'], stderr=STDOUT)
    jwtPrivate = open('jwt.pem', 'r').read()
    jwtPublic = open('jwt.pub', 'r').read()
    junk = check_output(['rm', 'jwt.pem', 'jwt.pub'], stderr=STDOUT)

    return {
        'public_key': pad_lines(jwtPublic, 16),
        'private_key': pad_lines(jwtPrivate, 16)
    }

# Generate OAuth credentials from GitHub.com
def generate_oauth(ip):
    print Template('''
    Please create a new OAuth application on GitHub.com
    Go to https://github.com/settings/applications/new to start the process
    For 'Homepage URL' put http://${ip}:9000
    For 'Authorization callback URL' put http://${ip}:9001/v4/auth/login
    When done, please provide the following values:
    ''').substitute(ip=ip)

    id = raw_input('    Client ID: ');
    secret = raw_input('    Client Secret: ');

    print ''
    return {
        'oauth_id': id,
        'oauth_secret': secret
    }

def check_component(component):
    if distutils.spawn.find_executable(component) == None:
        print '💀   Could not find {0}, please install and set path to {0}'.format(component)
        sys.exit(1)

def main():
    fields = {
        'ip': get_ip_address()
    }
    print '🎁   Boxing up Screwdriver'

    print '👀   Checking prerequisites'
    check_component('docker')
    check_component('docker-compose')
    check_component('openssl')

    print '🔐   Generating signing secrets'
    fields = dict(fields, **generate_jwt())

    print '📦   Generating OAuth credentials'
    fields = dict(fields, **generate_oauth(fields['ip']))

    print '💾   Writing Docker Compose file'
    compose = Template(dockerTemplate).substitute(fields)
    open('docker-compose.yml', 'w').write(compose)

    print '🚀   Screwdriver is ready to launch!'
    print Template('''
    Just run the following commands to get started!
      $ docker-compose pull
      $ docker-compose -p screwdriver up -d
      $ open http://${ip}:9000
    ''').safe_substitute(fields)
    prompt = raw_input('    Would you like to run them now? (y/n) ')
    if prompt.lower() == 'y':
        call('docker-compose pull', shell=True)
        call('docker-compose -p screwdriver up -d', shell=True)
        call(Template('open http://${ip}:9000').safe_substitute(fields), shell=True)
        print '\n👍   Launched!'
    else:
        print '\n👍   Skipping launch (for now)'

    print '''
    A few more things to note:
      - To stop/reset Screwdriver
        $ docker-compose -p screwdriver down
      - If your internal IP changes, update the docker-compose.yml and your GitHub OAuth application
      - In-a-box does not support Webhooks including PullRequests for triggering builds
      - For help with this and more, find us on Slack at https://slack.screwdriver.cd

❤   Screwdriver Crew
    '''

if __name__ == "__main__":
    main()
<commit_msg>fix: Obtain IP for Docker-Machine users<commit_after># coding=utf-8
import os
import socket
import sys
import distutils.spawn
from urlparse import urlparse
from string import Template
from subprocess import check_output, call, STDOUT, PIPE

dockerTemplate = '''
version: '2'
services:
    api:
        image: screwdrivercd/screwdriver:stable
        ports:
            - 9001:80
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock:rw
        environment:
            PORT: 80
            URI: http://${ip}:9001
            ECOSYSTEM_UI: http://${ip}:9000
            ECOSYSTEM_STORE: http://${ip}:9002
            DATASTORE_PLUGIN: sequelize
            DATASTORE_SEQUELIZE_DIALECT: sqlite
            EXECUTOR_PLUGIN: docker
            SECRET_WHITELIST: "[]"
            EXECUTOR_DOCKER_DOCKER: |
                {
                    "socketPath": "/var/run/docker.sock"
                }
            SECRET_OAUTH_CLIENT_ID: ${oauth_id}
            SECRET_OAUTH_CLIENT_SECRET: ${oauth_secret}
            SECRET_JWT_PRIVATE_KEY: |
${private_key}
            SECRET_JWT_PUBLIC_KEY: |
${public_key}
    ui:
        image: screwdrivercd/ui:stable
        ports:
            - 9000:80
        environment:
            ECOSYSTEM_API: http://${ip}:9001

    store:
        image: screwdrivercd/store:stable
        ports:
            - 9002:80
        environment:
            URI: http://${ip}:9002
            SECRET_JWT_PUBLIC_KEY: |
${public_key}
'''

# Check IP locally if running a docker-machine from docker-toolbox,
# otherwise, get IP by poking at Google's DNS.
# note: docker-for-mac does not set DOCKER environment variables
def get_ip_address():
    if os.environ.get('DOCKER_HOST'):
        url = urlparse(os.environ['DOCKER_HOST'])
        return url.hostname
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    return s.getsockname()[0]

# Left-Pad a set of lines (split by \n) with spaces
def pad_lines(lines, length):
    return '\n'.join(map((lambda row: ''.rjust(length, ' ') + row), lines.split('\n')))

# Generate a new JWT
def generate_jwt():
    junk = check_output(['openssl', 'genrsa', '-out', 'jwt.pem', '1024'], stderr=STDOUT)
    junk = check_output(['openssl', 'rsa', '-in', 'jwt.pem', '-pubout', '-out', 'jwt.pub'], stderr=STDOUT)
    jwtPrivate = open('jwt.pem', 'r').read()
    jwtPublic = open('jwt.pub', 'r').read()
    junk = check_output(['rm', 'jwt.pem', 'jwt.pub'], stderr=STDOUT)

    return {
        'public_key': pad_lines(jwtPublic, 16),
        'private_key': pad_lines(jwtPrivate, 16)
    }

# Generate OAuth credentials from GitHub.com
def generate_oauth(ip):
    print Template('''
    Please create a new OAuth application on GitHub.com
    Go to https://github.com/settings/applications/new to start the process
    For 'Homepage URL' put http://${ip}:9000
    For 'Authorization callback URL' put http://${ip}:9001/v4/auth/login
    When done, please provide the following values:
    ''').substitute(ip=ip)

    id = raw_input('    Client ID: ');
    secret = raw_input('    Client Secret: ');

    print ''
    return {
        'oauth_id': id,
        'oauth_secret': secret
    }

def check_component(component):
    if distutils.spawn.find_executable(component) == None:
        print '💀   Could not find {0}, please install and set path to {0}'.format(component)
        sys.exit(1)

def main():
    fields = {
        'ip': get_ip_address()
    }
    print '🎁   Boxing up Screwdriver'

    print '👀   Checking prerequisites'
    check_component('docker')
    check_component('docker-compose')
    check_component('openssl')

    print '🔐   Generating signing secrets'
    fields = dict(fields, **generate_jwt())

    print '📦   Generating OAuth credentials'
    fields = dict(fields, **generate_oauth(fields['ip']))

    print '💾   Writing Docker Compose file'
    compose = Template(dockerTemplate).substitute(fields)
    open('docker-compose.yml', 'w').write(compose)

    print '🚀   Screwdriver is ready to launch!'
    print Template('''
    Just run the following commands to get started!
      $ docker-compose pull
      $ docker-compose -p screwdriver up -d
      $ open http://${ip}:9000
    ''').safe_substitute(fields)
    prompt = raw_input('    Would you like to run them now? (y/n) ')
    if prompt.lower() == 'y':
        call('docker-compose pull', shell=True)
        call('docker-compose -p screwdriver up -d', shell=True)
        call(Template('open http://${ip}:9000').safe_substitute(fields), shell=True)
        print '\n👍   Launched!'
    else:
        print '\n👍   Skipping launch (for now)'

    print '''
    A few more things to note:
      - To stop/reset Screwdriver
        $ docker-compose -p screwdriver down
      - If your internal IP changes, update the docker-compose.yml and your GitHub OAuth application
      - In-a-box does not support Webhooks including PullRequests for triggering builds
      - For help with this and more, find us on Slack at https://slack.screwdriver.cd

❤   Screwdriver Crew
    '''

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
Custom Error and Warning names to improve readability
"""

# ----------
# Exceptions:
# ----------

class PlasmaPyError(Exception):
    r"""
    Base class of PlasmaPy custom errors.

    All custom exceptions raised by PlasmaPy should inherit from this
    class and be defined in this module.

    Custom exceptions can inherit from other exception types too.
    Thus, if code already knows how to handle a ValueError, it won't
    need any specific modification.
    """
    pass

class PhysicsError(PlasmaPyError, ValueError):
    r"""Error for use of a physics value outside PlasmaPy theoretical
    bounds"""
    pass

class RelativityError(PhysicsError):
    r"""Error for use of a speed greater than or equal to the speed of
    light"""
    pass

class AtomicError(PlasmaPyError):
    r"""An exception for errors occurring in the atomic subpackage."""
    pass

class MissingAtomicDataError(AtomicError):
    r"""An exception for when atomic data is missing."""
    pass

class ChargeError(AtomicError):
    r"""An exception for when charge information is incorrect or
    missing."""
    pass

class UnexpectedParticleError(AtomicError):
    r"""An exception for when a particle is not of the expected
    category."""
    pass

class InvalidIonError(UnexpectedParticleError):
    r"""An exception for when an argument is not an ion."""
    pass

class InvalidIsotopeError(UnexpectedParticleError):
    r"""An exception for when an argument is not an isotope."""
    pass

class InvalidElementError(UnexpectedParticleError):
    r"""An exception for when an argument is not an element."""
    pass

class InvalidParticleError(AtomicError):
    r"""An exception for when a particle is invalid."""
    pass

# ----------
# Warnings:
# ----------

class PlasmaPyWarning(Warning):
    r"""Base class of PlasmaPy custom warnings.

    All PlasmaPy custom warnings should inherit from this class and be defined
    in this module.

    Warnings should be issued using warnings.warn, which will not break
    execution if unhandled.
    """
    pass

class PhysicsWarning(PlasmaPyWarning):
    r"""Warning for using a mildly worrisome physics value"""
    pass

class RelativityWarning(PhysicsWarning):
    r"""Warning for use of a speed quantity approaching the speed of light"""
    pass

class AtomicWarning(PlasmaPyWarning):
    r"""Warnings for use in the atomic subpackage."""
    pass

class MissingAtomicDataWarning(AtomicWarning):
    r"""Warning for use when atomic data is missing."""
    pass
<commit_msg>Update ReST formatting in exceptions.py<commit_after>"""Exceptions and warnings specific to PlasmaPy."""

# ----------
# Exceptions
# ----------

class PlasmaPyError(Exception):
    """
    Base class of PlasmaPy custom errors.

    All custom exceptions raised by PlasmaPy should inherit from this
    class and be defined in this module.
    """
    pass

class PhysicsError(PlasmaPyError, ValueError):
    """
    The base exception for physics-related errors.
    """
    pass

class RelativityError(PhysicsError):
    """
    An exception for speeds greater than the speed of light.
    """
    pass

class AtomicError(PlasmaPyError):
    """An exception for errors in the `~plasmapy.atomic` subpackage."""
    pass

class MissingAtomicDataError(AtomicError):
    """An exception for missing atomic or particle data."""
    pass

class ChargeError(AtomicError):
    """An exception for incorrect or missing charge information."""
    pass

class UnexpectedParticleError(AtomicError):
    """An exception for when a particle is not of the expected category."""
    pass

class InvalidIonError(UnexpectedParticleError):
    """
    An exception for when an argument is a valid particle but not a
    valid ion.
    """
    pass

class InvalidIsotopeError(UnexpectedParticleError):
    """
    An exception for when an argument is a valid particle but not a
    valid isotope.
    """
    pass

class InvalidElementError(UnexpectedParticleError):
    """
    An exception for when an argument is a valid particle is not a
    valid element.
    """
    pass

class InvalidParticleError(AtomicError):
    """An exception for when a particle is invalid."""
    pass

# ----------
# Warnings:
# ----------

class PlasmaPyWarning(Warning):
    """
    Base class of PlasmaPy custom warnings.

    All PlasmaPy custom warnings should inherit from this class and be
    defined in this module.

    Warnings should be issued using `~warnings.warn`, which will not break
    execution if unhandled.

    """
    pass

class PhysicsWarning(PlasmaPyWarning):
    """The base warning for `~plasmapy.physics` related warnings."""
    pass

class RelativityWarning(PhysicsWarning):
    """
    A warning for when relativistic velocities are being used in or are
    returned by non-relativistic functionality.
    """
    pass

class AtomicWarning(PlasmaPyWarning):
    """The base warning for the `~plasmapy.atomic` subpackage."""
    pass

class MissingAtomicDataWarning(AtomicWarning):
    """Warning for use when atomic or particle data is missing."""
    pass
<|endoftext|>