<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2009 Edgewall Software
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.

"""Utilities for text translation with gettext."""

import re
import sys
try:
    import threading
except ImportError:
    import dummy_threading as threading

import pkg_resources

from genshi.builder import tag

__all__ = ['gettext', 'ngettext', 'gettext_noop', 'ngettext_noop',
           'tgettext', 'tgettext_noop', 'tngettext', 'tngettext_noop']

def safefmt(string, kwargs):
    if kwargs:
        try:
            return string % kwargs
        except KeyError:
            pass
    return string

def gettext_noop(string, **kwargs):
    return safefmt(string, kwargs)

def dgettext_noop(domain, string, **kwargs):
    return gettext_noop(string, **kwargs)

N_ = gettext_noop

def ngettext_noop(singular, plural, num, **kwargs):
    string = (plural, singular)[num == 1]
    kwargs.setdefault('num', num)
    return safefmt(string, kwargs)

def dngettext_noop(domain, singular, plural, num, **kwargs):
    return ngettext_noop(singular, plural, num, **kwargs)

_param_re = re.compile(r"%\((\w+)\)(?:s|[\d]*d|\d*.?\d*[fg])")
def _tag_kwargs(trans, kwargs):
    trans_elts = _param_re.split(trans)
    for i in xrange(1, len(trans_elts), 2):
        trans_elts[i] = kwargs.get(trans_elts[i], '???')
    return tag(*trans_elts)

def tgettext_noop(string, **kwargs):
    return kwargs and _tag_kwargs(string, kwargs) or string

def dtgettext_noop(domain, string, **kwargs):
    return tgettext_noop(string, **kwargs)

def tngettext_noop(singular, plural, num, **kwargs):
    string = (plural, singular)[num == 1]
    kwargs.setdefault('num', num)
    return _tag_kwargs(string, kwargs)

def dtngettext_noop(domain, singular, plural, num, **kwargs):
    return tngettext_noop(singular, plural, num, **kwargs)

def add_domain(domain, env_path, locale_dir):
    pass

def domain_functions(domain, *symbols):
    _functions = {
      'gettext': gettext_noop,
      '_': gettext_noop,
      'N_': gettext_noop,
      'ngettext': ngettext_noop,
      'tgettext': tgettext_noop,
      'tag_': tgettext_noop,
      'tngettext': tngettext_noop,
      'add_domain': lambda env_path, locale_dir: None,
      }
    return [_functions[s] for s in symbols]

try:
    from babel.support import LazyProxy, Translations
    from gettext import NullTranslations

    class TranslationsProxy(object):
        """Delegate Translations calls to the currently active Translations.

        If there's none, wrap those calls in LazyProxy objects.
        """

        def __init__(self):
            self._current = threading.local()
            self._null_translations = NullTranslations()
            self._plugin_domains = {}
            self._plugin_domains_lock = threading.RLock()

        # Public API

        def add_domain(self, domain, env_path, locales_dir):
            self._plugin_domains_lock.acquire()
            try:
                if env_path not in self._plugin_domains:
                    self._plugin_domains[env_path] = []
                self._plugin_domains[env_path].append((domain, locales_dir))
            finally:
                self._plugin_domains_lock.release()

        def activate(self, locale, env_path=None):
            locale_dir = pkg_resources.resource_filename(__name__, '../locale')
            t = Translations.load(locale_dir, locale)
            if env_path:
                self._plugin_domains_lock.acquire()
                try:
                    domains = list(self._plugin_domains.get(env_path, []))
                finally:
                    self._plugin_domains_lock.release()
                for domain, dirname in domains:
                    t.add(Translations.load(dirname, locale, domain))
            self._current.translations = t

        def deactivate(self):
            del self._current.translations

        @property
        def active(self):
            return getattr(self._current, 'translations',
                           self._null_translations)

        @property
        def isactive(self):
            return hasattr(self._current, 'translations')

        # Delegated methods

        def __getattr__(self, name):
            return getattr(self.active, name)

        def gettext(self, string, **kwargs):
            def _gettext():
                return safefmt(self.active.ugettext(string), kwargs)
            if not self.isactive:
                return LazyProxy(_gettext)
            return _gettext()

        def dgettext(self, domain, string, **kwargs):
            def _dgettext():
                return safefmt(self.active.dugettext(string), kwargs)
            if not self.isactive:
                return LazyProxy(_dgettext)
            return _dgettext()

        def ngettext(self, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            kwargs.setdefault('num', num)
            def _ngettext():
                trans = self.active.ungettext(singular, plural, num)
                return safefmt(trans, kwargs)
            if not self.isactive:
                return LazyProxy(_ngettext)
            return _ngettext()

        def dngettext(self, domain, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            kwargs.setdefault('num', num)
            def _dngettext():
                trans = self.active.dungettext(domain, singular, plural, num)
                return safefmt(trans, kwargs)
            if not self.isactive:
                return LazyProxy(_dngettext)
            return _dngettext()

        def tgettext(self, string, **kwargs):
            def _tgettext():
                trans = self.active.ugettext(string)
                return kwargs and _tag_kwargs(trans, kwargs) or trans
            if not self.isactive:
                return LazyProxy(_tgettext)
            return _tgettext()

        def dtgettext(self, domain, string, **kwargs):
            def _dtgettext():
                trans = self.active.dugettext(domain, string)
                return kwargs and _tag_kwargs(trans, kwargs) or trans
            if not self.isactive:
                return LazyProxy(_dtgettext)
            return _dtgettext()

        def tngettext(self, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            kwargs.setdefault('num', num)
            def _tngettext():
                trans = self.active.ungettext(singular, plural, num)
                return _tag_kwargs(trans, kwargs)
            if not self.isactive:
                return LazyProxy(_tngettext)
            return _tngettext()

        def dtngettext(self, domain, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            def _dtngettext():
                trans = self.active.dungettext(domain, singular, plural, num)
                if '%(num)' in trans:
                    kwargs.update(num=num)
                return kwargs and _tag_kwargs(trans, kwargs) or trans
            if not self.isactive:
                return LazyProxy(_dtngettext)
            return _dtngettext()

    translations = TranslationsProxy()

    def domain_functions(domain, *symbols):
        _functions = {
          'gettext': translations.dgettext,
          '_': translations.dgettext,
          'ngettext': translations.dngettext,
          'tgettext': translations.dtgettext,
          'tag_': translations.dtgettext,
          'tngettext': translations.dtngettext,
          'add_domain': translations.add_domain,
          }
        def wrapdomain(symbol):
            if symbol == 'N_':
                return gettext_noop
            return lambda *args, **kw: _functions[symbol](domain, *args, **kw)
        return [wrapdomain(s) for s in symbols]

    gettext = translations.gettext
    _ = gettext
    dgettext = translations.dgettext
    ngettext = translations.ngettext
    dngettext = translations.dngettext
    tgettext = translations.tgettext
    tag_ = tgettext
    dtgettext = translations.dtgettext
    tngettext = translations.tngettext
    dtngettext = translations.dtngettext

    def deactivate():
        translations.deactivate()

    def activate(locale, env_path=None):
        translations.activate(locale, env_path)

    def add_domain(domain, env_path, locale_dir):
        translations.add_domain(domain, env_path, locale_dir)

    def get_translations():
        return translations

    def get_available_locales():
        """Return a list of locale identifiers of the locales for which
        translations are available.
        """
        return [dirname for dirname
                in pkg_resources.resource_listdir(__name__, '../locale')
                if '.' not in dirname]

except ImportError: # fall back on 0.11 behavior, i18n functions are no-ops
    gettext = _ = gettext_noop
    dgettext = dgettext_noop
    ngettext = ngettext_noop
    dngettext = dngettext_noop
    tgettext = tag_ = tgettext_noop
    dtgettext = dtgettext_noop
    tngettext = tngettext_noop
    dtngettext = dtngettext_noop

    def activate(locale, env_path=None):
        pass

    def deactivate():
        pass

    def get_translations():
        # for correctness we should return a NullTranslations subclass,
        # like the DummyTranslations one can find in genshi/filters/i18n.py,
        # but this works just as well for our current needs:
        return gettext_noop

    def get_available_locales():
        return []
<commit_msg>Another follow-up to r8095, another missing argument...<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2009 Edgewall Software
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.

"""Utilities for text translation with gettext."""

import re
import sys
try:
    import threading
except ImportError:
    import dummy_threading as threading

import pkg_resources

from genshi.builder import tag

__all__ = ['gettext', 'ngettext', 'gettext_noop', 'ngettext_noop',
           'tgettext', 'tgettext_noop', 'tngettext', 'tngettext_noop']

def safefmt(string, kwargs):
    if kwargs:
        try:
            return string % kwargs
        except KeyError:
            pass
    return string

def gettext_noop(string, **kwargs):
    return safefmt(string, kwargs)

def dgettext_noop(domain, string, **kwargs):
    return gettext_noop(string, **kwargs)

N_ = gettext_noop

def ngettext_noop(singular, plural, num, **kwargs):
    string = (plural, singular)[num == 1]
    kwargs.setdefault('num', num)
    return safefmt(string, kwargs)

def dngettext_noop(domain, singular, plural, num, **kwargs):
    return ngettext_noop(singular, plural, num, **kwargs)

_param_re = re.compile(r"%\((\w+)\)(?:s|[\d]*d|\d*.?\d*[fg])")
def _tag_kwargs(trans, kwargs):
    trans_elts = _param_re.split(trans)
    for i in xrange(1, len(trans_elts), 2):
        trans_elts[i] = kwargs.get(trans_elts[i], '???')
    return tag(*trans_elts)

def tgettext_noop(string, **kwargs):
    return kwargs and _tag_kwargs(string, kwargs) or string

def dtgettext_noop(domain, string, **kwargs):
    return tgettext_noop(string, **kwargs)

def tngettext_noop(singular, plural, num, **kwargs):
    string = (plural, singular)[num == 1]
    kwargs.setdefault('num', num)
    return _tag_kwargs(string, kwargs)

def dtngettext_noop(domain, singular, plural, num, **kwargs):
    return tngettext_noop(singular, plural, num, **kwargs)

def add_domain(domain, env_path, locale_dir):
    pass

def domain_functions(domain, *symbols):
    _functions = {
      'gettext': gettext_noop,
      '_': gettext_noop,
      'N_': gettext_noop,
      'ngettext': ngettext_noop,
      'tgettext': tgettext_noop,
      'tag_': tgettext_noop,
      'tngettext': tngettext_noop,
      'add_domain': lambda env_path, locale_dir: None,
      }
    return [_functions[s] for s in symbols]

try:
    from babel.support import LazyProxy, Translations
    from gettext import NullTranslations

    class TranslationsProxy(object):
        """Delegate Translations calls to the currently active Translations.

        If there's none, wrap those calls in LazyProxy objects.
        """

        def __init__(self):
            self._current = threading.local()
            self._null_translations = NullTranslations()
            self._plugin_domains = {}
            self._plugin_domains_lock = threading.RLock()

        # Public API

        def add_domain(self, domain, env_path, locales_dir):
            self._plugin_domains_lock.acquire()
            try:
                if env_path not in self._plugin_domains:
                    self._plugin_domains[env_path] = []
                self._plugin_domains[env_path].append((domain, locales_dir))
            finally:
                self._plugin_domains_lock.release()

        def activate(self, locale, env_path=None):
            locale_dir = pkg_resources.resource_filename(__name__, '../locale')
            t = Translations.load(locale_dir, locale)
            if env_path:
                self._plugin_domains_lock.acquire()
                try:
                    domains = list(self._plugin_domains.get(env_path, []))
                finally:
                    self._plugin_domains_lock.release()
                for domain, dirname in domains:
                    t.add(Translations.load(dirname, locale, domain))
            self._current.translations = t

        def deactivate(self):
            del self._current.translations

        @property
        def active(self):
            return getattr(self._current, 'translations',
                           self._null_translations)

        @property
        def isactive(self):
            return hasattr(self._current, 'translations')

        # Delegated methods

        def __getattr__(self, name):
            return getattr(self.active, name)

        def gettext(self, string, **kwargs):
            def _gettext():
                return safefmt(self.active.ugettext(string), kwargs)
            if not self.isactive:
                return LazyProxy(_gettext)
            return _gettext()

        def dgettext(self, domain, string, **kwargs):
            def _dgettext():
                return safefmt(self.active.dugettext(domain, string), kwargs)
            if not self.isactive:
                return LazyProxy(_dgettext)
            return _dgettext()

        def ngettext(self, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            kwargs.setdefault('num', num)
            def _ngettext():
                trans = self.active.ungettext(singular, plural, num)
                return safefmt(trans, kwargs)
            if not self.isactive:
                return LazyProxy(_ngettext)
            return _ngettext()

        def dngettext(self, domain, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            kwargs.setdefault('num', num)
            def _dngettext():
                trans = self.active.dungettext(domain, singular, plural, num)
                return safefmt(trans, kwargs)
            if not self.isactive:
                return LazyProxy(_dngettext)
            return _dngettext()

        def tgettext(self, string, **kwargs):
            def _tgettext():
                trans = self.active.ugettext(string)
                return kwargs and _tag_kwargs(trans, kwargs) or trans
            if not self.isactive:
                return LazyProxy(_tgettext)
            return _tgettext()

        def dtgettext(self, domain, string, **kwargs):
            def _dtgettext():
                trans = self.active.dugettext(domain, string)
                return kwargs and _tag_kwargs(trans, kwargs) or trans
            if not self.isactive:
                return LazyProxy(_dtgettext)
            return _dtgettext()

        def tngettext(self, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            kwargs.setdefault('num', num)
            def _tngettext():
                trans = self.active.ungettext(singular, plural, num)
                return _tag_kwargs(trans, kwargs)
            if not self.isactive:
                return LazyProxy(_tngettext)
            return _tngettext()

        def dtngettext(self, domain, singular, plural, num, **kwargs):
            kwargs = kwargs.copy()
            def _dtngettext():
                trans = self.active.dungettext(domain, singular, plural, num)
                if '%(num)' in trans:
                    kwargs.update(num=num)
                return kwargs and _tag_kwargs(trans, kwargs) or trans
            if not self.isactive:
                return LazyProxy(_dtngettext)
            return _dtngettext()

    translations = TranslationsProxy()

    def domain_functions(domain, *symbols):
        _functions = {
          'gettext': translations.dgettext,
          '_': translations.dgettext,
          'ngettext': translations.dngettext,
          'tgettext': translations.dtgettext,
          'tag_': translations.dtgettext,
          'tngettext': translations.dtngettext,
          'add_domain': translations.add_domain,
          }
        def wrapdomain(symbol):
            if symbol == 'N_':
                return gettext_noop
            return lambda *args, **kw: _functions[symbol](domain, *args, **kw)
        return [wrapdomain(s) for s in symbols]

    gettext = translations.gettext
    _ = gettext
    dgettext = translations.dgettext
    ngettext = translations.ngettext
    dngettext = translations.dngettext
    tgettext = translations.tgettext
    tag_ = tgettext
    dtgettext = translations.dtgettext
    tngettext = translations.tngettext
    dtngettext = translations.dtngettext

    def deactivate():
        translations.deactivate()

    def activate(locale, env_path=None):
        translations.activate(locale, env_path)

    def add_domain(domain, env_path, locale_dir):
        translations.add_domain(domain, env_path, locale_dir)

    def get_translations():
        return translations

    def get_available_locales():
        """Return a list of locale identifiers of the locales for which
        translations are available.
        """
        return [dirname for dirname
                in pkg_resources.resource_listdir(__name__, '../locale')
                if '.' not in dirname]

except ImportError: # fall back on 0.11 behavior, i18n functions are no-ops
    gettext = _ = gettext_noop
    dgettext = dgettext_noop
    ngettext = ngettext_noop
    dngettext = dngettext_noop
    tgettext = tag_ = tgettext_noop
    dtgettext = dtgettext_noop
    tngettext = tngettext_noop
    dtngettext = dtngettext_noop

    def activate(locale, env_path=None):
        pass

    def deactivate():
        pass

    def get_translations():
        # for correctness we should return a NullTranslations subclass,
        # like the DummyTranslations one can find in genshi/filters/i18n.py,
        # but this works just as well for our current needs:
        return gettext_noop

    def get_available_locales():
        return []
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import contextlib
import logging
import math
import socket
import unittest
import unittest.mock
import urllib.parse

import PIL.Image
import requests
import web_cache

import sacad

web_cache.DISABLE_PERSISTENT_CACHING = True

def is_internet_reachable():
  try:
    # open TCP socket to Google DNS server
    with socket.create_connection(("8.8.8.8", 53)):
      pass
  except OSError as e:
    if e.errno == 101:
      return False
    raise
  return True

def download(url, filepath):
  with contextlib.closing(requests.get(url, timeout=5, verify=False, stream=True)) as response:
    response.raise_for_status()
    with open(filepath, "wb") as f:
      for chunk in response.iter_content(2 ** 14):
        f.write(chunk)

@unittest.skipUnless(is_internet_reachable(), "Need Internet access")
class TestSacad(unittest.TestCase):

  @staticmethod
  def getImgInfo(img_filepath):
    with open(img_filepath, "rb") as img_file:
      img = PIL.Image.open(img_file)
      format = img.format.lower()
      format = sacad.SUPPORTED_IMG_FORMATS[format]
      width, height = img.size
    return format, width, height

  def test_getMasterOfPuppetsCover(self):
    """ Search and download cover for 'Master of Puppets' with different parameters. """
    for format in sacad.cover.CoverImageFormat:
      for size in (300, 600, 1200):
        for size_tolerance in (0, 25, 50):
          with sacad.mkstemp_ctx.mkstemp(prefix="sacad_test_",
                                         suffix=".%s" % (format.name.lower())) as tmp_filepath:
            sacad.search_and_download("Master of Puppets",
                                      "Metallica",
                                      format,
                                      size,
                                      size_tolerance,
                                      (),
                                      False,
                                      tmp_filepath)
            out_format, out_width, out_height = __class__.getImgInfo(tmp_filepath)
            self.assertEqual(out_format, format)
            self.assertLessEqual(out_width, size * (100 + size_tolerance) / 100)
            self.assertGreaterEqual(out_width, size * (100 - size_tolerance) / 100)
            self.assertLessEqual(out_height, size * (100 + size_tolerance) / 100)
            self.assertGreaterEqual(out_height, size * (100 - size_tolerance) / 100)

  def test_getImageUrlMetadata(self):
    """ Download the beginning of image files to guess their format and resolution. """
    refs = {"https://www.nuclearblast.de/static/articles/152/152118.jpg/1000x1000.jpg": (sacad.cover.CoverImageFormat.JPEG,
                                                                                (700, 700),
                                                                                math.ceil(18000 / sacad.CoverSourceResult.METADATA_PEEK_SIZE_INCREMENT)),
            "http://img2-ak.lst.fm/i/u/55ad95c53e6043e3b150ba8a0a3b20a1.png": (sacad.cover.CoverImageFormat.PNG,
                                                                               (600, 600),
                                                                               1)}
    for url, (ref_fmt, ref_size, block_read) in refs.items():
      sacad.CoverSourceResult.getImageMetadata = unittest.mock.Mock(wraps=sacad.CoverSourceResult.getImageMetadata)
      source = unittest.mock.Mock()
      cover = sacad.CoverSourceResult(url,
                                      None,
                                      None,
                                      source=source,
                                      thumbnail_url=None,
                                      source_quality=None,
                                      check_metadata=sacad.cover.CoverImageMetadata.ALL)
      cover.updateImageMetadata()
      self.assertEqual(cover.size, ref_size)
      self.assertEqual(cover.format, ref_fmt)
      self.assertGreaterEqual(sacad.CoverSourceResult.getImageMetadata.call_count, 0)
      self.assertLessEqual(sacad.CoverSourceResult.getImageMetadata.call_count, block_read)

  def test_compareImageSignatures(self):
    """ Compare images using their signatures. """
    urls = ("http://wac.450f.edgecastcdn.net/80450F/kool1017.com/files/2013/09/cover_highway_to_hell_500x500.jpg",
            "http://www.jesus-is-savior.com/Evils%20in%20America/Rock-n-Roll/highway_to_hell-large.jpg",
            "http://i158.photobucket.com/albums/t113/gatershanks/Glee%20Alternative%20Song%20Covers/1x14%20Hell%20O/1x14Hell-O-HighwayToHell.jpg")
    with sacad.mkstemp_ctx.mkstemp(suffix="jpg") as temp_filepath1, \
            sacad.mkstemp_ctx.mkstemp(suffix="jpg") as temp_filepath2, \
            sacad.mkstemp_ctx.mkstemp(suffix="jpg") as temp_filepath3:
      img_sig = {}
      for i, (url, filepath) in enumerate(zip(urls, (temp_filepath1, temp_filepath2, temp_filepath3))):
        download(url, filepath)
        with open(filepath, "rb") as img_file:
          img_data = img_file.read()
          img_sig[i] = sacad.CoverSourceResult.computeImgSignature(img_data)
      self.assertTrue(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[0], img_sig[1]))
      self.assertTrue(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[1], img_sig[0]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[0], img_sig[2]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[1], img_sig[2]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[2], img_sig[0]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[2], img_sig[1]))

  def test_coverSources(self):
    """ Check all sources return valid results with different parameters. """
    for size in range(300, 1200 + 1, 300):
      source_args = (size, 0)
      sources = [sacad.sources.LastFmCoverSource(*source_args),
                 sacad.sources.GoogleImagesWebScrapeCoverSource(*source_args),
                 sacad.sources.AmazonDigitalCoverSource(*source_args)]
      sources.extend(sacad.sources.AmazonCdCoverSource(*source_args, tld=tld) for tld in sacad.sources.AmazonCdCoverSource.TLDS)
      for source in sources:
        for artist, album in zip(("Michael Jackson", "Björk"), ("Thriller", "Vespertine")):
          results = source.search(album, artist)
          results = sacad.CoverSourceResult.preProcessForComparison(results, size, 0)
          if not (((size > 500) and isinstance(source, sacad.sources.AmazonCdCoverSource)) or
                  ((size > 500) and isinstance(source, sacad.sources.LastFmCoverSource)) or
                  (isinstance(source, sacad.sources.AmazonCdCoverSource) and (artist == "Björk") and
                   (urllib.parse.urlsplit(source.base_url).netloc.rsplit(".", 1)[-1] == "cn"))):
            self.assertGreaterEqual(len(results), 1, "%s %s %s %u" % (source.__class__.__name__,
                                                                      artist,
                                                                      album,
                                                                      size))
          for result in results:
            self.assertTrue(result.urls)
            self.assertIn(result.format, sacad.cover.CoverImageFormat)
            self.assertGreaterEqual(result.size[0], size)

    # test for specific cover not available on amazon.com, but on amazon.de
    size = 290
    source = sacad.sources.AmazonCdCoverSource(size, 0, tld="de")
    results = source.search("Dream Dance 5", "Various")
    self.assertGreaterEqual(len(results), 1)
    for result in results:
      self.assertTrue(result.urls)
      self.assertIn(result.format, sacad.cover.CoverImageFormat)
      self.assertGreaterEqual(result.size[0], size)

  def test_unaccentuate(self):
    self.assertEqual(sacad.sources.base.CoverSource.unaccentuate("EéeAàaOöoIïi"), "EeeAaaOooIii")

  def test_is_square(self):
    for x in range(1, 100):
      if x in (1, 4, 9, 16, 25, 36, 49, 64, 81):
        self.assertTrue(sacad.cover.is_square(x), x)
      else:
        self.assertFalse(sacad.cover.is_square(x), x)

if __name__ == "__main__":
  # logging
  #logging.getLogger().setLevel(logging.DEBUG)
  logging.getLogger().setLevel(logging.CRITICAL + 1)

  # run tests
  unittest.main()
<commit_msg>Fix tests<commit_after>#!/usr/bin/env python3

import contextlib
import logging
import math
import socket
import unittest
import unittest.mock
import urllib.parse

import PIL.Image
import requests
import web_cache

import sacad

web_cache.DISABLE_PERSISTENT_CACHING = True

def is_internet_reachable():
  try:
    # open TCP socket to Google DNS server
    with socket.create_connection(("8.8.8.8", 53)):
      pass
  except OSError as e:
    if e.errno == 101:
      return False
    raise
  return True

def download(url, filepath):
  with contextlib.closing(requests.get(url, timeout=5, verify=False, stream=True)) as response:
    response.raise_for_status()
    with open(filepath, "wb") as f:
      for chunk in response.iter_content(2 ** 14):
        f.write(chunk)

@unittest.skipUnless(is_internet_reachable(), "Need Internet access")
class TestSacad(unittest.TestCase):

  @staticmethod
  def getImgInfo(img_filepath):
    with open(img_filepath, "rb") as img_file:
      img = PIL.Image.open(img_file)
      format = img.format.lower()
      format = sacad.SUPPORTED_IMG_FORMATS[format]
      width, height = img.size
    return format, width, height

  def test_getMasterOfPuppetsCover(self):
    """ Search and download cover for 'Master of Puppets' with different parameters. """
    for format in sacad.cover.CoverImageFormat:
      for size in (300, 600, 1200):
        for size_tolerance in (0, 25, 50):
          with sacad.mkstemp_ctx.mkstemp(prefix="sacad_test_",
                                         suffix=".%s" % (format.name.lower())) as tmp_filepath:
            sacad.search_and_download("Master of Puppets",
                                      "Metallica",
                                      format,
                                      size,
                                      size_tolerance,
                                      (),
                                      False,
                                      tmp_filepath)
            out_format, out_width, out_height = __class__.getImgInfo(tmp_filepath)
            self.assertEqual(out_format, format)
            self.assertLessEqual(out_width, size * (100 + size_tolerance) / 100)
            self.assertGreaterEqual(out_width, size * (100 - size_tolerance) / 100)
            self.assertLessEqual(out_height, size * (100 + size_tolerance) / 100)
            self.assertGreaterEqual(out_height, size * (100 - size_tolerance) / 100)

  def test_getImageUrlMetadata(self):
    """ Download the beginning of image files to guess their format and resolution. """
    refs = {"https://www.nuclearblast.de/static/articles/152/152118.jpg/1000x1000.jpg": (sacad.cover.CoverImageFormat.JPEG,
                                                                                (700, 700),
                                                                                math.ceil(18000 / sacad.CoverSourceResult.METADATA_PEEK_SIZE_INCREMENT)),
            "http://img2-ak.lst.fm/i/u/55ad95c53e6043e3b150ba8a0a3b20a1.png": (sacad.cover.CoverImageFormat.PNG,
                                                                               (600, 600),
                                                                               1)}
    for url, (ref_fmt, ref_size, block_read) in refs.items():
      sacad.CoverSourceResult.getImageMetadata = unittest.mock.Mock(wraps=sacad.CoverSourceResult.getImageMetadata)
      source = unittest.mock.Mock()
      source.http = sacad.http_helpers.Http(allow_session_cookies=False,
                                            min_delay_between_accesses=1 / 3)
      cover = sacad.CoverSourceResult(url,
                                      None,
                                      None,
                                      source=source,
                                      thumbnail_url=None,
                                      source_quality=None,
                                      check_metadata=sacad.cover.CoverImageMetadata.ALL)
      cover.updateImageMetadata()
      self.assertEqual(cover.size, ref_size)
      self.assertEqual(cover.format, ref_fmt)
      self.assertGreaterEqual(sacad.CoverSourceResult.getImageMetadata.call_count, 0)
      self.assertLessEqual(sacad.CoverSourceResult.getImageMetadata.call_count, block_read)

  def test_compareImageSignatures(self):
    """ Compare images using their signatures. """
    urls = ("http://wac.450f.edgecastcdn.net/80450F/kool1017.com/files/2013/09/cover_highway_to_hell_500x500.jpg",
            "http://www.jesus-is-savior.com/Evils%20in%20America/Rock-n-Roll/highway_to_hell-large.jpg",
            "http://i158.photobucket.com/albums/t113/gatershanks/Glee%20Alternative%20Song%20Covers/1x14%20Hell%20O/1x14Hell-O-HighwayToHell.jpg")
    with sacad.mkstemp_ctx.mkstemp(suffix="jpg") as temp_filepath1, \
            sacad.mkstemp_ctx.mkstemp(suffix="jpg") as temp_filepath2, \
            sacad.mkstemp_ctx.mkstemp(suffix="jpg") as temp_filepath3:
      img_sig = {}
      for i, (url, filepath) in enumerate(zip(urls, (temp_filepath1, temp_filepath2, temp_filepath3))):
        download(url, filepath)
        with open(filepath, "rb") as img_file:
          img_data = img_file.read()
          img_sig[i] = sacad.CoverSourceResult.computeImgSignature(img_data)
      self.assertTrue(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[0], img_sig[1]))
      self.assertTrue(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[1], img_sig[0]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[0], img_sig[2]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[1], img_sig[2]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[2], img_sig[0]))
      self.assertFalse(sacad.CoverSourceResult.areImageSigsSimilar(img_sig[2], img_sig[1]))

  def test_coverSources(self):
    """ Check all sources return valid results with different parameters. """
    for size in range(300, 1200 + 1, 300):
      source_args = (size, 0)
      sources = [sacad.sources.LastFmCoverSource(*source_args),
                 sacad.sources.GoogleImagesWebScrapeCoverSource(*source_args),
                 sacad.sources.AmazonDigitalCoverSource(*source_args)]
      sources.extend(sacad.sources.AmazonCdCoverSource(*source_args, tld=tld) for tld in sacad.sources.AmazonCdCoverSource.TLDS)
      for source in sources:
        for artist, album in zip(("Michael Jackson", "Björk"), ("Thriller", "Vespertine")):
          results = source.search(album, artist)
          results = sacad.CoverSourceResult.preProcessForComparison(results, size, 0)
          if not (((size > 500) and isinstance(source, sacad.sources.AmazonCdCoverSource)) or
                  ((size > 500) and isinstance(source, sacad.sources.LastFmCoverSource)) or
                  (isinstance(source, sacad.sources.AmazonCdCoverSource) and (artist == "Björk") and
                   (urllib.parse.urlsplit(source.base_url).netloc.rsplit(".", 1)[-1] == "cn"))):
            self.assertGreaterEqual(len(results), 1, "%s %s %s %u" % (source.__class__.__name__,
                                                                      artist,
                                                                      album,
                                                                      size))
          for result in results:
            self.assertTrue(result.urls)
            self.assertIn(result.format, sacad.cover.CoverImageFormat)
            self.assertGreaterEqual(result.size[0], size)

    # test for specific cover not available on amazon.com, but on amazon.de
    size = 290
    source = sacad.sources.AmazonCdCoverSource(size, 0, tld="de")
    results = source.search("Dream Dance 5", "Various")
    self.assertGreaterEqual(len(results), 1)
    for result in results:
      self.assertTrue(result.urls)
      self.assertIn(result.format, sacad.cover.CoverImageFormat)
      self.assertGreaterEqual(result.size[0], size)

  def test_unaccentuate(self):
    self.assertEqual(sacad.sources.base.CoverSource.unaccentuate("EéeAàaOöoIïi"), "EeeAaaOooIii")

  def test_is_square(self):
    for x in range(1, 100):
      if x in (1, 4, 9, 16, 25, 36, 49, 64, 81):
        self.assertTrue(sacad.cover.is_square(x), x)
      else:
        self.assertFalse(sacad.cover.is_square(x), x)

if __name__ == "__main__":
  # logging
  #logging.getLogger().setLevel(logging.DEBUG)
  logging.getLogger().setLevel(logging.CRITICAL + 1)

  # run tests
  unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Apache state

.. versionadded:: 2014.7.0

Allows for inputting a yaml dictionary into a file for apache configuration
files.

The variable ``this`` is special and signifies what should be included with
the above word between angle brackets (<>).

.. code-block:: yaml

    /etc/httpd/conf.d/website.com.conf:
      apache.configfile:
        - config:
          - VirtualHost:
              this: '*:80'
              ServerName:
                - website.com
              ServerAlias:
                - www.website.com
                - dev.website.com
              ErrorLog: logs/website.com-error_log
              CustomLog: logs/website.com-access_log combined
              DocumentRoot: /var/www/vhosts/website.com
              Directory:
                this: /var/www/vhosts/website.com
                Order: Deny,Allow
                Deny from: all
                Allow from:
                  - 127.0.0.1
                  - 192.168.100.0/24
                Options:
                  - +Indexes
                  - FollowSymlinks
                AllowOverride: All
'''

from __future__ import with_statement, print_function

# Import python libs
import os.path

# Import salt libs
import salt.utils.cloud

def __virtual__():
    return 'apache.config' in __salt__

def configfile(name, config):
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    configs = __salt__['apache.config'](name, config, edit=False)
    current_configs = ''
    if os.path.exists(name):
        with open(name) as config_file:
            current_configs = config_file.read()

    if configs == current_configs.strip():
        ret['result'] = True
        ret['comment'] = 'Configuration is up to date.'
        return ret
    elif __opts__['test']:
        ret['comment'] = 'Configuration will update.'
        ret['result'] = None
        return ret

    try:
        with open(name, 'w') as config_file:
            print(configs, file=config_file)
        ret['changes'] = {
            'old': current_configs,
            'new': configs
        }
        ret['result'] = True
        ret['comment'] = 'Successfully created configuration.'
    except Exception as exc:
        ret['result'] = False
        ret['comment'] = 'Failed to create apache configuration.'

    return ret
<commit_msg>Eliminate import of salt.utils.cloud.<commit_after># -*- coding: utf-8 -*-
'''
Apache state

.. versionadded:: 2014.7.0

Allows for inputting a yaml dictionary into a file for apache configuration
files.

The variable ``this`` is special and signifies what should be included with
the above word between angle brackets (<>).

.. code-block:: yaml

    /etc/httpd/conf.d/website.com.conf:
      apache.configfile:
        - config:
          - VirtualHost:
              this: '*:80'
              ServerName:
                - website.com
              ServerAlias:
                - www.website.com
                - dev.website.com
              ErrorLog: logs/website.com-error_log
              CustomLog: logs/website.com-access_log combined
              DocumentRoot: /var/www/vhosts/website.com
              Directory:
                this: /var/www/vhosts/website.com
                Order: Deny,Allow
                Deny from: all
                Allow from:
                  - 127.0.0.1
                  - 192.168.100.0/24
                Options:
                  - +Indexes
                  - FollowSymlinks
                AllowOverride: All
'''

from __future__ import with_statement, print_function

# Import python libs
import os.path

def __virtual__():
    return 'apache.config' in __salt__

def configfile(name, config):
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    configs = __salt__['apache.config'](name, config, edit=False)
    current_configs = ''
    if os.path.exists(name):
        with open(name) as config_file:
            current_configs = config_file.read()

    if configs == current_configs.strip():
        ret['result'] = True
        ret['comment'] = 'Configuration is up to date.'
        return ret
    elif __opts__['test']:
        ret['comment'] = 'Configuration will update.'
        ret['result'] = None
        return ret

    try:
        with open(name, 'w') as config_file:
            print(configs, file=config_file)
        ret['changes'] = {
            'old': current_configs,
            'new': configs
        }
        ret['result'] = True
        ret['comment'] = 'Successfully created configuration.'
    except Exception as exc:
        ret['result'] = False
        ret['comment'] = 'Failed to create apache configuration.'

    return ret
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import json
from operator import attrgetter

from dateutil.relativedelta import relativedelta

from mptt import models as mptt

from django.conf import settings
from django.db import models, transaction
from django.db.models import Count
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from tracpro.client import get_client
from tracpro.contacts.tasks import SyncOrgContacts

@python_2_unicode_compatible
class AbstractGroup(models.Model):
    """Corresponds to a RapidPro contact group."""

    uuid = models.CharField(max_length=36)
    org = models.ForeignKey(
        'orgs.Org', verbose_name=_("Organization"), related_name="%(class)ss")
    name = models.CharField(
        verbose_name=_("Name"), max_length=128, blank=True,
        help_text=_("The name of this panel"))
    is_active = models.BooleanField(
        default=True,
        help_text=_("Whether this item is active"))

    class Meta:
        abstract = True
        unique_together = [
            ('org', 'uuid'),
        ]

    def __str__(self):
        return self.name

    def deactivate(self):
        self.is_active = False
        self.save()

    @classmethod
    def sync_with_temba(cls, org, uuids):
        """Sync org groups with the selected UUIDs."""
        # De-activate any groups that are not specified.
        other_groups = cls.objects.filter(org=org, is_active=True)
        other_groups = other_groups.exclude(uuid__in=uuids)
        for group in other_groups:
            group.deactivate()

        # Fetch group details at once.
        temba_groups = get_client(org).get_groups()
        temba_groups = {g.uuid: g.name for g in temba_groups}

        for uuid in uuids:
            if uuid in temba_groups:
                # Create a new item or update an existing one.
                obj, _ = cls.objects.get_or_create(org=org, uuid=uuid)
                obj.is_active = True
                obj.name = temba_groups.get(uuid)
                obj.save()
            else:
                # The group was removed remotely and should be removed locally.
                obj = cls.objects.filter(org=org, uuid=uuid).first()
                if obj:
                    obj.deactivate()

        SyncOrgContacts().delay(org.pk)

    @classmethod
    def get_all(cls, org):
        return cls.objects.filter(org=org, is_active=True)

    @classmethod
    def get_response_counts(cls, org, include_empty=False):
        """Return response counts from the last 30 days."""
        from tracpro.polls.models import Response
        qs = Response.objects.filter(pollrun__poll__org=org, is_active=True)

        if not include_empty:
            qs = qs.exclude(status=Response.STATUS_EMPTY)

        window_max = timezone.now()
        window_min = window_max - relativedelta(days=30)
        qs = qs.filter(updated_on__gte=window_min, updated_on__lt=window_max)

        cls_str = cls.__name__.lower()
        if cls_str == 'region':
            field = 'contact__%s' % cls_str
            counts = qs.values(field).annotate(count=Count(field))
            return_counts = {count[field]: count['count'] for count in counts}
        else:
            # a potential issue is that there is no longer a one-to-one relationship,
            # so counts for groups here do not relate to the number of responses
            return_counts = {}
            for q in qs:
                for qcg in q.contact.groups.all():
                    if qcg.pk in return_counts:
                        return_counts[qcg.pk] += 1
                    else:
                        return_counts[qcg.pk] = 1

        return return_counts

    @classmethod
    def get_most_active(cls, org):
        count_by_id = cls.get_response_counts(org, include_empty=False)
        groups = []
        for group in cls.get_all(org):
            count = count_by_id.get(group.pk, 0)
            if count:
                group.response_count = count
                groups.append(group)

        return sorted(groups, key=lambda g: g.response_count, reverse=True)

    def get_contacts(self):
        return self.contacts.filter(is_active=True)

class Region(mptt.MPTTModel, AbstractGroup):
    """
    *In the user interface, this is now named a "Panel"*

    A geographical region modelled as a group.

    In RapidPro, this is a Group.
    """
    users = models.ManyToManyField(
        settings.AUTH_USER_MODEL, verbose_name=_("Users"), related_name='regions',
        help_text=_("Users who can access this panel"))
    parent = mptt.TreeForeignKey(
        'self', null=True, blank=True, related_name="children", db_index=True)
    boundary = models.ForeignKey(
        'groups.Boundary',
        null=True,
        verbose_name=_('boundary'),
        related_name='regions',
        on_delete=models.SET_NULL)

    class Meta:
        verbose_name = 'panel'

    class MPTTMeta:
        order_insertion_by = ['name']

    @transaction.atomic
    def deactivate(self):
        # Make this region's parent the parent of all of its children.
        with Region.objects.disable_mptt_updates():
            for child in self.get_children():
                child.parent = self.parent
                child.save()
        Region.objects.rebuild()

        # Move this node out of the tree.
        # If this region is re-activated, it will appear at the top level.
        self.parent = None

        super(Region, self).deactivate()

    def get_users(self):
        return self.users.filter(is_active=True).select_related('profile')

    @classmethod
    def sync_with_temba(cls, org, uuids):
        """Rebuild the tree hierarchy after new nodes are added."""
        super(Region, cls).sync_with_temba(org, uuids)
        Region.objects.rebuild()

class Group(AbstractGroup):
    """
    A data "reporting group".

    In RapidPro, this is a Group.
    """
    class Meta:
        verbose_name = 'cohort'

# === Boundaries === #

class BoundaryQuerySet(models.QuerySet):

    def by_org(self, org):
        return self.filter(org=org)

class BoundaryManager(models.Manager.from_queryset(BoundaryQuerySet)):

    def from_temba(self, org, temba_boundary):
        """Get the existing or create a new corresponding Boundary instance.

        Assumes that the boundary's parent, if any, has already been created.
        """
        boundary, _ = self.get_or_create(org=org, rapidpro_uuid=temba_boundary.osm_id)
        boundary.name = temba_boundary.name
        boundary.level = temba_boundary.level
        boundary.geometry = json.dumps(temba_boundary.geometry.serialize())
        boundary.parent = self.filter(org=org, rapidpro_uuid=temba_boundary.parent).first()
        boundary.save()
        return boundary

    def sync(self, org):
        """Update org Boundaries from RapidPro and delete ones that were removed."""
        # Retrieve current Boundaries known to RapidPro.
        temba_boundaries = list(get_client(org).get_boundaries())

        # Remove Boundaries that are no longer on RapidPro.
        uuids = [b.osm_id for b in temba_boundaries]
        Boundary.objects.by_org(org).exclude(rapidpro_uuid__in=uuids).delete()

        # Order boundaries from the highest level (country) to the lowest
        # (district). This ensures that each boundary's parent (if any)
        # has been  created before it is updated.
        temba_boundaries.sort(key=attrgetter('level'))

        # Create new or update existing Polls to match RapidPro data.
        for temba_boundary in temba_boundaries:
            Boundary.objects.from_temba(org, temba_boundary)

class Boundary(models.Model):
    """Corresponds with a RapidPro AdminBoundary."""

    LEVEL_COUNTRY = 0
    LEVEL_STATE = 1
    LEVEL_DISTRICT = 2
    LEVEL_CHOICES = (
        (LEVEL_COUNTRY, _("Country")),
        (LEVEL_STATE, _("State")),
        (LEVEL_DISTRICT, _("District")),
    )

    org = models.ForeignKey(
        'orgs.Org',
        verbose_name=_("org"))

    rapidpro_uuid = models.CharField(
        max_length=15,
        verbose_name=_("RapidPro UUID"),
        help_text=_("Not a standard UUID; rather, it is a variation of the OSM ID."))
    name = models.CharField(
        max_length=128,
        verbose_name=_("name"))
    level = models.IntegerField(
        choices=LEVEL_CHOICES,
        verbose_name=_("level"),
        default=LEVEL_COUNTRY)
    parent = models.ForeignKey(
        "groups.Boundary",
        null=True,
        related_name="children",
        verbose_name=_("parent"),
        on_delete=models.SET_NULL)
    geometry = models.TextField(
        help_text=_("The GeoJSON geometry of this boundary."),
        verbose_name=_("geojson"))

    objects = BoundaryManager()

    class Meta:
        unique_together = (
            ('org', 'rapidpro_uuid'),
        )

    def __str__(self):
        return self.name

    def as_geojson(self):
        if not hasattr(self, '_geojson'):
            self._geojson = {
                'type': "Feature",
                'geometry': json.loads(self.geometry),
                'properties': {
                    'id': self.id,
                    'level': self.level,
                    'name': self.name,
                },
            }
        return self._geojson
<commit_msg>added back in filtering method for regions<commit_after>from __future__ import absolute_import, unicode_literals

import json
from operator import attrgetter

from dateutil.relativedelta import relativedelta

from mptt import models as mptt

from django.conf import settings
from django.db import models, transaction
from django.db.models import Count
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from tracpro.client import get_client
from tracpro.contacts.tasks import SyncOrgContacts

@python_2_unicode_compatible
class AbstractGroup(models.Model):
    """Corresponds to a RapidPro contact group."""

    uuid = models.CharField(max_length=36)
    org = models.ForeignKey(
        'orgs.Org', verbose_name=_("Organization"), related_name="%(class)ss")
    name = models.CharField(
        verbose_name=_("Name"), max_length=128, blank=True,
        help_text=_("The name of this panel"))
    is_active = models.BooleanField(
        default=True,
        help_text=_("Whether this item is active"))

    class Meta:
        abstract = True
        unique_together = [
            ('org', 'uuid'),
        ]

    def __str__(self):
        return self.name

    def deactivate(self):
        self.is_active = False
        self.save()

    @classmethod
    def sync_with_temba(cls, org, uuids):
        """Sync org groups with the selected UUIDs."""
        # De-activate any groups that are not specified.
        other_groups = cls.objects.filter(org=org, is_active=True)
        other_groups = other_groups.exclude(uuid__in=uuids)
        for group in other_groups:
            group.deactivate()

        # Fetch group details at once.
        temba_groups = get_client(org).get_groups()
        temba_groups = {g.uuid: g.name for g in temba_groups}

        for uuid in uuids:
            if uuid in temba_groups:
                # Create a new item or update an existing one.
                obj, _ = cls.objects.get_or_create(org=org, uuid=uuid)
                obj.is_active = True
                obj.name = temba_groups.get(uuid)
                obj.save()
            else:
                # The group was removed remotely and should be removed locally.
                obj = cls.objects.filter(org=org, uuid=uuid).first()
                if obj:
                    obj.deactivate()

        SyncOrgContacts().delay(org.pk)

    @classmethod
    def get_all(cls, org):
        return cls.objects.filter(org=org, is_active=True)

    @classmethod
    def get_response_counts(cls, org, include_empty=False):
        """Return response counts from the last 30 days."""
        from tracpro.polls.models import Response
        qs = Response.objects.filter(pollrun__poll__org=org, is_active=True)

        if not include_empty:
            qs = qs.exclude(status=Response.STATUS_EMPTY)

        window_max = timezone.now()
        window_min = window_max - relativedelta(days=30)
        qs = qs.filter(updated_on__gte=window_min, updated_on__lt=window_max)

        cls_str = cls.__name__.lower()
        if cls_str == 'region':
            field = 'contact__%s' % cls_str
            qs = qs.filter(**{'%s__is_active' % field: True})
            counts = qs.values(field).annotate(count=Count(field))
            return_counts = {count[field]: count['count'] for count in counts}
        else:
            # a potential issue is that there is no longer a one-to-one relationship,
            # so counts for groups here do not relate to the number of responses
            return_counts = {}
            for q in qs:
                for qcg in q.contact.groups.all():
                    if qcg.pk in return_counts:
                        return_counts[qcg.pk] += 1
                    else:
                        return_counts[qcg.pk] = 1

        return return_counts

    @classmethod
    def get_most_active(cls, org):
        count_by_id = cls.get_response_counts(org, include_empty=False)
        groups = []
        for group in cls.get_all(org):
            count = count_by_id.get(group.pk, 0)
            if count:
                group.response_count = count
                groups.append(group)

        return sorted(groups, key=lambda g: g.response_count, reverse=True)

    def get_contacts(self):
        return self.contacts.filter(is_active=True)

class Region(mptt.MPTTModel, AbstractGroup):
    """
    *In the user interface, this is now named a "Panel"*

    A geographical region modelled as a group.

    In RapidPro, this is a Group.
    """
    users = models.ManyToManyField(
        settings.AUTH_USER_MODEL, verbose_name=_("Users"), related_name='regions',
        help_text=_("Users who can access this panel"))
    parent = mptt.TreeForeignKey(
        'self', null=True, blank=True, related_name="children", db_index=True)
    boundary = models.ForeignKey(
        'groups.Boundary',
        null=True,
        verbose_name=_('boundary'),
        related_name='regions',
        on_delete=models.SET_NULL)

    class Meta:
        verbose_name = 'panel'

    class MPTTMeta:
        order_insertion_by = ['name']

    @transaction.atomic
    def deactivate(self):
        # Make this region's parent the parent of all of its children.
        with Region.objects.disable_mptt_updates():
            for child in self.get_children():
                child.parent = self.parent
                child.save()
        Region.objects.rebuild()

        # Move this node out of the tree.
        # If this region is re-activated, it will appear at the top level.
        self.parent = None

        super(Region, self).deactivate()

    def get_users(self):
        return self.users.filter(is_active=True).select_related('profile')

    @classmethod
    def sync_with_temba(cls, org, uuids):
        """Rebuild the tree hierarchy after new nodes are added."""
        super(Region, cls).sync_with_temba(org, uuids)
        Region.objects.rebuild()

class Group(AbstractGroup):
    """
    A data "reporting group".

    In RapidPro, this is a Group.
    """
    class Meta:
        verbose_name = 'cohort'

# === Boundaries === #

class BoundaryQuerySet(models.QuerySet):

    def by_org(self, org):
        return self.filter(org=org)

class BoundaryManager(models.Manager.from_queryset(BoundaryQuerySet)):

    def from_temba(self, org, temba_boundary):
        """Get the existing or create a new corresponding Boundary instance.

        Assumes that the boundary's parent, if any, has already been created.
        """
        boundary, _ = self.get_or_create(org=org, rapidpro_uuid=temba_boundary.osm_id)
        boundary.name = temba_boundary.name
        boundary.level = temba_boundary.level
        boundary.geometry = json.dumps(temba_boundary.geometry.serialize())
        boundary.parent = self.filter(org=org, rapidpro_uuid=temba_boundary.parent).first()
        boundary.save()
        return boundary

    def sync(self, org):
        """Update org Boundaries from RapidPro and delete ones that were removed."""
        # Retrieve current Boundaries known to RapidPro.
        temba_boundaries = list(get_client(org).get_boundaries())

        # Remove Boundaries that are no longer on RapidPro.
        uuids = [b.osm_id for b in temba_boundaries]
        Boundary.objects.by_org(org).exclude(rapidpro_uuid__in=uuids).delete()

        # Order boundaries from the highest level (country) to the lowest
        # (district). This ensures that each boundary's parent (if any)
        # has been  created before it is updated.
        temba_boundaries.sort(key=attrgetter('level'))

        # Create new or update existing Polls to match RapidPro data.
        for temba_boundary in temba_boundaries:
            Boundary.objects.from_temba(org, temba_boundary)

class Boundary(models.Model):
    """Corresponds with a RapidPro AdminBoundary."""

    LEVEL_COUNTRY = 0
    LEVEL_STATE = 1
    LEVEL_DISTRICT = 2
    LEVEL_CHOICES = (
        (LEVEL_COUNTRY, _("Country")),
        (LEVEL_STATE, _("State")),
        (LEVEL_DISTRICT, _("District")),
    )

    org = models.ForeignKey(
        'orgs.Org',
        verbose_name=_("org"))

    rapidpro_uuid = models.CharField(
        max_length=15,
        verbose_name=_("RapidPro UUID"),
        help_text=_("Not a standard UUID; rather, it is a variation of the OSM ID."))
    name = models.CharField(
        max_length=128,
        verbose_name=_("name"))
    level = models.IntegerField(
        choices=LEVEL_CHOICES,
        verbose_name=_("level"),
        default=LEVEL_COUNTRY)
    parent = models.ForeignKey(
        "groups.Boundary",
        null=True,
        related_name="children",
        verbose_name=_("parent"),
        on_delete=models.SET_NULL)
    geometry = models.TextField(
        help_text=_("The GeoJSON geometry of this boundary."),
        verbose_name=_("geojson"))

    objects = BoundaryManager()

    class Meta:
        unique_together = (
            ('org', 'rapidpro_uuid'),
        )

    def __str__(self):
        return self.name

    def as_geojson(self):
        if not hasattr(self, '_geojson'):
            self._geojson = {
                'type': "Feature",
                'geometry': json.loads(self.geometry),
                'properties': {
                    'id': self.id,
                    'level': self.level,
                    'name': self.name,
                },
            }
        return self._geojson
<|endoftext|>


<commit_before>"""
Estimate Functional Connectivity using an estimator for Sparse Inverse Covariances
==================================================================================
This example constructs a functional connectome using the sparse penalized MLE
estimator implemented using QUIC.

This function extracts time-series from the ABIDE dataset, with nodes defined
using regions of interest from the
  Power-264 atlas (Power, 2011).
  Power, Jonathan D., et al. "Functional network organization of the human
  brain."
  Neuron 72.4 (2011): 665-678.

Then we estimate separate inverse covariance matrices for one subject
"""
##############################################################################

import numpy as np
import matplotlib.pyplot as plt
from nilearn import datasets, plotting, input_data

import sys
sys.path.append('..')
from inverse_covariance import (
    QuicGraphLasso,
    QuicGraphLassoCV,
    QuicGraphLassoEBIC,
    AdaptiveGraphLasso,
)

plt.ion()

# Fetch the coordinates of power atlas
power = datasets.fetch_coords_power_2011()
coords = np.vstack((
    power.rois['x'],
    power.rois['y'],
    power.rois['z'],
)).T

# Loading the functional datasets
abide = datasets.fetch_abide_pcp(n_subjects=1)
abide.func=abide.func_preproc

# print basic information on the dataset
# 4D data
print('First subject functional nifti images (4D) are at: %s' %abide.func[0])

###############################################################################
# Masking: taking the signal in a sphere of radius 5mm around Power coords

masker = input_data.NiftiSpheresMasker(seeds=coords,
                                       smoothing_fwhm=4,
                                       radius=5.,
                                       standardize=True,
                                       detrend=True,
                                       low_pass=0.1,
                                       high_pass=0.01,
                                       t_r=2.5)

timeseries = masker.fit_transform(abide.func[0])

###############################################################################
# Extract and plot sparse inverse covariance

estimator_type = 'QuicGraphLasso'

if estimator_type == 'QuicGraphLasso':
    # Compute the sparse inverse covariance via QuicGraphLasso
    estimator = QuicGraphLasso(
        init_method='cov',
        lam=0.5,
        mode='default',
        verbose=1)

elif estimator_type == 'QuicGraphLassoCV':
    # Compute the sparse inverse covariance via QuicGraphLassoCV
    estimator = QuicGraphLassoCV(
        init_method='cov',
        verbose=1)
    estimator.fit(timeseries)

elif estimator_type == 'QuicGraphLassoEBIC':
    # Compute the sparse inverse covariance via QuicGraphLassoEBIC
    estimator = QuicGraphLassoEBIC(
        init_method='cov',
        verbose=1)
    estimator.fit(timeseries)

elif estimator_type == 'AdaptiveQuicGraphLasso':
    # Compute the sparse inverse covariance via
    # AdaptiveGraphLasso + QuicGraphLassoEBIC + method='binary'
    model = AdaptiveGraphLasso(
            estimator=QuicGraphLassoEBIC(
                init_method='cov',
            ),
            method='binary',
        )
    model.fit(timeseries)
    estimator = model.estimator_

# Display the sparse inverse covariance
plt.figure(figsize=(7.5, 7.5))
plt.imshow(
    np.triu(-estimator.precision_, 1),
    interpolation="nearest",
    cmap=plt.cm.RdBu_r)
plt.title('Precision (Sparse Inverse Covariance) matrix')
plt.colorbar()

# And now display the corresponding graph
plotting.plot_connectome(-estimator.precision_, coords,
                         title='Functional Connectivity using Precision Matrix',
                         edge_threshold="99.2%",
                         node_size=20)
plotting.show()
raw_input()
<commit_msg>Added parent directory of pyquic<commit_after>"""
Estimate Functional Connectivity using an estimator for Sparse Inverse Covariances
==================================================================================
This example constructs a functional connectome using the sparse penalized MLE
estimator implemented using QUIC.

This function extracts time-series from the ABIDE dataset, with nodes defined
using regions of interest from the
  Power-264 atlas (Power, 2011).
  Power, Jonathan D., et al. "Functional network organization of the human
  brain."
  Neuron 72.4 (2011): 665-678.

Then we estimate separate inverse covariance matrices for one subject
"""
##############################################################################

import numpy as np
import matplotlib.pyplot as plt
from nilearn import datasets, plotting, input_data

import sys
sys.path.append('..')
sys.path.append('../inverse_covariance')
from inverse_covariance import (
    QuicGraphLasso,
    QuicGraphLassoCV,
    QuicGraphLassoEBIC,
    AdaptiveGraphLasso,
)

plt.ion()

# Fetch the coordinates of power atlas
power = datasets.fetch_coords_power_2011()
coords = np.vstack((
    power.rois['x'],
    power.rois['y'],
    power.rois['z'],
)).T

# Loading the functional datasets
abide = datasets.fetch_abide_pcp(n_subjects=1)
abide.func=abide.func_preproc

# print basic information on the dataset
# 4D data
print('First subject functional nifti images (4D) are at: %s' %abide.func[0])

###############################################################################
# Masking: taking the signal in a sphere of radius 5mm around Power coords

masker = input_data.NiftiSpheresMasker(seeds=coords,
                                       smoothing_fwhm=4,
                                       radius=5.,
                                       standardize=True,
                                       detrend=True,
                                       low_pass=0.1,
                                       high_pass=0.01,
                                       t_r=2.5)

timeseries = masker.fit_transform(abide.func[0])

###############################################################################
# Extract and plot sparse inverse covariance

estimator_type = 'QuicGraphLasso'

if estimator_type == 'QuicGraphLasso':
    # Compute the sparse inverse covariance via QuicGraphLasso
    estimator = QuicGraphLasso(
        init_method='cov',
        lam=0.5,
        mode='default',
        verbose=1)

elif estimator_type == 'QuicGraphLassoCV':
    # Compute the sparse inverse covariance via QuicGraphLassoCV
    estimator = QuicGraphLassoCV(
        init_method='cov',
        verbose=1)
    estimator.fit(timeseries)

elif estimator_type == 'QuicGraphLassoEBIC':
    # Compute the sparse inverse covariance via QuicGraphLassoEBIC
    estimator = QuicGraphLassoEBIC(
        init_method='cov',
        verbose=1)
    estimator.fit(timeseries)

elif estimator_type == 'AdaptiveQuicGraphLasso':
    # Compute the sparse inverse covariance via
    # AdaptiveGraphLasso + QuicGraphLassoEBIC + method='binary'
    model = AdaptiveGraphLasso(
            estimator=QuicGraphLassoEBIC(
                init_method='cov',
            ),
            method='binary',
        )
    model.fit(timeseries)
    estimator = model.estimator_

# Display the sparse inverse covariance
plt.figure(figsize=(7.5, 7.5))
plt.imshow(
    np.triu(-estimator.precision_, 1),
    interpolation="nearest",
    cmap=plt.cm.RdBu_r)
plt.title('Precision (Sparse Inverse Covariance) matrix')
plt.colorbar()

# And now display the corresponding graph
plotting.plot_connectome(-estimator.precision_, coords,
                         title='Functional Connectivity using Precision Matrix',
                         edge_threshold="99.2%",
                         node_size=20)
plotting.show()
raw_input()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Execution of Salt modules from within states
============================================

These states allow individual execution module calls to be made via states. To
call a single module function use a :mod:`module.run <salt.states.module.run>`
state:

.. code-block:: yaml

    mine.send:
      module.run:
        - name: network.interfaces

Note that this example is probably unnecessary to use in practice, since the
``mine_functions`` and ``mine_interval`` config parameters can be used to
schedule updates for the mine (see :doc:`here </topics/mine/index>` for more
info).

It is sometimes desirable to trigger a function call after a state is executed,
for this the :mod:`module.wait <salt.states.module.wait>` state can be used:

.. code-block:: yaml

    mine.send:
      module.wait:
        - name: network.interfaces
        - watch:
          - file: /etc/network/interfaces

All arguments that the ``module`` state does not consume are passed through to
the execution module function being executed:

.. code-block:: yaml

    fetch_out_of_band:
      module.run:
        - name: git.fetch
        - cwd: /path/to/my/repo
        - user: myuser
        - opts: '--all'

Due to how the state system works, if a module function accepts an
argument called, ``name``, then ``m_name`` must be used to specify that
argument, to avoid a collision with the ``name`` argument. For example:

.. code-block:: yaml

    disable_nfs:
      module.run:
        - name: service.disable
        - m_name: nfs

Note that some modules read all or some of the arguments from a list of keyword
arguments. For example:

.. code-block:: yaml

    mine.send:
      module.run:
        - func: network.ip_addrs
        - kwargs:
            interface: eth0
'''
# Import python libs
import datetime

# Import salt libs
import salt.loader
import salt.utils

def wait(name, **kwargs):
    '''
    Run a single module function only if the watch statement calls it

    ``name``
        The module function to execute

    ``**kwargs``
        Pass any arguments needed to execute the function

    .. note::
        Like the :mod:`cmd.run <salt.states.cmd.run>` state, this state will
        return ``True`` but not actually execute, unless one of the following
        two things happens:

        1. The state has a :doc:`watch requisite </ref/states/requisites>`, and
           the state which it is watching changes.

        2. Another state has a :doc:`watch_in requisite
           </ref/states/requisites>` which references this state, and the state
           wth the ``watch_in`` changes.
    '''
    return {'name': name,
            'changes': {},
            'result': True,
            'comment': ''}

# Alias module.watch to module.wait
watch = wait

def run(name, **kwargs):
    '''
    Run a single module function

    ``name``
        The module function to execute

    ``returner``
        Specify the returner to send the return of the module execution to

    ``**kwargs``
        Pass any arguments needed to execute the function
    '''
    ret = {'name': name,
           'changes': {},
           'comment': '',
           'result': None}
    if name not in __salt__:
        ret['comment'] = 'Module function {0} is not available'.format(name)
        ret['result'] = False
        return ret

    if __opts__['test']:
        ret['comment'] = 'Module function {0} is set to execute'.format(name)
        return ret

    aspec = salt.utils.get_function_argspec(__salt__[name])
    args = []
    defaults = {}

    arglen = 0
    deflen = 0
    if isinstance(aspec.args, list):
        arglen = len(aspec.args)
    if isinstance(aspec.defaults, tuple):
        deflen = len(aspec.defaults)
    # Match up the defaults with the respective args
    for ind in range(arglen - 1, -1, -1):
        minus = arglen - ind
        if deflen - minus > -1:
            defaults[aspec.args[ind]] = aspec.defaults[-minus]
    # overwrite passed default kwargs
    for arg in defaults:
        if arg == 'name':
            if 'm_name' in kwargs:
                defaults[arg] = kwargs.pop('m_name')
        elif arg == 'fun':
            if 'm_fun' in kwargs:
                defaults[arg] = kwargs.pop('m_fun')
        if arg in kwargs:
            defaults[arg] = kwargs.pop(arg)
    missing = set()
    for arg in aspec.args:
        if arg == 'name':
            rarg = 'm_name'
        elif arg == 'fun':
            rarg = 'm_fun'
        else:
            rarg = arg
        if rarg not in kwargs and arg not in defaults:
            missing.add(rarg)
            continue
        if arg in defaults:
            args.append(defaults[arg])
        else:
            args.append(kwargs.pop(rarg))
    if missing:
        comment = 'The following arguments are missing:'
        for arg in missing:
            comment += ' {0}'.format(arg)
        ret['comment'] = comment
        ret['result'] = False
        return ret

    if aspec.varargs and aspec.varargs in kwargs:
        varargs = kwargs.pop(aspec.varargs)

        if not isinstance(varargs, list):
            msg = "'{0}' must be a list."
            ret['comment'] = msg.format(aspec.varargs)
            ret['result'] = False
            return ret

        args.extend(varargs)

    nkwargs = {}
    if aspec.keywords and aspec.keywords in kwargs:
        nkwargs = kwargs.pop(aspec.keywords)

        if not isinstance(nkwargs, dict):
            msg = "'{0}' must be a dict."
            ret['comment'] = msg.format(aspec.keywords)
            ret['result'] = False
            return ret

    try:
        if aspec.keywords:
            mret = __salt__[name](*args, **nkwargs)
        else:
            mret = __salt__[name](*args)
    except Exception as e:
        ret['comment'] = 'Module function {0} threw an exception. Exception: {1}'.format(name, e)
        ret['result'] = False
        return ret
    else:
        if mret is not None:
            ret['changes']['ret'] = mret

    if 'returner' in kwargs:
        ret_ret = {
                'id': __opts__['id'],
                'ret': mret,
                'fun': name,
                'jid': '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())}
        returners = salt.loader.returners(__opts__, __salt__)
        if kwargs['returner'] in returners:
            returners[kwargs['returner']](ret_ret)
    ret['comment'] = 'Module function {0} executed'.format(name)

    ret['result'] = True
    # if mret is a dict and there is retcode and its non-zero
    if isinstance(mret, dict) and mret.get('retcode', 0) != 0:
        ret['result'] = False
    # if its a boolean, return that as the result
    elif isinstance(mret, bool):
        ret['result'] = mret
    else:
        changes_ret = ret['changes'].get('ret', {})
        if isinstance(changes_ret, dict) and changes_ret.get('retcode', 0) != 0:
            ret['result'] = False
    return ret

mod_watch = run  # pylint: disable=C0103
<commit_msg>Making salt.states.module python3 compatible<commit_after># -*- coding: utf-8 -*-
'''
Execution of Salt modules from within states
============================================

These states allow individual execution module calls to be made via states. To
call a single module function use a :mod:`module.run <salt.states.module.run>`
state:

.. code-block:: yaml

    mine.send:
      module.run:
        - name: network.interfaces

Note that this example is probably unnecessary to use in practice, since the
``mine_functions`` and ``mine_interval`` config parameters can be used to
schedule updates for the mine (see :doc:`here </topics/mine/index>` for more
info).

It is sometimes desirable to trigger a function call after a state is executed,
for this the :mod:`module.wait <salt.states.module.wait>` state can be used:

.. code-block:: yaml

    mine.send:
      module.wait:
        - name: network.interfaces
        - watch:
          - file: /etc/network/interfaces

All arguments that the ``module`` state does not consume are passed through to
the execution module function being executed:

.. code-block:: yaml

    fetch_out_of_band:
      module.run:
        - name: git.fetch
        - cwd: /path/to/my/repo
        - user: myuser
        - opts: '--all'

Due to how the state system works, if a module function accepts an
argument called, ``name``, then ``m_name`` must be used to specify that
argument, to avoid a collision with the ``name`` argument. For example:

.. code-block:: yaml

    disable_nfs:
      module.run:
        - name: service.disable
        - m_name: nfs

Note that some modules read all or some of the arguments from a list of keyword
arguments. For example:

.. code-block:: yaml

    mine.send:
      module.run:
        - func: network.ip_addrs
        - kwargs:
            interface: eth0
'''
from __future__ import absolute_import
# Import python libs
import datetime

# Import salt libs
import salt.loader
import salt.utils
from six.moves import range

def wait(name, **kwargs):
    '''
    Run a single module function only if the watch statement calls it

    ``name``
        The module function to execute

    ``**kwargs``
        Pass any arguments needed to execute the function

    .. note::
        Like the :mod:`cmd.run <salt.states.cmd.run>` state, this state will
        return ``True`` but not actually execute, unless one of the following
        two things happens:

        1. The state has a :doc:`watch requisite </ref/states/requisites>`, and
           the state which it is watching changes.

        2. Another state has a :doc:`watch_in requisite
           </ref/states/requisites>` which references this state, and the state
           wth the ``watch_in`` changes.
    '''
    return {'name': name,
            'changes': {},
            'result': True,
            'comment': ''}

# Alias module.watch to module.wait
watch = wait

def run(name, **kwargs):
    '''
    Run a single module function

    ``name``
        The module function to execute

    ``returner``
        Specify the returner to send the return of the module execution to

    ``**kwargs``
        Pass any arguments needed to execute the function
    '''
    ret = {'name': name,
           'changes': {},
           'comment': '',
           'result': None}
    if name not in __salt__:
        ret['comment'] = 'Module function {0} is not available'.format(name)
        ret['result'] = False
        return ret

    if __opts__['test']:
        ret['comment'] = 'Module function {0} is set to execute'.format(name)
        return ret

    aspec = salt.utils.get_function_argspec(__salt__[name])
    args = []
    defaults = {}

    arglen = 0
    deflen = 0
    if isinstance(aspec.args, list):
        arglen = len(aspec.args)
    if isinstance(aspec.defaults, tuple):
        deflen = len(aspec.defaults)
    # Match up the defaults with the respective args
    for ind in range(arglen - 1, -1, -1):
        minus = arglen - ind
        if deflen - minus > -1:
            defaults[aspec.args[ind]] = aspec.defaults[-minus]
    # overwrite passed default kwargs
    for arg in defaults:
        if arg == 'name':
            if 'm_name' in kwargs:
                defaults[arg] = kwargs.pop('m_name')
        elif arg == 'fun':
            if 'm_fun' in kwargs:
                defaults[arg] = kwargs.pop('m_fun')
        if arg in kwargs:
            defaults[arg] = kwargs.pop(arg)
    missing = set()
    for arg in aspec.args:
        if arg == 'name':
            rarg = 'm_name'
        elif arg == 'fun':
            rarg = 'm_fun'
        else:
            rarg = arg
        if rarg not in kwargs and arg not in defaults:
            missing.add(rarg)
            continue
        if arg in defaults:
            args.append(defaults[arg])
        else:
            args.append(kwargs.pop(rarg))
    if missing:
        comment = 'The following arguments are missing:'
        for arg in missing:
            comment += ' {0}'.format(arg)
        ret['comment'] = comment
        ret['result'] = False
        return ret

    if aspec.varargs and aspec.varargs in kwargs:
        varargs = kwargs.pop(aspec.varargs)

        if not isinstance(varargs, list):
            msg = "'{0}' must be a list."
            ret['comment'] = msg.format(aspec.varargs)
            ret['result'] = False
            return ret

        args.extend(varargs)

    nkwargs = {}
    if aspec.keywords and aspec.keywords in kwargs:
        nkwargs = kwargs.pop(aspec.keywords)

        if not isinstance(nkwargs, dict):
            msg = "'{0}' must be a dict."
            ret['comment'] = msg.format(aspec.keywords)
            ret['result'] = False
            return ret

    try:
        if aspec.keywords:
            mret = __salt__[name](*args, **nkwargs)
        else:
            mret = __salt__[name](*args)
    except Exception as e:
        ret['comment'] = 'Module function {0} threw an exception. Exception: {1}'.format(name, e)
        ret['result'] = False
        return ret
    else:
        if mret is not None:
            ret['changes']['ret'] = mret

    if 'returner' in kwargs:
        ret_ret = {
                'id': __opts__['id'],
                'ret': mret,
                'fun': name,
                'jid': '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())}
        returners = salt.loader.returners(__opts__, __salt__)
        if kwargs['returner'] in returners:
            returners[kwargs['returner']](ret_ret)
    ret['comment'] = 'Module function {0} executed'.format(name)

    ret['result'] = True
    # if mret is a dict and there is retcode and its non-zero
    if isinstance(mret, dict) and mret.get('retcode', 0) != 0:
        ret['result'] = False
    # if its a boolean, return that as the result
    elif isinstance(mret, bool):
        ret['result'] = mret
    else:
        changes_ret = ret['changes'].get('ret', {})
        if isinstance(changes_ret, dict) and changes_ret.get('retcode', 0) != 0:
            ret['result'] = False
    return ret

mod_watch = run  # pylint: disable=C0103
<|endoftext|>


<commit_before># Copyright 2020 Google LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Preprocessing and Infeeding Tools For Redial and MovieLens Data."""

import tensorflow.compat.v1 as tf
import tensorflow_datasets as tfds
import functools
import os
import json

BASE_DIR = "gs://e2e_central"
DATA_DIR = os.path.join(BASE_DIR, "data")

rd_tsv_path = {
    "train": os.path.join(DATA_DIR, "rd-train.tsv"),
    "validation": os.path.join(DATA_DIR, "rd-validation.tsv")
}

def rd_jsonl_to_tsv(in_fname, out_fname):
  """Converts the redial jsonl to a tsv."""
  print("Reading: " + in_fname)
  def fix_spacing(text):
    """Removes extra spaces."""
    # Remove incorrect spacing around punctuation.
    text = text.replace(" ,", ",").replace(" .", ".").replace(" %", "%")
    text = text.replace(" - ", "-").replace(" : ", ":").replace(" / ", "/")
    text = text.replace("( ", "(").replace(" )", ")")
    text = text.replace("`` ", "\"").replace(" ''", "\"")
    text = text.replace(" 's", "'s").replace("s ' ", "s' ")
    return text

  count = 0
  with tf.io.gfile.GFile(in_fname, "rb") as infile,\
      tf.io.gfile.GFile(out_fname, "w") as outfile:
    for line in infile:
      ex = json.loads(line)
      print(line)
      conversation = fix_spacing(ex["conversation"])
      response = fix_spacing(ex["response"])
      # Write this line as <conversation>\t<response>
      outfile.write("%s\t%s\n" % (conversation, response))
      count += 1
      tf.logging.log_every_n(
          tf.logging.INFO,
          "Wrote %d examples to %s." % (count, out_fname),
          1000)
    return count

def rd_dataset_fn(split, shuffle_files=False):
  """Returns a tf dataset of (conversation, response) pairs for redial."""
  # We only have one file for each split.
  del shuffle_files

  # Load lines from the text file as examples.
  ds = tf.data.TextLineDataset(rd_tsv_path[split])
  # Split each "<conversation>\t<response>" example into (conversation, response) tuple.
  ds = ds.map(
      functools.partial(tf.io.decode_csv, record_defaults=["", ""],
                        field_delim="\t", use_quote_delim=False),
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  # Map each tuple to a {"conversation": ... "response": ...} dict.
  ds = ds.map(lambda *ex: dict(zip(["conversation", "response"], ex)))
  return ds

def conversation_preprocessor(ds):
  """Prepares text for input into model."""
  def normalize_text(text):
    """Lowercase and remove quotes from a TensorFlow string."""
    text = tf.strings.lower(text)
    text = tf.strings.regex_replace(text,"'(.*)'", r"\1")
    return text

  def to_inputs_and_targets(ex):
    """Map {"conversation": ..., "response": ...}->{"inputs": ..., "targets": ...}."""
    return {
        "inputs":
            tf.strings.join(
                ["movie recommendation: ", normalize_text(ex["conversation"])]),
        "targets": normalize_text(ex["response"])
    }
  return ds.map(to_inputs_and_targets,
                num_parallel_calls=tf.data.experimental.AUTOTUNE)<commit_msg>remove print<commit_after># Copyright 2020 Google LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Preprocessing and Infeeding Tools For Redial and MovieLens Data."""

import tensorflow.compat.v1 as tf
import tensorflow_datasets as tfds
import functools
import os
import json

BASE_DIR = "gs://e2e_central"
DATA_DIR = os.path.join(BASE_DIR, "data")

rd_tsv_path = {
    "train": os.path.join(DATA_DIR, "rd-train.tsv"),
    "validation": os.path.join(DATA_DIR, "rd-validation.tsv")
}

def rd_jsonl_to_tsv(in_fname, out_fname):
  """Converts the redial jsonl to a tsv."""
  print("Reading: " + in_fname)
  def fix_spacing(text):
    """Removes extra spaces."""
    # Remove incorrect spacing around punctuation.
    text = text.replace(" ,", ",").replace(" .", ".").replace(" %", "%")
    text = text.replace(" - ", "-").replace(" : ", ":").replace(" / ", "/")
    text = text.replace("( ", "(").replace(" )", ")")
    text = text.replace("`` ", "\"").replace(" ''", "\"")
    text = text.replace(" 's", "'s").replace("s ' ", "s' ")
    return text

  count = 0
  with tf.io.gfile.GFile(in_fname, "rb") as infile,\
      tf.io.gfile.GFile(out_fname, "w") as outfile:
    for line in infile:
      ex = json.loads(line)
      conversation = fix_spacing(ex["conversation"])
      response = fix_spacing(ex["response"])
      # Write this line as <conversation>\t<response>
      outfile.write("%s\t%s\n" % (conversation, response))
      count += 1
      tf.logging.log_every_n(
          tf.logging.INFO,
          "Wrote %d examples to %s." % (count, out_fname),
          1000)
    return count

def rd_dataset_fn(split, shuffle_files=False):
  """Returns a tf dataset of (conversation, response) pairs for redial."""
  # We only have one file for each split.
  del shuffle_files

  # Load lines from the text file as examples.
  ds = tf.data.TextLineDataset(rd_tsv_path[split])
  # Split each "<conversation>\t<response>" example into (conversation, response) tuple.
  ds = ds.map(
      functools.partial(tf.io.decode_csv, record_defaults=["", ""],
                        field_delim="\t", use_quote_delim=False),
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  # Map each tuple to a {"conversation": ... "response": ...} dict.
  ds = ds.map(lambda *ex: dict(zip(["conversation", "response"], ex)))
  return ds

def conversation_preprocessor(ds):
  """Prepares text for input into model."""
  def normalize_text(text):
    """Lowercase and remove quotes from a TensorFlow string."""
    text = tf.strings.lower(text)
    text = tf.strings.regex_replace(text,"'(.*)'", r"\1")
    return text

  def to_inputs_and_targets(ex):
    """Map {"conversation": ..., "response": ...}->{"inputs": ..., "targets": ...}."""
    return {
        "inputs":
            tf.strings.join(
                ["movie recommendation: ", normalize_text(ex["conversation"])]),
        "targets": normalize_text(ex["response"])
    }
  return ds.map(to_inputs_and_targets,
                num_parallel_calls=tf.data.experimental.AUTOTUNE)<|endoftext|>


<commit_before>"""Provides convenient access to an SQLObject-managed database.

This module enables easy use of an SQLObject database by providing an auto-connect
hub that will utilize the db uri string given in the Paste conf file called
``sqlobject.dburi``

It is based heavily (if not 99%) on the TurboGears file of the same name.
"""

import sqlobject
from sqlobject.dbconnection import ConnectionHub, Transaction, TheURIOpener
from paste.deploy.config import CONFIG

class AutoConnectHub(ConnectionHub):
    """Connects to the database once per thread.

    The AutoConnectHub also provides convenient methods for managing transactions.
    """
    uri = None
    params = {}

    def __init__(self, uri=None):
        if not uri:
            uri = CONFIG['app_conf'].get('sqlobject.dburi')
        self.uri = uri
        ConnectionHub.__init__(self)

    def getConnection(self):
        try:
            conn = self.threadingLocal.connection
            return conn
        except AttributeError:
            if self.uri:
                conn = sqlobject.connectionForURI(self.uri)
                # the following line effectively turns off the DBAPI connection
                # cache. We're already holding on to a connection per thread,
                # and the cache causes problems with sqlite.
                if self.uri.startswith("sqlite"):
                    TheURIOpener.cachedURIs = {}
                self.threadingLocal.connection = conn
                return conn
            try:
                return self.processConnection
            except AttributeError:
                raise AttributeError(
                    "No connection has been defined for this thread "
                    "or process")

    def begin(self):
        """Starts a transaction."""
        conn = self.getConnection()
        if isinstance(conn, Transaction):
            if conn._obsolete:
                conn.begin()
            return
        self.threadingLocal.old_conn = conn
        self.threadingLocal.connection = conn.transaction()

    def commit(self):
        """Commits the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction):
            self.threadingLocal.connection.commit()

    def rollback(self):
        """Rolls back the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction) and not conn._obsolete:
            self.threadingLocal.connection.rollback()

    def end(self):
        """Ends the transaction, returning to a standard connection."""
        conn = self.threadingLocal.connection
        if not isinstance(conn, Transaction):
            return
        if not conn._obsolete:
            conn.rollback()
        self.threadingLocal.connection = self.threadingLocal.old_conn
        del self.threadingLocal.old_conn
        self.threadingLocal.connection.cache.clear()

# This dictionary stores the AutoConnectHubs used for each
# connection URI
_hubs = dict()

class PackageHub(object):
    """Transparently proxies to an AutoConnectHub for the URI
    that is appropriate for this package. A package URI is
    configured via "packagename.dburi" in the global CherryPy
    settings. If there is no package DB URI configured, the
    default (provided by "sqlobject.dburi") is used.

    The hub is not instantiated until an attempt is made to
    use the database.
    """
    def __init__(self, packagename, dburi=None):
        self.packagename = packagename
        self.hub = None
        self.dburi = dburi

    def __get__(self, obj, type):
        if not self.hub:
            self.set_hub()
        return self.hub.__get__(obj, type)

    def __set__(self, obj, type):
        if not self.hub:
            self.set_hub()
        return self.hub.__set__(obj, type)

    def __getattr__(self, name):
        if not self.hub:
            self.set_hub()
        return getattr(self.hub, name)

    def set_hub(self):
        dburi = self.dburi
        if not dburi:
            appconf = CONFIG['app_conf']
            dburi = appconf.get("%s.dburi" % self.packagename)
            if not dburi:
                dburi = appconf.get("sqlobject.dburi")
        if not dburi:
            raise KeyError, "No database configuration found!"
        hub = _hubs.get(dburi)
        if not hub:
            hub = AutoConnectHub(dburi)
            _hubs[dburi] = hub
        self.hub = hub

__all__ = ["PackageHub", "AutoConnectHub"]
<commit_msg>[svn] Code cleanup<commit_after>"""Provides convenient access to an SQLObject-managed database.

This module enables easy use of an SQLObject database by providing an
auto-connect hub that will utilize the db uri string given in the Paste conf
file called ``sqlobject.dburi``.

It is based heavily (if not 99%) on the TurboGears file of the same name.
"""

import sqlobject
from sqlobject.dbconnection import ConnectionHub, Transaction, TheURIOpener
from paste.deploy.config import CONFIG

class AutoConnectHub(ConnectionHub):
    """Connects to the database once per thread.

    The AutoConnectHub also provides convenient methods for managing transactions.
    """
    uri = None
    params = {}

    def __init__(self, uri=None):
        if not uri:
            uri = CONFIG['app_conf'].get('sqlobject.dburi')
        self.uri = uri
        ConnectionHub.__init__(self)

    def get_connection(self):
        try:
            conn = self.threadingLocal.connection
            return conn
        except AttributeError:
            if self.uri:
                conn = sqlobject.connectionForURI(self.uri)
                # the following line effectively turns off the DBAPI connection
                # cache. We're already holding on to a connection per thread,
                # and the cache causes problems with sqlite.
                if self.uri.startswith("sqlite"):
                    TheURIOpener.cachedURIs = {}
                self.threadingLocal.connection = conn
                return conn
            try:
                return self.processConnection
            except AttributeError:
                raise AttributeError(
                    "No connection has been defined for this thread "
                    "or process")

    def begin(self):
        """Starts a transaction."""
        conn = self.get_connection()
        if isinstance(conn, Transaction):
            if conn._obsolete:
                conn.begin()
            return
        self.threadingLocal.old_conn = conn
        self.threadingLocal.connection = conn.transaction()

    def commit(self):
        """Commits the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction):
            self.threadingLocal.connection.commit()

    def rollback(self):
        """Rolls back the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction) and not conn._obsolete:
            self.threadingLocal.connection.rollback()

    def end(self):
        """Ends the transaction, returning to a standard connection."""
        conn = self.threadingLocal.connection
        if not isinstance(conn, Transaction):
            return
        if not conn._obsolete:
            conn.rollback()
        self.threadingLocal.connection = self.threadingLocal.old_conn
        del self.threadingLocal.old_conn
        self.threadingLocal.connection.cache.clear()

# This dictionary stores the AutoConnectHubs used for each
# connection URI
_hubs = dict()

class PackageHub(object):
    """Transparently proxies to an AutoConnectHub for the URI
    that is appropriate for this package. A package URI is
    configured via "packagename.dburi" in the Paste ini file
    settings. If there is no package DB URI configured, the
    default (provided by "sqlobject.dburi") is used.

    The hub is not instantiated until an attempt is made to
    use the database.
    """
    def __init__(self, packagename, dburi=None):
        self.packagename = packagename
        self.hub = None
        self.dburi = dburi

    def __get__(self, obj, type):
        if not self.hub:
            self.set_hub()
        return self.hub.__get__(obj, type)

    def __set__(self, obj, type):
        if not self.hub:
            self.set_hub()
        return self.hub.__set__(obj, type)

    def __getattr__(self, name):
        if not self.hub:
            self.set_hub()
        return getattr(self.hub, name)

    def set_hub(self):
        dburi = self.dburi
        if not dburi:
            appconf = CONFIG['app_conf']
            dburi = appconf.get("%s.dburi" % self.packagename)
            if not dburi:
                dburi = appconf.get("sqlobject.dburi")
        if not dburi:
            raise KeyError, "No database configuration found!"
        hub = _hubs.get(dburi)
        if not hub:
            hub = AutoConnectHub(dburi)
            _hubs[dburi] = hub
        self.hub = hub

__all__ = ["PackageHub", "AutoConnectHub"]
<|endoftext|>