<commit_before># -*- coding: utf-8 -*-
"""
The MIT License (MIT)

Copyright (c) 2014 Cuble Desarrollo S.L.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""
from django.db import models
from easy_thumbnails.fields import ThumbnailerImageField
from filer.fields.file import FilerFileField
from core.files import readable_name_to_path

class Quote(models.Model):
    """
    """

    author = models.CharField(max_length=256)
    picture = ThumbnailerImageField(upload_to=readable_name_to_path('images'), null=True, blank=True)
    text = models.TextField()
<commit_msg>filer dependencies<commit_after># -*- coding: utf-8 -*-
"""
The MIT License (MIT)

Copyright (c) 2014 Cuble Desarrollo S.L.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""
from django.db import models
from easy_thumbnails.fields import ThumbnailerImageField

from core.files import readable_name_to_path

class Quote(models.Model):
    """
    """

    author = models.CharField(max_length=256)
    picture = ThumbnailerImageField(upload_to=readable_name_to_path('images'), null=True, blank=True)
    text = models.TextField()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
import os
import re
import warnings
import io
import sys

import typing
from typing import Any, Dict, List, Optional, Text

from rasa_nlu import utils
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.featurizers import Featurizer
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData

import numpy as np

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa_nlu.model import Metadata

REGEX_FEATURIZER_FILE_NAME = "regex_featurizer.json"

class RegexFeaturizer(Featurizer):
    name = "intent_entity_featurizer_regex"

    provides = ["text_features"]

    requires = ["tokens"]

    def __init__(self, component_config=None,
                 known_patterns=None, lookup_tables=[]):

        super(RegexFeaturizer, self).__init__(component_config)

        self.known_patterns = known_patterns if known_patterns else []
        self._add_lookup_table_regexes(lookup_tables)

    def train(self, training_data, config, **kwargs):
        # type: (TrainingData, RasaNLUModelConfig, **Any) -> None

        self.known_patterns = training_data.regex_features
        self._add_lookup_table_regexes(training_data.lookup_tables)

        for example in training_data.training_examples:
            updated = self._text_features_with_regex(example)
            example.set("text_features", updated)

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None

        updated = self._text_features_with_regex(message)
        message.set("text_features", updated)

    def _text_features_with_regex(self, message):
        if self.known_patterns:
            extras = self.features_for_patterns(message)
            return self._combine_with_existing_text_features(message, extras)
        else:
            return message.get("text_features")

    def _add_lookup_table_regexes(self, lookup_tables):
        # appends the regex features from the lookup tables to
        # self.known_patterns
        for table in lookup_tables:
            regex_pattern = self._generate_lookup_regex(table)
            lookup_regex = {'name': table['name'],
                            'pattern': regex_pattern}
            self.known_patterns.append(lookup_regex)

    def features_for_patterns(self, message):
        """Checks which known patterns match the message.

        Given a sentence, returns a vector of {1,0} values indicating which
        regexes did match. Furthermore, if the
        message is tokenized, the function will mark all tokens with a dict
        relating the name of the regex to whether it was matched."""

        matches = []
        for i, exp in enumerate(self.known_patterns):
            match = re.search(exp["pattern"], message.text)
            matches.append(match)
            for token_index, t in enumerate(message.get("tokens", [])):
                patterns = t.get("pattern", default={})
                if match is not None:
                    if t.offset < match.end() and t.end > match.start():
                        patterns[exp["name"]] = True
                    else:
                        patterns[exp["name"]] = False
                else:
                    patterns[exp["name"]] = False
                t.set("pattern", patterns)
        found = [1.0 if m is not None else 0.0 for m in matches]
        return np.array(found)

    def _generate_lookup_regex(self, lookup_table):
        """creates a regex out of the contents of a lookup table file"""
        lookup_elements = lookup_table['elements']
        elements_to_regex = []

        # if it's a list, it should be the elements directly
        if isinstance(lookup_elements, list):
            elements_to_regex = lookup_elements

        # otherwise it's a file path.
        else:

            try:
                f = io.open(lookup_elements, 'r')
            except IOError:
                raise ValueError("Could not load lookup table {}".format(lookup_elements))

            with f:
                for line in f:
                    new_element = line.strip()
                    if new_element:
                        elements_to_regex.append(new_element)

        # regex matching elements with word boundaries on either side
        regex_string = '(?i)(\\b' + '\\b|\\b'.join(elements_to_regex) + '\\b)'
        return regex_string

    @classmethod
    def load(cls,
             model_dir=None,   # type: Optional[Text]
             model_metadata=None,   # type: Optional[Metadata]
             cached_component=None,   # type: Optional[RegexFeaturizer]
             **kwargs  # type: **Any
             ):
        # type: (...) -> RegexFeaturizer

        meta = model_metadata.for_component(cls.name)
        file_name = meta.get("regex_file", REGEX_FEATURIZER_FILE_NAME)
        regex_file = os.path.join(model_dir, file_name)

        if os.path.exists(regex_file):
            known_patterns = utils.read_json_file(regex_file)
            return RegexFeaturizer(meta, known_patterns=known_patterns)
        else:
            return RegexFeaturizer(meta)

    def persist(self, model_dir):
        # type: (Text) -> Optional[Dict[Text, Any]]
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again."""

        regex_file = os.path.join(model_dir, REGEX_FEATURIZER_FILE_NAME)
        utils.write_json_to_file(regex_file, self.known_patterns, indent=4)

        return {"regex_file": REGEX_FEATURIZER_FILE_NAME}
<commit_msg>lookup_tables=None as default argument<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
import os
import re
import warnings
import io
import sys

import typing
from typing import Any, Dict, List, Optional, Text

from rasa_nlu import utils
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.featurizers import Featurizer
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData

import numpy as np

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa_nlu.model import Metadata

REGEX_FEATURIZER_FILE_NAME = "regex_featurizer.json"

class RegexFeaturizer(Featurizer):
    name = "intent_entity_featurizer_regex"

    provides = ["text_features"]

    requires = ["tokens"]

    def __init__(self, component_config=None,
                 known_patterns=None, lookup_tables=None):

        super(RegexFeaturizer, self).__init__(component_config)

        self.known_patterns = known_patterns if known_patterns else []
        lookup_tables = lookup_tables or []
        self._add_lookup_table_regexes(lookup_tables)

    def train(self, training_data, config, **kwargs):
        # type: (TrainingData, RasaNLUModelConfig, **Any) -> None

        self.known_patterns = training_data.regex_features
        self._add_lookup_table_regexes(training_data.lookup_tables)

        for example in training_data.training_examples:
            updated = self._text_features_with_regex(example)
            example.set("text_features", updated)

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None

        updated = self._text_features_with_regex(message)
        message.set("text_features", updated)

    def _text_features_with_regex(self, message):
        if self.known_patterns:
            extras = self.features_for_patterns(message)
            return self._combine_with_existing_text_features(message, extras)
        else:
            return message.get("text_features")

    def _add_lookup_table_regexes(self, lookup_tables):
        # appends the regex features from the lookup tables to
        # self.known_patterns
        for table in lookup_tables:
            regex_pattern = self._generate_lookup_regex(table)
            lookup_regex = {'name': table['name'],
                            'pattern': regex_pattern}
            self.known_patterns.append(lookup_regex)

    def features_for_patterns(self, message):
        """Checks which known patterns match the message.

        Given a sentence, returns a vector of {1,0} values indicating which
        regexes did match. Furthermore, if the
        message is tokenized, the function will mark all tokens with a dict
        relating the name of the regex to whether it was matched."""

        matches = []
        for i, exp in enumerate(self.known_patterns):
            match = re.search(exp["pattern"], message.text)
            matches.append(match)
            for token_index, t in enumerate(message.get("tokens", [])):
                patterns = t.get("pattern", default={})
                if match is not None:
                    if t.offset < match.end() and t.end > match.start():
                        patterns[exp["name"]] = True
                    else:
                        patterns[exp["name"]] = False
                else:
                    patterns[exp["name"]] = False
                t.set("pattern", patterns)
        found = [1.0 if m is not None else 0.0 for m in matches]
        return np.array(found)

    def _generate_lookup_regex(self, lookup_table):
        """creates a regex out of the contents of a lookup table file"""
        lookup_elements = lookup_table['elements']
        elements_to_regex = []

        # if it's a list, it should be the elements directly
        if isinstance(lookup_elements, list):
            elements_to_regex = lookup_elements

        # otherwise it's a file path.
        else:

            try:
                f = io.open(lookup_elements, 'r')
            except IOError:
                raise ValueError("Could not load lookup table {}".format(lookup_elements))

            with f:
                for line in f:
                    new_element = line.strip()
                    if new_element:
                        elements_to_regex.append(new_element)

        # regex matching elements with word boundaries on either side
        regex_string = '(?i)(\\b' + '\\b|\\b'.join(elements_to_regex) + '\\b)'
        return regex_string

    @classmethod
    def load(cls,
             model_dir=None,   # type: Optional[Text]
             model_metadata=None,   # type: Optional[Metadata]
             cached_component=None,   # type: Optional[RegexFeaturizer]
             **kwargs  # type: **Any
             ):
        # type: (...) -> RegexFeaturizer

        meta = model_metadata.for_component(cls.name)
        file_name = meta.get("regex_file", REGEX_FEATURIZER_FILE_NAME)
        regex_file = os.path.join(model_dir, file_name)

        if os.path.exists(regex_file):
            known_patterns = utils.read_json_file(regex_file)
            return RegexFeaturizer(meta, known_patterns=known_patterns)
        else:
            return RegexFeaturizer(meta)

    def persist(self, model_dir):
        # type: (Text) -> Optional[Dict[Text, Any]]
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again."""

        regex_file = os.path.join(model_dir, REGEX_FEATURIZER_FILE_NAME)
        utils.write_json_to_file(regex_file, self.known_patterns, indent=4)

        return {"regex_file": REGEX_FEATURIZER_FILE_NAME}
<|endoftext|>


<commit_before>#!/usr/bin/env python
""" Extraction parsers for structured data embedded into HTML or XML files. The former may include
RDFa or microdata. The syntax and the extraction procedures are based on:

* The RDFa specifications: http://www.w3.org/TR/#tr_RDFa
* The microdata specification: http://www.w3.org/TR/microdata/
* The specification of the microdata to RDF conversion: http://www.w3.org/TR/microdata-rdf/

License: W3C Software License, http://www.w3.org/Consortium/Legal/copyright-software
Author: Ivan Herman

"""
import sys, imp

from rdflib.parser import Parser, StringInputSource, URLInputSource, FileInputSource

try:
	import html5lib
	html5lib = True
except ImportError:
	import warnings
	warnings.warn('html5lib not found! RDFa and Microdata parsers will not be available.')
	html5lib = False

def _get_orig_source(source) :
	"""A bit of a hack; the RDFa/microdata parsers need more than what the upper layers of RDFLib provide...
	This method returns the original source references.
	"""
	if isinstance(source, StringInputSource) :
		orig_source = source.getByteStream()
	elif isinstance(source, URLInputSource) :
		orig_source = source.url
	elif isinstance(source, FileInputSource) :
		orig_source = source.file.name
		source.file.close()
	else:
		orig_source = source.getByteStream()
	baseURI = source.getPublicId()
	return (baseURI, orig_source)

def _check_error(graph) :
	from .pyRdfa import RDFA_Error, ns_rdf
	for (s,p,o) in graph.triples((None, ns_rdf["type"], RDFA_Error)) :
		for (x,y,msg) in graph.triples((s,None,None)) :
			raise Exception(msg)

# This is the parser interface as it would look when called from the rest of RDFLib
class RDFaParser(Parser) :
	"""
	Wrapper around the RDFa 1.1 parser. For further details on the RDFa 1.1 processing, see the
	relevant W3C documents at http://www.w3.org/TR/#tr_RDFa. RDFa 1.1 is defined for XHTML, HTML5, SVG and,
	in general, for any XML language.

	Note that the parser can also handle RDFa 1.0 if the extra parameter is used and/or the input source uses
	RDFa 1.0 specific @version or DTD-s.
	"""
	def parse(self, source, graph,
			  pgraph                 = None,
			  media_type             = "",
			  rdfa_version           = None,
			  embedded_rdf           = False,
			  vocab_expansion        = False, vocab_cache = False) :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@type graph: RDFLib Graph
		@keyword pgraph: target for error and warning triples; processor graph, in RDFa spec. parlance. If set to None, these triples are ignored
		@type pgraph: RDFLib Graph
		@keyword media_type: explicit setting of the preferred media type (a.k.a. content type) of the the RDFa source. None means the content type of the HTTP result is used, or a guess is made based on the suffix of a file
		@type media_type: string
		@keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by default, 1.1 is used unless the source has explicit signals to use 1.0 (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
		@type rdfa_version: string
		@keyword embedded_rdf: some formats allow embedding RDF in other formats: (X)HTML can contain turtle in a special <script> element, SVG can have RDF/XML embedded in a <metadata> element. This flag controls whether those triples should be interpreted and added to the output graph. Some languages (e.g., SVG) require this, and the flag is ignored.
		@type embedded_rdf: Boolean
		@keyword vocab_expansion: whether the RDFa @vocab attribute should also mean vocabulary expansion (see the RDFa 1.1 spec for further details)
		@type vocab_expansion: Boolean
		@keyword vocab_cache: in case vocab expansion is used, whether the expansion data (i.e., vocabulary) should be cached locally. This requires the ability for the local application to write on the local file system
		@type vocab_chache: Boolean
		"""

		if html5lib == False:
			raise ImportError('html5lib is not installed, cannot use RDFa and Microdata parsers.')

		(baseURI, orig_source) = _get_orig_source(source)
		self._process(graph, pgraph, baseURI, orig_source,
			          media_type      = media_type,
			          rdfa_version    = rdfa_version,
			          embedded_rdf    = embedded_rdf,
			          vocab_expansion = vocab_expansion, vocab_cache = vocab_cache)

	def _process(self, graph, pgraph, baseURI,  orig_source,
			  media_type      = "",
			  rdfa_version    = None,
			  embedded_rdf    = False,
			  vocab_expansion = False, vocab_cache     = False) :
		from .pyRdfa import pyRdfa, Options
		self.options = Options(output_processor_graph = (pgraph != None),
							   embedded_rdf           = embedded_rdf,
							   vocab_expansion        = vocab_expansion,
							   vocab_cache            = vocab_cache)

		if media_type == None : media_type = ""
		processor    = pyRdfa(self.options,
							  base = baseURI,
							  media_type = media_type,
							  rdfa_version = rdfa_version)
		processor.graph_from_source(orig_source, graph=graph, pgraph=pgraph, rdfOutput=False)
		# This may result in an exception if the graph parsing led to an error
		_check_error(graph)

class RDFa10Parser(Parser) :
	"""
	This is just a convenience class to wrap around the RDFa 1.0 parser.
	"""
	def parse(self, source, graph, pgraph = None, media_type = "") :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@type graph: RDFLib Graph
		@keyword pgraph: target for error and warning triples; processor graph, in RDFa spec. parlance. If set to None, these triples are ignored
		@type pgraph: RDFLib Graph
		@keyword media_type: explicit setting of the preferred media type (a.k.a. content type) of the the RDFa source. None means the content type of the HTTP result is used, or a guess is made based on the suffix of a file
		@type media_type: string
		@keyword rdfOutput: whether Exceptions should be catched and added, as triples, to the processor graph, or whether they should be raised.
		@type rdfOutput: Boolean
		"""
		RDFaParser().parse(source, graph, pgraph = pgraph, media_type = media_type, rdfa_version = "1.0")

class MicrodataParser(Parser) :
	"""
	Wrapper around an HTML5 microdata, extracted and converted into RDF. For the specification of microdata,
	see the relevant section of the HTML5 spec: http://www.w3.org/TR/microdata/; for the algorithm used
	to extract microdata into RDF, see http://www.w3.org/TR/microdata-rdf/.
	"""
	def parse(self, source, graph, vocab_expansion = False, vocab_cache = False) :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@type graph: RDFLib Graph
		@keyword vocab_expansion: whether the RDFa @vocab attribute should also mean vocabulary expansion (see the RDFa 1.1 spec for further details)
		@type vocab_expansion: Boolean
		@keyword vocab_cache: in case vocab expansion is used, whether the expansion data (i.e., vocabulary) should be cached locally. This requires the ability for the local application to write on the local file system
		@type vocab_chache: Boolean
		@keyword rdfOutput: whether Exceptions should be catched and added, as triples, to the processor graph, or whether they should be raised.
		@type rdfOutput: Boolean
		"""
		if html5lib == False:
			raise ImportError('html5lib is not installed, cannot use RDFa and Microdata parsers.')

		(baseURI, orig_source) = _get_orig_source(source)
		self._process(graph, baseURI, orig_source,
			          vocab_expansion = vocab_expansion,
			          vocab_cache     = vocab_cache)

	def _process(self, graph, baseURI, orig_source, vocab_expansion = False, vocab_cache  = False) :
		from .pyMicrodata import pyMicrodata
		processor    = pyMicrodata(base = baseURI, vocab_expansion = vocab_expansion, vocab_cache = vocab_cache)
		processor.graph_from_source(orig_source, graph=graph, rdfOutput = False)

class StructuredDataParser(Parser) :
	"""
	Convenience parser to extract both RDFa (including embedded Turtle) and microdata from an HTML file.
	It is simply a wrapper around the specific parsers.
	"""
	def parse(self, source, graph,
			  pgraph                 = None,
			  rdfa_version           = "",
			  vocab_expansion        = False,
			  vocab_cache            = False) :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by default, 1.1 is used unless the source has explicit signals to use 1.0 (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
		@type rdfa_version: string
		@type graph: RDFLib Graph
		@keyword pgraph: target for error and warning triples; processor graph, in RDFa spec. parlance. If set to None, these triples are ignored
		@type pgraph: RDFLib Graph
		@keyword vocab_expansion: whether the RDFa @vocab attribute should also mean vocabulary expansion (see the RDFa 1.1 spec for further details)
		@type vocab_expansion: Boolean
		@keyword vocab_cache: in case vocab expansion is used, whether the expansion data (i.e., vocabulary) should be cached locally. This requires the ability for the local application to write on the local file system
		@type vocab_chache: Boolean
		@keyword rdfOutput: whether Exceptions should be catched and added, as triples, to the processor graph, or whether they should be raised.
		@type rdfOutput: Boolean
		"""
		(baseURI, orig_source) = _get_orig_source(source)
		RDFaParser()._process(graph, pgraph, baseURI, orig_source,
			          media_type      = 'text/html',
			          rdfa_version    = rdfa_version,
			          vocab_expansion = vocab_expansion,
			          vocab_cache     = vocab_cache)
		MicrodataParser()._process(graph, baseURI, orig_source,
			          vocab_expansion = vocab_expansion,
			          vocab_cache     = vocab_cache)
		from .hturtle import HTurtleParser
		HTurtleParser()._process(graph, baseURI, orig_source, media_type = 'text/html')

<commit_msg>Improved the way errors are handled. What is defined strictly as an Error in RDFa (which is, essentially, a parsing error in the XML or HTML5 parsing) raises an exception right away.<commit_after>#!/usr/bin/env python
""" Extraction parsers for structured data embedded into HTML or XML files. The former may include
RDFa or microdata. The syntax and the extraction procedures are based on:

* The RDFa specifications: http://www.w3.org/TR/#tr_RDFa
* The microdata specification: http://www.w3.org/TR/microdata/
* The specification of the microdata to RDF conversion: http://www.w3.org/TR/microdata-rdf/

License: W3C Software License, http://www.w3.org/Consortium/Legal/copyright-software
Author: Ivan Herman

"""
import sys, imp

from rdflib.parser import Parser, StringInputSource, URLInputSource, FileInputSource

try:
	import html5lib
	html5lib = True
except ImportError:
	import warnings
	warnings.warn('html5lib not found! RDFa and Microdata parsers will not be available.')
	html5lib = False

def _get_orig_source(source) :
	"""A bit of a hack; the RDFa/microdata parsers need more than what the upper layers of RDFLib provide...
	This method returns the original source references.
	"""
	if isinstance(source, StringInputSource) :
		orig_source = source.getByteStream()
	elif isinstance(source, URLInputSource) :
		orig_source = source.url
	elif isinstance(source, FileInputSource) :
		orig_source = source.file.name
		source.file.close()
	else:
		orig_source = source.getByteStream()
	baseURI = source.getPublicId()
	return (baseURI, orig_source)

def _check_error(graph) :
	from .pyRdfa import RDFA_Error, ns_rdf
	from .pyRdfa.options import ns_dc
	for (s,p,o) in graph.triples((None, ns_rdf["type"], RDFA_Error)) :
		for (x,y,msg) in graph.triples((s,ns_dc["description"],None)) :
			raise Exception("RDFa parsing Error! %s" % msg)

# This is the parser interface as it would look when called from the rest of RDFLib
class RDFaParser(Parser) :
	"""
	Wrapper around the RDFa 1.1 parser. For further details on the RDFa 1.1 processing, see the
	relevant W3C documents at http://www.w3.org/TR/#tr_RDFa. RDFa 1.1 is defined for XHTML, HTML5, SVG and,
	in general, for any XML language.

	Note that the parser can also handle RDFa 1.0 if the extra parameter is used and/or the input source uses
	RDFa 1.0 specific @version or DTD-s.
	"""
	def parse(self, source, graph,
			  pgraph                 = None,
			  media_type             = "",
			  rdfa_version           = None,
			  embedded_rdf           = False,
			  vocab_expansion        = False, vocab_cache = False) :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@type graph: RDFLib Graph
		@keyword pgraph: target for error and warning triples; processor graph, in RDFa spec. parlance. If set to None, these triples are ignored
		@type pgraph: RDFLib Graph
		@keyword media_type: explicit setting of the preferred media type (a.k.a. content type) of the the RDFa source. None means the content type of the HTTP result is used, or a guess is made based on the suffix of a file
		@type media_type: string
		@keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by default, 1.1 is used unless the source has explicit signals to use 1.0 (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
		@type rdfa_version: string
		@keyword embedded_rdf: some formats allow embedding RDF in other formats: (X)HTML can contain turtle in a special <script> element, SVG can have RDF/XML embedded in a <metadata> element. This flag controls whether those triples should be interpreted and added to the output graph. Some languages (e.g., SVG) require this, and the flag is ignored.
		@type embedded_rdf: Boolean
		@keyword vocab_expansion: whether the RDFa @vocab attribute should also mean vocabulary expansion (see the RDFa 1.1 spec for further details)
		@type vocab_expansion: Boolean
		@keyword vocab_cache: in case vocab expansion is used, whether the expansion data (i.e., vocabulary) should be cached locally. This requires the ability for the local application to write on the local file system
		@type vocab_chache: Boolean
		"""

		if html5lib == False:
			raise ImportError('html5lib is not installed, cannot use RDFa and Microdata parsers.')

		(baseURI, orig_source) = _get_orig_source(source)
		self._process(graph, pgraph, baseURI, orig_source,
			          media_type      = media_type,
			          rdfa_version    = rdfa_version,
			          embedded_rdf    = embedded_rdf,
			          vocab_expansion = vocab_expansion, vocab_cache = vocab_cache)

	def _process(self, graph, pgraph, baseURI,  orig_source,
			  media_type      = "",
			  rdfa_version    = None,
			  embedded_rdf    = False,
			  vocab_expansion = False, vocab_cache     = False) :
		from .pyRdfa import pyRdfa, Options
		from rdflib import Graph
		processor_graph = pgraph if pgraph != None else Graph()
		self.options = Options(output_processor_graph = True,
							   embedded_rdf           = embedded_rdf,
							   vocab_expansion        = vocab_expansion,
							   vocab_cache            = vocab_cache)

		if media_type == None : media_type = ""
		processor    = pyRdfa(self.options,
							  base = baseURI,
							  media_type = media_type,
							  rdfa_version = rdfa_version)
		processor.graph_from_source(orig_source, graph=graph, pgraph=processor_graph, rdfOutput=False)
		# This may result in an exception if the graph parsing led to an error
		# At this moment, the only Error that is defined is a parsing error for, say, XHTML, and this is
		# actually returned as part of the output graph. So the extra check for processor_graph is
		# not strictly necessary, but it leaves the door open if other types of errors are added.
		_check_error(processor_graph)
		_check_error(graph)

class RDFa10Parser(Parser) :
	"""
	This is just a convenience class to wrap around the RDFa 1.0 parser.
	"""
	def parse(self, source, graph, pgraph = None, media_type = "") :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@type graph: RDFLib Graph
		@keyword pgraph: target for error and warning triples; processor graph, in RDFa spec. parlance. If set to None, these triples are ignored
		@type pgraph: RDFLib Graph
		@keyword media_type: explicit setting of the preferred media type (a.k.a. content type) of the the RDFa source. None means the content type of the HTTP result is used, or a guess is made based on the suffix of a file
		@type media_type: string
		@keyword rdfOutput: whether Exceptions should be catched and added, as triples, to the processor graph, or whether they should be raised.
		@type rdfOutput: Boolean
		"""
		RDFaParser().parse(source, graph, pgraph = pgraph, media_type = media_type, rdfa_version = "1.0")

class MicrodataParser(Parser) :
	"""
	Wrapper around an HTML5 microdata, extracted and converted into RDF. For the specification of microdata,
	see the relevant section of the HTML5 spec: http://www.w3.org/TR/microdata/; for the algorithm used
	to extract microdata into RDF, see http://www.w3.org/TR/microdata-rdf/.
	"""
	def parse(self, source, graph, vocab_expansion = False, vocab_cache = False) :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@type graph: RDFLib Graph
		@keyword vocab_expansion: whether the RDFa @vocab attribute should also mean vocabulary expansion (see the RDFa 1.1 spec for further details)
		@type vocab_expansion: Boolean
		@keyword vocab_cache: in case vocab expansion is used, whether the expansion data (i.e., vocabulary) should be cached locally. This requires the ability for the local application to write on the local file system
		@type vocab_chache: Boolean
		@keyword rdfOutput: whether Exceptions should be catched and added, as triples, to the processor graph, or whether they should be raised.
		@type rdfOutput: Boolean
		"""
		if html5lib == False:
			raise ImportError('html5lib is not installed, cannot use RDFa and Microdata parsers.')

		(baseURI, orig_source) = _get_orig_source(source)
		self._process(graph, baseURI, orig_source,
			          vocab_expansion = vocab_expansion,
			          vocab_cache     = vocab_cache)

	def _process(self, graph, baseURI, orig_source, vocab_expansion = False, vocab_cache  = False) :
		from .pyMicrodata import pyMicrodata
		processor    = pyMicrodata(base = baseURI, vocab_expansion = vocab_expansion, vocab_cache = vocab_cache)
		processor.graph_from_source(orig_source, graph=graph, rdfOutput = False)

class StructuredDataParser(Parser) :
	"""
	Convenience parser to extract both RDFa (including embedded Turtle) and microdata from an HTML file.
	It is simply a wrapper around the specific parsers.
	"""
	def parse(self, source, graph,
			  pgraph                 = None,
			  rdfa_version           = "",
			  vocab_expansion        = False,
			  vocab_cache            = False) :
		"""
		@param source: one of the input sources that the RDFLib package defined
		@type source: InputSource class instance
		@param graph: target graph for the triples; output graph, in RDFa spec. parlance
		@keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by default, 1.1 is used unless the source has explicit signals to use 1.0 (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
		@type rdfa_version: string
		@type graph: RDFLib Graph
		@keyword pgraph: target for error and warning triples; processor graph, in RDFa spec. parlance. If set to None, these triples are ignored
		@type pgraph: RDFLib Graph
		@keyword vocab_expansion: whether the RDFa @vocab attribute should also mean vocabulary expansion (see the RDFa 1.1 spec for further details)
		@type vocab_expansion: Boolean
		@keyword vocab_cache: in case vocab expansion is used, whether the expansion data (i.e., vocabulary) should be cached locally. This requires the ability for the local application to write on the local file system
		@type vocab_chache: Boolean
		@keyword rdfOutput: whether Exceptions should be catched and added, as triples, to the processor graph, or whether they should be raised.
		@type rdfOutput: Boolean
		"""
		(baseURI, orig_source) = _get_orig_source(source)
		RDFaParser()._process(graph, pgraph, baseURI, orig_source,
			          media_type      = 'text/html',
			          rdfa_version    = rdfa_version,
			          vocab_expansion = vocab_expansion,
			          vocab_cache     = vocab_cache)
		MicrodataParser()._process(graph, baseURI, orig_source,
			          vocab_expansion = vocab_expansion,
			          vocab_cache     = vocab_cache)
		from .hturtle import HTurtleParser
		HTurtleParser()._process(graph, baseURI, orig_source, media_type = 'text/html')

<|endoftext|>


<commit_before>"""
Created on Apr 18, 2019

@author: lunik1
"""
import glob
import json
from io import BytesIO
from random import choice, sample

import requests
from PIL import Image, ImageDraw, ImageFont
from twisted.plugin import IPlugin
from zope.interface import implementer

from desertbot.message import IRCMessage
from desertbot.moduleinterface import IModule
from desertbot.modules.commandinterface import BotCommand
from desertbot.response import IRCResponse, ResponseType
from desertbot.utils import string

@implementer(IPlugin, IModule)
class Comic(BotCommand):
    # just hardcode a limit on max messages for now, should limit comic size pretty effecively?
    messageLimit = 8

    def actions(self):
        # define additional actions so all PRIVMSGs and ACTIONs in the channel get handled by storeMessage
        return super(Comic, self).actions() + [("message-channel", 1, self.storeMessage),
                                               ("action-channel", 1, self.storeMessage)]

    def onLoad(self) -> None:
        # messages need to be stored for comics to work, store in a list as (nick, message)
        # keys are channel names, as each channel kinda should have its own messages
        self.messageStore = {}

    def triggers(self):
        return ['comic']

    def help(self, query):
        return 'comic - make a comic'

    def execute(self, message: IRCMessage):
        messages = self.getMessages(message.replyTo)
        if messages:
            comicBytes = self.makeComic(messages)
            return IRCResponse(ResponseType.Say, self.postComic(comicBytes), message.replyTo)

    def getMessages(self, channel: str):
        """
        Get the messages for the given channel
        Returns [] if there are no messages for that channel yet
        """
        return self.messageStore.get(channel, [])

    def storeMessage(self, message: IRCMessage):
        """
        Store the message into the messageStore, and prune it to the limit if needed
        """
        # TODO just store the IRCMessage objects, so we can use other things than just the nick and message string
        if message.command:  # don't store messages containing bot commands
            return

        # fetch the message list for the channel this message belongs to and append the message data
        messages = self.getMessages(message.replyTo)
        if message.type == "ACTION":
            messages.append((message.user.nick, "*{}*".format(string.stripFormatting(message.messageString))))
        else:
            messages.append((message.user.nick, string.stripFormatting(message.messageString)))

        if len(messages) > self.messageLimit:
            messages.pop(0)     # remove the first (oldest) message in the list if we're now above the limit

        # store the new message list into the messageStore
        self.messageStore[message.replyTo] = messages

    def postComic(self, comicObject):
        apiUrl = 'https://dbco.link/'
        postData = {'c': ('comic.png', comicObject, 'application/octet-stream')}
        headers = {'Accept': 'application/json'}

        try:
            response = requests.post(apiUrl, files=postData, headers=headers)
            responseJson = response.json()
            return responseJson['url']
        except requests.exceptions.RequestException:
            self.logger.exception("dbco.link POST url error {}".format(comicObject))
        except json.decoder.JSONDecodeError:
            self.logger.exception("dbco.link json response decode error, {} (at {})".format(
                response.content, comicObject))

    def makeComic(self, messages):
        chars = set()  # chars is a set of the "characters" involved, at this point these are message.user.nick
        panels = []  # panels is a list of comic "panels", each involving 1-2 "characters", each with a message spoken
        panelHeight = 300
        panelWidth = 450

        # one "panel" is a list of the (message.user.nick, message.messageString) pairs to be drawn in that panel
        panel = []
        lastChar = None
        for message in messages:
            char = message[0]
            chars.add(char)

            # Start a new panel if the panel is full or the same user speaks twice in a row
            if len(panel) == 2 or len(panel) == 1 and char == lastChar:
                panels.append(panel)
                panel = []

            panel.append(message)
            lastChar = char
        panels.append(panel)

        # Randomly associate a character image to each user
        filenames = glob.glob('data/comics/chars/*')
        charmap_filenames = zip(chars, sample(filenames, len(chars)))
        charmap = {ch: Image.open(f).convert("RGBA") for ch, f in charmap_filenames}

        self.logger.debug("Character images used for comic: {}".format(", ".join([f for ch, f in charmap_filenames])))
        # charmap is now a dict of message.user.nick to their randomly picked "character" image

        # How big is the whole comic?
        imgWidth = panelWidth
        imgHeight = panelHeight * len(panels)

        # this will be the background image for each separate panel
        background_file = choice(glob.glob('data/comics/backgrounds/*'))
        background = Image.open(background_file).convert("RGBA")
        self.logger.debug("Background image used for comic: {}".format(background_file))
        background = Comic.fitbkg(background, panelWidth, panelHeight)

        # comicImage is our entire comic, to be filled with our panels
        comicImage = Image.new("RGBA", (imgWidth, imgHeight), (0xff, 0xff, 0xff, 0xff))
        font = ImageFont.truetype('data/comics/fonts/ComicRelief.ttf', 14)

        for i, panel in enumerate(panels):
            # paste the panel Image object onto our comic Image object, using the index i to offset height
            comicImage.paste(Comic.makePanel(panel, panelWidth, panelHeight, charmap, background, font),
                             (0, panelHeight * i))

        comicByteArray = BytesIO()
        comicImage.save(comicByteArray, format="PNG", quality=85)
        return comicByteArray.getvalue()

    @staticmethod
    def makePanel(panel, panelWidth, panelHeight, charmap, background, font):
        # for each panel, create an Image object
        panelImage = Image.new("RGBA", (panelWidth, panelHeight), (0xff, 0xff, 0xff, 0xff))

        # paste the bg image into our panel Image
        panelImage.paste(background, (0, 0))
        draw = ImageDraw.Draw(panelImage)

        # call the wrap function to get a formatted string to be drawn onto the image
        # use 2/3rds panel width as the "max width" for the text
        (lines, (_, string1Height)) = Comic.wrap(panel[0][1], font, draw, 2 * panelWidth / 3)
        # then draw that string onto the image at 10 from the top, 10 from the left edge
        Comic.rendertext(lines, font, draw, (10, 10))

        string2Height = 0
        if len(panel) == 2:
            # if there is a second message in this panel, draw it differently from the first
            # call the wrap function again to get the second string (again with 2/3rds panel width as "max width")
            (string2, (string2Width, string2Height)) = Comic.wrap(panel[1][1], font, draw, 2 * panelWidth / 3.0)
            # then draw that string onto the image as close to the right edge as it can fit, 10 below the 1st string
            Comic.rendertext(string2, font, draw, (panelWidth - 10 - string2Width, string1Height + 10))

        # calculate the "height" of the text, with some spacing (used for scaling character images?)
        textHeight = string1Height + 10
        if string2Height > 0:
            textHeight += string2Height + 10 + 5

        # scale the character image for the 1st message and paste it into the panel Image object
        maxch = panelHeight - textHeight
        char1Image = Comic.fitimg(charmap[panel[0][0]], 2 * panelWidth / 5.0 - 10, maxch)
        panelImage.paste(char1Image, (10, panelHeight - char1Image.size[1]), char1Image)

        # if there is a second character, also scale and paste that into the panel Image object
        if len(panel) == 2:
            char2Image = Comic.fitimg(charmap[panel[1][0]], 2 * panelWidth / 5.0 - 10, maxch)
            char2Image = char2Image.transpose(
                Image.FLIP_LEFT_RIGHT)  # flip the character image, so it "faces" the first character image
            panelImage.paste(char2Image, (panelWidth - char2Image.size[0] - 10, panelHeight - char2Image.size[1]),
                             char2Image)

        # draw a small black line at the top? of the panel Image
        draw.line([(0, 0), (0, panelHeight - 1), (panelWidth - 1, panelHeight - 1), (panelWidth - 1, 0), (0, 0)],
                  (0, 0, 0, 0xff))

        return panelImage

    @staticmethod
    def wrap(message, font, draw, maxWidth):
        """
        Work out how much space `message` will take up on the image. Returns a list of strings representing each line
        of the message split into lines after wrapping and the wrapped width and height as a tuple
        """
        messageWords = message.split()
        wrappedWidth = 0
        wrappedHeight = 0
        lines = []

        while messageWords:
            numWords = 1
            while numWords < len(messageWords):
                # Try fitting the next numWords words on the line
                width, _ = draw.textsize(" ".join(messageWords[:numWords]), font=font)
                if width > maxWidth:
                    numWords -= 1
                    break
                else:
                    # There's still space, try another word
                    numWords += 1

            if numWords == 0 and messageWords:  # we've hit a case where the current word is wider than the screen
                numWords = 1

            # How big is this line?
            lineWidth, lineHeight = draw.textsize(" ".join(messageWords[:numWords]), font=font)

            wrappedWidth = max(wrappedWidth, lineWidth)  # wrappedWidth should be the length of the longest line
            wrappedHeight += lineHeight

            lines.append(" ".join(messageWords[:numWords]))
            messageWords = messageWords[numWords:]  # drop the words wrapped so far and continue

        return lines, (wrappedWidth, wrappedHeight)

    @staticmethod
    def rendertext(lines, font, draw, position):
        """
        This function renders the given `lines` at the given position in the given "draw" object - ImageDraw.Draw()
        """
        # draw black outline first
        for offset in [(-1, -1), (-1, 1), (1, -1), (1, 1)]:
            draw.text((position[0] + offset[0], position[1] + offset[1]), '\n'.join(lines),
                      font=font, fill=(0x00, 0x00, 0x00, 0xff))

        draw.text(position, '\n'.join(lines), font=font, fill=(0xff, 0xff, 0xff, 0xff))

    @staticmethod
    def fitimg(img, width, height):
        """
        Scale img proprotionally so that it's new width is `width` or height is `height`, whichever comes first.
        """

        # Calculate required scale factor to match width
        sf = width / img.size[0]
        # if this would make the image taller than the desired height, instead use the scale factor to match height
        if img.size[1] * sf > height:
            sf = height / img.size[1]

        # Resize image, round pixel count to nearest integer
        return img.resize((int(sf * img.size[0] + 0.5), int(sf * img.size[1] + 0.5)), Image.LANCZOS)

    @staticmethod
    def fitbkg(img, width, height):
        """
        Scale img to (`width`, `height`), cropping to the correct aspect ratio beforehand
        """

        targetAspectRatio = width / height
        imgAspectRatio = img.size[0] / img.size[1]

        if targetAspectRatio > imgAspectRatio:  # img too tall
            # How tall should it be?
            h = img.size[0] / targetAspectRatio

            # crop from top and bottom
            crop = img.size[1] - h / 2
            img = img.crop((0, crop, 0, crop))
        elif targetAspectRatio < imgAspectRatio:  # img too wide
            # How wide should it be?
            w = img.size[1] * targetAspectRatio

            # crop from left and right
            crop = img.size[1] - w / 2
            img = img.crop((crop, 0, crop, 0))

        return img.resize((width, height), Image.LANCZOS)

comic = Comic()
<commit_msg>[Comic] Fix background crop calculation<commit_after>"""
Created on Apr 18, 2019

@author: lunik1
"""
import glob
import json
from io import BytesIO
from random import choice, sample

import requests
from PIL import Image, ImageDraw, ImageFont
from twisted.plugin import IPlugin
from zope.interface import implementer

from desertbot.message import IRCMessage
from desertbot.moduleinterface import IModule
from desertbot.modules.commandinterface import BotCommand
from desertbot.response import IRCResponse, ResponseType
from desertbot.utils import string

@implementer(IPlugin, IModule)
class Comic(BotCommand):
    # just hardcode a limit on max messages for now, should limit comic size pretty effecively?
    messageLimit = 8

    def actions(self):
        # define additional actions so all PRIVMSGs and ACTIONs in the channel get handled by storeMessage
        return super(Comic, self).actions() + [("message-channel", 1, self.storeMessage),
                                               ("action-channel", 1, self.storeMessage)]

    def onLoad(self) -> None:
        # messages need to be stored for comics to work, store in a list as (nick, message)
        # keys are channel names, as each channel kinda should have its own messages
        self.messageStore = {}

    def triggers(self):
        return ['comic']

    def help(self, query):
        return 'comic - make a comic'

    def execute(self, message: IRCMessage):
        messages = self.getMessages(message.replyTo)
        if messages:
            comicBytes = self.makeComic(messages)
            return IRCResponse(ResponseType.Say, self.postComic(comicBytes), message.replyTo)

    def getMessages(self, channel: str):
        """
        Get the messages for the given channel
        Returns [] if there are no messages for that channel yet
        """
        return self.messageStore.get(channel, [])

    def storeMessage(self, message: IRCMessage):
        """
        Store the message into the messageStore, and prune it to the limit if needed
        """
        # TODO just store the IRCMessage objects, so we can use other things than just the nick and message string
        if message.command:  # don't store messages containing bot commands
            return

        # fetch the message list for the channel this message belongs to and append the message data
        messages = self.getMessages(message.replyTo)
        if message.type == "ACTION":
            messages.append((message.user.nick, "*{}*".format(string.stripFormatting(message.messageString))))
        else:
            messages.append((message.user.nick, string.stripFormatting(message.messageString)))

        if len(messages) > self.messageLimit:
            messages.pop(0)     # remove the first (oldest) message in the list if we're now above the limit

        # store the new message list into the messageStore
        self.messageStore[message.replyTo] = messages

    def postComic(self, comicObject):
        apiUrl = 'https://dbco.link/'
        postData = {'c': ('comic.png', comicObject, 'application/octet-stream')}
        headers = {'Accept': 'application/json'}

        try:
            response = requests.post(apiUrl, files=postData, headers=headers)
            responseJson = response.json()
            return responseJson['url']
        except requests.exceptions.RequestException:
            self.logger.exception("dbco.link POST url error {}".format(comicObject))
        except json.decoder.JSONDecodeError:
            self.logger.exception("dbco.link json response decode error, {} (at {})".format(
                response.content, comicObject))

    def makeComic(self, messages):
        chars = set()  # chars is a set of the "characters" involved, at this point these are message.user.nick
        panels = []  # panels is a list of comic "panels", each involving 1-2 "characters", each with a message spoken
        panelHeight = 300
        panelWidth = 450

        # one "panel" is a list of the (message.user.nick, message.messageString) pairs to be drawn in that panel
        panel = []
        lastChar = None
        for message in messages:
            char = message[0]
            chars.add(char)

            # Start a new panel if the panel is full or the same user speaks twice in a row
            if len(panel) == 2 or len(panel) == 1 and char == lastChar:
                panels.append(panel)
                panel = []

            panel.append(message)
            lastChar = char
        panels.append(panel)

        # Randomly associate a character image to each user
        filenames = glob.glob('data/comics/chars/*')
        charmap_filenames = zip(chars, sample(filenames, len(chars)))
        charmap = {ch: Image.open(f).convert("RGBA") for ch, f in charmap_filenames}

        self.logger.debug("Character images used for comic: {}".format(", ".join([f for ch, f in charmap_filenames])))
        # charmap is now a dict of message.user.nick to their randomly picked "character" image

        # How big is the whole comic?
        imgWidth = panelWidth
        imgHeight = panelHeight * len(panels)

        # this will be the background image for each separate panel
        background_file = choice(glob.glob('data/comics/backgrounds/*'))
        background = Image.open(background_file).convert("RGBA")
        self.logger.debug("Background image used for comic: {}".format(background_file))
        background = Comic.fitbkg(background, panelWidth, panelHeight)

        # comicImage is our entire comic, to be filled with our panels
        comicImage = Image.new("RGBA", (imgWidth, imgHeight), (0xff, 0xff, 0xff, 0xff))
        font = ImageFont.truetype('data/comics/fonts/ComicRelief.ttf', 14)

        for i, panel in enumerate(panels):
            # paste the panel Image object onto our comic Image object, using the index i to offset height
            comicImage.paste(Comic.makePanel(panel, panelWidth, panelHeight, charmap, background, font),
                             (0, panelHeight * i))

        comicByteArray = BytesIO()
        comicImage.save(comicByteArray, format="PNG", quality=85)
        return comicByteArray.getvalue()

    @staticmethod
    def makePanel(panel, panelWidth, panelHeight, charmap, background, font):
        # for each panel, create an Image object
        panelImage = Image.new("RGBA", (panelWidth, panelHeight), (0xff, 0xff, 0xff, 0xff))

        # paste the bg image into our panel Image
        panelImage.paste(background, (0, 0))
        draw = ImageDraw.Draw(panelImage)

        # call the wrap function to get a formatted string to be drawn onto the image
        # use 2/3rds panel width as the "max width" for the text
        (lines, (_, string1Height)) = Comic.wrap(panel[0][1], font, draw, 2 * panelWidth / 3)
        # then draw that string onto the image at 10 from the top, 10 from the left edge
        Comic.rendertext(lines, font, draw, (10, 10))

        string2Height = 0
        if len(panel) == 2:
            # if there is a second message in this panel, draw it differently from the first
            # call the wrap function again to get the second string (again with 2/3rds panel width as "max width")
            (string2, (string2Width, string2Height)) = Comic.wrap(panel[1][1], font, draw, 2 * panelWidth / 3.0)
            # then draw that string onto the image as close to the right edge as it can fit, 10 below the 1st string
            Comic.rendertext(string2, font, draw, (panelWidth - 10 - string2Width, string1Height + 10))

        # calculate the "height" of the text, with some spacing (used for scaling character images?)
        textHeight = string1Height + 10
        if string2Height > 0:
            textHeight += string2Height + 10 + 5

        # scale the character image for the 1st message and paste it into the panel Image object
        maxch = panelHeight - textHeight
        char1Image = Comic.fitimg(charmap[panel[0][0]], 2 * panelWidth / 5.0 - 10, maxch)
        panelImage.paste(char1Image, (10, panelHeight - char1Image.size[1]), char1Image)

        # if there is a second character, also scale and paste that into the panel Image object
        if len(panel) == 2:
            char2Image = Comic.fitimg(charmap[panel[1][0]], 2 * panelWidth / 5.0 - 10, maxch)
            char2Image = char2Image.transpose(
                Image.FLIP_LEFT_RIGHT)  # flip the character image, so it "faces" the first character image
            panelImage.paste(char2Image, (panelWidth - char2Image.size[0] - 10, panelHeight - char2Image.size[1]),
                             char2Image)

        # draw a small black line at the top? of the panel Image
        draw.line([(0, 0), (0, panelHeight - 1), (panelWidth - 1, panelHeight - 1), (panelWidth - 1, 0), (0, 0)],
                  (0, 0, 0, 0xff))

        return panelImage

    @staticmethod
    def wrap(message, font, draw, maxWidth):
        """
        Work out how much space `message` will take up on the image. Returns a list of strings representing each line
        of the message split into lines after wrapping and the wrapped width and height as a tuple
        """
        messageWords = message.split()
        wrappedWidth = 0
        wrappedHeight = 0
        lines = []

        while messageWords:
            numWords = 1
            while numWords < len(messageWords):
                # Try fitting the next numWords words on the line
                width, _ = draw.textsize(" ".join(messageWords[:numWords]), font=font)
                if width > maxWidth:
                    numWords -= 1
                    break
                else:
                    # There's still space, try another word
                    numWords += 1

            if numWords == 0 and messageWords:  # we've hit a case where the current word is wider than the screen
                numWords = 1

            # How big is this line?
            lineWidth, lineHeight = draw.textsize(" ".join(messageWords[:numWords]), font=font)

            wrappedWidth = max(wrappedWidth, lineWidth)  # wrappedWidth should be the length of the longest line
            wrappedHeight += lineHeight

            lines.append(" ".join(messageWords[:numWords]))
            messageWords = messageWords[numWords:]  # drop the words wrapped so far and continue

        return lines, (wrappedWidth, wrappedHeight)

    @staticmethod
    def rendertext(lines, font, draw, position):
        """
        This function renders the given `lines` at the given position in the given "draw" object - ImageDraw.Draw()
        """
        # draw black outline first
        for offset in [(-1, -1), (-1, 1), (1, -1), (1, 1)]:
            draw.text((position[0] + offset[0], position[1] + offset[1]), '\n'.join(lines),
                      font=font, fill=(0x00, 0x00, 0x00, 0xff))

        draw.text(position, '\n'.join(lines), font=font, fill=(0xff, 0xff, 0xff, 0xff))

    @staticmethod
    def fitimg(img, width, height):
        """
        Scale img proprotionally so that it's new width is `width` or height is `height`, whichever comes first.
        """

        # Calculate required scale factor to match width
        sf = width / img.size[0]
        # if this would make the image taller than the desired height, instead use the scale factor to match height
        if img.size[1] * sf > height:
            sf = height / img.size[1]

        # Resize image, round pixel count to nearest integer
        return img.resize((int(sf * img.size[0] + 0.5), int(sf * img.size[1] + 0.5)), Image.LANCZOS)

    @staticmethod
    def fitbkg(img, width, height):
        """
        Scale img to (`width`, `height`), cropping to the correct aspect ratio beforehand
        """

        targetAspectRatio = width / height
        imgAspectRatio = img.size[0] / img.size[1]

        if targetAspectRatio > imgAspectRatio:  # img too tall
            # How tall should it be?
            h = img.size[0] / targetAspectRatio

            # crop from top and bottom
            crop = (img.size[1] - h) / 2
            img = img.crop((0, crop, 0, crop))
        elif targetAspectRatio < imgAspectRatio:  # img too wide
            # How wide should it be?
            w = img.size[1] * targetAspectRatio

            # crop from left and right
            crop = (img.size[0] - w) / 2
            img = img.crop((crop, 0, crop, 0))

        return img.resize((width, height), Image.LANCZOS)

comic = Comic()
<|endoftext|>


<commit_before>"""
Created on Mar 5, 2018

@author: StarlitGhost
"""
from twisted.plugin import IPlugin
from desertbot.moduleinterface import IModule, BotModule
from zope.interface import implementer

import requests
from requests import Response
from bs4 import BeautifulSoup

import html
import ipaddress
import socket
from urllib.parse import urlparse
import re

from typing import Any, Dict, Optional

from apiclient.discovery import build

from desertbot.utils.api_keys import load_key

@implementer(IPlugin, IModule)
class WebUtils(BotModule):
    def actions(self):
        return super(WebUtils, self).actions() + [('fetch-url', 1, self.fetchURL),
                                                  ('post-url', 1, self.postURL),
                                                  ('get-html-title', 1, self.getPageTitle),
                                                  ('shorten-url', 1, self.shortenURL),
                                                  ('search-web', 1, self.googleSearch),
                                                  ('upload-pasteee', 1, self.pasteEE)]

    def onLoad(self):
        self.ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
        self.accept = ("text/*, "
                       "application/xml, application/xhtml+xml, "
                       "application/rss+xml, application/atom+xml, application/rdf+xml, "
                       "application/json")

    def isPublicURL(self, url: str) -> bool:
        parsed = urlparse(url)
        host = socket.gethostbyname(parsed.hostname)
        ip = ipaddress.ip_address(host)
        return ip.is_global

    def fetchURL(self, url: str,
                 params: Any=None,
                 extraHeaders: Optional[Dict[str, str]]=None) -> Optional[Response]:
        # check the requested url is public
        if not self.isPublicURL(url):
            return

        headers = {"User-agent": self.ua, "Accept": self.accept}
        # Make sure we don't download any unwanted things
        check = (r"^("
                 r"text/.*|"  # text
                 r"application/((rss|atom|rdf)\+)?xml(;.*)?|"  # rss/xml
                 r"application/(.*)json(;.*)?"  # json
                 r")$")
        check = re.compile(check)
        if extraHeaders:
            headers.update(extraHeaders)
        try:
            response = requests.get(url, params=params, headers=headers, timeout=10)
            pageType = response.headers["content-type"]

            if check.match(pageType):
                return response
            else:
                response.close()

        except requests.exceptions.RequestException:
            self.logger.exception("GET from {!r} failed!".format(url))

    # mostly taken directly from Heufneutje's PyHeufyBot
    # https://github.com/Heufneutje/PyHeufyBot/blob/eb10b5218cd6b9247998d8795d93b8cd0af45024/pyheufybot/utils/webutils.py#L43
    def postURL(self, url: str,
                data: Any=None,
                json: Any=None,
                extraHeaders: Optional[Dict[str, str]]=None) -> Optional[Response]:
        # check the requested url is public
        if not self.isPublicURL(url):
            return

        headers = {"User-agent": self.ua, "Accept": self.accept}
        if extraHeaders:
            headers.update(extraHeaders)

        try:
            response = requests.post(url, data=data, json=json, headers=headers, timeout=10)

            return response

        except requests.exceptions.RequestException:
            self.logger.exception("POST to {!r} failed!".format(url))

    def getPageTitle(self, webpage: str) -> Optional[str]:
        def cleanTitle(title: str) -> str:
            title = re.sub('[\r\n]+', '', title)  # strip any newlines
            title = title.strip()  # strip all whitespace either side
            title = ' '.join(title.split())  # replace multiple whitespace with single space
            title = html.unescape(title)  # unescape html entities

            # Split on the first space before 300 characters, and replace the rest with '...'
            if len(title) > 300:
                title = title[:300].rsplit(' ', 1)[0] + " ..."

            return title

        soup = BeautifulSoup(webpage, 'lxml')
        # look for a meta title first, it will probably be more relevant if there is one
        title = soup.find('meta', {'property': 'og:title'})
        if title:
            return cleanTitle(title['content'])
        # fall back to the html title tag
        title = soup.title
        if title:
            return cleanTitle(title.text)

        return

    def shortenURL(self, url: str) -> str:
        apiURL = 'https://dbco.link/u'
        post = {'content': url}
        headers = {'Content-Type': 'application/json',
                   'Accept': 'application/json'}

        try:
            response = requests.post(apiURL, json=post, headers=headers)
            responseJson = response.json()
            return responseJson['url']
        except requests.exceptions.RequestException:
            self.logger.exception("dbco.link url error")

    def googleSearch(self, query: str) -> Optional[Dict[str, Any]]:
        googleKey = load_key('Google')
        if not googleKey:
            return None

        service = build('customsearch', 'v1', developerKey=googleKey)
        res = service.cse().list(
            q=query,
            cx='002603151577378558984:xiv3qbttad0'
        ).execute()
        return res

    # mostly taken directly from Heufneutje's PyHeufyBot
    # https://github.com/Heufneutje/PyHeufyBot/blob/eb10b5218cd6b9247998d8795d93b8cd0af45024/pyheufybot/utils/webutils.py#L74
    def pasteEE(self, data: str, description: str, expire: int, raw: bool=True) -> str:
        # pasteEEKey = load_key('Paste.ee') (Heufneutje, 2018-4-27): Unused?

        values = {"key": "public",
                  "description": description,
                  "paste": data,
                  "expiration": expire,
                  "format": "json"}
        result = self.postURL("http://paste.ee/api", values)
        if result:
            jsonResult = result.json()
            if jsonResult["status"] == "success":
                linkType = "raw" if raw else "link"
                return jsonResult["paste"][linkType]
            elif jsonResult["status"] == "error":
                return ("An error occurred while posting to Paste.ee, code: {}, reason: {}"
                        .format(jsonResult["errorcode"], jsonResult["error"]))

web = WebUtils()
<commit_msg>[WebUtils-getPageTitle] look for <h1> inside <article> first<commit_after>"""
Created on Mar 5, 2018

@author: StarlitGhost
"""
from twisted.plugin import IPlugin
from desertbot.moduleinterface import IModule, BotModule
from zope.interface import implementer

import requests
from requests import Response
from bs4 import BeautifulSoup

import html
import ipaddress
import socket
from urllib.parse import urlparse
import re

from typing import Any, Dict, Optional

from apiclient.discovery import build

from desertbot.utils.api_keys import load_key

@implementer(IPlugin, IModule)
class WebUtils(BotModule):
    def actions(self):
        return super(WebUtils, self).actions() + [('fetch-url', 1, self.fetchURL),
                                                  ('post-url', 1, self.postURL),
                                                  ('get-html-title', 1, self.getPageTitle),
                                                  ('shorten-url', 1, self.shortenURL),
                                                  ('search-web', 1, self.googleSearch),
                                                  ('upload-pasteee', 1, self.pasteEE)]

    def onLoad(self):
        self.ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
        self.accept = ("text/*, "
                       "application/xml, application/xhtml+xml, "
                       "application/rss+xml, application/atom+xml, application/rdf+xml, "
                       "application/json")

    def isPublicURL(self, url: str) -> bool:
        parsed = urlparse(url)
        host = socket.gethostbyname(parsed.hostname)
        ip = ipaddress.ip_address(host)
        return ip.is_global

    def fetchURL(self, url: str,
                 params: Any=None,
                 extraHeaders: Optional[Dict[str, str]]=None) -> Optional[Response]:
        # check the requested url is public
        if not self.isPublicURL(url):
            return

        headers = {"User-agent": self.ua, "Accept": self.accept}
        # Make sure we don't download any unwanted things
        check = (r"^("
                 r"text/.*|"  # text
                 r"application/((rss|atom|rdf)\+)?xml(;.*)?|"  # rss/xml
                 r"application/(.*)json(;.*)?"  # json
                 r")$")
        check = re.compile(check)
        if extraHeaders:
            headers.update(extraHeaders)
        try:
            response = requests.get(url, params=params, headers=headers, timeout=10)
            pageType = response.headers["content-type"]

            if check.match(pageType):
                return response
            else:
                response.close()

        except requests.exceptions.RequestException:
            self.logger.exception("GET from {!r} failed!".format(url))

    # mostly taken directly from Heufneutje's PyHeufyBot
    # https://github.com/Heufneutje/PyHeufyBot/blob/eb10b5218cd6b9247998d8795d93b8cd0af45024/pyheufybot/utils/webutils.py#L43
    def postURL(self, url: str,
                data: Any=None,
                json: Any=None,
                extraHeaders: Optional[Dict[str, str]]=None) -> Optional[Response]:
        # check the requested url is public
        if not self.isPublicURL(url):
            return

        headers = {"User-agent": self.ua, "Accept": self.accept}
        if extraHeaders:
            headers.update(extraHeaders)

        try:
            response = requests.post(url, data=data, json=json, headers=headers, timeout=10)

            return response

        except requests.exceptions.RequestException:
            self.logger.exception("POST to {!r} failed!".format(url))

    def getPageTitle(self, webpage: str) -> Optional[str]:
        def cleanTitle(title: str) -> str:
            title = re.sub('[\r\n]+', '', title)  # strip any newlines
            title = title.strip()  # strip all whitespace either side
            title = ' '.join(title.split())  # replace multiple whitespace with single space
            title = html.unescape(title)  # unescape html entities

            # Split on the first space before 300 characters, and replace the rest with '...'
            if len(title) > 300:
                title = title[:300].rsplit(' ', 1)[0] + " ..."

            return title

        soup = BeautifulSoup(webpage, 'lxml')
        # look for an h1 inside an article tag first, this should be the actual title on the page
        article = soup.find('article')
        if article:
            title = article.find('h1')
            if title:
                return cleanTitle(title.text)
        # look for a meta title next, it will probably be more relevant if there is one
        title = soup.find('meta', {'property': 'og:title'})
        if title:
            return cleanTitle(title['content'])
        # fall back to the html title tag
        title = soup.title
        if title:
            return cleanTitle(title.text)

        return

    def shortenURL(self, url: str) -> str:
        apiURL = 'https://dbco.link/u'
        post = {'content': url}
        headers = {'Content-Type': 'application/json',
                   'Accept': 'application/json'}

        try:
            response = requests.post(apiURL, json=post, headers=headers)
            responseJson = response.json()
            return responseJson['url']
        except requests.exceptions.RequestException:
            self.logger.exception("dbco.link url error")

    def googleSearch(self, query: str) -> Optional[Dict[str, Any]]:
        googleKey = load_key('Google')
        if not googleKey:
            return None

        service = build('customsearch', 'v1', developerKey=googleKey)
        res = service.cse().list(
            q=query,
            cx='002603151577378558984:xiv3qbttad0'
        ).execute()
        return res

    # mostly taken directly from Heufneutje's PyHeufyBot
    # https://github.com/Heufneutje/PyHeufyBot/blob/eb10b5218cd6b9247998d8795d93b8cd0af45024/pyheufybot/utils/webutils.py#L74
    def pasteEE(self, data: str, description: str, expire: int, raw: bool=True) -> str:
        # pasteEEKey = load_key('Paste.ee') (Heufneutje, 2018-4-27): Unused?

        values = {"key": "public",
                  "description": description,
                  "paste": data,
                  "expiration": expire,
                  "format": "json"}
        result = self.postURL("http://paste.ee/api", values)
        if result:
            jsonResult = result.json()
            if jsonResult["status"] == "success":
                linkType = "raw" if raw else "link"
                return jsonResult["paste"][linkType]
            elif jsonResult["status"] == "error":
                return ("An error occurred while posting to Paste.ee, code: {}, reason: {}"
                        .format(jsonResult["errorcode"], jsonResult["error"]))

web = WebUtils()
<|endoftext|>


<commit_before>"""Converts a TAPE9 file to a sparse matrix on disk."""
import os
from argparse import ArgumentParser

from .util import save_sparse_csr
from .tape9utils import tape9_to_sparse, THRESHOLD

def make_parser():
    p = ArgumentParser('tape9sparse')
    p.add_argument('tape9', help="path to the TAPE9 file.")
    p.add_argument('phi', help='the neutron flux in [n/cm^2/sec]',
                   type=float)
    p.add_argument('-f', '--format', help='The sparse matrix format',
                   default='csr', dest='format')
    p.add_argument('-d', '--decay', help='path to the decay file, if needed',
                   default='decay.lib', dest='decaylib')
    p.add_argument('--include-fission', action='store_true', default=True,
                   dest='include_fission',
                   help='Include fission reactions in the matrix.')
    p.add_argument('--dont-include-fission', action='store_false',
                   dest='include_fission',
                   help="Don't include fission reactions in the matrix.")
    p.add_argument('-t', '--threshold', default=THRESHOLD, dest='threshold',
                   help='cutoff for ignoring reactions', type=float)
    p.add_argument('-o', '--output', dest='output', default=None,
                   help='The filename to write the matrix to, in npz format.')
    return p

def main(args=None):
    p = make_parser()
    ns = p.parse_args(args=args)
    if ns.output is None:
        base = os.path.basename(ns.tape9)
        base, _ = os.path.splitext(base)
        ns.output = base + '.npz'
    if ns.format != 'csr':
        raise ValueError('Only the CSR format is currently available from the '
                         'command line interface.')
    mat, nucs = tape9_to_sparse(ns.tape9, ns.phi, format=ns.format,
                                decaylib=ns.decaylib,
                                include_fission=ns.include_fission,
                                threshold=ns.threshold)
    save_sparse_csr(ns.output, mat, nucs)

if __name__ == '__main__':
    main()
<commit_msg>Show the defaults in python -m transmutagen.tape9sparse --help<commit_after>"""Converts a TAPE9 file to a sparse matrix on disk."""
import os
import argparse

from .util import save_sparse_csr
from .tape9utils import tape9_to_sparse, THRESHOLD

def make_parser():
    p = argparse.ArgumentParser('tape9sparse', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument('tape9', help="path to the TAPE9 file.")
    p.add_argument('phi', help='the neutron flux in [n/cm^2/sec]',
                   type=float)
    p.add_argument('-f', '--format', help='The sparse matrix format',
                   default='csr', dest='format')
    p.add_argument('-d', '--decay', help='path to the decay file, if needed',
                   default='decay.lib', dest='decaylib')
    p.add_argument('--include-fission', action='store_true', default=True,
                   dest='include_fission',
                   help='Include fission reactions in the matrix.')
    p.add_argument('--dont-include-fission', action='store_false',
                   dest='include_fission',
                   help="Don't include fission reactions in the matrix.")
    p.add_argument('-t', '--threshold', default=THRESHOLD, dest='threshold',
                   help='cutoff for ignoring reactions', type=float)
    p.add_argument('-o', '--output', dest='output', default=None,
                   help='The filename to write the matrix to, in npz format.')
    return p

def main(args=None):
    p = make_parser()
    ns = p.parse_args(args=args)
    if ns.output is None:
        base = os.path.basename(ns.tape9)
        base, _ = os.path.splitext(base)
        ns.output = base + '.npz'
    if ns.format != 'csr':
        raise ValueError('Only the CSR format is currently available from the '
                         'command line interface.')
    mat, nucs = tape9_to_sparse(ns.tape9, ns.phi, format=ns.format,
                                decaylib=ns.decaylib,
                                include_fission=ns.include_fission,
                                threshold=ns.threshold)
    save_sparse_csr(ns.output, mat, nucs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>def transform_scalars(dataset):

    from tomviz import utils
    import numpy as np

    # Get the current volume as a numpy array.
    array = utils.get_array(dataset)

    # Create 3D hanning window.
    for axis, axis_size in enumerate(array.shape):
        # Set up shape for numpy broadcasting.
        filter_shape = [1, ] * array.ndim
        filter_shape[axis] = axis_size
        window = np.hanning(axis_size).reshape(filter_shape)
        array *= window

    # This is where you operate on your data
    result = array
    #result = window

    # This is where the transformed data is set, it will display in tomviz.
    utils.set_array(dataset, result)
<commit_msg>Handle type incompatibility<commit_after>def transform_scalars(dataset):

    from tomviz import utils
    import numpy as np

    # Get the current volume as a numpy array.
    array = utils.get_array(dataset)

    # Save the type
    input_dtype = array.dtype
    array = array.astype(np.float32)

    # Create 3D hanning window.
    for axis, axis_size in enumerate(array.shape):
        # Set up shape for numpy broadcasting.
        filter_shape = [1, ] * array.ndim
        filter_shape[axis] = axis_size
        window = np.hanning(axis_size).reshape(filter_shape)
        array *= window

    # Recast the data to the input type
    result = array.astype(input_dtype)
    utils.set_array(dataset, result)
<|endoftext|>


<commit_before>from org.transcrypt.stubs.browser import __pragma__

__pragma__ ('tconv')

def run (autoTester):
    autoTester.check (len ({1:2}))

    autoTester.check ('Select nonemtpy container, if any
')

    autoTester.check ((0) or (1, 2, 3))
    autoTester.check (() or (1, 2, 3))
    autoTester.check (() or ())

    autoTester.check ((-1) or (0) or (1, 2, 3))
    autoTester.check (() or (0) or (1, 2, 3))
    autoTester.check (() or () or (1, 2, 3))
    autoTester.check (() or () or ())

    autoTester.check ([0] or [1, 2, 3])
    autoTester.check ([] or [1, 2, 3])
    autoTester.check ([] or [])

    autoTester.check ([-1] or [0] or [1, 2, 3])
    autoTester.check ([] or [0] or [1, 2, 3])
    autoTester.check ([] or [] or [1, 2, 3])
    autoTester.check ([] or [] or [])

    autoTester.check ({0} or {1, 2, 3, 4})
    autoTester.check (set () or {1, 2, 3, 4})
    autoTester.check (set () or set ())

    autoTester.check ({-1} or {0} or {1, 2, 3, 5})
    autoTester.check (set () or {0} or {1, 2, 3, 6})
    autoTester.check (set () or set () or {1, 2, 3, 7})
    autoTester.check (set () or set () or set ())

    autoTester.check ({0:10} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {1, 2, 3, 8})
    autoTester.check ({} or {})

    autoTester.check ({-1:-11} or {0:10} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {0:10} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {} or {})

    autoTester.check ('

')
    autoTester.check ('Boolean evaluations')
    for expression in (
        '
 -- falsy --
',
        (),
        [],
        set (),
        {},
        0,
        '',
        3 > 5,
        False,
        '
 -- truthy --
',
        (1, 2, 3),
        [1, 2, 3],
        {1, 2, 3},
        {'a': 1, 'b': 2, 'c': 3},
        3,
        'hello',
        5 > 3,
        True
    ):
        if expression in ('
 -- falsy --
', '
 -- truthy --
'):
            autoTester.check (expression)
        else:
            autoTester.check (expression, ' . . . ')
            autoTester.check ('operators')
            autoTester.check (not not expression)
            autoTester.check (not not (True and expression))
            autoTester.check (not not (False or expression))
            autoTester.check (not not (expression and True))
            autoTester.check (not not (expression and False))

            autoTester.check ('if')
            if expression:
                autoTester.check (True)
            else:
                autoTester.check (False)

            if expression or expression:
                autoTester.check (True)
            else:
                autoTester.check (False)

            if False:
                autoTester.check ('if')
            elif expression:
                autoTester.check ('elif')
            else:
                autoTester.check ('else')

            autoTester.check ('while')
            while expression:
                autoTester.check (True)
                break

            autoTester.check ('condex')
            autoTester.check (True if expression else False)

    if (0.0):
        autoTester.check ('0.0')
    elif (0.1):
        autoTester.check ('0.1')
    else:
        autoTester.check ('Shouldn\'t be here...')

    class A:
        pass

    class B:
        def __bool__(self):
            return False

    class C:
        def __bool__(self):
            return True

        def __len__(self):
            return 0

    class D:
        def __len__(self):
            return 0

    class E:
        def __len__(self):
            return 1

    autoTester.check ('instances of a custom class')
    autoTester.check (not not A ())
    autoTester.check (not not B ())
    autoTester.check (not not C ())
    autoTester.check (not not D ())
    autoTester.check (not not E ())
<commit_msg>Minor nits<commit_after>from org.transcrypt.stubs.browser import __pragma__

__pragma__ ('tconv')

def run (autoTester):
    autoTester.check (len ({1:2}))

    autoTester.check ('Select nonemtpy container, if any
')

    autoTester.check ((0) or (1, 2, 3))
    autoTester.check (() or (1, 2, 3))
    autoTester.check (() or ())

    autoTester.check ((-1) or (0) or (1, 2, 3))
    autoTester.check (() or (0) or (1, 2, 3))
    autoTester.check (() or () or (1, 2, 3))
    autoTester.check (() or () or ())

    autoTester.check ([0] or [1, 2, 3])
    autoTester.check ([] or [1, 2, 3])
    autoTester.check ([] or [])

    autoTester.check ([-1] or [0] or [1, 2, 3])
    autoTester.check ([] or [0] or [1, 2, 3])
    autoTester.check ([] or [] or [1, 2, 3])
    autoTester.check ([] or [] or [])

    autoTester.check ({0} or {1, 2, 3, 4})
    autoTester.check (set () or {1, 2, 3, 4})
    autoTester.check (set () or set ())

    autoTester.check ({-1} or {0} or {1, 2, 3, 5})
    autoTester.check (set () or {0} or {1, 2, 3, 6})
    autoTester.check (set () or set () or {1, 2, 3, 7})
    autoTester.check (set () or set () or set ())

    autoTester.check ({0:10} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {1, 2, 3, 8})
    autoTester.check ({} or {})

    autoTester.check ({-1:-11} or {0:10} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {0:10} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {} or {1:11, 2:12, 3:13})
    autoTester.check ({} or {} or {})

    autoTester.check ('

')
    autoTester.check ('Boolean evaluations')
    for expression in (
        '
 -- falsy --
',
        (),
        [],
        set (),
        {},
        0,
        '',
        3 > 5,
        False,
        '
 -- truthy --
',
        (1, 2, 3),
        [1, 2, 3],
        {1, 2, 3},
        {'a': 1, 'b': 2, 'c': 3},
        3,
        'hello',
        5 > 3,
        True
    ):
        if expression in ('
 -- falsy --
', '
 -- truthy --
'):
            autoTester.check (expression)
        else:
            autoTester.check (expression, ' . . . ')
            autoTester.check ('operators')
            autoTester.check (not not expression)
            autoTester.check (not not (True and expression))
            autoTester.check (not not (False or expression))
            autoTester.check (not not (expression and True))
            autoTester.check (not not (expression and False))

            autoTester.check ('if')
            if expression:
                autoTester.check (True)
            else:
                autoTester.check (False)

            if expression or expression:
                autoTester.check (True)
            else:
                autoTester.check (False)

            if False:
                autoTester.check ('if')
            elif expression:
                autoTester.check ('elif')
            else:
                autoTester.check ('else')

            autoTester.check ('while')
            while expression:
                autoTester.check (True)
                break

            autoTester.check ('condex')
            autoTester.check (True if expression else False)

    if (0.0):
        autoTester.check ('0.0')
    elif (0.1):
        autoTester.check ('0.1')
    else:
        autoTester.check ('Shouldn\'t be here...')

    class A:
        pass

    class B:
        def __bool__ ( self):
            return False

    class C:
        def __bool__ (self):
            return True

        def __len__ (self):
            return 0

    class D:
        def __len__ (self):
            return 0

    class E:
        def __len__ (self):
            return 1

    autoTester.check ('instances of custom classes')
    autoTester.check (not not A ())
    autoTester.check (not not B ())
    autoTester.check (not not C ())
    autoTester.check (not not D ())
    autoTester.check (not not E ())

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2015 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#

from OSEncryptionState import *

class SelinuxState(OSEncryptionState):
    def __init__(self, context):
        super(SelinuxState, self).__init__('SelinuxState', context)

    def enter(self):
        if not super(SelinuxState, self).should_enter():
            return

        self.context.logger.log("Entering selinux state")

    def should_exit(self):
        self.context.logger.log("Verifying if machine should exit selinux state")

        return super(SelinuxState, self).should_exit()
<commit_msg>Disable SELinux before OS disk encryption<commit_after>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2015 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#

from OSEncryptionState import *

class SelinuxState(OSEncryptionState):
    def __init__(self, context):
        super(SelinuxState, self).__init__('SelinuxState', context)

    def enter(self):
        if not super(SelinuxState, self).should_enter():
            return

        self.context.logger.log("Entering selinux state")

        se_linux_status = self.context.encryption_environment.get_se_linux()
        if(se_linux_status.lower() == 'enforcing'):
            self.context.logger.log("SELinux is in enforcing mode, disabling")
            self.context.encryption_environment.disable_se_linux()

    def should_exit(self):
        self.context.logger.log("Verifying if machine should exit selinux state")

        return super(SelinuxState, self).should_exit()
<|endoftext|>