<commit_before>from encodings.aliases import aliases
from hashlib import sha256
from json import dumps
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union

from .constant import NOT_PRINTABLE_PATTERN, TOO_BIG_SEQUENCE
from .utils import iana_name, is_multi_byte_encoding, unicode_range

class CharsetMatch:
    def __init__(
        self,
        payload: bytes,
        guessed_encoding: str,
        mean_mess_ratio: float,
        has_sig_or_bom: bool,
        languages: "CoherenceMatches",
        decoded_payload: Optional[str] = None,
    ):
        self._payload: bytes = payload

        self._encoding: str = guessed_encoding
        self._mean_mess_ratio: float = mean_mess_ratio
        self._languages: CoherenceMatches = languages
        self._has_sig_or_bom: bool = has_sig_or_bom
        self._unicode_ranges: Optional[List[str]] = None

        self._leaves: List[CharsetMatch] = []
        self._mean_coherence_ratio: float = 0.0

        self._output_payload: Optional[bytes] = None
        self._output_encoding: Optional[str] = None

        self._string: Optional[str] = decoded_payload

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, CharsetMatch):
            raise TypeError(
                "__eq__ cannot be invoked on {} and {}.".format(
                    str(other.__class__), str(self.__class__)
                )
            )
        return self.encoding == other.encoding and self.fingerprint == other.fingerprint

    def __lt__(self, other: object) -> bool:
        """
        Implemented to make sorted available upon CharsetMatches items.
        """
        if not isinstance(other, CharsetMatch):
            raise ValueError

        chaos_difference: float = abs(self.chaos - other.chaos)
        coherence_difference: float = abs(self.coherence - other.coherence)

        # Bellow 1% difference --> Use Coherence
        if chaos_difference < 0.01 and coherence_difference > 0.02:
            # When having a tough decision, use the result that decoded as many multi-byte as possible.
            if chaos_difference == 0.0 and self.coherence == other.coherence:
                return self.multi_byte_usage > other.multi_byte_usage
            return self.coherence > other.coherence

        return self.chaos < other.chaos

    @property
    def multi_byte_usage(self) -> float:
        return 1.0 - len(str(self)) / len(self.raw)

    def __str__(self) -> str:
        # Lazy Str Loading
        if self._string is None:
            self._string = str(self._payload, self._encoding, "strict")
        return self._string

    def __repr__(self) -> str:
        return "<CharsetMatch '{}' bytes({})>".format(self.encoding, self.fingerprint)

    def add_submatch(self, other: "CharsetMatch") -> None:
        if not isinstance(other, CharsetMatch) or other == self:
            raise ValueError(
                "Unable to add instance <{}> as a submatch of a CharsetMatch".format(
                    other.__class__
                )
            )

        other._string = None  # Unload RAM usage; dirty trick.
        self._leaves.append(other)

    @property
    def encoding(self) -> str:
        return self._encoding

    @property
    def encoding_aliases(self) -> List[str]:
        """
        Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
        """
        also_known_as: List[str] = []
        for u, p in aliases.items():
            if self.encoding == u:
                also_known_as.append(p)
            elif self.encoding == p:
                also_known_as.append(u)
        return also_known_as

    @property
    def bom(self) -> bool:
        return self._has_sig_or_bom

    @property
    def byte_order_mark(self) -> bool:
        return self._has_sig_or_bom

    @property
    def languages(self) -> List[str]:
        """
        Return the complete list of possible languages found in decoded sequence.
        Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
        """
        return [e[0] for e in self._languages]

    @property
    def language(self) -> str:
        """
        Most probable language found in decoded sequence. If none were detected or inferred, the property will return
        "Unknown".
        """
        if not self._languages:
            # Trying to infer the language based on the given encoding
            # Its either English or we should not pronounce ourselves in certain cases.
            if "ascii" in self.could_be_from_charset:
                return "English"

            # doing it there to avoid circular import
            from charset_normalizer.cd import encoding_languages, mb_encoding_languages

            languages = (
                mb_encoding_languages(self.encoding)
                if is_multi_byte_encoding(self.encoding)
                else encoding_languages(self.encoding)
            )

            if len(languages) == 0 or "Latin Based" in languages:
                return "Unknown"

            return languages[0]

        return self._languages[0][0]

    @property
    def chaos(self) -> float:
        return self._mean_mess_ratio

    @property
    def coherence(self) -> float:
        if not self._languages:
            return 0.0
        return self._languages[0][1]

    @property
    def percent_chaos(self) -> float:
        return round(self.chaos * 100, ndigits=3)

    @property
    def percent_coherence(self) -> float:
        return round(self.coherence * 100, ndigits=3)

    @property
    def raw(self) -> bytes:
        """
        Original untouched bytes.
        """
        return self._payload

    @property
    def submatch(self) -> List["CharsetMatch"]:
        return self._leaves

    @property
    def has_submatch(self) -> bool:
        return len(self._leaves) > 0

    @property
    def alphabets(self) -> List[str]:
        if self._unicode_ranges is not None:
            return self._unicode_ranges
        # list detected ranges
        detected_ranges: List[Optional[str]] = [
            unicode_range(char) for char in str(self)
        ]
        # filter and sort
        self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
        return self._unicode_ranges

    @property
    def could_be_from_charset(self) -> List[str]:
        """
        The complete list of encoding that output the exact SAME str result and therefore could be the originating
        encoding.
        This list does include the encoding available in property 'encoding'.
        """
        return [self._encoding] + [m.encoding for m in self._leaves]

    def first(self) -> "CharsetMatch":
        """
        Kept for BC reasons. Will be removed in 3.0.
        """
        return self

    def best(self) -> "CharsetMatch":
        """
        Kept for BC reasons. Will be removed in 3.0.
        """
        return self

    def output(self, encoding: str = "utf_8") -> bytes:
        """
        Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
        Any errors will be simply ignored by the encoder NOT replaced.
        """
        if self._output_encoding is None or self._output_encoding != encoding:
            self._output_encoding = encoding
            self._output_payload = str(self).encode(encoding, "replace")

        return self._output_payload  # type: ignore

    @property
    def fingerprint(self) -> str:
        """
        Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one.
        """
        return sha256(self.output()).hexdigest()

class CharsetMatches:
    """
    Container with every CharsetMatch items ordered by default from most probable to the less one.
    Act like a list(iterable) but does not implements all related methods.
    """

    def __init__(self, results: Optional[List[CharsetMatch]] = None):
        self._results: List[CharsetMatch] = sorted(results) if results else []

    def __iter__(self) -> Iterator[CharsetMatch]:
        yield from self._results

    def __getitem__(self, item: Union[int, str]) -> CharsetMatch:
        """
        Retrieve a single item either by its position or encoding name (alias may be used here).
        Raise KeyError upon invalid index or encoding not present in results.
        """
        if isinstance(item, int):
            return self._results[item]
        if isinstance(item, str):
            item = iana_name(item, False)
            for result in self._results:
                if item in result.could_be_from_charset:
                    return result
        raise KeyError

    def __len__(self) -> int:
        return len(self._results)

    def __bool__(self) -> bool:
        return len(self._results) > 0

    def append(self, item: CharsetMatch) -> None:
        """
        Insert a single match. Will be inserted accordingly to preserve sort.
        Can be inserted as a submatch.
        """
        if not isinstance(item, CharsetMatch):
            raise ValueError(
                "Cannot append instance '{}' to CharsetMatches".format(
                    str(item.__class__)
                )
            )
        # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
        if len(item.raw) <= TOO_BIG_SEQUENCE:
            for match in self._results:
                if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
                    match.add_submatch(item)
                    return
        self._results.append(item)
        self._results = sorted(self._results)

    def best(self) -> Optional["CharsetMatch"]:
        """
        Simply return the first match. Strict equivalent to matches[0].
        """
        if not self._results:
            return None
        return self._results[0]

    def first(self) -> Optional["CharsetMatch"]:
        """
        Redundant method, call the method best(). Kept for BC reasons.
        """
        return self.best()

CoherenceMatch = Tuple[str, float]
CoherenceMatches = List[CoherenceMatch]

class CliDetectionResult:
    def __init__(
        self,
        path: str,
        encoding: Optional[str],
        encoding_aliases: List[str],
        alternative_encodings: List[str],
        language: str,
        alphabets: List[str],
        has_sig_or_bom: bool,
        chaos: float,
        coherence: float,
        unicode_path: Optional[str],
        is_preferred: bool,
    ):
        self.path: str = path
        self.unicode_path: Optional[str] = unicode_path
        self.encoding: Optional[str] = encoding
        self.encoding_aliases: List[str] = encoding_aliases
        self.alternative_encodings: List[str] = alternative_encodings
        self.language: str = language
        self.alphabets: List[str] = alphabets
        self.has_sig_or_bom: bool = has_sig_or_bom
        self.chaos: float = chaos
        self.coherence: float = coherence
        self.is_preferred: bool = is_preferred

    @property
    def __dict__(self) -> Dict[str, Any]:  # type: ignore
        return {
            "path": self.path,
            "encoding": self.encoding,
            "encoding_aliases": self.encoding_aliases,
            "alternative_encodings": self.alternative_encodings,
            "language": self.language,
            "alphabets": self.alphabets,
            "has_sig_or_bom": self.has_sig_or_bom,
            "chaos": self.chaos,
            "coherence": self.coherence,
            "unicode_path": self.unicode_path,
            "is_preferred": self.is_preferred,
        }

    def to_json(self) -> str:
        return dumps(self.__dict__, ensure_ascii=True, indent=4)
<commit_msg>:fire: remove NOT_PRINTABLE_PATTERN import in models.py<commit_after>from encodings.aliases import aliases
from hashlib import sha256
from json import dumps
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union

from .constant import TOO_BIG_SEQUENCE
from .utils import iana_name, is_multi_byte_encoding, unicode_range

class CharsetMatch:
    def __init__(
        self,
        payload: bytes,
        guessed_encoding: str,
        mean_mess_ratio: float,
        has_sig_or_bom: bool,
        languages: "CoherenceMatches",
        decoded_payload: Optional[str] = None,
    ):
        self._payload: bytes = payload

        self._encoding: str = guessed_encoding
        self._mean_mess_ratio: float = mean_mess_ratio
        self._languages: CoherenceMatches = languages
        self._has_sig_or_bom: bool = has_sig_or_bom
        self._unicode_ranges: Optional[List[str]] = None

        self._leaves: List[CharsetMatch] = []
        self._mean_coherence_ratio: float = 0.0

        self._output_payload: Optional[bytes] = None
        self._output_encoding: Optional[str] = None

        self._string: Optional[str] = decoded_payload

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, CharsetMatch):
            raise TypeError(
                "__eq__ cannot be invoked on {} and {}.".format(
                    str(other.__class__), str(self.__class__)
                )
            )
        return self.encoding == other.encoding and self.fingerprint == other.fingerprint

    def __lt__(self, other: object) -> bool:
        """
        Implemented to make sorted available upon CharsetMatches items.
        """
        if not isinstance(other, CharsetMatch):
            raise ValueError

        chaos_difference: float = abs(self.chaos - other.chaos)
        coherence_difference: float = abs(self.coherence - other.coherence)

        # Bellow 1% difference --> Use Coherence
        if chaos_difference < 0.01 and coherence_difference > 0.02:
            # When having a tough decision, use the result that decoded as many multi-byte as possible.
            if chaos_difference == 0.0 and self.coherence == other.coherence:
                return self.multi_byte_usage > other.multi_byte_usage
            return self.coherence > other.coherence

        return self.chaos < other.chaos

    @property
    def multi_byte_usage(self) -> float:
        return 1.0 - len(str(self)) / len(self.raw)

    def __str__(self) -> str:
        # Lazy Str Loading
        if self._string is None:
            self._string = str(self._payload, self._encoding, "strict")
        return self._string

    def __repr__(self) -> str:
        return "<CharsetMatch '{}' bytes({})>".format(self.encoding, self.fingerprint)

    def add_submatch(self, other: "CharsetMatch") -> None:
        if not isinstance(other, CharsetMatch) or other == self:
            raise ValueError(
                "Unable to add instance <{}> as a submatch of a CharsetMatch".format(
                    other.__class__
                )
            )

        other._string = None  # Unload RAM usage; dirty trick.
        self._leaves.append(other)

    @property
    def encoding(self) -> str:
        return self._encoding

    @property
    def encoding_aliases(self) -> List[str]:
        """
        Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
        """
        also_known_as: List[str] = []
        for u, p in aliases.items():
            if self.encoding == u:
                also_known_as.append(p)
            elif self.encoding == p:
                also_known_as.append(u)
        return also_known_as

    @property
    def bom(self) -> bool:
        return self._has_sig_or_bom

    @property
    def byte_order_mark(self) -> bool:
        return self._has_sig_or_bom

    @property
    def languages(self) -> List[str]:
        """
        Return the complete list of possible languages found in decoded sequence.
        Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
        """
        return [e[0] for e in self._languages]

    @property
    def language(self) -> str:
        """
        Most probable language found in decoded sequence. If none were detected or inferred, the property will return
        "Unknown".
        """
        if not self._languages:
            # Trying to infer the language based on the given encoding
            # Its either English or we should not pronounce ourselves in certain cases.
            if "ascii" in self.could_be_from_charset:
                return "English"

            # doing it there to avoid circular import
            from charset_normalizer.cd import encoding_languages, mb_encoding_languages

            languages = (
                mb_encoding_languages(self.encoding)
                if is_multi_byte_encoding(self.encoding)
                else encoding_languages(self.encoding)
            )

            if len(languages) == 0 or "Latin Based" in languages:
                return "Unknown"

            return languages[0]

        return self._languages[0][0]

    @property
    def chaos(self) -> float:
        return self._mean_mess_ratio

    @property
    def coherence(self) -> float:
        if not self._languages:
            return 0.0
        return self._languages[0][1]

    @property
    def percent_chaos(self) -> float:
        return round(self.chaos * 100, ndigits=3)

    @property
    def percent_coherence(self) -> float:
        return round(self.coherence * 100, ndigits=3)

    @property
    def raw(self) -> bytes:
        """
        Original untouched bytes.
        """
        return self._payload

    @property
    def submatch(self) -> List["CharsetMatch"]:
        return self._leaves

    @property
    def has_submatch(self) -> bool:
        return len(self._leaves) > 0

    @property
    def alphabets(self) -> List[str]:
        if self._unicode_ranges is not None:
            return self._unicode_ranges
        # list detected ranges
        detected_ranges: List[Optional[str]] = [
            unicode_range(char) for char in str(self)
        ]
        # filter and sort
        self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
        return self._unicode_ranges

    @property
    def could_be_from_charset(self) -> List[str]:
        """
        The complete list of encoding that output the exact SAME str result and therefore could be the originating
        encoding.
        This list does include the encoding available in property 'encoding'.
        """
        return [self._encoding] + [m.encoding for m in self._leaves]

    def first(self) -> "CharsetMatch":
        """
        Kept for BC reasons. Will be removed in 3.0.
        """
        return self

    def best(self) -> "CharsetMatch":
        """
        Kept for BC reasons. Will be removed in 3.0.
        """
        return self

    def output(self, encoding: str = "utf_8") -> bytes:
        """
        Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
        Any errors will be simply ignored by the encoder NOT replaced.
        """
        if self._output_encoding is None or self._output_encoding != encoding:
            self._output_encoding = encoding
            self._output_payload = str(self).encode(encoding, "replace")

        return self._output_payload  # type: ignore

    @property
    def fingerprint(self) -> str:
        """
        Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one.
        """
        return sha256(self.output()).hexdigest()

class CharsetMatches:
    """
    Container with every CharsetMatch items ordered by default from most probable to the less one.
    Act like a list(iterable) but does not implements all related methods.
    """

    def __init__(self, results: Optional[List[CharsetMatch]] = None):
        self._results: List[CharsetMatch] = sorted(results) if results else []

    def __iter__(self) -> Iterator[CharsetMatch]:
        yield from self._results

    def __getitem__(self, item: Union[int, str]) -> CharsetMatch:
        """
        Retrieve a single item either by its position or encoding name (alias may be used here).
        Raise KeyError upon invalid index or encoding not present in results.
        """
        if isinstance(item, int):
            return self._results[item]
        if isinstance(item, str):
            item = iana_name(item, False)
            for result in self._results:
                if item in result.could_be_from_charset:
                    return result
        raise KeyError

    def __len__(self) -> int:
        return len(self._results)

    def __bool__(self) -> bool:
        return len(self._results) > 0

    def append(self, item: CharsetMatch) -> None:
        """
        Insert a single match. Will be inserted accordingly to preserve sort.
        Can be inserted as a submatch.
        """
        if not isinstance(item, CharsetMatch):
            raise ValueError(
                "Cannot append instance '{}' to CharsetMatches".format(
                    str(item.__class__)
                )
            )
        # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
        if len(item.raw) <= TOO_BIG_SEQUENCE:
            for match in self._results:
                if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
                    match.add_submatch(item)
                    return
        self._results.append(item)
        self._results = sorted(self._results)

    def best(self) -> Optional["CharsetMatch"]:
        """
        Simply return the first match. Strict equivalent to matches[0].
        """
        if not self._results:
            return None
        return self._results[0]

    def first(self) -> Optional["CharsetMatch"]:
        """
        Redundant method, call the method best(). Kept for BC reasons.
        """
        return self.best()

CoherenceMatch = Tuple[str, float]
CoherenceMatches = List[CoherenceMatch]

class CliDetectionResult:
    def __init__(
        self,
        path: str,
        encoding: Optional[str],
        encoding_aliases: List[str],
        alternative_encodings: List[str],
        language: str,
        alphabets: List[str],
        has_sig_or_bom: bool,
        chaos: float,
        coherence: float,
        unicode_path: Optional[str],
        is_preferred: bool,
    ):
        self.path: str = path
        self.unicode_path: Optional[str] = unicode_path
        self.encoding: Optional[str] = encoding
        self.encoding_aliases: List[str] = encoding_aliases
        self.alternative_encodings: List[str] = alternative_encodings
        self.language: str = language
        self.alphabets: List[str] = alphabets
        self.has_sig_or_bom: bool = has_sig_or_bom
        self.chaos: float = chaos
        self.coherence: float = coherence
        self.is_preferred: bool = is_preferred

    @property
    def __dict__(self) -> Dict[str, Any]:  # type: ignore
        return {
            "path": self.path,
            "encoding": self.encoding,
            "encoding_aliases": self.encoding_aliases,
            "alternative_encodings": self.alternative_encodings,
            "language": self.language,
            "alphabets": self.alphabets,
            "has_sig_or_bom": self.has_sig_or_bom,
            "chaos": self.chaos,
            "coherence": self.coherence,
            "unicode_path": self.unicode_path,
            "is_preferred": self.is_preferred,
        }

    def to_json(self) -> str:
        return dumps(self.__dict__, ensure_ascii=True, indent=4)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from cStringIO import StringIO
from os.path import getsize
from struct import pack, unpack, error as StructError
from .log import log
from .structures import fields

class DBHeader(object):
	"""
	Base DBFile header class
	"""
	def __repr__(self):
		return "%s(%s)" % (self.__class__.__name__, ", ".join("%s=%r" % (k, self.__dict__[k]) for k in self.__dict__))

class DBFile(object):
	"""
	Base class for WDB and DBC files
	"""

	def __init__(self, file, build, structure, environment):
		if isinstance(file, basestring):
			file = open(file, "r")
		self.file = file
		self.build = build
		self._addresses = {}
		self._values = {}
		self.environment = environment

		self._rowDynamicFields = 0 # Dynamic fields index, used when parsing a row

	def __repr__(self):
		return "%s(file=%r, build=%r)" % (self.__class__.__name__, self.file, self.build)

	def __contains__(self, id):
		return id in self._addresses

	def __getitem__(self, item):
		if isinstance(item, slice):
			keys = sorted(self._addresses.keys())[item]
			return [self[k] for k in keys]

		if item not in self._values:
			self._parse_row(item)

		return self._values[item]

	def __setitem__(self, item, value):
		if not isinstance(item, int):
			raise TypeError("DBFile indices must be integers, not %s" % (type(item)))

		if isinstance(value, DBRow):
			self._values[item] = value
			self._addresses[item] = -1
		else:
			# FIXME technically we should allow DBRow, but this is untested and will need resetting parent
			raise TypeError("Unsupported type for DBFile.__setitem__: %s" % (type(value)))

	def __delitem__(self, item):
		if item in self._values:
			del self._values[item]
		del self._addresses[item]

	def __iter__(self):
		return self._addresses.__iter__()

	def __len__(self):
		return len(self._addresses)

	def _add_row(self, id, address, reclen):
		if id in self._addresses: # Something's wrong here
			log.warning("Multiple instances of row %r found" % (id))
		self._addresses[id] = (address, reclen)

	def _parse_field(self, data, field, row=None):
		"""
		Parse a single field in stream.
		"""
		if field.dyn > self._rowDynamicFields:
			return None # The column doesn't exist in this row, we set it to None

		ret = None
		try:
			if isinstance(field, fields.StringField):
				ret = self._parse_string(data)

			elif isinstance(field, fields.DataField): # wowcache.wdb
				length = getattr(row, field.master)
				ret = data.read(length)

			elif isinstance(field, fields.DynamicMaster):
				ret, = unpack("<I", data.read(4))
				self._rowDynamicFields = ret

			else:
				ret, = unpack("<%s" % (field.char), data.read(field.size))
		except StructError:
			log.warning("Field %s could not be parsed properly" % (field))
			ret = None

		return ret

	def supportsSeeking(self):
		return hasattr(self.file, "seek")

	def append(self, row):
		"""
		Append a row at the end of the file.
		If the row does not have an id, one is automatically assigned.
		"""
		i = len(self) + 1 # FIXME this wont work properly in incomplete files
		if "_id" not in row:
			row["_id"] = i
		self[i] = row

	def clear(self):
		"""
		Delete every row in the file
		"""
		for k in self.keys(): # Use key, otherwise we get RuntimeError: dictionary changed size during iteration
			del self[k]

	def keys(self):
		return self._addresses.keys()

	def items(self):
		return [(k, self[k]) for k in self]

	def parse_row(self, data, reclen=0):
		"""
		Assign data to a DBRow instance
		"""
		return DBRow(self, data=data, reclen=reclen)

	def values(self):
		"""
		Return a list of the file's values
		"""
		return [self[id] for id in self]

	def setRow(self, key, **values):
		self.__setitem__(key, DBRow(self, columns=values))

	def update(self, other):
		"""
		Update file from iterable other
		"""
		for k in other:
			self[k] = other[k]

	def write(self, filename=""):
		"""
		Write the file data on disk. If filename is not given, use currently opened file.
		"""
		_filename = filename or self.file.name

		data = self.header.data() + self.data() + self.eof()

		f = open(_filename, "wb") # Don't open before calling data() as uncached rows would be empty
		f.write(data)
		f.close()
		log.info("Written %i bytes at %s" % (getsize(f.name), f.name))

		if not filename: # Reopen self.file, we modified it
			# XXX do we need to wipe self._values here?
			self.file.close()
			self.file = open(f.name, "rb")

class DBRow(list):
	"""
	A database row.
	Names of the variables of that class should not be used in field names of structures
	"""
	initialized = False

	def __init__(self, parent, data=None, columns=None, reclen=0):
		self._parent = parent
		self._values = {} # Columns values storage
		self.structure = parent.structure

		self.initialized = True # needed for __setattr__

		if columns:
			if type(columns) == list:
				self.extend(columns)

			elif type(columns) == dict:
				self._default()
				_cols = [k.name for k in self.structure]
				for k in columns:
					try:
						self[_cols.index(k)] = columns[k]
					except ValueError:
						log.warning("Column %r not found" % (k))

		elif data:
			dynfields = 0
			data = StringIO(data)
			for field in self.structure:
				_data = parent._parse_field(data, field, self)
				self.append(_data)

			if reclen:
				real_reclen = reclen + self._parent.row_header_size
				if data.tell() != real_reclen:
					log.warning("Reclen not respected for row %r. Expected %i, read %i. (%+i)" % (self.id, real_reclen, data.tell(), real_reclen-data.tell()))

	def __dir__(self):
		result = self.__dict__.keys()
		result.extend(self.structure.column_names)
		return result

	def __getattr__(self, attr):
		if attr in self.structure:
			return self._get_value(attr)

		if attr in self.structure._abstractions: # Union abstractions etc
			field, func = self.structure._abstractions[attr]
			return func(field, self)

		if "__" in attr:
			return self._get_deep_relation(attr)

		return super(DBRow, self).__getattribute__(attr)

	def __int__(self):
		return self.id

	def __setattr__(self, attr, value):
		# Do not preserve the value in DBRow! Use the save method to save.
		if self.initialized and attr in self.structure:
			self._set_value(attr, value)
		return super(DBRow, self).__setattr__(attr, value)

	def __setitem__(self, index, value):
		if not isinstance(index, int):
			raise TypeError("Expected int instance, got %s instead (%r)" % (type(index), index))
		list.__setitem__(self, index, value)
		col = self.structure[index]
		try:
			self._values[col.name] = col.to_python(value, row=self)
		except fields.UnresolvedRelation:
			self._values[col.name] = value

	def _get_reverse_relation(self, table, field):
		"""
		Return a list of rows matching the reverse relation
		"""
		if not hasattr(self._parent, "_reverse_relation_cache"):
			self._parent._reverse_relation_cache = {}
		cache = self._parent._reverse_relation_cache

		tfield = table + "__" + field
		if tfield not in cache:
			cache[tfield] = {}
			# First time lookup, let's build the cache
			table = self._parent.environment[table]
			for row in table:
				row = table[row]
				id = row._raw(field)
				if id not in cache[tfield]:
					cache[tfield][id] = []
				cache[tfield][id].append(row)

		return cache[tfield].get(self.id, None)

	def _get_deep_relation(self, rel):
		"""
		Parse a django-like multilevel relationship
		"""
		rels = rel.split("__")
		if "" in rels: # empty string
			raise ValueError("Invalid relation string")

		first = rels[0]
		if not hasattr(self, first):
			if first in self._parent.environment:
				remainder = rel[len(first + "__"):]
				return self._get_reverse_relation(first, remainder)
			raise ValueError("Invalid relation string")

		ret = self
		rels = rels[::-1]
		while rels:
			ret = getattr(ret, rels.pop())

		return ret

	def _set_value(self, name, value):
		index = self.structure.index(name)
		col = self.structure[index]
		try:
			self._values[name] = col.to_python(value, self)
		except fields.UnresolvedRelation:
			self._values[name] = value
		self[index] = value

	def _get_value(self, name):
		if name not in self._values:
			raw_value = self[self.structure.index(name)]

			try:
				self._set_value(name, raw_value)
			except fields.UnresolvedRelation as e:
				return fields.UnresolvedObjectRef(e.reference)
			except fields.RelationError:
				return None # Key doesn't exist, or equals 0

		return self._values[name]

	def _raw(self, name):
		"""
		Returns the raw value from field 'name'
		"""
		index = self.structure.index(name)
		return self[index]

	def _save(self):
		for name in self._values:
			index = self.structure.index(name)
			col = self.structure[index]
			self[index] = col.from_python(self._values[name])

	def _field(self, name):
		"""
		Returns the field 'name'
		"""
		index = self.structure.index(name)
		return self.structure[index]

	def _default(self):
		"""
		Change all fields to their default values
		"""
		del self[:]
		self._values = {}
		for col in self.structure:
			char = col.char
			if col.dyn:
				self.append(None)
			elif char == "s":
				self.append("")
			elif char == "f":
				self.append(0.0)
			else:
				self.append(0)

	def dict(self):
		"""
		Return a dict of the row as colname: value
		"""
		return dict(zip(self.structure.column_names, self))

	def update(self, other):
		for k in other:
			self[k] = other[k]

	@property
	def id(self):
		"Temporary hack to transition between _id and id"
		return self._id
<commit_msg>wdbc: Remove some overzealous error checking/fallback behaviour. I'll never not follow the Zen of Python again.<commit_after># -*- coding: utf-8 -*-

from cStringIO import StringIO
from os.path import getsize
from struct import pack, unpack, error as StructError
from .log import log
from .structures import fields

class DBHeader(object):
	"""
	Base DBFile header class
	"""
	def __repr__(self):
		return "%s(%s)" % (self.__class__.__name__, ", ".join("%s=%r" % (k, self.__dict__[k]) for k in self.__dict__))

class DBFile(object):
	"""
	Base class for WDB and DBC files
	"""

	def __init__(self, file, build, structure, environment):
		if isinstance(file, basestring):
			file = open(file, "r")
		self.file = file
		self.build = build
		self._addresses = {}
		self._values = {}
		self.environment = environment

		self._rowDynamicFields = 0 # Dynamic fields index, used when parsing a row

	def __repr__(self):
		return "%s(file=%r, build=%r)" % (self.__class__.__name__, self.file, self.build)

	def __contains__(self, id):
		return id in self._addresses

	def __getitem__(self, item):
		if isinstance(item, slice):
			keys = sorted(self._addresses.keys())[item]
			return [self[k] for k in keys]

		if item not in self._values:
			self._parse_row(item)

		return self._values[item]

	def __setitem__(self, item, value):
		if not isinstance(item, int):
			raise TypeError("DBFile indices must be integers, not %s" % (type(item)))

		if isinstance(value, DBRow):
			self._values[item] = value
			self._addresses[item] = -1
		else:
			# FIXME technically we should allow DBRow, but this is untested and will need resetting parent
			raise TypeError("Unsupported type for DBFile.__setitem__: %s" % (type(value)))

	def __delitem__(self, item):
		if item in self._values:
			del self._values[item]
		del self._addresses[item]

	def __iter__(self):
		return self._addresses.__iter__()

	def __len__(self):
		return len(self._addresses)

	def _add_row(self, id, address, reclen):
		if id in self._addresses: # Something's wrong here
			log.warning("Multiple instances of row %r found" % (id))
		self._addresses[id] = (address, reclen)

	def _parse_field(self, data, field, row=None):
		"""
		Parse a single field in stream.
		"""
		if field.dyn > self._rowDynamicFields:
			return None # The column doesn't exist in this row, we set it to None

		ret = None
		try:
			if isinstance(field, fields.StringField):
				ret = self._parse_string(data)

			elif isinstance(field, fields.DataField): # wowcache.wdb
				length = getattr(row, field.master)
				ret = data.read(length)

			elif isinstance(field, fields.DynamicMaster):
				ret, = unpack("<I", data.read(4))
				self._rowDynamicFields = ret

			else:
				ret, = unpack("<%s" % (field.char), data.read(field.size))
		except StructError:
			log.warning("Field %s could not be parsed properly" % (field))
			ret = None

		return ret

	def supportsSeeking(self):
		return hasattr(self.file, "seek")

	def append(self, row):
		"""
		Append a row at the end of the file.
		If the row does not have an id, one is automatically assigned.
		"""
		i = len(self) + 1 # FIXME this wont work properly in incomplete files
		if "_id" not in row:
			row["_id"] = i
		self[i] = row

	def clear(self):
		"""
		Delete every row in the file
		"""
		for k in self.keys(): # Use key, otherwise we get RuntimeError: dictionary changed size during iteration
			del self[k]

	def keys(self):
		return self._addresses.keys()

	def items(self):
		return [(k, self[k]) for k in self]

	def parse_row(self, data, reclen=0):
		"""
		Assign data to a DBRow instance
		"""
		return DBRow(self, data=data, reclen=reclen)

	def values(self):
		"""
		Return a list of the file's values
		"""
		return [self[id] for id in self]

	def setRow(self, key, **values):
		self.__setitem__(key, DBRow(self, columns=values))

	def update(self, other):
		"""
		Update file from iterable other
		"""
		for k in other:
			self[k] = other[k]

	def write(self, filename=""):
		"""
		Write the file data on disk. If filename is not given, use currently opened file.
		"""
		_filename = filename or self.file.name

		data = self.header.data() + self.data() + self.eof()

		f = open(_filename, "wb") # Don't open before calling data() as uncached rows would be empty
		f.write(data)
		f.close()
		log.info("Written %i bytes at %s" % (getsize(f.name), f.name))

		if not filename: # Reopen self.file, we modified it
			# XXX do we need to wipe self._values here?
			self.file.close()
			self.file = open(f.name, "rb")

class DBRow(list):
	"""
	A database row.
	Names of the variables of that class should not be used in field names of structures
	"""
	initialized = False

	def __init__(self, parent, data=None, columns=None, reclen=0):
		self._parent = parent
		self._values = {} # Columns values storage
		self.structure = parent.structure

		self.initialized = True # needed for __setattr__

		if columns:
			if type(columns) == list:
				self.extend(columns)

			elif type(columns) == dict:
				self._default()
				_cols = [k.name for k in self.structure]
				for k in columns:
					try:
						self[_cols.index(k)] = columns[k]
					except ValueError:
						log.warning("Column %r not found" % (k))

		elif data:
			dynfields = 0
			data = StringIO(data)
			for field in self.structure:
				_data = parent._parse_field(data, field, self)
				self.append(_data)

			if reclen:
				real_reclen = reclen + self._parent.row_header_size
				if data.tell() != real_reclen:
					log.warning("Reclen not respected for row %r. Expected %i, read %i. (%+i)" % (self.id, real_reclen, data.tell(), real_reclen-data.tell()))

	def __dir__(self):
		result = self.__dict__.keys()
		result.extend(self.structure.column_names)
		return result

	def __getattr__(self, attr):
		if attr in self.structure:
			return self._get_value(attr)

		if attr in self.structure._abstractions: # Union abstractions etc
			field, func = self.structure._abstractions[attr]
			return func(field, self)

		if "__" in attr:
			return self._get_deep_relation(attr)

		return super(DBRow, self).__getattribute__(attr)

	def __int__(self):
		return self.id

	def __setattr__(self, attr, value):
		# Do not preserve the value in DBRow! Use the save method to save.
		if self.initialized and attr in self.structure:
			self._set_value(attr, value)
		return super(DBRow, self).__setattr__(attr, value)

	def __setitem__(self, index, value):
		if not isinstance(index, int):
			raise TypeError("Expected int instance, got %s instead (%r)" % (type(index), index))
		list.__setitem__(self, index, value)
		col = self.structure[index]
		try:
			self._values[col.name] = col.to_python(value, row=self)
		except fields.UnresolvedRelation:
			self._values[col.name] = value

	def _get_reverse_relation(self, table, field):
		"""
		Return a list of rows matching the reverse relation
		"""
		if not hasattr(self._parent, "_reverse_relation_cache"):
			self._parent._reverse_relation_cache = {}
		cache = self._parent._reverse_relation_cache

		tfield = table + "__" + field
		if tfield not in cache:
			cache[tfield] = {}
			# First time lookup, let's build the cache
			table = self._parent.environment[table]
			for row in table:
				row = table[row]
				id = row._raw(field)
				if id not in cache[tfield]:
					cache[tfield][id] = []
				cache[tfield][id].append(row)

		return cache[tfield].get(self.id, None)

	def _get_deep_relation(self, rel):
		"""
		Parse a django-like multilevel relationship
		"""
		rels = rel.split("__")
		if "" in rels: # empty string
			raise ValueError("Invalid relation string")

		first = rels[0]
		if not hasattr(self, first):
			if first in self._parent.environment:
				remainder = rel[len(first + "__"):]
				return self._get_reverse_relation(first, remainder)
			raise ValueError("Invalid relation string")

		ret = self
		rels = rels[::-1]
		while rels:
			ret = getattr(ret, rels.pop())

		return ret

	def _set_value(self, name, value):
		index = self.structure.index(name)
		col = self.structure[index]
		self._values[name] = col.to_python(value, self)
		self[index] = value

	def _get_value(self, name):
		if name not in self._values:
			raw_value = self[self.structure.index(name)]

			self._set_value(name, raw_value)

		return self._values[name]

	def _raw(self, name):
		"""
		Returns the raw value from field 'name'
		"""
		index = self.structure.index(name)
		return self[index]

	def _save(self):
		for name in self._values:
			index = self.structure.index(name)
			col = self.structure[index]
			self[index] = col.from_python(self._values[name])

	def _field(self, name):
		"""
		Returns the field 'name'
		"""
		index = self.structure.index(name)
		return self.structure[index]

	def _default(self):
		"""
		Change all fields to their default values
		"""
		del self[:]
		self._values = {}
		for col in self.structure:
			char = col.char
			if col.dyn:
				self.append(None)
			elif char == "s":
				self.append("")
			elif char == "f":
				self.append(0.0)
			else:
				self.append(0)

	def dict(self):
		"""
		Return a dict of the row as colname: value
		"""
		return dict(zip(self.structure.column_names, self))

	def update(self, other):
		for k in other:
			self[k] = other[k]

	@property
	def id(self):
		"Temporary hack to transition between _id and id"
		return self._id
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Test suite for cross-python compatibility helpers."""

from __future__ import absolute_import, division, print_function
__metaclass__ = type

import pytest
import six

from cheroot._compat import ntob, ntou, bton

@pytest.mark.parametrize(
    'func,inp,out',
    [
        (ntob, 'bar', b'bar'),
        (ntou, 'bar', u'bar'),
        (bton, b'bar', 'bar'),
    ],
)
def test_compat_functions_positive(func, inp, out):
    """Check that compat functions work with correct input."""
    assert func(inp, encoding='utf-8') == out

@pytest.mark.parametrize(
    'func',
    [
        ntob,
        ntou,
    ],
)
def test_compat_functions_negative_nonnative(func):
    """Check that compat functions fail loudly for incorrect input."""
    non_native_test_str = b'bar' if six.PY3 else u'bar'
    with pytest.raises(TypeError):
        func(non_native_test_str, encoding='utf-8')

def test_ntou_escape():
    """Check that ntou supports escape-encoding under Python 2."""
    expected = u'hišřії'
    actual = ntou('hi\u0161\u0159\u0456\u0457', encoding='escape')
    assert actual == expected
<commit_msg>✅ Add tests for cheroot._compat.extract_bytes<commit_after># -*- coding: utf-8 -*-
"""Test suite for cross-python compatibility helpers."""

from __future__ import absolute_import, division, print_function
__metaclass__ = type

import pytest
import six

from cheroot._compat import extract_bytes, ntob, ntou, bton

@pytest.mark.parametrize(
    'func,inp,out',
    [
        (ntob, 'bar', b'bar'),
        (ntou, 'bar', u'bar'),
        (bton, b'bar', 'bar'),
    ],
)
def test_compat_functions_positive(func, inp, out):
    """Check that compat functions work with correct input."""
    assert func(inp, encoding='utf-8') == out

@pytest.mark.parametrize(
    'func',
    [
        ntob,
        ntou,
    ],
)
def test_compat_functions_negative_nonnative(func):
    """Check that compat functions fail loudly for incorrect input."""
    non_native_test_str = b'bar' if six.PY3 else u'bar'
    with pytest.raises(TypeError):
        func(non_native_test_str, encoding='utf-8')

def test_ntou_escape():
    """Check that ntou supports escape-encoding under Python 2."""
    expected = u'hišřії'
    actual = ntou('hi\u0161\u0159\u0456\u0457', encoding='escape')
    assert actual == expected

@pytest.mark.parametrize(
    'input_argument,expected_result',
    [
        (b'qwerty', b'qwerty'),
        (memoryview(b'asdfgh'), b'asdfgh'),
    ],
)
def test_extract_bytes(input_argument, expected_result):
    """Check that legitimate inputs produce bytes."""
    assert extract_bytes(input_argument) == expected_result

def test_extract_bytes_invalid():
    """Ensure that invalid input causes exception to be raised."""
    with pytest.raises(ValueError):
        extract_bytes(1)
<|endoftext|>


<commit_before>from cherrypy.test import test
test.prefer_parent_path()

import os
curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
has_space_filepath = os.path.join(curdir, 'static', 'has space.html')
import threading

import cherrypy

def setup_server():
    if not os.path.exists(has_space_filepath):
        file(has_space_filepath, 'wb').write('Hello, world\r\n')

    class Root:
        pass

    class Static:

        def index(self):
            return 'You want the Baron? You can have the Baron!'
        index.exposed = True

        def dynamic(self):
            return "This is a DYNAMIC page"
        dynamic.exposed = True

    root = Root()
    root.static = Static()

    conf = {
        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'static',
            'tools.staticdir.root': curdir,
        },
        '/style.css': {
            'tools.staticfile.on': True,
            'tools.staticfile.filename': os.path.join(curdir, 'style.css'),
        },
        '/docroot': {
            'tools.staticdir.on': True,
            'tools.staticdir.root': curdir,
            'tools.staticdir.dir': 'static',
            'tools.staticdir.index': 'index.html',
        },
        '/error': {
            'tools.staticdir.on': True,
            'request.show_tracebacks': True,
        },
        }

    cherrypy.tree.mount(root, config=conf)
    cherrypy.config.update({'environment': 'test_suite'})

def teardown_server():
    if os.path.exists(has_space_filepath):
        try:
            os.unlink(has_space_filepath)
        except:
            pass

from cherrypy.test import helper

class StaticTest(helper.CPWebCase):

    def testStatic(self):
        self.getPage("/static/index.html")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')

        # Using a staticdir.root value in a subdir...
        self.getPage("/docroot/index.html")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')

        # Check a filename with spaces in it
        self.getPage("/static/has%20space.html")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')

        self.getPage("/style.css")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/css')
        # Note: The body should be exactly 'Dummy stylesheet\n', but
        #   unfortunately some tools such as WinZip sometimes turn \n
        #   into \r\n on Windows when extracting the CherryPy tarball so
        #   we just check the content
        self.assertMatchesBody('^Dummy stylesheet')

        # Test that NotFound will then try dynamic handlers (see [878]).
        self.getPage("/static/dynamic")
        self.assertBody("This is a DYNAMIC page")

        # Check a directory via fall-through to dynamic handler.
        self.getPage("/static/")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('You want the Baron? You can have the Baron!')

        # Check a directory via "staticdir.index".
        self.getPage("/docroot/")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')
        # The same page should be returned even if redirected.
        self.getPage("/docroot")
        self.assertStatus((302, 303))
        self.assertHeader('Location', '%s/docroot/' % self.base())
        self.assertBody("This resource can be found at "
                        "%s/docroot/." % (self.base(), self.base()))

        # Check that we get an error if no .file or .dir
        self.getPage("/error/thing.html")
        self.assertErrorPage(500)
        self.assertInBody("TypeError: staticdir() takes at least 2 "
                          "arguments (0 given)")

        # Test up-level security
        self.getPage("/static/../../test/style.css")
        self.assertStatus((400, 403))

        # Test modified-since on a reasonably-large file
        self.getPage("/static/dirback.jpg")
        self.assertStatus("200 OK")
        lastmod = ""
        for k, v in self.headers:
            if k == 'Last-Modified':
                lastmod = v
        ims = ("If-Modified-Since", lastmod)
        self.getPage("/static/dirback.jpg", headers=[ims])
        self.assertStatus(304)
        self.assertNoHeader("Content-Type")
        self.assertNoHeader("Content-Length")
        self.assertNoHeader("Content-Disposition")
        self.assertBody("")

if __name__ == "__main__":
    setup_server()
    helper.testmain()
<commit_msg>Test for #755 (trailing_slash not working for static files in 3.1 beta).<commit_after>from cherrypy.test import test
test.prefer_parent_path()

import os
curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
has_space_filepath = os.path.join(curdir, 'static', 'has space.html')
import threading

import cherrypy

def setup_server():
    if not os.path.exists(has_space_filepath):
        file(has_space_filepath, 'wb').write('Hello, world\r\n')

    class Root:
        pass

    class Static:

        def index(self):
            return 'You want the Baron? You can have the Baron!'
        index.exposed = True

        def dynamic(self):
            return "This is a DYNAMIC page"
        dynamic.exposed = True

    cherrypy.config.update({'environment': 'test_suite'})

    root = Root()
    root.static = Static()

    rootconf = {
        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'static',
            'tools.staticdir.root': curdir,
        },
        '/style.css': {
            'tools.staticfile.on': True,
            'tools.staticfile.filename': os.path.join(curdir, 'style.css'),
        },
        '/docroot': {
            'tools.staticdir.on': True,
            'tools.staticdir.root': curdir,
            'tools.staticdir.dir': 'static',
            'tools.staticdir.index': 'index.html',
        },
        '/error': {
            'tools.staticdir.on': True,
            'request.show_tracebacks': True,
        },
        }
    rootApp = cherrypy.Application(root)
    rootApp.merge(rootconf)

    test_app_conf = {
        '/test': {
            'tools.staticdir.index': 'index.html',
            'tools.staticdir.on': True,
            'tools.staticdir.root': curdir,
            'tools.staticdir.dir': 'static',
            },
        }
    testApp = cherrypy.Application(Static())
    testApp.merge(test_app_conf)

    vhost = cherrypy._cpwsgi.VirtualHost(rootApp, {'virt.net': testApp})
    cherrypy.tree.graft(vhost)

def teardown_server():
    if os.path.exists(has_space_filepath):
        try:
            os.unlink(has_space_filepath)
        except:
            pass

from cherrypy.test import helper

class StaticTest(helper.CPWebCase):

    def testStatic(self):
        self.getPage("/static/index.html")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')

        # Using a staticdir.root value in a subdir...
        self.getPage("/docroot/index.html")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')

        # Check a filename with spaces in it
        self.getPage("/static/has%20space.html")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')

        self.getPage("/style.css")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/css')
        # Note: The body should be exactly 'Dummy stylesheet\n', but
        #   unfortunately some tools such as WinZip sometimes turn \n
        #   into \r\n on Windows when extracting the CherryPy tarball so
        #   we just check the content
        self.assertMatchesBody('^Dummy stylesheet')

    def test_fallthrough(self):
        # Test that NotFound will then try dynamic handlers (see [878]).
        self.getPage("/static/dynamic")
        self.assertBody("This is a DYNAMIC page")

        # Check a directory via fall-through to dynamic handler.
        self.getPage("/static/")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('You want the Baron? You can have the Baron!')

    def test_index(self):
        # Check a directory via "staticdir.index".
        self.getPage("/docroot/")
        self.assertStatus('200 OK')
        self.assertHeader('Content-Type', 'text/html')
        self.assertBody('Hello, world\r\n')
        # The same page should be returned even if redirected.
        self.getPage("/docroot")
        self.assertStatus((302, 303))
        self.assertHeader('Location', '%s/docroot/' % self.base())
        self.assertBody("This resource can be found at "
                        "%s/docroot/." % (self.base(), self.base()))

    def test_config_errors(self):
        # Check that we get an error if no .file or .dir
        self.getPage("/error/thing.html")
        self.assertErrorPage(500)
        self.assertInBody("TypeError: staticdir() takes at least 2 "
                          "arguments (0 given)")

    def test_security(self):
        # Test up-level security
        self.getPage("/static/../../test/style.css")
        self.assertStatus((400, 403))

    def test_modif(self):
        # Test modified-since on a reasonably-large file
        self.getPage("/static/dirback.jpg")
        self.assertStatus("200 OK")
        lastmod = ""
        for k, v in self.headers:
            if k == 'Last-Modified':
                lastmod = v
        ims = ("If-Modified-Since", lastmod)
        self.getPage("/static/dirback.jpg", headers=[ims])
        self.assertStatus(304)
        self.assertNoHeader("Content-Type")
        self.assertNoHeader("Content-Length")
        self.assertNoHeader("Content-Disposition")
        self.assertBody("")

    def test_755_vhost(self):
        self.getPage("/test/", [('Host', 'virt.net')])
        self.assertStatus(200)
        self.getPage("/test", [('Host', 'virt.net')])
        self.assertStatus((302, 303))
        self.assertHeader('Location', 'http://virt.net/test/')

if __name__ == "__main__":
    setup_server()
    helper.testmain()
<|endoftext|>


<commit_before>from django.template import RequestContext
from django.shortcuts import render_to_response, render, get_object_or_404, redirect
from api.models import Project
from .form import ProjectForm
from django.contrib.auth import logout as auth_logout
from api.models import Project
from api.models import UserProfile

def index(request):
	return render_to_response('pages/index.html',{})

def projects(request):
	project_list = Project.objects.all()
	return render(request, 'pages/projects.html',{'project_list': project_list})

def about(request):
	return render(request, 'pages/about.html',{})

def members(request):
	members_list = UserProfile.objects.filter(active=True).order_by('fullname')
	return render(request, 'pages/members.html',{'members_list': members_list})

def events(request):
	return render(request, 'pages/events.html',{})

def gallery(request):
	return render(request, 'pages/gallery.html',{})

def login(request):
	return render(request, 'pages/login.html',{})

def links(request):
	return render(request, 'pages/links.html',{})

def edituser(request):
	return render(request, 'pages/edituser.html',{})

def privacy(request):
	return render(request, 'pages/privacy.html',{})

def project_detail(request,project_id):
	project = get_object_or_404(Project, project_id=project_id)
	return render(request, 'pages/project_detail.html', {'project': project})

def project_new(request):
	if request.method == "POST":
		projectform = ProjectForm (request.POST)
		if projectform.is_valid():
			project = projectform.save()
			return redirect('project-detail', project_id=project.project_id)
	else:
		projectform = ProjectForm()

	return render(request, 'pages/project_edit.html', {'projectform': projectform})

def project_edit(request, project_id):
	project = get_object_or_404(Project, project_id=project_id)
	if request.method == "POST":
		projectform = ProjectForm(request.POST, instance=project)
		if projectform.is_valid():
			project = projectform.save()
			return redirect('project-detail', project_id=project.project_id)
	else:
		projectform = ProjectForm(instance=project)

	return render(request, 'pages/project_edit.html', {'projectform': projectform})

def logout(request):
    auth_logout(request)
    return redirect('/')
<commit_msg>Getting rid of get_object_or_404<commit_after>from django.template import RequestContext
from django.shortcuts import render_to_response, render, get_object_or_404, redirect
from api.models import Project
from .form import ProjectForm
from django.contrib.auth import logout as auth_logout
from api.models import Project
from api.models import UserProfile

def index(request):
	return render_to_response('pages/index.html',{})

def projects(request):
	project_list = Project.objects.all()
	return render(request, 'pages/projects.html',{'project_list': project_list})

def about(request):
	return render(request, 'pages/about.html',{})

def members(request):
	members_list = UserProfile.objects.filter(active=True).order_by('fullname')
	return render(request, 'pages/members.html',{'members_list': members_list})

def events(request):
	return render(request, 'pages/events.html',{})

def gallery(request):
	return render(request, 'pages/gallery.html',{})

def login(request):
	return render(request, 'pages/login.html',{})

def links(request):
	return render(request, 'pages/links.html',{})

def edituser(request):
	return render(request, 'pages/edituser.html',{})

def privacy(request):
	return render(request, 'pages/privacy.html',{})

def project_detail(request, project_id):
	project = Project.objects.get(project_id=project_id)
	return render(request, 'pages/project_detail.html', {'project': project})

def project_new(request):
	if request.method == "POST":
		projectform = ProjectForm (request.POST)
		if projectform.is_valid():
			project = projectform.save()
			return redirect('project-detail', project_id=project.project_id)
	else:
		projectform = ProjectForm()

	return render(request, 'pages/project_edit.html', {'projectform': projectform})

def project_edit(request, project_id):
	project = Project.objects.get(project_id=project_id)
	if request.method == "POST":
		projectform = ProjectForm(request.POST, instance=project)
		if projectform.is_valid():
			project = projectform.save()
			return redirect('project-detail', project_id=project.project_id)
	else:
		projectform = ProjectForm(instance=project)

	return render(request, 'pages/project_edit.html', {'projectform': projectform})

def logout(request):
    auth_logout(request)
    return redirect('/')
<|endoftext|>


<commit_before>import contextlib
import json
import os
import re
import shutil
import subprocess
import zipfile
import settings
from projectinfo import Resource

class NPMInstallError(Exception):
    pass

@contextlib.contextmanager
def temporary_package_json(root_dir, dependencies):
    package_path = os.path.join(root_dir, 'package.json')
    try:
        with open(package_path, 'w') as f:
            json.dump({
                "name": "cloudpebble-ycmd-proxy",
                "version": "1.0.0",
                "description": "A temporary package.json",
                "repository": None,
                "license": "",
                "dependencies": dependencies
            }, f)
        yield
    finally:
        os.unlink(package_path)

def validate_dependencies(dependencies):
    """ Check that none of the version strings in a dictionary of dependencies reference local paths. """
    # CloudPebble performs identical checks for this, so hopefully it should never actually get triggered.
    for version in dependencies.values():
        if re.match(r'^file:|(\.*|~)/', version):
            raise ValueError("Dependencies are not allowed to reference paths")

def install_dependencies(dependencies, root_dir):
    """ Install some npm dependencies into a directory and then extract the headers
    :param dependencies: a dictionary of name->version
    :param root_dir: the directory to install into
    """
    validate_dependencies(dependencies)
    # Make a minimal package.json file
    with temporary_package_json(root_dir, dependencies):
        try:
            # Install all the dependencies
            # TODO: Should NPM itself have resource limits?
            subprocess.check_output([settings.NPM_BINARY, "prune"], stderr=subprocess.STDOUT, cwd=root_dir)
            if dependencies:
                subprocess.check_output([settings.NPM_BINARY, "install", "--ignore-scripts"], stderr=subprocess.STDOUT, cwd=root_dir)
                subprocess.check_output([settings.NPM_BINARY, "dedupe"], stderr=subprocess.STDOUT, cwd=root_dir)
        except subprocess.CalledProcessError:
            # Setting the error message to e.output here would let the user see their lovely NPM error output.
            raise NPMInstallError("One or more of your dependencies cannot be installed. Please check that the names and versions for all of your dependencies are valid.")

def search_node_modules(base, filename, folder='node_modules'):
    search_root = os.path.join(base, folder)
    for dirname, _, filenames in os.walk(search_root):
        if filename in filenames:
            path = os.path.join(dirname, filename)
            libname = os.path.dirname(os.path.relpath(path, search_root))
            yield libname, path

def get_package_metadata(root_dir):
    """ Given a directory with a node_modules directory full of pebble modules, get all of their messageKeys and resource type/name information
    :param root_dir:
    :return: (array of resources, array of (type, name) tuples
    """
    resources = []
    messagekeys = []
    versions = {}
    for libname, package_path in search_node_modules(root_dir, 'package.json'):
        with open(package_path, 'r') as f:
            data = json.load(f)
            if 'pebble' not in data:
                continue
            messagekeys.extend(data.get('pebble', {}).get('messageKeys', []))
            resources.extend(Resource(r['type'], r['name']) for r in data.get('pebble', {}).get('resources', {}).get('media', []))
            versions[libname] = data.get('version', None)
    return resources, messagekeys, versions

def extract_library_headers(root_dir):
    """ Given a directory with a node_modules directory full of Pebble modules, extract all of their header files into ./librares/include/<module_name>
    :param root_dir: base directory
    """
    # Make the libraries directory if it does not exist
    libs_path = os.path.join(root_dir, 'libraries')
    if os.path.isdir(libs_path):
        shutil.rmtree(libs_path)
    os.mkdir(libs_path)
    # Look for C modules with dist.zip files
    for libname, zip_path in search_node_modules(root_dir, 'dist.zip'):
        try:
            # Construct the expected path to the library's headers based on its name
            includes_path = os.path.join('include', libname)
            with zipfile.ZipFile(zip_path) as z:
                # Extract any header files which are inside 'include/<module_name>'
                for zip_entry in z.infolist():
                    if zip_entry.filename.startswith(includes_path) and zip_entry.filename.endswith('.h'):
                        extracted_to = z.extract(zip_entry, libs_path)
                        if not extracted_to.endswith('auto.h'):
                            yield os.path.relpath(extracted_to, os.path.join(libs_path, 'include'))

        except Exception:
            raise NPMInstallError("One or more of your dependencies is not a valid pebble library.")

def _startswith_any(str, options):
    for x in options:
        if str.startswith(x):
            return x
    return None

def make_library_info(dependencies, versions, headers):
    libs = {}
    for name, version in versions.iteritems():
        if version and name in dependencies:
            libs[name] = {
                'headers': [],
                'version': version
            }

    for header in headers:
        dep = _startswith_any(header, dependencies)
        if dep and dep in libs:
            libs[dep]['headers'].append(header)
    return libs

def setup_dependencies(dependencies, root_dir):
    install_dependencies(dependencies, root_dir)
    lib_headers = extract_library_headers(root_dir)
    lib_resources, lib_messagekeys, lib_versions = get_package_metadata(root_dir)
    lib_info = make_library_info(dependencies, lib_versions, lib_headers)
    return lib_info, lib_messagekeys, lib_resources

def try_setup_dependencies(dependencies, root_dir):
    try:
        return setup_dependencies(dependencies, root_dir), None
    except NPMInstallError as e:
        return ({}, [], []), str(e)
<commit_msg>Removed _startswith_any<commit_after>import contextlib
import json
import os
import re
import shutil
import subprocess
import zipfile
import settings
from projectinfo import Resource

class NPMInstallError(Exception):
    pass

@contextlib.contextmanager
def temporary_package_json(root_dir, dependencies):
    package_path = os.path.join(root_dir, 'package.json')
    try:
        with open(package_path, 'w') as f:
            json.dump({
                "name": "cloudpebble-ycmd-proxy",
                "version": "1.0.0",
                "description": "A temporary package.json",
                "repository": None,
                "license": "",
                "dependencies": dependencies
            }, f)
        yield
    finally:
        os.unlink(package_path)

def validate_dependencies(dependencies):
    """ Check that none of the version strings in a dictionary of dependencies reference local paths. """
    # CloudPebble performs identical checks for this, so hopefully it should never actually get triggered.
    for version in dependencies.values():
        if re.match(r'^file:|(\.*|~)/', version):
            raise ValueError("Dependencies are not allowed to reference paths")

def install_dependencies(dependencies, root_dir):
    """ Install some npm dependencies into a directory and then extract the headers
    :param dependencies: a dictionary of name->version
    :param root_dir: the directory to install into
    """
    validate_dependencies(dependencies)
    # Make a minimal package.json file
    with temporary_package_json(root_dir, dependencies):
        try:
            # Install all the dependencies
            # TODO: Should NPM itself have resource limits?
            subprocess.check_output([settings.NPM_BINARY, "prune"], stderr=subprocess.STDOUT, cwd=root_dir)
            if dependencies:
                subprocess.check_output([settings.NPM_BINARY, "install", "--ignore-scripts"], stderr=subprocess.STDOUT, cwd=root_dir)
                subprocess.check_output([settings.NPM_BINARY, "dedupe"], stderr=subprocess.STDOUT, cwd=root_dir)
        except subprocess.CalledProcessError:
            # Setting the error message to e.output here would let the user see their lovely NPM error output.
            raise NPMInstallError("One or more of your dependencies cannot be installed. Please check that the names and versions for all of your dependencies are valid.")

def search_node_modules(base, filename, folder='node_modules'):
    search_root = os.path.join(base, folder)
    for dirname, _, filenames in os.walk(search_root):
        if filename in filenames:
            path = os.path.join(dirname, filename)
            libname = os.path.dirname(os.path.relpath(path, search_root))
            yield libname, path

def get_package_metadata(root_dir):
    """ Given a directory with a node_modules directory full of pebble modules, get all of their messageKeys and resource type/name information
    :param root_dir:
    :return: (array of resources, array of (type, name) tuples
    """
    resources = []
    messagekeys = []
    versions = {}
    for libname, package_path in search_node_modules(root_dir, 'package.json'):
        with open(package_path, 'r') as f:
            data = json.load(f)
            if 'pebble' not in data:
                continue
            messagekeys.extend(data.get('pebble', {}).get('messageKeys', []))
            resources.extend(Resource(r['type'], r['name']) for r in data.get('pebble', {}).get('resources', {}).get('media', []))
            versions[libname] = data.get('version', None)
    return resources, messagekeys, versions

def extract_library_headers(root_dir):
    """ Given a directory with a node_modules directory full of Pebble modules, extract all of their header files into ./librares/include/<module_name>
    :param root_dir: base directory
    """
    # Make the libraries directory if it does not exist
    libs_path = os.path.join(root_dir, 'libraries')
    if os.path.isdir(libs_path):
        shutil.rmtree(libs_path)
    os.mkdir(libs_path)
    # Look for C modules with dist.zip files
    for libname, zip_path in search_node_modules(root_dir, 'dist.zip'):
        try:
            # Construct the expected path to the library's headers based on its name
            includes_path = os.path.join('include', libname)
            with zipfile.ZipFile(zip_path) as z:
                # Extract any header files which are inside 'include/<module_name>'
                for zip_entry in z.infolist():
                    if zip_entry.filename.startswith(includes_path) and zip_entry.filename.endswith('.h'):
                        extracted_to = z.extract(zip_entry, libs_path)
                        if not extracted_to.endswith('auto.h'):
                            yield os.path.relpath(extracted_to, os.path.join(libs_path, 'include'))

        except Exception:
            raise NPMInstallError("One or more of your dependencies is not a valid pebble library.")

def make_library_info(dependencies, versions, headers):
    libs = {}
    for name, version in versions.iteritems():
        if version and name in dependencies:
            libs[name] = {
                'headers': [],
                'version': version
            }

    for header in headers:
        dep = header.startswith(tuple(dependencies))
        if dep and dep in libs:
            libs[dep]['headers'].append(header)
    return libs

def setup_dependencies(dependencies, root_dir):
    install_dependencies(dependencies, root_dir)
    lib_headers = extract_library_headers(root_dir)
    lib_resources, lib_messagekeys, lib_versions = get_package_metadata(root_dir)
    lib_info = make_library_info(dependencies, lib_versions, lib_headers)
    return lib_info, lib_messagekeys, lib_resources

def try_setup_dependencies(dependencies, root_dir):
    try:
        return setup_dependencies(dependencies, root_dir), None
    except NPMInstallError as e:
        return ({}, [], []), str(e)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import inspect
import itertools
import random
import warnings

import numpy as np

from .gd import GradientDescent
from .bfgs import Lbfgs
from .cg import NonlinearConjugateGradient
from .rprop import Rprop
from .rmsprop import RmsProp
from .adadelta import Adadelta
from .adam import Adam

try:
    from sklearn.grid_search import ParameterSampler
except ImportError:
    pass

def is_garray(cand):
    return hasattr(cand, 'as_numpy_array')

def is_array(cand):
    return is_garray(cand) or isinstance(cand, np.ndarray)

def clear_info(info):
    """Clean up contents of info dictionary for better use.

    Keys to be removed are ``args``, ``kwargs`` and any non-scalar numpy or
    gnumpy arrays. Numpy scalars are converted to floats.

    Examples
    --------

    >>> import numpy as np
    >>> info = {'args': None, 'foo': np.zeros(3), 'bar': np.array(1),
    ...         'loss': 1.}
    >>> cleared = clear_info(info)
    >>> cleared == {'bar': 1.0, 'loss': 1.0}
    True
    """
    items = info.iteritems()
    items = ((k, float(v.reshape((1,))[0]) if is_array(v) and v.size == 1 else v)
             for k, v in items)
    items = ((k, v) for k, v in items if not is_array(v))
    items = ((k, v) for k, v in items if k not in ('args', 'kwargs'))

    return dict(items)

def coroutine(f):
    """Turn a generator function into a coroutine by calling .next() once."""
    def started(*args, **kwargs):
        cr = f(*args, **kwargs)
        next(cr)
        return cr
    return started

def aslist(item):
    if not isinstance(item, (list, tuple)):
        item = [item]
    return item

def mini_slices(n_samples, batch_size):
    """Yield slices of size `batch_size` that work with a container of length
    `n_samples`."""
    n_batches, rest = divmod(n_samples, batch_size)
    if rest != 0:
        n_batches += 1

    return [slice(i * batch_size, (i + 1) * batch_size) for i in range(n_batches)]

def draw_mini_slices(n_samples, batch_size, with_replacement=False):
    slices = mini_slices(n_samples, batch_size)
    idxs = list(range(len(slices)))

    if with_replacement:
        yield random.choice(slices)
    else:
        while True:
            random.shuffle(idxs)
            for i in idxs:
                yield slices[i]

def draw_mini_indices(n_samples, batch_size):
    assert n_samples > batch_size
    idxs = list(range(n_samples))
    random.shuffle(idxs)
    pos = 0

    while True:
        while pos + batch_size <= n_samples:
             yield idxs[pos:pos + batch_size]
             pos += batch_size

        batch = idxs[pos:]
        needed = batch_size - len(batch)
        random.shuffle(idxs)
        batch += idxs[0:needed]
        yield batch
        pos = needed

def optimizer(identifier, wrt, *args, **kwargs):
    """Return an optimizer with the desired configuration.

    This is a convenience function if one wants to try out different optimizers
    but wants to change as little code as possible.

    Additional arguments and keyword arguments will be passed to the constructor
    of the class. If the found class does not take the arguments supplied, this
    will `not` throw an error, but pass silently.

    :param identifier: String identifying the optimizer to use. Can be either
        ``asgd``, ``gd``, ``lbfgs``, ``ncg``, ``rprop``, ``adadelta`` or
        ``smd``.
    :param wrt: Numpy array pointing to the data to optimize.
    """
    klass_map = {
        'gd': GradientDescent,
        'lbfgs': Lbfgs,
        'ncg': NonlinearConjugateGradient,
        'rprop': Rprop,
        'rmsprop': RmsProp,
        'adadelta': Adadelta,
        'adam': Adam,
    }
    # Find out which arguments to pass on.
    klass = klass_map[identifier]
    argspec = inspect.getargspec(klass.__init__)
    if argspec.keywords is None:
        # Issue a warning for each of the arguments that have been passed
        # to this optimizer but were not used.
        expected_keys = set(argspec.args)
        given_keys = set(kwargs.keys())
        unused_keys = given_keys - expected_keys
        for i in unused_keys:
            warnings.warn('Argument named %s is not expected by %s'
                          % (i, klass))

        # We need to filter stuff out.
        used_keys = expected_keys & given_keys
        kwargs = dict((k, kwargs[k]) for k in used_keys)
    try:
        opt = klass(wrt, *args, **kwargs)
    except TypeError:
        raise TypeError('required arguments for %s: %s' % (klass, argspec.args))

    return opt

def shaped_from_flat(flat, shapes):
    """Given a one dimensional array ``flat``, return a list of views of shapes
    ``shapes`` on that array.

    Each view will point to a distinct memory region, consecutively allocated
    in flat.

    Parameters
    ----------

    flat : array_like
        Array of one dimension.

    shapes : list of tuples of ints
        Each entry of this list specifies the shape of the corresponding view
        into ``flat``.

    Returns
    -------

    views : list of arrays
        Each entry has the shape given in ``shapes`` and points as a view into
        ``flat``.
    """
    shapes = [(i,) if isinstance(i, int) else i for i in shapes]
    sizes = [np.prod(i) for i in shapes]

    n_used = 0
    views = []
    for size, shape in zip(sizes, shapes):
        this = flat[n_used:n_used + size]
        n_used += size
        this.shape = shape
        views.append(this)

    return views

def empty_with_views(shapes, empty_func=np.empty):
    """Create an array and views shaped according to ``shapes``.

    The ``shapes`` parameter is a list of tuples of ints.  Each tuple
    represents a desired shape for an array which will be allocated in a bigger
    memory region. This memory region will be represented by an array as well.

    For example, the shape speciciation ``[2, (3, 2)]`` will create an array
    ``flat`` of size 8. The first view will have a size of ``(2,)`` and point
    to the first two entries, i.e. ``flat`[:2]`, while the second array will
    have a shape of ``(3, 2)`` and point to the elements ``flat[2:8]``.

    Parameters
    ----------

    spec : list of tuples of ints
        Specification of the desired shapes.

    empty_func : callable
        function that returns a memory region given an integer of the desired
        size. (Examples include ``numpy.empty``, which is the default,
        ``gnumpy.empty`` and ``theano.tensor.empty``.

    Returns
    -------

    flat : array_like (depending on ``empty_func``)
        Memory region containing all the views.

    views : list of array_like
        Variable number of results. Each contains a view into the array
        ``flat``.

    Examples
    --------

    >>> from climin.util import empty_with_views
    >>> flat, (w, b) = empty_with_views([(3, 2), 2])
    >>> w[...] = 1
    >>> b[...] = 2
    >>> flat
    array([ 1.,  1.,  1.,  1.,  1.,  1.,  2.,  2.])
    >>> flat[0] = 3
    >>> w
    array([[ 3.,  1.],
           [ 1.,  1.],
           [ 1.,  1.]])

    """
    shapes = [(i,) if isinstance(i, int) else i for i in shapes]
    sizes = [np.prod(i) for i in shapes]
    n_pars = sum(sizes)
    flat = empty_func(n_pars)

    views = shaped_from_flat(flat, shapes)

    return flat, views

def arbitrary_slice(arr, start, stop=None, axis=0):
    """Return a slice from `start` to `stop` in dimension `axis` of array `arr`.

    Parameters
    ----------

    arr : array_like
        Can be numpy ndarray, hdf5 dataset, or list.
        If ``arr`` is a list, ``axis`` must be 0.

    start : int
        Index at which to start slicing.

    stop : int, optional [default: None]
        Index at which to stop slicing.
        If not specified, the axis is sliced until its end.

    axis : int, optional [default: 0]
        Axis along which should be sliced

    Returns
    -------

    slice : array_like
        The respective slice of ``arr``
    """

    if type(arr) is list:
        if axis == 0:
            return arr[start:stop]
        else:
            raise ValueError("Cannot slice a list in non-zero axis {}"
                             .format(axis))

    n_axes = len(arr.shape)

    if axis >= n_axes:
        raise IndexError('Argument `axis` with value {} out of range. '
                         'Must be smaller than rank {} of `arr`.'
                         .format(axis, n_axes))

    this_slice = [slice(None) for _ in range(n_axes)]
    this_slice[axis] = slice(start, stop)

    return arr[tuple(this_slice)]

def minibatches(arr, batch_size, d=0):
    """Return a list of views of the given arr.

    Each view represents a mini bach of the data.

    Parameters
    ----------

    arr : array_like
        Array to obtain batches from. Needs to be slicable. If ``d > 0``, needs
        to have a ``.shape`` attribute from which the number of samples can
        be obtained.

    batch_size : int
        Size of a batch. Last batch might be smaller if ``batch_size`` is not a
        divisor of ``arr``.

    d : int, optional, default: 0
        Dimension along which the data samples are separated and thus slicing
        should be done.

    Returns
    -------

    mini_batches : list
        Each item of the list is a view of ``arr``. Views are ordered.
    """
    # This alternative is to make this work with lists in the case of d == 0.
    if d == 0:
        n_batches, rest = divmod(len(arr), batch_size)
    else:
        n_batches, rest = divmod(arr.shape[d], batch_size)
    if rest:
        n_batches += 1

    slices = (slice(i * batch_size, (i + 1) * batch_size)
              for i in range(n_batches))
    if d == 0:
        res = [arr[i] for i in slices]
    elif d == 1:
        res = [arr[:, i] for i in slices]
    elif d == 2:
        res = [arr[:, :, i] for i in slices]

    return res

def iter_minibatches(lst, batch_size, dims, n_cycles=False, random_state=None):
    """Return an iterator that successively yields tuples containing aligned
    minibatches of size `batch_size` from slicable objects given in `lst`, in
    random order without replacement.

    Because different containers might require slicing over different
    dimensions, the dimension of each container has to be givens as a list
    `dims`.

    Parameters
    ----------

    lst : list of array_like
        Each item of the list will be sliced into mini batches in alignment
        with the others.

    batch_size : int
        Size of each batch. Last batch might be smaller.

    dims : list
        Aligned with ``lst``, gives the dimension along which the data samples
        are separated.

    n_cycles : int or False, optional [default: False]
        Number of cycles after which to stop the iterator. If ``False``, will
        yield forever.

    random_state : a numpy.random.RandomState object, optional [default : None]
        Random number generator that will act as a seed for the minibatch order

    Returns
    -------

    batches : iterator
        Infinite iterator of mini batches in random order (without
        replacement).
    """

    if any([d > 2 for d in dims]):
        raise ValueError("cannot slice along a dimension larger than 2")

    if any([len(arr.shape) <= d for (arr, d) in zip(lst, dims)]):
        raise ValueError("cannot slice along that dimension (shape, dim): {}"
                         .format([(a.shape, d) for (a, d) in zip(lst, dims)]))

    if any([arr.shape[d] != lst[0].shape[dims[0]]
            for (arr, d) in zip(lst, dims)]):
        raise ValueError("containers to be batched have different lengths: {}"
                         .format([a.shape[d] for (a, d) in zip(lst, dims)]))

    # This alternative is to make this work with lists in the case of d == 0.
    if dims[0] == 0:
        n_batches, rest = divmod(len(lst[0]), batch_size)
    else:
        n_batches, rest = divmod(lst[0].shape[dims[0]], batch_size)
    if rest:
        n_batches += 1

    counter = itertools.count()
    if random_state is not None:
        random.seed(random_state.normal())
    while True:
        indices = range(n_batches)
        while True:
            random.shuffle(indices)
            for i in indices:
                start = i * batch_size
                end = (i + 1) * batch_size
                batch = []
                for (arr, d) in zip(lst, dims):
                    if d == 0:
                        batch.append(arr[start:end])
                    elif d == 1:
                        batch.append(arr[:, start:end])
                    elif d == 2:
                        batch.append(arr[:, :, start:end])
                yield tuple(batch)

            count = next(counter)
            if n_cycles and count >= n_cycles:
                raise StopIteration()

class OptimizerDistribution(object):
    """OptimizerDistribution class.

    Can be used for specifying optimizers in scikit-learn's randomized parameter
    search.

    Attributes
    ----------

    options : dict
        Maps an optimizer key to a grid to sample from.
    """

    def __init__(self, **options):
        """Create an OptimizerDistribution object.

        Parameters
        ----------

        options : dict
            Maps an optimizer key to a grid to sample from.
        """
        self.options = options

    def rvs(self):
        opt = random.choice(list(self.options.keys()))
        grid = self.options[opt]
        sample = list(ParameterSampler(grid, n_iter=1))[0]
        return opt, sample
<commit_msg>major refactoring of iter_minibatches<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import inspect
import itertools
import random
import warnings

import numpy as np

from .gd import GradientDescent
from .bfgs import Lbfgs
from .cg import NonlinearConjugateGradient
from .rprop import Rprop
from .rmsprop import RmsProp
from .adadelta import Adadelta
from .adam import Adam

try:
    from sklearn.grid_search import ParameterSampler
except ImportError:
    pass

def is_garray(cand):
    return hasattr(cand, 'as_numpy_array')

def is_array(cand):
    return is_garray(cand) or isinstance(cand, np.ndarray)

def clear_info(info):
    """Clean up contents of info dictionary for better use.

    Keys to be removed are ``args``, ``kwargs`` and any non-scalar numpy or
    gnumpy arrays. Numpy scalars are converted to floats.

    Examples
    --------

    >>> import numpy as np
    >>> info = {'args': None, 'foo': np.zeros(3), 'bar': np.array(1),
    ...         'loss': 1.}
    >>> cleared = clear_info(info)
    >>> cleared == {'bar': 1.0, 'loss': 1.0}
    True
    """
    items = info.iteritems()
    items = ((k, float(v.reshape((1,))[0]) if is_array(v) and v.size == 1 else v)
             for k, v in items)
    items = ((k, v) for k, v in items if not is_array(v))
    items = ((k, v) for k, v in items if k not in ('args', 'kwargs'))

    return dict(items)

def coroutine(f):
    """Turn a generator function into a coroutine by calling .next() once."""
    def started(*args, **kwargs):
        cr = f(*args, **kwargs)
        next(cr)
        return cr
    return started

def aslist(item):
    if not isinstance(item, (list, tuple)):
        item = [item]
    return item

def mini_slices(n_samples, batch_size):
    """Yield slices of size `batch_size` that work with a container of length
    `n_samples`."""
    n_batches, rest = divmod(n_samples, batch_size)
    if rest != 0:
        n_batches += 1

    return [slice(i * batch_size, (i + 1) * batch_size) for i in range(n_batches)]

def draw_mini_slices(n_samples, batch_size, with_replacement=False):
    slices = mini_slices(n_samples, batch_size)
    idxs = list(range(len(slices)))

    if with_replacement:
        yield random.choice(slices)
    else:
        while True:
            random.shuffle(idxs)
            for i in idxs:
                yield slices[i]

def draw_mini_indices(n_samples, batch_size):
    assert n_samples > batch_size
    idxs = list(range(n_samples))
    random.shuffle(idxs)
    pos = 0

    while True:
        while pos + batch_size <= n_samples:
             yield idxs[pos:pos + batch_size]
             pos += batch_size

        batch = idxs[pos:]
        needed = batch_size - len(batch)
        random.shuffle(idxs)
        batch += idxs[0:needed]
        yield batch
        pos = needed

def optimizer(identifier, wrt, *args, **kwargs):
    """Return an optimizer with the desired configuration.

    This is a convenience function if one wants to try out different optimizers
    but wants to change as little code as possible.

    Additional arguments and keyword arguments will be passed to the constructor
    of the class. If the found class does not take the arguments supplied, this
    will `not` throw an error, but pass silently.

    :param identifier: String identifying the optimizer to use. Can be either
        ``asgd``, ``gd``, ``lbfgs``, ``ncg``, ``rprop``, ``adadelta`` or
        ``smd``.
    :param wrt: Numpy array pointing to the data to optimize.
    """
    klass_map = {
        'gd': GradientDescent,
        'lbfgs': Lbfgs,
        'ncg': NonlinearConjugateGradient,
        'rprop': Rprop,
        'rmsprop': RmsProp,
        'adadelta': Adadelta,
        'adam': Adam,
    }
    # Find out which arguments to pass on.
    klass = klass_map[identifier]
    argspec = inspect.getargspec(klass.__init__)
    if argspec.keywords is None:
        # Issue a warning for each of the arguments that have been passed
        # to this optimizer but were not used.
        expected_keys = set(argspec.args)
        given_keys = set(kwargs.keys())
        unused_keys = given_keys - expected_keys
        for i in unused_keys:
            warnings.warn('Argument named %s is not expected by %s'
                          % (i, klass))

        # We need to filter stuff out.
        used_keys = expected_keys & given_keys
        kwargs = dict((k, kwargs[k]) for k in used_keys)
    try:
        opt = klass(wrt, *args, **kwargs)
    except TypeError:
        raise TypeError('required arguments for %s: %s' % (klass, argspec.args))

    return opt

def shaped_from_flat(flat, shapes):
    """Given a one dimensional array ``flat``, return a list of views of shapes
    ``shapes`` on that array.

    Each view will point to a distinct memory region, consecutively allocated
    in flat.

    Parameters
    ----------

    flat : array_like
        Array of one dimension.

    shapes : list of tuples of ints
        Each entry of this list specifies the shape of the corresponding view
        into ``flat``.

    Returns
    -------

    views : list of arrays
        Each entry has the shape given in ``shapes`` and points as a view into
        ``flat``.
    """
    shapes = [(i,) if isinstance(i, int) else i for i in shapes]
    sizes = [np.prod(i) for i in shapes]

    n_used = 0
    views = []
    for size, shape in zip(sizes, shapes):
        this = flat[n_used:n_used + size]
        n_used += size
        this.shape = shape
        views.append(this)

    return views

def empty_with_views(shapes, empty_func=np.empty):
    """Create an array and views shaped according to ``shapes``.

    The ``shapes`` parameter is a list of tuples of ints.  Each tuple
    represents a desired shape for an array which will be allocated in a bigger
    memory region. This memory region will be represented by an array as well.

    For example, the shape speciciation ``[2, (3, 2)]`` will create an array
    ``flat`` of size 8. The first view will have a size of ``(2,)`` and point
    to the first two entries, i.e. ``flat`[:2]`, while the second array will
    have a shape of ``(3, 2)`` and point to the elements ``flat[2:8]``.

    Parameters
    ----------

    spec : list of tuples of ints
        Specification of the desired shapes.

    empty_func : callable
        function that returns a memory region given an integer of the desired
        size. (Examples include ``numpy.empty``, which is the default,
        ``gnumpy.empty`` and ``theano.tensor.empty``.

    Returns
    -------

    flat : array_like (depending on ``empty_func``)
        Memory region containing all the views.

    views : list of array_like
        Variable number of results. Each contains a view into the array
        ``flat``.

    Examples
    --------

    >>> from climin.util import empty_with_views
    >>> flat, (w, b) = empty_with_views([(3, 2), 2])
    >>> w[...] = 1
    >>> b[...] = 2
    >>> flat
    array([ 1.,  1.,  1.,  1.,  1.,  1.,  2.,  2.])
    >>> flat[0] = 3
    >>> w
    array([[ 3.,  1.],
           [ 1.,  1.],
           [ 1.,  1.]])

    """
    shapes = [(i,) if isinstance(i, int) else i for i in shapes]
    sizes = [np.prod(i) for i in shapes]
    n_pars = sum(sizes)
    flat = empty_func(n_pars)

    views = shaped_from_flat(flat, shapes)

    return flat, views

def arbitrary_slice(arr, start, stop=None, axis=0):
    """Return a slice from `start` to `stop` in dimension `axis` of array `arr`.

    Parameters
    ----------

    arr : array_like
        Can be numpy ndarray, hdf5 dataset, or list.
        If ``arr`` is a list, ``axis`` must be 0.

    start : int
        Index at which to start slicing.

    stop : int, optional [default: None]
        Index at which to stop slicing.
        If not specified, the axis is sliced until its end.

    axis : int, optional [default: 0]
        Axis along which should be sliced

    Returns
    -------

    slice : array_like
        The respective slice of ``arr``
    """

    if type(arr) is list:
        if axis == 0:
            return arr[start:stop]
        else:
            raise ValueError("Cannot slice a list in non-zero axis {}"
                             .format(axis))

    n_axes = len(arr.shape)

    if axis >= n_axes:
        raise IndexError('Argument `axis` with value {} out of range. '
                         'Must be smaller than rank {} of `arr`.'
                         .format(axis, n_axes))

    this_slice = [slice(None) for _ in range(n_axes)]
    this_slice[axis] = slice(start, stop)

    return arr[tuple(this_slice)]

def minibatches(arr, batch_size, d=0):
    """Return a list of views of the given arr.

    Each view represents a mini bach of the data.

    Parameters
    ----------

    arr : array_like
        Array to obtain batches from. Needs to be slicable. If ``d > 0``, needs
        to have a ``.shape`` attribute from which the number of samples can
        be obtained.

    batch_size : int
        Size of a batch. Last batch might be smaller if ``batch_size`` is not a
        divisor of ``arr``.

    d : int, optional, default: 0
        Dimension along which the data samples are separated and thus slicing
        should be done.

    Returns
    -------

    mini_batches : list
        Each item of the list is a view of ``arr``. Views are ordered.
    """
    # This alternative is to make this work with lists in the case of d == 0.
    if d == 0:
        n_batches, rest = divmod(len(arr), batch_size)
    else:
        n_batches, rest = divmod(arr.shape[d], batch_size)
    if rest:
        n_batches += 1

    slices = (slice(i * batch_size, (i + 1) * batch_size)
              for i in range(n_batches))
    if d == 0:
        res = [arr[i] for i in slices]
    elif d == 1:
        res = [arr[:, i] for i in slices]
    elif d == 2:
        res = [arr[:, :, i] for i in slices]

    return res

def iter_minibatches(lst, batch_size, dims, n_cycles=None, random_state=None,
                     discard_illsized_batch=False):
    """Return an iterator that successively yields tuples containing aligned
    minibatches of size `batch_size` from slicable objects given in `lst`, in
    random order without replacement.
    Because different containers might require slicing over different
    dimensions, the dimension of each container has to be givens as a list
    `dims`.

    Parameters
    ----------

    lst : list of array_like
        Each item of the list will be sliced into mini batches in alignment
        with the others.

    batch_size : int
        Size of each batch. Last batch might be smaller.

    dims : list
        Aligned with ``lst``, gives the dimension along which the data samples
        are separated.

    n_cycles : int, optional [default: None]
        Number of cycles after which to stop the iterator. If ``None``, will
        yield forever.

    random_state : a numpy.random.RandomState object, optional [default : None]
        Random number generator that will act as a seed for the minibatch order.

    discard_illsized_batch : bool, optional [default : False]
        If ``True`` and the length of the sliced dimension is not divisible by
        ``batch_size``, the leftover samples are discarded.

    Returns
    -------
    batches : iterator
        Infinite iterator of mini batches in random order (without replacement).
    """

    try:
        # case distinction for handling lists
        dm_result = [divmod(len(arr), batch_size)
                     if d == 0 else divmod(arr.shape[d], batch_size)
                     for (arr, d) in zip(lst, dims)]
    except AttributeError:
        raise AttributeError("'list' object has no attribute 'shape'. "
                             "Trying to slice a list in a non-zero axis.")
    except IndexError:
        raise IndexError("tuple index out of range. "
                         "Trying to slice along a non-existing dimension.")

    # check if all to-be-sliced dimensions have the same length
    if dm_result.count(dm_result[0]) == len(dm_result):
        n_batches, rest = dm_result[0]
        if rest and not discard_illsized_batch:
            n_batches += 1
    else:
        raise ValueError("The axes along which to slice have unequal lengths.")

    if random_state is not None:
        random.seed(random_state.normal())

    counter = itertools.count()
    count = next(counter)

    while True:
        indices = range(n_batches)
        while True:
            if n_cycles is not None and count >= n_cycles:
                raise StopIteration()
            count = next(counter)

            random.shuffle(indices)
            for i in indices:
                start = i * batch_size
                stop = (i + 1) * batch_size
                batch = [arbitrary_slice(arr, start, stop, axis) for (arr, axis)
                         in zip(lst, dims)]
                yield tuple(batch)

class OptimizerDistribution(object):
    """OptimizerDistribution class.

    Can be used for specifying optimizers in scikit-learn's randomized parameter
    search.

    Attributes
    ----------

    options : dict
        Maps an optimizer key to a grid to sample from.
    """

    def __init__(self, **options):
        """Create an OptimizerDistribution object.

        Parameters
        ----------

        options : dict
            Maps an optimizer key to a grid to sample from.
        """
        self.options = options

    def rvs(self):
        opt = random.choice(list(self.options.keys()))
        grid = self.options[opt]
        sample = list(ParameterSampler(grid, n_iter=1))[0]
        return opt, sample
<|endoftext|>