<commit_before># Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Miscellaneous methods to support testing.
"""

import os
import random
import string
import subprocess
import sys
import time
import unittest

import psutil  # pylint: disable=wrong-import-order

from stratis_cli import run

try:
    _STRATISD = os.environ["STRATISD"]
except KeyError:
    message = "STRATISD environment variable must be set to absolute path of stratisd executable"
    sys.exit(message)

def device_name_list(min_devices=0, max_devices=10):
    """
    Return a function that returns a random list of device names based on
    parameters.
    """

    def the_func():
        return [
            "/dev/%s"
            % "".join(
                random.choice(string.ascii_uppercase + string.digits) for _ in range(4)
            )
            for _ in range(random.randrange(min_devices, max_devices + 1))
        ]

    return the_func

class _Service:
    """
    Handle starting and stopping the Rust service.
    """

    def setUp(self):
        """
        Start the stratisd daemon with the simulator.
        """
        self._stratisd = subprocess.Popen([os.path.join(_STRATISD), "--sim"])
        time.sleep(1)

    def tearDown(self):
        """
        Stop the stratisd simulator and daemon.
        """
        self._stratisd.terminate()
        self._stratisd.wait()

    def cleanup(self):
        """
        Stop the daemon if it has been started.
        """
        if hasattr(self, "_stratisd"):
            self.tearDown()

class SimTestCase(unittest.TestCase):
    """
    A SimTestCase must always start and stop stratisd (simulator vesion).
    """

    @classmethod
    def setUpClass(cls):
        """
        Assert that there are no other stratisd processes running.
        """
        for pid in psutil.pids():
            try:
                assert psutil.Process(pid).name() != "stratisd", (
                    "Evidently a stratisd process with process id %u is running" % pid
                )
            except psutil.NoSuchProcess:
                pass

    def setUp(self):
        """
        Start the stratisd daemon with the simulator.
        """
        self._service = _Service()
        self.addCleanup(self._service.cleanup)
        self._service.setUp()

RUNNER = run()
<commit_msg>Remove an unnecessary pylint disable<commit_after># Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Miscellaneous methods to support testing.
"""

import os
import random
import string
import subprocess
import sys
import time
import unittest

import psutil

from stratis_cli import run

try:
    _STRATISD = os.environ["STRATISD"]
except KeyError:
    message = "STRATISD environment variable must be set to absolute path of stratisd executable"
    sys.exit(message)

def device_name_list(min_devices=0, max_devices=10):
    """
    Return a function that returns a random list of device names based on
    parameters.
    """

    def the_func():
        return [
            "/dev/%s"
            % "".join(
                random.choice(string.ascii_uppercase + string.digits) for _ in range(4)
            )
            for _ in range(random.randrange(min_devices, max_devices + 1))
        ]

    return the_func

class _Service:
    """
    Handle starting and stopping the Rust service.
    """

    def setUp(self):
        """
        Start the stratisd daemon with the simulator.
        """
        self._stratisd = subprocess.Popen([os.path.join(_STRATISD), "--sim"])
        time.sleep(1)

    def tearDown(self):
        """
        Stop the stratisd simulator and daemon.
        """
        self._stratisd.terminate()
        self._stratisd.wait()

    def cleanup(self):
        """
        Stop the daemon if it has been started.
        """
        if hasattr(self, "_stratisd"):
            self.tearDown()

class SimTestCase(unittest.TestCase):
    """
    A SimTestCase must always start and stop stratisd (simulator vesion).
    """

    @classmethod
    def setUpClass(cls):
        """
        Assert that there are no other stratisd processes running.
        """
        for pid in psutil.pids():
            try:
                assert psutil.Process(pid).name() != "stratisd", (
                    "Evidently a stratisd process with process id %u is running" % pid
                )
            except psutil.NoSuchProcess:
                pass

    def setUp(self):
        """
        Start the stratisd daemon with the simulator.
        """
        self._service = _Service()
        self.addCleanup(self._service.cleanup)
        self._service.setUp()

RUNNER = run()
<|endoftext|>


<commit_before>"""Implementation of a sampler for the Robust Soliton Distribution.

This is the distribution on the `degree` of blocks encoded in the
Luby Transform code. Blocks of data transmitted are generated by
sampling degree `d` from the Robust Soliton Distrubution, then
sampling `d` blocks uniformly from the sequence of blocks in the
file to be transmitted. These are XOR'ed together, and the result
is transmitted.

Critically, the state of the PRNG when the degree of a block was
sampled is transmitted with the block as metadata, so the
receiver can reconstruct the sampling of source blocks given the
same PRNG parameters below.
"""
from math import log, floor, sqrt

DEFAULT_C = 0.1
DEFAULT_DELTA = 0.5

# Parameters for Pseudorandom Number Generator
PRNG_A = 16807
PRNG_M = (1 << 31) - 1
PRNG_MAX_RAND = PRNG_M - 1

def gen_tau(S, K, delta):
    """The Robust part of the RSD, we precompute an
    array for speed
    """
    pivot = floor(K/S)
    return [S/K * 1/d for d in range(1, pivot)] \
            + [S/K * log(S/delta)] \
            + [0 for d in range(pivot, K)]

def gen_rho(K):
    """The Ideal Soliton Distribution, we precompute
    an array for speed
    """
    return [1/K] + [1/(d*(d-1)) for d in range(2, K+1)]

def gen_mu(K, delta, c):
    """The Robust Soliton Distribution on the degree of
    transmitted blocks
    """

    S = c * log(K/delta) * sqrt(K)
    tau = gen_tau(S, K, delta)
    rho = gen_rho(K)
    normalizer = sum(rho) + sum(tau)
    return [(rho[d] + tau[d])/normalizer for d in range(K)]

def gen_rsd_cdf(K, delta, c):
    """The CDF of the RSD on block degree, precomputed for
    sampling speed"""

    mu = gen_mu(K, delta, c)
    return [sum(mu[:d+1]) for d in range(K)]

class PRNG(object):
    """A Pseudorandom Number Generator that yields samples
    from the set of source blocks using the RSD degree
    distribution described above.
    """

    def __init__(self, params):
        """Provide RSD parameters on construction
        """

        self.seed = None # Seed is set by interfacing code using set_seed
        K, delta, c = params
        self.K = K
        self.cdf = gen_rsd_cdf(K, delta, c)

    def set_seed(self, seed):
        """Reset the state of the PRNG to the
        given seed
        """

        self.seed = seed
        self.state = seed

    def get_next(self):
        """Executes the next iteration of the PRNG
        evolution process, and returns the result
        """

        self.state = PRNG_A * self.state % PRNG_M
        return self.state

    def get_state(self):
        """Returns current state of the linear PRNG
        """

        return self.state

    def get_src_blocks(self, seed=None):
        """Returns the indices of a set of `d` source blocks
        sampled from indices i = 1, ..., K-1 uniformly, where
        `d` is sampled from the RSD described above.
        """

        if seed:
            self.state = seed

        blockseed = self.state
        d = self._sample_d()
        have = 0
        nums = set()
        while have < d:
            num = self.get_next() % self.K
            if num not in nums:
                nums.add(num)
                have += 1
        return blockseed, d, nums

    # Samples from the CDF of mu
    def _sample_d(self):
        """Samples degree given the precomputed
        distributions above and the linear PRNG output
        """

        p = self.get_next() / PRNG_MAX_RAND
        for ix, v in enumerate(self.cdf):
            if v > p:
                return ix + 1
        return ix + 1
<commit_msg>seeding parameterization is weird, gonna leave for now<commit_after>"""Implementation of a sampler for the Robust Soliton Distribution.

This is the distribution on the `degree` of blocks encoded in the
Luby Transform code. Blocks of data transmitted are generated by
sampling degree `d` from the Robust Soliton Distrubution, then
sampling `d` blocks uniformly from the sequence of blocks in the
file to be transmitted. These are XOR'ed together, and the result
is transmitted.

Critically, the state of the PRNG when the degree of a block was
sampled is transmitted with the block as metadata, so the
receiver can reconstruct the sampling of source blocks given the
same PRNG parameters below.
"""
from math import log, floor, sqrt

DEFAULT_C = 0.1
DEFAULT_DELTA = 0.5

# Parameters for Pseudorandom Number Generator
PRNG_A = 16807
PRNG_M = (1 << 31) - 1
PRNG_MAX_RAND = PRNG_M - 1

def gen_tau(S, K, delta):
    """The Robust part of the RSD, we precompute an
    array for speed
    """
    pivot = floor(K/S)
    return [S/K * 1/d for d in range(1, pivot)] \
            + [S/K * log(S/delta)] \
            + [0 for d in range(pivot, K)]

def gen_rho(K):
    """The Ideal Soliton Distribution, we precompute
    an array for speed
    """
    return [1/K] + [1/(d*(d-1)) for d in range(2, K+1)]

def gen_mu(K, delta, c):
    """The Robust Soliton Distribution on the degree of
    transmitted blocks
    """

    S = c * log(K/delta) * sqrt(K)
    tau = gen_tau(S, K, delta)
    rho = gen_rho(K)
    normalizer = sum(rho) + sum(tau)
    return [(rho[d] + tau[d])/normalizer for d in range(K)]

def gen_rsd_cdf(K, delta, c):
    """The CDF of the RSD on block degree, precomputed for
    sampling speed"""

    mu = gen_mu(K, delta, c)
    return [sum(mu[:d+1]) for d in range(K)]

class PRNG(object):
    """A Pseudorandom Number Generator that yields samples
    from the set of source blocks using the RSD degree
    distribution described above.
    """

    def __init__(self, params):
        """Provide RSD parameters on construction
        """

        self.state = None  # Seed is set by interfacing code using set_seed
        K, delta, c = params
        self.K = K
        self.cdf = gen_rsd_cdf(K, delta, c)

    def set_seed(self, seed):
        """Reset the state of the PRNG to the
        given seed
        """

        self.state = seed

    def get_next(self):
        """Executes the next iteration of the PRNG
        evolution process, and returns the result
        """

        self.state = PRNG_A * self.state % PRNG_M
        return self.state

    def get_state(self):
        """Returns current state of the linear PRNG
        """

        return self.state

    def get_src_blocks(self, seed=None):
        """Returns the indices of a set of `d` source blocks
        sampled from indices i = 1, ..., K-1 uniformly, where
        `d` is sampled from the RSD described above.
        """

        if seed:
            self.state = seed

        blockseed = self.state
        d = self._sample_d()
        have = 0
        nums = set()
        while have < d:
            num = self.get_next() % self.K
            if num not in nums:
                nums.add(num)
                have += 1
        return blockseed, d, nums

    # Samples from the CDF of mu
    def _sample_d(self):
        """Samples degree given the precomputed
        distributions above and the linear PRNG output
        """

        p = self.get_next() / PRNG_MAX_RAND
        for ix, v in enumerate(self.cdf):
            if v > p:
                return ix + 1
        return ix + 1
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#python -m unittest tests.tests

import unittest
import tika.parser

class CreateTest(unittest.TestCase):
    "test for file types"

    def test_remote_pdf(self):
        'parse remote PDF'
        self.assertTrue(tika.parser.from_file(
            'http://appsrv.achd.net/reports/rwservlet?food_rep_insp&P_ENCOUNTER=201504160015'))
    def test_remote_html(self):
        'parse remote HTML'
        self.assertTrue(tika.parser.from_file(
            'http://philadelphia.pa.gegov.com/_templates/551/RetailFood/_report_full.cfm?inspectionID=8B2C8CA4-8039-EC2C-F94DBD247613E5CC&domainID=551&userID=0'))
    def test_remote_mp3(self):
        'parese remote mp3'
        self.assertTrue(tika.parser.from_file(
            'http://tindeck.com/download/pro/viqis/theCalling-Lemming-Version.mp3'))
    def test_remote_jpg(self):
        'parse remote jpg'
        self.assertTrue(tika.parser.from_file(
            'https://www.nasa.gov/sites/default/files/thumbnails/image/j2m-shareable.jpg'))

if __name__ == '__main__':
    unittest.main()
<commit_msg>remove URL that no longer works from tests.<commit_after>#!/usr/bin/env python
# encoding: utf-8
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#python -m unittest tests.tests

import unittest
import tika.parser

class CreateTest(unittest.TestCase):
    "test for file types"

    def test_remote_pdf(self):
        'parse remote PDF'
        self.assertTrue(tika.parser.from_file(
            'http://appsrv.achd.net/reports/rwservlet?food_rep_insp&P_ENCOUNTER=201504160015'))
    def test_remote_html(self):
        'parse remote HTML'
        self.assertTrue(tika.parser.from_file(
            'https://www.nasa.gov/index.html'))
    def test_remote_mp3(self):
        'parese remote mp3'
        self.assertTrue(tika.parser.from_file(
            'http://tindeck.com/download/pro/viqis/theCalling-Lemming-Version.mp3'))
    def test_remote_jpg(self):
        'parse remote jpg'
        self.assertTrue(tika.parser.from_file(
            'https://www.nasa.gov/sites/default/files/thumbnails/image/j2m-shareable.jpg'))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import dateutil.parser

from datetime import datetime
from dateutil.tz import tz

class TimeWarriorInterval(object):
    def __init__(self, start, end, tags):
        self.__start = self.__get_local_datetime(start)
        self.__end = self.__get_local_datetime(end)
        self.__tags = tags

    def __eq__(self, other):
        return self.__start == other.get_start() \
            and self.__end == other.get_end() \
            and self.__tags == other.get_tags()

    def get_start(self):
        return self.__start

    def get_end(self):
        return self.__end

    def get_tags(self):
        return self.__tags

    def get_date(self):
        return datetime(self.__start.year, self.__start.month, self.__start.day)

    def __get_local_datetime(self, datetime_string):
        from_zone = tz.tzutc()
        to_zone = tz.tzlocal()

        date = dateutil.parser.parse(datetime_string)
        date.replace(tzinfo=from_zone)

        return date.astimezone(to_zone)
<commit_msg>Add method to query if interval is open<commit_after>import dateutil.parser

from datetime import datetime
from dateutil.tz import tz

class TimeWarriorInterval(object):
    def __init__(self, start, end, tags):
        self.__start = self.__get_local_datetime(start)
        self.__end = self.__get_local_datetime(end)
        self.__tags = tags

    def __eq__(self, other):
        return self.__start == other.get_start() \
            and self.__end == other.get_end() \
            and self.__tags == other.get_tags()

    def get_start(self):
        return self.__start

    def get_end(self):
        return self.__end

    def get_tags(self):
        return self.__tags

    def is_open(self):
        return False

    def get_date(self):
        return datetime(self.__start.year, self.__start.month, self.__start.day)

    def __get_local_datetime(self, datetime_string):
        from_zone = tz.tzutc()
        to_zone = tz.tzlocal()

        date = dateutil.parser.parse(datetime_string)
        date.replace(tzinfo=from_zone)

        return date.astimezone(to_zone)
<|endoftext|>


<commit_before>import torch
import torchvision.transforms.functional as F
try:
    from torchvision.transforms.functional import InterpolationMode
    has_interpolation_mode = True
except ImportError:
    has_interpolation_mode = False
from PIL import Image
import warnings
import math
import random
import numpy as np

class ToNumpy:

    def __call__(self, pil_img):
        np_img = np.array(pil_img, dtype=np.uint8)
        if np_img.ndim < 3:
            np_img = np.expand_dims(np_img, axis=-1)
        np_img = np.rollaxis(np_img, 2)  # HWC to CHW
        return np_img

class ToTensor:

    def __init__(self, dtype=torch.float32):
        self.dtype = dtype

    def __call__(self, pil_img):
        np_img = np.array(pil_img, dtype=np.uint8)
        if np_img.ndim < 3:
            np_img = np.expand_dims(np_img, axis=-1)
        np_img = np.rollaxis(np_img, 2)  # HWC to CHW
        return torch.from_numpy(np_img).to(dtype=self.dtype)

_pil_interpolation_to_str = {
    Image.NEAREST: 'nearest',
    Image.BILINEAR: 'bilinear',
    Image.BICUBIC: 'bicubic',
    Image.BOX: 'box',
    Image.HAMMING: 'hamming',
    Image.LANCZOS: 'lanczos',
}
_str_to_pil_interpolation = {b: a for a, b in _pil_interpolation_to_str.items()}

if has_interpolation_mode:
    _torch_interpolation_to_str = {
        InterpolationMode.NEAREST: 'nearest',
        InterpolationMode.BILINEAR: 'bilinear',
        InterpolationMode.BICUBIC: 'bicubic',
        InterpolationMode.BOX: 'box',
        InterpolationMode.HAMMING: 'hamming',
        InterpolationMode.LANCZOS: 'lanczos',
    }
    _str_to_torch_interpolation = {b: a for a, b in _torch_interpolation_to_str.items()}
else:
    _pil_interpolation_to_torch = {}
    _torch_interpolation_to_str = {}

def str_to_pil_interp(mode_str):
    return _str_to_pil_interpolation[mode_str]

def str_to_interp_mode(mode_str):
    if has_interpolation_mode:
        return _str_to_torch_interpolation[mode_str]
    else:
        return _str_to_pil_interpolation[mode_str]

def interp_mode_to_str(mode):
    if has_interpolation_mode:
        return _torch_interpolation_to_str[mode]
    else:
        return _pil_interpolation_to_str[mode]

_RANDOM_INTERPOLATION = (str_to_interp_mode('bilinear'), str_to_interp_mode('bicubic'))

class RandomResizedCropAndInterpolation:
    """Crop the given PIL Image to random size and aspect ratio with random interpolation.

    A crop of random size (default: of 0.08 to 1.0) of the original size and a random
    aspect ratio (default: of 3/4 to 4/3) of the original aspect ratio is made. This crop
    is finally resized to given size.
    This is popularly used to train the Inception networks.

    Args:
        size: expected output size of each edge
        scale: range of size of the origin size cropped
        ratio: range of aspect ratio of the origin aspect ratio cropped
        interpolation: Default: PIL.Image.BILINEAR
    """

    def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.),
                 interpolation='bilinear'):
        if isinstance(size, (list, tuple)):
            self.size = tuple(size)
        else:
            self.size = (size, size)
        if (scale[0] > scale[1]) or (ratio[0] > ratio[1]):
            warnings.warn("range should be of kind (min, max)")

        if interpolation == 'random':
            self.interpolation = _RANDOM_INTERPOLATION
        else:
            self.interpolation = str_to_interp_mode(interpolation)
        self.scale = scale
        self.ratio = ratio

    @staticmethod
    def get_params(img, scale, ratio):
        """Get parameters for ``crop`` for a random sized crop.

        Args:
            img (PIL Image): Image to be cropped.
            scale (tuple): range of size of the origin size cropped
            ratio (tuple): range of aspect ratio of the origin aspect ratio cropped

        Returns:
            tuple: params (i, j, h, w) to be passed to ``crop`` for a random
                sized crop.
        """
        area = img.size[0] * img.size[1]

        for attempt in range(10):
            target_area = random.uniform(*scale) * area
            log_ratio = (math.log(ratio[0]), math.log(ratio[1]))
            aspect_ratio = math.exp(random.uniform(*log_ratio))

            w = int(round(math.sqrt(target_area * aspect_ratio)))
            h = int(round(math.sqrt(target_area / aspect_ratio)))

            if w <= img.size[0] and h <= img.size[1]:
                i = random.randint(0, img.size[1] - h)
                j = random.randint(0, img.size[0] - w)
                return i, j, h, w

        # Fallback to central crop
        in_ratio = img.size[0] / img.size[1]
        if in_ratio < min(ratio):
            w = img.size[0]
            h = int(round(w / min(ratio)))
        elif in_ratio > max(ratio):
            h = img.size[1]
            w = int(round(h * max(ratio)))
        else:  # whole image
            w = img.size[0]
            h = img.size[1]
        i = (img.size[1] - h) // 2
        j = (img.size[0] - w) // 2
        return i, j, h, w

    def __call__(self, img):
        """
        Args:
            img (PIL Image): Image to be cropped and resized.

        Returns:
            PIL Image: Randomly cropped and resized image.
        """
        i, j, h, w = self.get_params(img, self.scale, self.ratio)
        if isinstance(self.interpolation, (tuple, list)):
            interpolation = random.choice(self.interpolation)
        else:
            interpolation = self.interpolation
        return F.resized_crop(img, i, j, h, w, self.size, interpolation)

    def __repr__(self):
        if isinstance(self.interpolation, (tuple, list)):
            interpolate_str = ' '.join([interp_mode_to_str(x) for x in self.interpolation])
        else:
            interpolate_str = interp_mode_to_str(self.interpolation)
        format_string = self.__class__.__name__ + '(size={0}'.format(self.size)
        format_string += ', scale={0}'.format(tuple(round(s, 4) for s in self.scale))
        format_string += ', ratio={0}'.format(tuple(round(r, 4) for r in self.ratio))
        format_string += ', interpolation={0})'.format(interpolate_str)
        return format_string

<commit_msg>use `Image.Resampling` namespace for PIL mapping (#1256)<commit_after>import torch
import torchvision.transforms.functional as F
try:
    from torchvision.transforms.functional import InterpolationMode
    has_interpolation_mode = True
except ImportError:
    has_interpolation_mode = False
from PIL import Image
import warnings
import math
import random
import numpy as np

class ToNumpy:

    def __call__(self, pil_img):
        np_img = np.array(pil_img, dtype=np.uint8)
        if np_img.ndim < 3:
            np_img = np.expand_dims(np_img, axis=-1)
        np_img = np.rollaxis(np_img, 2)  # HWC to CHW
        return np_img

class ToTensor:

    def __init__(self, dtype=torch.float32):
        self.dtype = dtype

    def __call__(self, pil_img):
        np_img = np.array(pil_img, dtype=np.uint8)
        if np_img.ndim < 3:
            np_img = np.expand_dims(np_img, axis=-1)
        np_img = np.rollaxis(np_img, 2)  # HWC to CHW
        return torch.from_numpy(np_img).to(dtype=self.dtype)

# Pillow is deprecating the top-level resampling attributes (e.g., Image.BILINEAR) in
# favor of the Image.Resampling enum. The top-level resampling attributes will be
# removed in Pillow 10.
if hasattr(Image, "Resampling"):
    _pil_interpolation_to_str = {
        Image.Resampling.NEAREST: 'nearest',
        Image.Resampling.BILINEAR: 'bilinear',
        Image.Resampling.BICUBIC: 'bicubic',
        Image.Resampling.BOX: 'box',
        Image.Resampling.HAMMING: 'hamming',
        Image.Resampling.LANCZOS: 'lanczos',
    }
else:
    _pil_interpolation_to_str = {
        Image.NEAREST: 'nearest',
        Image.BILINEAR: 'bilinear',
        Image.BICUBIC: 'bicubic',
        Image.BOX: 'box',
        Image.HAMMING: 'hamming',
        Image.LANCZOS: 'lanczos',
    }

_str_to_pil_interpolation = {b: a for a, b in _pil_interpolation_to_str.items()}

if has_interpolation_mode:
    _torch_interpolation_to_str = {
        InterpolationMode.NEAREST: 'nearest',
        InterpolationMode.BILINEAR: 'bilinear',
        InterpolationMode.BICUBIC: 'bicubic',
        InterpolationMode.BOX: 'box',
        InterpolationMode.HAMMING: 'hamming',
        InterpolationMode.LANCZOS: 'lanczos',
    }
    _str_to_torch_interpolation = {b: a for a, b in _torch_interpolation_to_str.items()}
else:
    _pil_interpolation_to_torch = {}
    _torch_interpolation_to_str = {}

def str_to_pil_interp(mode_str):
    return _str_to_pil_interpolation[mode_str]

def str_to_interp_mode(mode_str):
    if has_interpolation_mode:
        return _str_to_torch_interpolation[mode_str]
    else:
        return _str_to_pil_interpolation[mode_str]

def interp_mode_to_str(mode):
    if has_interpolation_mode:
        return _torch_interpolation_to_str[mode]
    else:
        return _pil_interpolation_to_str[mode]

_RANDOM_INTERPOLATION = (str_to_interp_mode('bilinear'), str_to_interp_mode('bicubic'))

class RandomResizedCropAndInterpolation:
    """Crop the given PIL Image to random size and aspect ratio with random interpolation.

    A crop of random size (default: of 0.08 to 1.0) of the original size and a random
    aspect ratio (default: of 3/4 to 4/3) of the original aspect ratio is made. This crop
    is finally resized to given size.
    This is popularly used to train the Inception networks.

    Args:
        size: expected output size of each edge
        scale: range of size of the origin size cropped
        ratio: range of aspect ratio of the origin aspect ratio cropped
        interpolation: Default: PIL.Image.BILINEAR
    """

    def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.),
                 interpolation='bilinear'):
        if isinstance(size, (list, tuple)):
            self.size = tuple(size)
        else:
            self.size = (size, size)
        if (scale[0] > scale[1]) or (ratio[0] > ratio[1]):
            warnings.warn("range should be of kind (min, max)")

        if interpolation == 'random':
            self.interpolation = _RANDOM_INTERPOLATION
        else:
            self.interpolation = str_to_interp_mode(interpolation)
        self.scale = scale
        self.ratio = ratio

    @staticmethod
    def get_params(img, scale, ratio):
        """Get parameters for ``crop`` for a random sized crop.

        Args:
            img (PIL Image): Image to be cropped.
            scale (tuple): range of size of the origin size cropped
            ratio (tuple): range of aspect ratio of the origin aspect ratio cropped

        Returns:
            tuple: params (i, j, h, w) to be passed to ``crop`` for a random
                sized crop.
        """
        area = img.size[0] * img.size[1]

        for attempt in range(10):
            target_area = random.uniform(*scale) * area
            log_ratio = (math.log(ratio[0]), math.log(ratio[1]))
            aspect_ratio = math.exp(random.uniform(*log_ratio))

            w = int(round(math.sqrt(target_area * aspect_ratio)))
            h = int(round(math.sqrt(target_area / aspect_ratio)))

            if w <= img.size[0] and h <= img.size[1]:
                i = random.randint(0, img.size[1] - h)
                j = random.randint(0, img.size[0] - w)
                return i, j, h, w

        # Fallback to central crop
        in_ratio = img.size[0] / img.size[1]
        if in_ratio < min(ratio):
            w = img.size[0]
            h = int(round(w / min(ratio)))
        elif in_ratio > max(ratio):
            h = img.size[1]
            w = int(round(h * max(ratio)))
        else:  # whole image
            w = img.size[0]
            h = img.size[1]
        i = (img.size[1] - h) // 2
        j = (img.size[0] - w) // 2
        return i, j, h, w

    def __call__(self, img):
        """
        Args:
            img (PIL Image): Image to be cropped and resized.

        Returns:
            PIL Image: Randomly cropped and resized image.
        """
        i, j, h, w = self.get_params(img, self.scale, self.ratio)
        if isinstance(self.interpolation, (tuple, list)):
            interpolation = random.choice(self.interpolation)
        else:
            interpolation = self.interpolation
        return F.resized_crop(img, i, j, h, w, self.size, interpolation)

    def __repr__(self):
        if isinstance(self.interpolation, (tuple, list)):
            interpolate_str = ' '.join([interp_mode_to_str(x) for x in self.interpolation])
        else:
            interpolate_str = interp_mode_to_str(self.interpolation)
        format_string = self.__class__.__name__ + '(size={0}'.format(self.size)
        format_string += ', scale={0}'.format(tuple(round(s, 4) for s in self.scale))
        format_string += ', ratio={0}'.format(tuple(round(r, 4) for r in self.ratio))
        format_string += ', interpolation={0})'.format(interpolate_str)
        return format_string
<|endoftext|>


<commit_before>import binascii
from OpenSSL import crypto
from twisted.trial.unittest import SynchronousTestCase
import getdns
from tlsep import _dane

class TLSADomainNameTests(SynchronousTestCase):
    def test_tlsaDomainName(self):
        """
        L{_dane.tlsaDomainName} returns the port, proto and parent domain as
        labels of a new domain name string.
        """
        self.assertEqual(
            "_443._tcp.example.com",
            _dane.tlsaDomainName('example.com', 443, 'tcp')
        )

class FakeGetdns(object):
    """
    An in memory fake of the getdns api for testing.
    """

    def __init__(self, generalResult=None):
        self._generalResult = generalResult

        for k, v in getdns.__dict__.items():
            if k.startswith('GETDNS_'):
                setattr(self, k, v)

    def context_create(self):
        """
        """

    def general(self, context, name, request_type, extensions):
        """
        """
        return self._generalResult

class TLSATests(SynchronousTestCase):
    def test_tlsaCert(self):
        """
        L{_dane.tlsa} returns a TLSACert instance if the domain name exists and
        a verified record is found and the record selector type is CERT.
        """
        fakeGetdns = FakeGetdns(
            generalResult=createResults(status=getdns.GETDNS_RESPSTATUS_GOOD,
                                        selector=0,
                                        certificate_association_data=CERT_FULL))
        res = _dane.lookup_tlsa_records('example.com', 443, 'tcp', getdns=fakeGetdns)
        self.assertEqual(
            (0, crypto.X509),
            (res.type, type(res.cert))
        )

    def test_tlsaSPKI(self):
        """
        L{_dane.tlsa} returns a TLSASPKI instance if the domain name exists and
        a verfied record is found and the record selector type is SPKI.
        """
        expected = b'FOOBAR'
        fakeGetdns = FakeGetdns(
            generalResult=createResults(status=getdns.GETDNS_RESPSTATUS_GOOD,
                                        selector=1,
                                        certificate_association_data=expected))
        self.assertEqual(
            binascii.hexlify(expected),
            _dane.lookup_tlsa_records('example.com', 443, 'tcp', getdns=fakeGetdns).spki
        )

    def test_tlsaNoname(self):
        """
        L{_dane.tlsa} raises LookupError if the domain name does not exist.
        """
        e = self.assertRaises(
            _dane.GetdnsResponseError,
            _dane.lookup_tlsa_records, 'example.com', 443, 'tcp', getdns=FakeGetdns(
                generalResult=createResults(
                    status=getdns.GETDNS_RESPSTATUS_NO_NAME
                )
            )
        )
        self.assertEqual(
            getdns.GETDNS_RESPSTATUS_NO_NAME,
            e.errorCode
        )

# A full certificate from full.cert.getdnsapi.net
CERT_FULL = binascii.unhexlify("308205d7308203bfa00302010202030e9593300d06092a864886f70d01010d050030793110300e060355040a1307526f6f74204341311e301c060355040b1315687474703a2f2f7777772e6361636572742e6f7267312230200603550403131943412043657274205369676e696e6720417574686f726974793121301f06092a864886f70d0109011612737570706f7274406361636572742e6f7267301e170d3134303232373135303435385a170d3134303832363135303435385a3018311630140603550403130d676574646e736170692e6e657430820122300d06092a864886f70d01010105000382010f003082010a0282010100dc0226b2c9e7aa079992730d09530e8d14fd74bead887f1039a8baa7a28b68cc358c455364a6d4f6393831072f8bbb28ea04256833779ea3c841867b14a659fe2495829b7ae1c21c6bc1f5ce63e3421fd851c5cd339b77166d2bf4cb738a86908d0439e1b1dffb7cd0a3ea91faa36dfed2f4fc5315a96764e082df8cc0f698b6278f1b1f58ab27cf53ac1827cd69b979c45733fcc8c9532065f3f1c35e933f7c3b6caf56176df852076b0195dc0b848a4cbc7c4a38e07ea39b1a55104888bdcbba69d0dc8f3298002e36e1c082bb1a48cb47829678c3c6e753e810565cbd1b68e263046839935e2d2631f4f85469540e50ea01c54de8d97de1f929b6e1a8c1810203010001a38201c7308201c3300c0603551d130101ff04023000300e0603551d0f0101ff0404030203a830340603551d25042d302b06082b0601050507030206082b0601050507030106096086480186f8420401060a2b0601040182370a0303303306082b0601050507010104273025302306082b060105050730018617687474703a2f2f6f6373702e6361636572742e6f72672f30310603551d1f042a30283026a024a0228620687474703a2f2f63726c2e6361636572742e6f72672f7265766f6b652e63726c308201030603551d110481fb3081f8820d676574646e736170692e6e6574a01b06082b06010505070805a00f0c0d676574646e736170692e6e657482117777772e676574646e736170692e6e6574a01f06082b06010505070805a0130c117777772e676574646e736170692e6e6574820d676574646e736170692e6f7267a01b06082b06010505070805a00f0c0d676574646e736170692e6f726782117777772e676574646e736170692e6f7267a01f06082b06010505070805a0130c117777772e676574646e736170692e6f72678213676574646e732e6e6c6e65746c6162732e6e6ca02106082b06010505070805a0150c13676574646e732e6e6c6e65746c6162732e6e6c300d06092a864886f70d01010d05000382020100694130bffe0f84062c5dbc9526688cddef65ea3ba9f882bf171504a6de3dda34e371ab82fc5b2ffd18598c4b48cec6e4ec9ba3d7a4bbd5016101694c8f2abcb27fae79b392eb6eae3639fa611245206411bef8682eadf0060c44c929221396a79f3320cc847ca4aad2b7e65c34455a2fc471b7876d848d3ee0a9c7fb323237d8549ab9356c8aac16c90daa0f1f9eb3cd74a6b6214e28a82c7e6fea230e2d687a7122c8f3b1313bfaa1ec0a7eaa89b8de535d1a414f4d5b4e998d5212096817c85d01e13b4cd65b5ee8eb273f5c68213702874333768ad8a5ee641f34ea2a3486d9960cff0ac3860470987bef2bd0aa16c1a442c70473c9d3ca139861548b18e28e9771d11ff0c359cf67360a8efc15ef6c0bb540f2805a746e77ab420661719dbc2a351e70f26f6153880c43da3cd317e2140ac965fd80cf784fa3ae9db6dcc516080242b61b45bdda8a434e99c634f3b13071b43e1777485768ca83a109a035d8a57499779c947340b7b68e4da1647ceabf1f7fce94d9048e47d699739f497705fd80073de2df0686dff06d8ed5e97860a15fe80cae0e9e928dad667419bab6ddac25a36438e6eace45f6c06f1b18b96e68c28774e52e14a82630c17466b18d08d6799a7af9870c94202a43086e0c96a3d384b054136c097c47f3fe1634d7c27e2a8ae013247690bef2be96c8e663d268865e979426dfdc2d4972b37a64bb1d")

def createResults(status=getdns.GETDNS_RESPSTATUS_GOOD,
                  selector=None,
                  certificate_association_data=b"",):
    return {'answer_type': 800,
            'canonical_name': '_443._tcp.getdnsapi.org.',
            'just_address_answers': [],
#             'replies_full': [<read-only buffer ptr 0x7fe2e0029e80, size 636 at 0x7fe2e4e58fb0>],
            'replies_tree': [{'answer': [{'class': 1,
                                          'name': '_443._tcp.getdnsapi.org.',
                                          'rdata': {
                                              'certificate_association_data': certificate_association_data,
                                              'certificate_usage': 3,
                                              'matching_type': 1,
                                              # 'rdata_raw': "",
                                              'selector': selector
                                          },
                                          # 'ttl': 450,
                                          # 'type': 52},
                                         # {'class': 1,
                                         #  'name': '_443._tcp.getdnsapi.org.',
                                         #  'rdata': {'algorithm': 7,
                                         #            'key_tag': 49262,
                                         #            'labels': 4,
                                         #            'original_ttl': 450,
                                         #            'rdata_raw': <read-only buffer ptr 0x7fe2e0261b70, size 161 at 0x7fe2e4e60130>,
                                         #            'signature': <read-only buffer ptr 0x7fe2e0254c40, size 128 at 0x7fe2e4e60170>,
                                         #            'signature_expiration': 1399325172,
                                         #            'signature_inception': 1398100703,
                                         #            'signers_name': 'getdnsapi.org.',
                                         #            'type_covered': 52},
                                         #  'ttl': 450,
                                         #  'type': 46
                                      }
                                     ],
                               'answer_type': 800,
                              # 'authority': [{'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'nsdname': 'ns.secret-wg.org.',
                              #                          'rdata_raw': 'ns.secret-wg.org.'},
                              #                'ttl': 450,
                              #                'type': 2},
                              #               {'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'nsdname': 'mcvax.nlnetlabs.nl.',
                              #                          'rdata_raw': 'mcvax.nlnetlabs.nl.'},
                              #                'ttl': 450,
                              #                'type': 2},
                              #               {'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'nsdname': 'open.nlnetlabs.nl.',
                              #                          'rdata_raw': 'open.nlnetlabs.nl.'},
                              #                'ttl': 450,
                              #                'type': 2},
                              #               {'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'algorithm': 7,
                              #                          'key_tag': 49262,
                              #                          'labels': 2,
                              #                          'original_ttl': 450,
                              #                          'rdata_raw': <read-only buffer ptr 0x7fe2e0261f90, size 161 at 0x7fe2e4e601f0>,
                              #                          'signature': <read-only buffer ptr 0x7fe2e0028120, size 128 at 0x7fe2e4e60230>,
                              #                          'signature_expiration': 1399278072,
                              #                          'signature_inception': 1398093503,
                              #                          'signers_name': 'getdnsapi.org.',
                              #                          'type_covered': 2},
                              #                'ttl': 450,
                              #                'type': 46}],
                              'canonical_name': '_443._tcp.getdnsapi.org.',
                              'dnssec_status': 400,
                              'header': {'aa': 0,
                                         'ad': 1,
                                         'ancount': 2,
                                         'arcount': 0,
                                         'cd': 0,
                                         'id': 0,
                                         'nscount': 4,
                                         'opcode': 0,
                                         'qdcount': 1,
                                         'qr': 1,
                                         'ra': 1,
                                         'rcode': 0,
                                         'rd': 1,
                                         'tc': 0,
                                         'z': 0},
                              'question': {'qclass': 1,
                                           'qname': '_443._tcp.getdnsapi.org.',
                                           'qtype': 52}}],
            'status': status}
<commit_msg>Fix remaining tests<commit_after>import binascii
from OpenSSL import crypto
from twisted.trial.unittest import SynchronousTestCase
import getdns
from tlsep import _dane

class TLSADomainNameTests(SynchronousTestCase):
    def test_tlsaDomainName(self):
        """
        L{_dane.tlsaDomainName} returns the port, proto and parent domain as
        labels of a new domain name string.
        """
        self.assertEqual(
            "_443._tcp.example.com",
            _dane.tlsaDomainName('example.com', 443, 'tcp')
        )

class FakeGetdns(object):
    """
    An in memory fake of the getdns api for testing.
    """

    def __init__(self, generalResult=None):
        self._generalResult = generalResult

        for k, v in getdns.__dict__.items():
            if k.startswith('GETDNS_'):
                setattr(self, k, v)

    def context_create(self):
        """
        """

    def general(self, context, name, request_type, extensions):
        """
        """
        return self._generalResult

class TLSATests(SynchronousTestCase):
    def test_tlsaCert(self):
        """
        L{_dane.lookup_tlsa_records} returns a L{_dane.TLSARecord} instance if
        the domain name exists and a verified record is found and the record
        selector type is CERT.
        """
        fakeGetdns = FakeGetdns(
            generalResult=createResults(status=getdns.GETDNS_RESPSTATUS_GOOD,
                                        selector=_dane.SELECTOR.CERT.value,
                                        certificate_association_data=b'FOOBAR'))
        res = _dane.lookup_tlsa_records(
            'example.com', 443, 'tcp', getdns=fakeGetdns)
        self.assertEqual(
            (_dane.SELECTOR.CERT, b'FOOBAR'),
            (res.selector, res.payload)
        )

    def test_tlsaSPKI(self):
        """
        L{_dane.lookup_tlsa_records} returns a L{_dane.TLSARecord} instance if
        the domain name exists and a verfied record is found and the record
        selector type is SPKI.
        """
        fakeGetdns = FakeGetdns(
            generalResult=createResults(status=getdns.GETDNS_RESPSTATUS_GOOD,
                                        selector=_dane.SELECTOR.SPKI.value,
                                        certificate_association_data=b'FOOBAR'))
        res = _dane.lookup_tlsa_records(
            'example.com', 443, 'tcp', getdns=fakeGetdns)
        self.assertEqual(
            (_dane.SELECTOR.SPKI, b'FOOBAR'),
            (res.selector, res.payload)
        )

    def test_tlsaNoname(self):
        """
        L{_dane.lookup_tlsa_records} raises LookupError if the domain name does
        not exist.
        """
        e = self.assertRaises(
            _dane.GetdnsResponseError,
            _dane.lookup_tlsa_records, 'example.com', 443, 'tcp',
            getdns=FakeGetdns(
                generalResult=createResults(
                    status=getdns.GETDNS_RESPSTATUS_NO_NAME
                )
            )
        )
        self.assertEqual(
            getdns.GETDNS_RESPSTATUS_NO_NAME,
            e.errorCode
        )

def createResults(status=getdns.GETDNS_RESPSTATUS_GOOD,
                  selector=None,
                  certificate_association_data=b"",):
    return {'answer_type': 800,
            'canonical_name': '_443._tcp.getdnsapi.org.',
            'just_address_answers': [],
            # 'replies_full': [<read-only buffer ptr 0x7fe2e0029e80, size 636 at 0x7fe2e4e58fb0>],
            'replies_tree': [{'answer': [{'class': 1,
                                          'name': '_443._tcp.getdnsapi.org.',
                                          'rdata': {
                                              'certificate_association_data': certificate_association_data,
                                              'certificate_usage': 3,
                                              'matching_type': 1,
                                              # 'rdata_raw': "",
                                              'selector': selector
                                          },
                                          # 'ttl': 450,
                                          # 'type': 52},
                                         # {'class': 1,
                                         #  'name': '_443._tcp.getdnsapi.org.',
                                         #  'rdata': {'algorithm': 7,
                                         #            'key_tag': 49262,
                                         #            'labels': 4,
                                         #            'original_ttl': 450,
                                         #            'rdata_raw': <read-only buffer ptr 0x7fe2e0261b70, size 161 at 0x7fe2e4e60130>,
                                         #            'signature': <read-only buffer ptr 0x7fe2e0254c40, size 128 at 0x7fe2e4e60170>,
                                         #            'signature_expiration': 1399325172,
                                         #            'signature_inception': 1398100703,
                                         #            'signers_name': 'getdnsapi.org.',
                                         #            'type_covered': 52},
                                         #  'ttl': 450,
                                         #  'type': 46
                                      }
                                     ],
                               'answer_type': 800,
                              # 'authority': [{'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'nsdname': 'ns.secret-wg.org.',
                              #                          'rdata_raw': 'ns.secret-wg.org.'},
                              #                'ttl': 450,
                              #                'type': 2},
                              #               {'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'nsdname': 'mcvax.nlnetlabs.nl.',
                              #                          'rdata_raw': 'mcvax.nlnetlabs.nl.'},
                              #                'ttl': 450,
                              #                'type': 2},
                              #               {'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'nsdname': 'open.nlnetlabs.nl.',
                              #                          'rdata_raw': 'open.nlnetlabs.nl.'},
                              #                'ttl': 450,
                              #                'type': 2},
                              #               {'class': 1,
                              #                'name': 'getdnsapi.org.',
                              #                'rdata': {'algorithm': 7,
                              #                          'key_tag': 49262,
                              #                          'labels': 2,
                              #                          'original_ttl': 450,
                              #                          'rdata_raw': <read-only buffer ptr 0x7fe2e0261f90, size 161 at 0x7fe2e4e601f0>,
                              #                          'signature': <read-only buffer ptr 0x7fe2e0028120, size 128 at 0x7fe2e4e60230>,
                              #                          'signature_expiration': 1399278072,
                              #                          'signature_inception': 1398093503,
                              #                          'signers_name': 'getdnsapi.org.',
                              #                          'type_covered': 2},
                              #                'ttl': 450,
                              #                'type': 46}],
                              'canonical_name': '_443._tcp.getdnsapi.org.',
                              'dnssec_status': 400,
                              'header': {'aa': 0,
                                         'ad': 1,
                                         'ancount': 2,
                                         'arcount': 0,
                                         'cd': 0,
                                         'id': 0,
                                         'nscount': 4,
                                         'opcode': 0,
                                         'qdcount': 1,
                                         'qr': 1,
                                         'ra': 1,
                                         'rcode': 0,
                                         'rd': 1,
                                         'tc': 0,
                                         'z': 0},
                              'question': {'qclass': 1,
                                           'qname': '_443._tcp.getdnsapi.org.',
                                           'qtype': 52}}],
            'status': status}
<|endoftext|>


<commit_before>VERSION = '0.10-SNAPSHOT'
<commit_msg>v0.9.1<commit_after>#0.9
VERSION = '0.9.1'
<|endoftext|>


<commit_before>import asyncio
import traceback
from functools import partial
from httptools import HttpParserUpgrade
from httptools import HttpRequestParser, parse_url
from httptools.parser.errors import HttpParserError

from mach9.exceptions import (
    RequestTimeout, PayloadTooLarge, InvalidUsage, ServerError)
from mach9.response import ALL_STATUS_CODES
from mach9.websocket import upgrade_to_websocket

class BodyChannel(asyncio.Queue):

    def __init__(self, transport):
        super().__init__()
        self.transport = transport

    def send(self, body_chunk):
        return self.put(body_chunk)

    def receive(self):
        return self.get()

class ReplyChannel:

    def __init__(self, protocol):
        super().__init__()
        self._protocol = protocol

    async def send(self, message):
        self._protocol.send(message)

class HttpProtocol(asyncio.Protocol):
    __slots__ = (
        # exceptions
        '_request_timeout', '_payload_too_large', '_invalid_usage',
        '_server_error',
        # event loop, connection
        'loop', 'transport', 'connections', 'signal',
        # request params
        'parser', 'url', 'headers',
        # request config
        'request_handler', 'request_timeout', 'request_max_size',
        'request_class',
        # enable or disable access log / error log purpose
        'has_log', 'log', 'netlog',
        # connection management
        '_is_upgrade', '_tasks',
        '_total_request_size', '_timeout_handler', '_last_request_time',
        'get_current_time', 'body_channel', 'message')

    def __init__(self, *, loop, request_handler, error_handler, log=None,
                 signal=None, connections=set(), request_timeout=60,
                 request_max_size=None, has_log=True,
                 keep_alive=True, request_class=None,
                 netlog=None, get_current_time=None, _request_timeout=None,
                 _payload_too_large=None, _invalid_usage=None,
                 _server_error=None):
        '''signal is shared'''
        self.loop = loop
        self.transport = None
        self.parser = None
        self.url = None
        self.headers = None
        self.body_channel = None
        self.message = None
        self.signal = signal
        self.has_log = has_log
        self.log = log
        self.netlog = netlog
        self.connections = connections
        self.request_handler = request_handler
        self.request_class = request_class
        self.error_handler = error_handler
        self.request_timeout = request_timeout
        self.request_max_size = request_max_size
        self.get_current_time = get_current_time
        self._total_request_size = 0
        self._timeout_handler = None
        self._last_request_time = None
        # for task management
        self._tasks = []
        self._is_upgrade = False
        # config.KEEP_ALIVE or not check_headers()['connection_close']
        self._keep_alive = keep_alive
        self._request_timeout = _request_timeout or RequestTimeout
        self._payload_too_large = _payload_too_large or PayloadTooLarge
        self._invalid_usage = _invalid_usage or InvalidUsage
        self._server_error = _server_error or ServerError

    @property
    def keep_alive(self):
        return (self._keep_alive
                and not self.signal.stopped
                and self.parser
                and self.parser.should_keep_alive())

    def cancel_tasks(self):
        for t in self._tasks:
            t.cancel()
        self._tasks = []

    # -------------------------------------------- #
    # Connection
    # -------------------------------------------- #

    def connection_made(self, transport):
        self.connections.add(self)
        self._timeout_handler = self.loop.call_later(
            self.request_timeout, self.connection_timeout)
        self.transport = transport
        self._last_request_time = self.get_current_time()

    def connection_lost(self, exc):
        self.connections.discard(self)
        self._timeout_handler.cancel()
        self.cancel_tasks()

    def connection_timeout(self):
        # Check if
        time_elapsed = self.get_current_time() - self._last_request_time
        if time_elapsed < self.request_timeout:
            time_left = self.request_timeout - time_elapsed
            self._timeout_handler = (
                self.loop.call_later(time_left, self.connection_timeout))
        else:
            self.cancel_tasks()
            exception = self._request_timeout('Request Timeout')
            self.write_error(exception)

    # -------------------------------------------- #
    # Parsing
    # -------------------------------------------- #

    def data_received(self, data: bytes):
        # Check for the request itself getting too large and exceeding
        # memory limits
        self._total_request_size += len(data)
        if self._total_request_size > self.request_max_size:
            exception = self._payload_too_large('Payload Too Large')
            self.write_error(exception)

        # Create parser if this is the first time we're receiving data
        if self.parser is None:
            self.headers = []
            self.parser = HttpRequestParser(self)

        # Parse request chunk or close connection
        try:
            self.parser.feed_data(data)
        except HttpParserUpgrade:
            upgrade_to_websocket(self)
        except HttpParserError:
            exception = self._invalid_usage('Bad Request')
            self.write_error(exception)

    def on_url(self, url: bytes):
        self.url = url

    def on_header(self, name: bytes, value: bytes):
        # for websocket
        name = name.lower()
        if name == b'content-length' and int(value) > self.request_max_size:
            exception = self._payload_too_large('Payload Too Large')
            self.write_error(exception)
        if name == b'upgrade':
            self._is_upgrade = True
        self.headers.append([name, value])

    def register_task(self, coro):
        task = self.loop.create_task(coro)
        self._tasks.append(task)

        def pop_task(self, task):
            try:
                self._tasks.remove(task)
            except ValueError:
                pass
        task.add_done_callback(partial(pop_task, self))

    def on_headers_complete(self):
        if self._is_upgrade:
            return
        channels = {}
        self.message = self.get_message(
            self.transport, self.parser.get_http_version(),
            self.parser.get_method(), self.url, self.headers)
        channels['body'] = BodyChannel(self.transport)
        channels['reply'] = ReplyChannel(self)
        self.body_channel = channels['body']
        self.register_task(self.request_handler(self.message, channels))

    def on_body(self, body: bytes):
        if self._is_upgrade:
            return
        body_chunk = self.get_request_body_chunk(body, False, True)
        self.register_task(self.body_channel.send(body_chunk))

    def on_message_complete(self):
        if self._is_upgrade:
            return
        body_chunk = self.get_request_body_chunk(b'', False, False)
        self.register_task(self.body_channel.send(body_chunk))

    def get_request_body_chunk(self, content: bytes, closed: bool,
                               more_content: bool) -> dict:
        '''
        http://channels.readthedocs.io/en/stable/asgi/www.html#request-body-chunk
        '''
        return {
            'content': content,
            'closed': closed,
            'more_content': more_content
        }

    def get_message(self, transport, http_version: str, method: bytes,
                    url: bytes, headers: list) -> dict:
        '''
        http://channels.readthedocs.io/en/stable/asgi/www.html#request
        '''
        url_obj = parse_url(url)
        if url_obj.schema is None:
            if transport.get_extra_info('sslcontext'):
                scheme = 'https'
            else:
                scheme = 'http'
        else:
            scheme = url_obj.schema.decode()
        path = '' if url_obj.path is None else url_obj.path.decode('utf-8')
        query = b'' if url_obj.query is None else url_obj.query
        return {
            'channel': 'http.request',
            'reply_channel': None,
            'http_version': http_version,
            'method': method.decode(),
            'scheme': scheme,
            'path': path,
            'query_string': query,
            'root_path': '',
            'headers': headers,
            'body': b'',
            'body_channel': None,
            'client': transport.get_extra_info('peername'),
            'server': transport.get_extra_info('sockname')
        }

    def check_headers(self, headers: list) -> dict:
        connection_close = False
        content_length = False
        for key, value in headers:
            if key == b'Connection' and value == b'close':
                connection_close = True
            if key == b'Content-Length':
                content_length = True
        return {
            'connection_close': connection_close,
            'content_length': content_length
        }

    def after_write(self, more_content, keep_alive):
        if not more_content and not keep_alive:
            self.transport.close()
        elif not more_content and keep_alive:
            self._last_request_time = self.get_current_time()
            self.cleanup()

    def is_response_chunk(self, message):
        return 'status' not in message and 'headers' not in message

    def make_header_content(self, headers, result_headers,
                            content, more_content):
        header_content = b''
        if headers is not None:
            _header_content = []
            if not more_content and not result_headers['content_length']:
                _header_content.extend([b'Content-Length: ',
                                        str(len(content)).encode(), b'\r\n'])
            for key, value in headers:
                if key == b'Connection':
                    continue
                _header_content.extend([key, b': ', value, b'\r\n'])
            header_content = b''.join(_header_content)
        return header_content

    def send(self, message):
        transport = self.transport
        status = message.get('status')
        headers = message.get('headers')
        content = message.get('content')
        more_content = message.get('more_content', False)
        if headers is not None:
            result_headers = self.check_headers(headers)
            if result_headers['connection_close'] is True:
                self._keep_alive = False
        keep_alive = self.keep_alive

        if self.is_response_chunk(message):
            content_length = len(content)
            if more_content and content_length > 0:
                transport.write(b'%x\r\n%b\r\n' % (content_length, content))
                self.after_write(more_content, keep_alive)
            elif more_content is False:
                transport.write(b'0\r\n\r\n')
                self.after_write(more_content, keep_alive)
            return

        keep_alive_timeout = self.request_timeout
        timeout_header = b''
        if keep_alive and keep_alive_timeout is not None:
            timeout_header = b'Keep-Alive: %d\r\n' % keep_alive_timeout

        header_content = self.make_header_content(headers, result_headers,
                                                  content, more_content)

        response = (
            b'HTTP/1.1 %d %b\r\n'
            b'Connection: %b\r\n'
            b'%b'
            b'%b\r\n'
            b'%b') % (
                status,
                ALL_STATUS_CODES[status],
                b'keep-alive' if keep_alive else b'close',
                timeout_header,
                header_content,
                content
            )
        transport.write(response)
        self.after_write(more_content, keep_alive)

    def write_error(self, exception):
        try:
            request = None
            if self.message:
                request = self.request_class(self.message)
            response = self.error_handler.response(request, exception)
            self.send(response.get_message(False))
            if self.has_log:
                extra = {
                    'status': response.status,
                    'host': '',
                    'request': str(request) + str(self.url)
                }
                if response and hasattr(response, 'body'):
                    extra['byte'] = len(response.body)
                else:
                    extra['byte'] = -1
                if request:
                    extra['host'] = '%s:%d' % request.ip,
                    extra['request'] = '%s %s' % (request.method,
                                                  self.url)
                self.netlog.info('', extra=extra)
        except RuntimeError:
            self.log.error(
                'Connection lost before error written @ {}'.format(
                    request.ip if self.request else 'Unknown'))
        except Exception as e:
            self.bail_out(
                'Writing error failed, connection closed {}'.format(repr(e)),
                from_error=True)
        finally:
            self.transport.close()

    def bail_out(self, message, from_error=False):
        if from_error or self.transport.is_closing():
            self.log.error(
                ('Transport closed @ {} and exception '
                 'experienced during error handling').format(
                    self.transport.get_extra_info('peername')))
            self.log.debug(
                'Exception:\n{}'.format(traceback.format_exc()))
        else:
            exception = self._server_error(message)
            self.write_error(exception)
            self.log.error(message)

    def cleanup(self):
        self.cancel_tasks()
        self.parser = None
        self.url = None
        self.headers = None
        self._total_request_size = 0
        self.body_channel = None
        self.message = None

    def close_if_idle(self):
        '''Close the connection if a request is not being sent or received

        :return: boolean - True if closed, false if staying open
        '''
        if not self.parser:
            self.transport.close()
            return True
        return False
<commit_msg>Revert (#40)<commit_after>import asyncio
import traceback
from httptools import HttpParserUpgrade
from httptools import HttpRequestParser, parse_url
from httptools.parser.errors import HttpParserError

from mach9.exceptions import (
    RequestTimeout, PayloadTooLarge, InvalidUsage, ServerError)
from mach9.response import ALL_STATUS_CODES
from mach9.websocket import upgrade_to_websocket

class BodyChannel(asyncio.Queue):

    def __init__(self, transport):
        super().__init__()
        self.transport = transport

    def send(self, body_chunk):
        return self.put(body_chunk)

    def receive(self):
        return self.get()

class ReplyChannel:

    def __init__(self, protocol):
        super().__init__()
        self._protocol = protocol

    async def send(self, message):
        self._protocol.send(message)

class HttpProtocol(asyncio.Protocol):
    __slots__ = (
        # exceptions
        '_request_timeout', '_payload_too_large', '_invalid_usage',
        '_server_error',
        # event loop, connection
        'loop', 'transport', 'connections', 'signal',
        # request params
        'parser', 'url', 'headers',
        # request config
        'request_handler', 'request_timeout', 'request_max_size',
        'request_class',
        # enable or disable access log / error log purpose
        'has_log', 'log', 'netlog',
        # connection management
        '_is_upgrade',
        '_total_request_size', '_timeout_handler', '_last_request_time',
        'get_current_time', 'body_channel', 'message')

    def __init__(self, *, loop, request_handler, error_handler, log=None,
                 signal=None, connections=set(), request_timeout=60,
                 request_max_size=None, has_log=True,
                 keep_alive=True, request_class=None,
                 netlog=None, get_current_time=None, _request_timeout=None,
                 _payload_too_large=None, _invalid_usage=None,
                 _server_error=None):
        '''signal is shared'''
        self.loop = loop
        self.transport = None
        self.parser = None
        self.url = None
        self.headers = None
        self.body_channel = None
        self.message = None
        self.signal = signal
        self.has_log = has_log
        self.log = log
        self.netlog = netlog
        self.connections = connections
        self.request_handler = request_handler
        self.request_class = request_class
        self.error_handler = error_handler
        self.request_timeout = request_timeout
        self.request_max_size = request_max_size
        self.get_current_time = get_current_time
        self._total_request_size = 0
        self._timeout_handler = None
        self._last_request_time = None
        self._request_handler_task = None
        self._request_stream_task = None
        self._is_upgrade = False
        # config.KEEP_ALIVE or not check_headers()['connection_close']
        self._keep_alive = keep_alive
        self._request_timeout = _request_timeout or RequestTimeout
        self._payload_too_large = _payload_too_large or PayloadTooLarge
        self._invalid_usage = _invalid_usage or InvalidUsage
        self._server_error = _server_error or ServerError

    @property
    def keep_alive(self):
        return (self._keep_alive
                and not self.signal.stopped
                and self.parser
                and self.parser.should_keep_alive())

    # -------------------------------------------- #
    # Connection
    # -------------------------------------------- #

    def connection_made(self, transport):
        self.connections.add(self)
        self._timeout_handler = self.loop.call_later(
            self.request_timeout, self.connection_timeout)
        self.transport = transport
        self._last_request_time = self.get_current_time()

    def connection_lost(self, exc):
        self.connections.discard(self)
        self._timeout_handler.cancel()

    def connection_timeout(self):
        # Check if
        time_elapsed = self.get_current_time() - self._last_request_time
        if time_elapsed < self.request_timeout:
            time_left = self.request_timeout - time_elapsed
            self._timeout_handler = (
                self.loop.call_later(time_left, self.connection_timeout))
        else:
            if self._request_stream_task:
                self._request_stream_task.cancel()
            if self._request_handler_task:
                self._request_handler_task.cancel()
            exception = self._request_timeout('Request Timeout')
            self.write_error(exception)

    # -------------------------------------------- #
    # Parsing
    # -------------------------------------------- #

    def data_received(self, data: bytes):
        # Check for the request itself getting too large and exceeding
        # memory limits
        self._total_request_size += len(data)
        if self._total_request_size > self.request_max_size:
            exception = self._payload_too_large('Payload Too Large')
            self.write_error(exception)

        # Create parser if this is the first time we're receiving data
        if self.parser is None:
            self.headers = []
            self.parser = HttpRequestParser(self)

        # Parse request chunk or close connection
        try:
            self.parser.feed_data(data)
        except HttpParserUpgrade:
            upgrade_to_websocket(self)
        except HttpParserError:
            exception = self._invalid_usage('Bad Request')
            self.write_error(exception)

    def on_url(self, url: bytes):
        self.url = url

    def on_header(self, name: bytes, value: bytes):
        # for websocket
        name = name.lower()
        if name == b'content-length' and int(value) > self.request_max_size:
            exception = self._payload_too_large('Payload Too Large')
            self.write_error(exception)
        if name == b'upgrade':
            self._is_upgrade = True
        self.headers.append([name, value])

    def on_headers_complete(self):
        if self._is_upgrade:
            return
        channels = {}
        self.message = self.get_message(
            self.transport, self.parser.get_http_version(),
            self.parser.get_method(), self.url, self.headers)
        channels['body'] = BodyChannel(self.transport)
        channels['reply'] = ReplyChannel(self)
        self.body_channel = channels['body']
        self._request_handler_task = self.loop.create_task(
            self.request_handler(self.message, channels))

    def on_body(self, body: bytes):
        if self._is_upgrade:
            return
        body_chunk = self.get_request_body_chunk(body, False, True)
        self._request_stream_task = self.loop.create_task(
            self.body_channel.send(body_chunk))

    def on_message_complete(self):
        if self._is_upgrade:
            return
        body_chunk = self.get_request_body_chunk(b'', False, False)
        self._request_stream_task = self.loop.create_task(
            self.body_channel.send(body_chunk))

    def get_request_body_chunk(self, content: bytes, closed: bool,
                               more_content: bool) -> dict:
        '''
        http://channels.readthedocs.io/en/stable/asgi/www.html#request-body-chunk
        '''
        return {
            'content': content,
            'closed': closed,
            'more_content': more_content
        }

    def get_message(self, transport, http_version: str, method: bytes,
                    url: bytes, headers: list) -> dict:
        '''
        http://channels.readthedocs.io/en/stable/asgi/www.html#request
        '''
        url_obj = parse_url(url)
        if url_obj.schema is None:
            if transport.get_extra_info('sslcontext'):
                scheme = 'https'
            else:
                scheme = 'http'
        else:
            scheme = url_obj.schema.decode()
        path = '' if url_obj.path is None else url_obj.path.decode('utf-8')
        query = b'' if url_obj.query is None else url_obj.query
        return {
            'channel': 'http.request',
            'reply_channel': None,
            'http_version': http_version,
            'method': method.decode(),
            'scheme': scheme,
            'path': path,
            'query_string': query,
            'root_path': '',
            'headers': headers,
            'body': b'',
            'body_channel': None,
            'client': transport.get_extra_info('peername'),
            'server': transport.get_extra_info('sockname')
        }

    def check_headers(self, headers: list) -> dict:
        connection_close = False
        content_length = False
        for key, value in headers:
            if key == b'Connection' and value == b'close':
                connection_close = True
            if key == b'Content-Length':
                content_length = True
        return {
            'connection_close': connection_close,
            'content_length': content_length
        }

    def after_write(self, more_content, keep_alive):
        if not more_content and not keep_alive:
            self.transport.close()
        elif not more_content and keep_alive:
            self._last_request_time = self.get_current_time()
            self.cleanup()

    def is_response_chunk(self, message):
        return 'status' not in message and 'headers' not in message

    def make_header_content(self, headers, result_headers,
                            content, more_content):
        header_content = b''
        if headers is not None:
            _header_content = []
            if not more_content and not result_headers['content_length']:
                _header_content.extend([b'Content-Length: ',
                                        str(len(content)).encode(), b'\r\n'])
            for key, value in headers:
                if key == b'Connection':
                    continue
                _header_content.extend([key, b': ', value, b'\r\n'])
            header_content = b''.join(_header_content)
        return header_content

    def send(self, message):
        transport = self.transport
        status = message.get('status')
        headers = message.get('headers')
        content = message.get('content')
        more_content = message.get('more_content', False)
        if headers is not None:
            result_headers = self.check_headers(headers)
            if result_headers['connection_close'] is True:
                self._keep_alive = False
        keep_alive = self.keep_alive

        if self.is_response_chunk(message):
            content_length = len(content)
            if more_content and content_length > 0:
                transport.write(b'%x\r\n%b\r\n' % (content_length, content))
                self.after_write(more_content, keep_alive)
            elif more_content is False:
                transport.write(b'0\r\n\r\n')
                self.after_write(more_content, keep_alive)
            return

        keep_alive_timeout = self.request_timeout
        timeout_header = b''
        if keep_alive and keep_alive_timeout is not None:
            timeout_header = b'Keep-Alive: %d\r\n' % keep_alive_timeout

        header_content = self.make_header_content(headers, result_headers,
                                                  content, more_content)

        response = (
            b'HTTP/1.1 %d %b\r\n'
            b'Connection: %b\r\n'
            b'%b'
            b'%b\r\n'
            b'%b') % (
                status,
                ALL_STATUS_CODES[status],
                b'keep-alive' if keep_alive else b'close',
                timeout_header,
                header_content,
                content
            )
        transport.write(response)
        self.after_write(more_content, keep_alive)

    def write_error(self, exception):
        try:
            request = None
            if self.message:
                request = self.request_class(self.message)
            response = self.error_handler.response(request, exception)
            self.send(response.get_message(False))
            if self.has_log:
                extra = {
                    'status': response.status,
                    'host': '',
                    'request': str(request) + str(self.url)
                }
                if response and hasattr(response, 'body'):
                    extra['byte'] = len(response.body)
                else:
                    extra['byte'] = -1
                if request:
                    extra['host'] = '%s:%d' % request.ip,
                    extra['request'] = '%s %s' % (request.method,
                                                  self.url)
                self.netlog.info('', extra=extra)
        except RuntimeError:
            self.log.error(
                'Connection lost before error written @ {}'.format(
                    request.ip if self.request else 'Unknown'))
        except Exception as e:
            self.bail_out(
                'Writing error failed, connection closed {}'.format(repr(e)),
                from_error=True)
        finally:
            self.transport.close()

    def bail_out(self, message, from_error=False):
        if from_error or self.transport.is_closing():
            self.log.error(
                ('Transport closed @ {} and exception '
                 'experienced during error handling').format(
                    self.transport.get_extra_info('peername')))
            self.log.debug(
                'Exception:\n{}'.format(traceback.format_exc()))
        else:
            exception = self._server_error(message)
            self.write_error(exception)
            self.log.error(message)

    def cleanup(self):
        self.parser = None
        self.url = None
        self.headers = None
        self._request_handler_task = None
        self._request_stream_task = None
        self._total_request_size = 0
        self.body_channel = None
        self.message = None

    def close_if_idle(self):
        '''Close the connection if a request is not being sent or received

        :return: boolean - True if closed, false if staying open
        '''
        if not self.parser:
            self.transport.close()
            return True
        return False
<|endoftext|>


<commit_before>from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.core.paginator import Paginator
from django.http import HttpResponse, HttpResponseRedirect
from urllib import unquote
from haystack.query import SearchQuerySet
from mail.models import *
from django.db.models import Q
from django.core.urlresolvers import reverse
from django.core.cache import cache
import re

RESULTS_PER_PAGE = 50

def _search_string(request):
    return request.GET.get('q', None)

def _search_tokens(request):
    s = _search_string(request)
    if s is None:
        return []

    # protection!
    re_sanitize = re.compile(r'[^\w\d\s\'"\,\.\?\$]', re.I)
    s = re_sanitize.sub('', s)

    tokens = []
    re_quotes = re.compile(r'\"([^\"]+)\"')
    for m in re_quotes.findall(s):
        tokens.append(m.replace('"','').strip())
        s = s.replace('"%s"' % m, '')
    for t in s.split(' '):
        tokens.append(t.strip())

    while '' in tokens:
        tokens.remove('')

    return tokens

def _highlight(text, tokens):
    regexes = []
    sorted_tokens = sorted(tokens, key=lambda x: len(x))
    for t in sorted_tokens:
        regexes.append(re.compile(r'(%s)' % t.replace(' ', r'\s+'), re.I))
    for r in regexes:
        text = r.sub('\\1', text)
    return text

def _prepare_ids_from_cookie(request, cookie_name):
    cookie = unquote(request.COOKIES.get(cookie_name,'')).replace(',,', ',')
    if len(cookie)>1:
        if cookie[0]==',':
            cookie = cookie[1:]
        if cookie[-1]==',':
            cookie = cookie[:-1]

    try:
        id_list = map(lambda x: (x!='') and int(x) or 0, cookie.split(','))
    except:
        id_list = []

    return id_list

def _annotate_emails(emails, search=[]):
    r = []
    for email in emails:
        email.text = _highlight(email.text, search)
        r.append({ 'creator_html': email.creator_html(), 'to_html': email.to_html(), 'cc_html': email.cc_html(), 'obj': email })
    return r

def index(request, search=[], threads=None):

    if threads is None:
        kagan = Person.objects.elena_kagan()
        threads = Thread.objects.exclude(creator=kagan).order_by('-date')

    threads_count = threads.count()

    p = Paginator(threads, RESULTS_PER_PAGE)

    page_num = 1
    try:
        page_num = int(request.GET.get('page', 1))
    except:
        pass
    page = p.page(page_num)

    highlighted_threads = []
    for thread in page.object_list:
        if (threads is not None) and type(threads) is SearchQuerySet: # deal with searchqueryset objects
            thread = thread.object

        thread.name = _highlight(thread.name, search)
        highlighted_threads.append(thread)

    template_vars = {
        'range': "%d - %d of %d" % (page.start_index(), page.end_index(), threads_count),
        'num_pages': p.num_pages ,
        'next': page_num
1 and max(1, page_num-1) or False,
        'first': '1',
        'last': p.num_pages,
        'current_page': page_num,
        'threads': highlighted_threads,
        'search': " ".join(search),
        'search_orig': (_search_string(request) is not None) and _search_string(request) or '',
        'path': request.path,
    }

    return render_to_response('index.html', template_vars, context_instance=RequestContext(request))

def sent(request):
    kagan = Person.objects.elena_kagan()
    emails = Thread.objects.filter(creator=kagan).order_by('-date')
    return index(request, threads=emails)

def contact_by_id(request, contact_id, suppress_redirect=False):
    cache_key = 'contact_%d' % int(contact_id)
    threads = cache.get(cache_key)
    if threads is None:
        try:
            person = Person.objects.get(id=contact_id)
        except Person.DoesNotExist, e:
            return HttpResponseRedirect(reverse('mail.views.index'))

        if person.merged_into is not None:
            return HttpResponseRedirect('/contact/%d/' % person.merged_into.id)

        threads = []
        emails = Email.objects.filter(Q(to=person)|Q(cc=person))
        for e in emails:
            if e.email_thread is not None:
                threads.append(e.email_thread.id)
        threads = Thread.objects.filter(id__in=threads).order_by('-date')

        cache.set(cache_key, threads)

    return index(request, threads=threads)

def contact_by_name(request, contact_name):
    try:
        contact = Person.objects.get(slug=contact_name)
    except Person.DoesNotExist, e:
        return HttpResponseRedirect(reverse('mail.views.contacts_index'))
    except Thread.MultipleObjectsReturned, e:
        return HttpResponseRedirect(reverse('mail.views.contacts_index'))

    return contact_by_id(request, contact.id, suppress_redirect=True)

def contacts_index(request):
    return index(request)

def thread_by_id(request, thread_id, suppress_redirect=False):
    try:
        thread = Thread.objects.get(id=thread_id)
    except Thread.DoesNotExist, e:
        return HttpResponseRedirect(reverse('mail.views.index'))

    # if merged thread, redirect
    if thread.merged_into is not None:
        return HttpResponseRedirect('/thread/%d/' % thread.merged_into.id)

    # if access by ID, redirect to descriptive URL
    if (not suppress_redirect) and (len(thread.slug.strip())>3):
        return HttpResponseRedirect('/thread/%s/' % thread.slug)

    search = _search_tokens(request)
    thread_starred = thread.id in _prepare_ids_from_cookie(request, 'kagan_star')
    emails = _annotate_emails(Email.objects.filter(email_thread=thread).order_by('creation_date_time'), search)

    return render_to_response('thread.html', {'thread': thread, 'thread_starred': thread_starred, 'emails': emails }, context_instance=RequestContext(request))

def thread_by_name(request, thread_name):
    try:
        thread = Thread.objects.get(slug=thread_name)
    except Thread.DoesNotExist, e:
        return HttpResponseRedirect(reverse('mail.views.index'))
    except Thread.MultipleObjectsReturned, e:
        return HttpResponseRedirect(reverse('mail.views.index'))

    return thread_by_id(request, thread.id, suppress_redirect=True)

def search(request):
    tokens = _search_tokens(request)
    if len(tokens) is None:
        return HttpResponseRedirect(reverse('mail.views.index'))

    sqs = SearchQuerySet().models(Thread)
    for t in tokens:
        sqs = sqs.filter_or(text_and_recipients=t)
    sqs = sqs.order_by('-date')

    if sqs.count()==0:
        return render_to_response('search_empty.html', { 'path': request.path }, context_instance=RequestContext(request))

    return index(request, search=tokens, threads=sqs)

def star_record_ajax(request, thread_id, action):
    try:
        thread = Thread.objects.get(id=thread_id)
    except Thread.DoesNotExist, e:
        return HttpResponse('{ status: \'not_found\'}');

    if thread.star_count is None:
        thread.star_count = 0
    if action=='add':
        thread.star_count += 1
    elif action=='remove':
        thread.star_count -= 1
    thread.save()

    return HttpResponse('{ status: \'success\'}')

def starred(request):
    starred_ids = _prepare_ids_from_cookie(request, 'kagan_star')
    if len(starred_ids)==0:
        return HttpResponseRedirect(reverse('mail.views.index'))

    starred = Thread.objects.filter(id__in=starred_ids).order_by('-date')

    if starred.count()==0:
        return render_to_response('search_empty.html', { 'path': request.path }, context_instance=RequestContext(request))
    else:
        return index(request, threads=starred)

    return index(request, threads=starred)

def starred_all(request):
    starred = Thread.objects.filter(star_count__gt=0).order_by('-star_count','-date')
    if starred.count()==0:
        return render_to_response('search_empty.html', { 'path': request.path }, context_instance=RequestContext(request))
    else:
        return index(request, threads=starred)
<commit_msg>Change views to show Palin e-mails<commit_after>from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.core.paginator import Paginator
from django.http import HttpResponse, HttpResponseRedirect
from urllib import unquote
from haystack.query import SearchQuerySet
from mail.models import *
from django.db.models import Q
from django.core.urlresolvers import reverse
from django.core.cache import cache
import re

RESULTS_PER_PAGE = 50

def _search_string(request):
    return request.GET.get('q', None)

def _search_tokens(request):
    s = _search_string(request)
    if s is None:
        return []

    # protection!
    re_sanitize = re.compile(r'[^\w\d\s\'"\,\.\?\$]', re.I)
    s = re_sanitize.sub('', s)

    tokens = []
    re_quotes = re.compile(r'\"([^\"]+)\"')
    for m in re_quotes.findall(s):
        tokens.append(m.replace('"','').strip())
        s = s.replace('"%s"' % m, '')
    for t in s.split(' '):
        tokens.append(t.strip())

    while '' in tokens:
        tokens.remove('')

    return tokens

def _highlight(text, tokens):
    regexes = []
    sorted_tokens = sorted(tokens, key=lambda x: len(x))
    for t in sorted_tokens:
        regexes.append(re.compile(r'(%s)' % t.replace(' ', r'\s+'), re.I))
    for r in regexes:
        text = r.sub('\\1', text)
    return text

def _prepare_ids_from_cookie(request, cookie_name):
    cookie = unquote(request.COOKIES.get(cookie_name,'')).replace(',,', ',')
    if len(cookie)>1:
        if cookie[0]==',':
            cookie = cookie[1:]
        if cookie[-1]==',':
            cookie = cookie[:-1]

    try:
        id_list = map(lambda x: (x!='') and int(x) or 0, cookie.split(','))
    except:
        id_list = []

    return id_list

def _annotate_emails(emails, search=[]):
    r = []
    for email in emails:
        email.text = _highlight(email.text, search)
        r.append({ 'creator_html': email.creator_html(), 'to_html': email.to_html(), 'cc_html': email.cc_html(), 'obj': email })
    return r

def index(request, search=[], threads=None):

    if threads is None:
        palin = Person.objects.sarah_palin()
        threads = Thread.objects.exclude(creator__in=palin).order_by('-date')

    threads_count = threads.count()

    p = Paginator(threads, RESULTS_PER_PAGE)

    page_num = 1
    try:
        page_num = int(request.GET.get('page', 1))
    except:
        pass
    page = p.page(page_num)

    highlighted_threads = []
    for thread in page.object_list:
        if (threads is not None) and type(threads) is SearchQuerySet: # deal with searchqueryset objects
            thread = thread.object

        thread.name = _highlight(thread.name, search)
        highlighted_threads.append(thread)

    template_vars = {
        'range': "%d - %d of %d" % (page.start_index(), page.end_index(), threads_count),
        'num_pages': p.num_pages ,
        'next': page_num
1 and max(1, page_num-1) or False,
        'first': '1',
        'last': p.num_pages,
        'current_page': page_num,
        'threads': highlighted_threads,
        'search': " ".join(search),
        'search_orig': (_search_string(request) is not None) and _search_string(request) or '',
        'path': request.path,
    }

    return render_to_response('index.html', template_vars, context_instance=RequestContext(request))

def sent(request):
    kagan = Person.objects.elena_kagan()
    emails = Thread.objects.filter(creator=kagan).order_by('-date')
    return index(request, threads=emails)

def contact_by_id(request, contact_id, suppress_redirect=False):
    cache_key = 'contact_%d' % int(contact_id)
    threads = cache.get(cache_key)
    if threads is None:
        try:
            person = Person.objects.get(id=contact_id)
        except Person.DoesNotExist, e:
            return HttpResponseRedirect(reverse('mail.views.index'))

        if person.merged_into is not None:
            return HttpResponseRedirect('/contact/%d/' % person.merged_into.id)

        threads = []
        emails = Email.objects.filter(Q(to=person)|Q(cc=person))
        for e in emails:
            if e.email_thread is not None:
                threads.append(e.email_thread.id)
        threads = Thread.objects.filter(id__in=threads).order_by('-date')

        cache.set(cache_key, threads)

    return index(request, threads=threads)

def contact_by_name(request, contact_name):
    try:
        contact = Person.objects.get(slug=contact_name)
    except Person.DoesNotExist, e:
        return HttpResponseRedirect(reverse('mail.views.contacts_index'))
    except Thread.MultipleObjectsReturned, e:
        return HttpResponseRedirect(reverse('mail.views.contacts_index'))

    return contact_by_id(request, contact.id, suppress_redirect=True)

def contacts_index(request):
    return index(request)

def thread_by_id(request, thread_id, suppress_redirect=False):
    try:
        thread = Thread.objects.get(id=thread_id)
    except Thread.DoesNotExist, e:
        return HttpResponseRedirect(reverse('mail.views.index'))

    # if merged thread, redirect
    if thread.merged_into is not None:
        return HttpResponseRedirect('/thread/%d/' % thread.merged_into.id)

    # if access by ID, redirect to descriptive URL
    if (not suppress_redirect) and (len(thread.slug.strip())>3):
        return HttpResponseRedirect('/thread/%s/' % thread.slug)

    search = _search_tokens(request)
    thread_starred = thread.id in _prepare_ids_from_cookie(request, 'kagan_star')
    emails = _annotate_emails(Email.objects.filter(email_thread=thread).order_by('creation_date_time'), search)

    return render_to_response('thread.html', {'thread': thread, 'thread_starred': thread_starred, 'emails': emails }, context_instance=RequestContext(request))

def thread_by_name(request, thread_name):
    try:
        thread = Thread.objects.get(slug=thread_name)
    except Thread.DoesNotExist, e:
        return HttpResponseRedirect(reverse('mail.views.index'))
    except Thread.MultipleObjectsReturned, e:
        return HttpResponseRedirect(reverse('mail.views.index'))

    return thread_by_id(request, thread.id, suppress_redirect=True)

def search(request):
    tokens = _search_tokens(request)
    if len(tokens) is None:
        return HttpResponseRedirect(reverse('mail.views.index'))

    sqs = SearchQuerySet().models(Thread)
    for t in tokens:
        sqs = sqs.filter_or(text_and_recipients=t)
    sqs = sqs.order_by('-date')

    if sqs.count()==0:
        return render_to_response('search_empty.html', { 'path': request.path }, context_instance=RequestContext(request))

    return index(request, search=tokens, threads=sqs)

def star_record_ajax(request, thread_id, action):
    try:
        thread = Thread.objects.get(id=thread_id)
    except Thread.DoesNotExist, e:
        return HttpResponse('{ status: \'not_found\'}');

    if thread.star_count is None:
        thread.star_count = 0
    if action=='add':
        thread.star_count += 1
    elif action=='remove':
        thread.star_count -= 1
    thread.save()

    return HttpResponse('{ status: \'success\'}')

def starred(request):
    starred_ids = _prepare_ids_from_cookie(request, 'kagan_star')
    if len(starred_ids)==0:
        return HttpResponseRedirect(reverse('mail.views.index'))

    starred = Thread.objects.filter(id__in=starred_ids).order_by('-date')

    if starred.count()==0:
        return render_to_response('search_empty.html', { 'path': request.path }, context_instance=RequestContext(request))
    else:
        return index(request, threads=starred)

    return index(request, threads=starred)

def starred_all(request):
    starred = Thread.objects.filter(star_count__gt=0).order_by('-star_count','-date')
    if starred.count()==0:
        return render_to_response('search_empty.html', { 'path': request.path }, context_instance=RequestContext(request))
    else:
        return index(request, threads=starred)
<|endoftext|>