<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Module for interacting with lectio.dk.

Currently supplies the Period class, which represents a period of a lectio
timetable.
"""

__version__ = (0, 0, 2)

# This is by no means beautiful code - scrapers rarely are.

import requests
from bs4 import BeautifulSoup as bs

import html.parser
import re

from enum import Enum

import datetime

LECTIO_URL = (u"https://www.lectio.dk/lectio/{SCHOOL_ID}/SkemaNy.aspx"
              u"?type=elev&elevid={STUDENT_ID}&week={WEEK_ID}")

def _craft_week_id(week, year):
    """
    Returns a WeekID.
    """
    return str(week).zfill(2) + str(year)

def craft_url(school_id, student_id, week, year):
    """
    Returns a Lectio URL for a given student at a given school for the proper
    week.
    """
    week_id = _craft_week_id(week, year)

    return LECTIO_URL.format(SCHOOL_ID=school_id, STUDENT_ID=student_id,
                             WEEK_ID=week_id)

class PeriodStatuses(Enum):
    """
    Enumerates the statuses that a ``Period`` can have.
    """
    NOTHING = 0
    CANCELLED = 1
    CHANGED = 2

class Period(object):
    """
    Represents a Period in a student's Lectio timetable.
    """
    CHANGED = "Ændret!"
    CANCELLED = "Aflyst!"

    def __init__(self, raw_tag):
        self.status = None
        self.starttime = None
        self.endtime = None
        self.topic = None
        self.groups = None
        self.teachers = None
        self.rooms = None
        self.resources = None
        self.links = None
        self.homework = None
        self.note = None

        self.data = raw_tag["title"]

        # Remove empty elements, remove left and right whitespacing
        self.lines = [x.lstrip().rstrip() for x in self.data.split("\n") if x]

        # Decode xml chars
        parser = html.parser.HTMLParser()
        self.lines = [parser.unescape(x) for x in self.lines]

        # Status line, this isn't always present
        if self.lines[0] in (self.CHANGED, self.CANCELLED):
            status_line = self.lines.pop(0)

            if status_line == self.CHANGED:
                self.status = PeriodStatuses.CHANGED

            elif status_line == self.CANCELLED:
                self.status = PeriodStatuses.CANCELLED

        else:
            self.status = PeriodStatuses.NOTHING

        # Date line, always present
        p = r'^(?P<startday>\d+)\/(?P<startmonth>\d+)-(?P<startyear>\d{4}) (?P<starthour>\d{2}):(?P<startminute>\d{2})'
        p += r' til '
        p += r'(?P<endday>\d*)(?:\/*)(?P<endmonth>\d*)(?:-*)(?P<endyear>\d*)(?: *)(?P<endhour>\d{2}):(?P<endminute>\d{2})$'
        DATE_LINE_PATTERN = re.compile(p)
        date_line = self.lines.pop(0)
        self.result = re.match(DATE_LINE_PATTERN, date_line)

        startday = int(self.result.group("startday"))
        startmonth = int(self.result.group("startmonth"))
        startyear = int(self.result.group("startyear"))
        starthour = int(self.result.group("starthour"))
        startminute = int(self.result.group("startminute"))

        endday = self.result.group("endday")
        if not endday:
            endday = self.result.group("startday")
        endday = int(endday)

        endmonth = self.result.group("endmonth")
        if not endmonth:
            endmonth = self.result.group("startmonth")
        endmonth = int(endmonth)

        endyear = self.result.group("endyear")
        if not endyear:
            endyear = self.result.group("startyear")
        endyear = int(endyear)

        endhour = int(self.result.group("endhour"))
        endminute = int(self.result.group("endminute"))

        self.starttime = datetime.datetime(startyear, startmonth, startday,
                                           starthour, startminute)
        self.endtime = datetime.datetime(endyear, endmonth, endday, endhour,
                                         endminute)

        # topic line, rarely present
        if not self.lines[0].startswith("Hold: "):
            topic_line = self.lines.pop(0)
            self.topic = topic_line

        # group line, always present
        group_line = self.lines.pop(0)
        self.groups = [x.lstrip().rstrip() for x in group_line[len("Hold: "):].split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # teachers line, almost always present
        if (self.lines[0].startswith("Lærer: ")
                or self.lines[0].startswith("Lærere: ")):

            teachers_line = self.lines.pop(0)
            # remove lead
            teachers_line = teachers_line.split(" ", 1)[-1]
            self.teachers = [x.lstrip().rstrip() for x in teachers_line.split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # room line, always present if teacher present
        if (self.lines[0].startswith("Lokale: ")
                or self.lines[0].startswith("Lokaler: ")):

            room_line = self.lines.pop(0)
            # remove lead
            room_line = room_line.split(" ", 1)[-1]
            self.rooms = [x.lstrip().rstrip() for x in room_line.split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Resource line
        if (self.lines[0].startswith("Resource: ")
                or self.lines[0].startswith("Ressourcer: ")):

            resource_line = self.lines.pop(0)
            # remove lead
            resource_line = resource_line.split(" ", 1)[-1]
            self.resources = [x.lstrip().rstrip() for x in resource_line.split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Links line
        if (self.lines[0].startswith("Link: ")
                or self.lines[0].startswith("Links: ")):

            link_line = self.lines.pop(0)
            # remove lead
            link_line = link_line.split(" ", 1)[-1]

            LINK_PATTERN = re.compile(r'^(?P<link>\d+).*$')
            result = re.match(LINK_PATTERN, link_line)
            self.links = int(result.group("link"))

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Homework lines
        if self.lines[0].startswith("Lektier:"):
            self.lines.pop(0)  # Delete this line
            homework_lines = []
            while not self.lines[0].startswith("Note:"):
                homework_lines.append(self.lines.pop(0))
                if not self.lines:
                    break

            self.homework = "\n".join(homework_lines)

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Note lines
        if self.lines[0].startswith("Note:"):
            self.lines.pop(0)  # Delete this line
            note_lines = []
            while True:
                note_lines.append(self.lines.pop(0))
                if not self.lines:
                    break

            self.note = "\n".join(note_lines)

        return

    @staticmethod
    def is_period(tag):
        """Used by BeautifulSoup to determine whether a tag is a period."""
        return all((tag.name == "a",
                    "s2skemabrik" in tag.get("class", []),
                    "s2bgbox" in tag.get("class", [])))

    def __repr__(self):
        indent = "\t"

        attributes = ["status", "starttime", "endtime", "topic", "groups",
                      "teachers", "rooms", "resources", "links", "homework",
                      "note"]

        x = "<Period>"

        for attribute in attributes:
            attr_line = "{}: {}".format(attribute, getattr(self, attribute))
            x += "\n{}{}".format(indent, attr_line)

        return x

def get_periods(school_id, student_id, week, year):
    """
    Returns a list of ``Period``s for a given week and year.
    """
    week = str(week)
    year = str(year)

    url = craft_url(school_id, student_id, week, year)
    page = requests.get(url)
    soup = bs(page.text)

    raw_periods = soup.find_all(Period.is_period)

    periods = [Period(raw) for raw in raw_periods]

    return periods
<commit_msg>version: 0.0.3<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Module for interacting with lectio.dk.

Currently supplies the Period class, which represents a period of a lectio
timetable.
"""

__version__ = (0, 0, 3)

# This is by no means beautiful code - scrapers rarely are.

import requests
from bs4 import BeautifulSoup as bs

import html.parser
import re

from enum import Enum

import datetime

LECTIO_URL = (u"https://www.lectio.dk/lectio/{SCHOOL_ID}/SkemaNy.aspx"
              u"?type=elev&elevid={STUDENT_ID}&week={WEEK_ID}")

def _craft_week_id(week, year):
    """
    Returns a WeekID.
    """
    return str(week).zfill(2) + str(year)

def craft_url(school_id, student_id, week, year):
    """
    Returns a Lectio URL for a given student at a given school for the proper
    week.
    """
    week_id = _craft_week_id(week, year)

    return LECTIO_URL.format(SCHOOL_ID=school_id, STUDENT_ID=student_id,
                             WEEK_ID=week_id)

class PeriodStatuses(Enum):
    """
    Enumerates the statuses that a ``Period`` can have.
    """
    NOTHING = 0
    CANCELLED = 1
    CHANGED = 2

class Period(object):
    """
    Represents a Period in a student's Lectio timetable.
    """
    CHANGED = "Ændret!"
    CANCELLED = "Aflyst!"

    def __init__(self, raw_tag):
        self.status = None
        self.starttime = None
        self.endtime = None
        self.topic = None
        self.groups = None
        self.teachers = None
        self.rooms = None
        self.resources = None
        self.links = None
        self.homework = None
        self.note = None

        self.data = raw_tag["title"]

        # Remove empty elements, remove left and right whitespacing
        self.lines = [x.lstrip().rstrip() for x in self.data.split("\n") if x]

        # Decode xml chars
        parser = html.parser.HTMLParser()
        self.lines = [parser.unescape(x) for x in self.lines]

        # Status line, this isn't always present
        if self.lines[0] in (self.CHANGED, self.CANCELLED):
            status_line = self.lines.pop(0)

            if status_line == self.CHANGED:
                self.status = PeriodStatuses.CHANGED

            elif status_line == self.CANCELLED:
                self.status = PeriodStatuses.CANCELLED

        else:
            self.status = PeriodStatuses.NOTHING

        # Date line, always present
        p = r'^(?P<startday>\d+)\/(?P<startmonth>\d+)-(?P<startyear>\d{4}) (?P<starthour>\d{2}):(?P<startminute>\d{2})'
        p += r' til '
        p += r'(?P<endday>\d*)(?:\/*)(?P<endmonth>\d*)(?:-*)(?P<endyear>\d*)(?: *)(?P<endhour>\d{2}):(?P<endminute>\d{2})$'
        DATE_LINE_PATTERN = re.compile(p)
        date_line = self.lines.pop(0)
        self.result = re.match(DATE_LINE_PATTERN, date_line)

        startday = int(self.result.group("startday"))
        startmonth = int(self.result.group("startmonth"))
        startyear = int(self.result.group("startyear"))
        starthour = int(self.result.group("starthour"))
        startminute = int(self.result.group("startminute"))

        endday = self.result.group("endday")
        if not endday:
            endday = self.result.group("startday")
        endday = int(endday)

        endmonth = self.result.group("endmonth")
        if not endmonth:
            endmonth = self.result.group("startmonth")
        endmonth = int(endmonth)

        endyear = self.result.group("endyear")
        if not endyear:
            endyear = self.result.group("startyear")
        endyear = int(endyear)

        endhour = int(self.result.group("endhour"))
        endminute = int(self.result.group("endminute"))

        self.starttime = datetime.datetime(startyear, startmonth, startday,
                                           starthour, startminute)
        self.endtime = datetime.datetime(endyear, endmonth, endday, endhour,
                                         endminute)

        # topic line, rarely present
        if not self.lines[0].startswith("Hold: "):
            topic_line = self.lines.pop(0)
            self.topic = topic_line

        # group line, always present
        group_line = self.lines.pop(0)
        self.groups = [x.lstrip().rstrip() for x in group_line[len("Hold: "):].split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # teachers line, almost always present
        if (self.lines[0].startswith("Lærer: ")
                or self.lines[0].startswith("Lærere: ")):

            teachers_line = self.lines.pop(0)
            # remove lead
            teachers_line = teachers_line.split(" ", 1)[-1]
            self.teachers = [x.lstrip().rstrip() for x in teachers_line.split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # room line, always present if teacher present
        if (self.lines[0].startswith("Lokale: ")
                or self.lines[0].startswith("Lokaler: ")):

            room_line = self.lines.pop(0)
            # remove lead
            room_line = room_line.split(" ", 1)[-1]
            self.rooms = [x.lstrip().rstrip() for x in room_line.split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Resource line
        if (self.lines[0].startswith("Resource: ")
                or self.lines[0].startswith("Ressourcer: ")):

            resource_line = self.lines.pop(0)
            # remove lead
            resource_line = resource_line.split(" ", 1)[-1]
            self.resources = [x.lstrip().rstrip() for x in resource_line.split(",")]

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Links line
        if (self.lines[0].startswith("Link: ")
                or self.lines[0].startswith("Links: ")):

            link_line = self.lines.pop(0)
            # remove lead
            link_line = link_line.split(" ", 1)[-1]

            LINK_PATTERN = re.compile(r'^(?P<link>\d+).*$')
            result = re.match(LINK_PATTERN, link_line)
            self.links = int(result.group("link"))

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Homework lines
        if self.lines[0].startswith("Lektier:"):
            self.lines.pop(0)  # Delete this line
            homework_lines = []
            while not self.lines[0].startswith("Note:"):
                homework_lines.append(self.lines.pop(0))
                if not self.lines:
                    break

            self.homework = "\n".join(homework_lines)

        # We need to return if we have no more data to process
        if not self.lines:
            return

        # Note lines
        if self.lines[0].startswith("Note:"):
            self.lines.pop(0)  # Delete this line
            note_lines = []
            while True:
                note_lines.append(self.lines.pop(0))
                if not self.lines:
                    break

            self.note = "\n".join(note_lines)

        return

    @staticmethod
    def is_period(tag):
        """Used by BeautifulSoup to determine whether a tag is a period."""
        return all((tag.name == "a",
                    "s2skemabrik" in tag.get("class", []),
                    "s2bgbox" in tag.get("class", [])))

    def __repr__(self):
        indent = "\t"

        attributes = ["status", "starttime", "endtime", "topic", "groups",
                      "teachers", "rooms", "resources", "links", "homework",
                      "note"]

        x = "<Period>"

        for attribute in attributes:
            attr_line = "{}: {}".format(attribute, getattr(self, attribute))
            x += "\n{}{}".format(indent, attr_line)

        return x

def get_periods(school_id, student_id, week, year):
    """
    Returns a list of ``Period``s for a given week and year.
    """
    week = str(week)
    year = str(year)

    url = craft_url(school_id, student_id, week, year)
    page = requests.get(url)
    soup = bs(page.text)

    raw_periods = soup.find_all(Period.is_period)

    periods = [Period(raw) for raw in raw_periods]

    return periods
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

from unittest import TestCase

import keras.backend as K
from bigdl.keras.converter import WeightLoader
from zoo.common.nncontext import *

import logging

np.random.seed(1337)  # for reproducibility

class ZooTestCase(TestCase):
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('py4j').setLevel(logging.INFO)

    def setup_method(self, method):
        """
        Setup any state tied to the execution of the given method in a class.
        It is invoked for every test method of a class.
        """
        K.set_image_dim_ordering("th")
        sparkConf = init_spark_conf().setMaster("local[4]").setAppName("zoo test case")
        assert str(sparkConf.get("spark.shuffle.reduceLocality.enabled")) == "false"
        assert str(sparkConf.get("spark.serializer")) \
            == "org.apache.spark.serializer.JavaSerializer"
        assert SparkContext._active_spark_context is None
        self.sc = init_nncontext(sparkConf)
        self.sc.setLogLevel("ERROR")
        self.sqlContext = SQLContext(self.sc)

    def teardown_method(self, method):
        """
        Teardown any state that was previously setup with a setup_method call.
        """
        K.set_image_dim_ordering("th")
        self.sc.stop()

    def assert_allclose(self, a, b, rtol=1e-6, atol=1e-6, msg=None):
        # from tensorflow
        self.assertEqual(a.shape, b.shape, "Shape mismatch: expected %s, got %s." %
                         (a.shape, b.shape))
        if not np.allclose(a, b, rtol=rtol, atol=atol):
            cond = np.logical_or(
                np.abs(a - b) > atol + rtol * np.abs(b), np.isnan(a) != np.isnan(b))
            if a.ndim:
                x = a[np.where(cond)]
                y = b[np.where(cond)]
                print("not close where = ", np.where(cond))
            else:
                # np.where is broken for scalars
                x, y = a, b
            print("not close lhs = ", x)
            print("not close rhs = ", y)
            print("not close dif = ", np.abs(x - y))
            print("not close tol = ", atol + rtol * np.abs(y))
            print("dtype = %s, shape = %s" % (a.dtype, a.shape))
            np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, err_msg=msg)

    def compare_loss(self, y_a, y_b, kloss, zloss, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras loss against Zoo loss.

        # Arguments
        y_a: input/y_pred
        y_b: target/y_true
        """
        keras_output = np.mean(K.eval(kloss(K.variable(y_b), K.variable(y_a))))
        zoo_output = zloss.forward(y_a, y_b)
        np.testing.assert_allclose(zoo_output, keras_output, rtol=rtol, atol=atol)

    def compare_layer(self, klayer, zlayer, input_data, weight_converter=None,
                      is_training=False, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras layer against Zoo Keras API layer.
        """
        from keras.models import Sequential as KSequential
        from zoo.pipeline.api.keras.models import Sequential as ZSequential
        zmodel = ZSequential()
        zmodel.add(zlayer)
        kmodel = KSequential()
        kmodel.add(klayer)
        koutput = kmodel.predict(input_data)
        from zoo.pipeline.api.keras.layers import BatchNormalization
        if isinstance(zlayer, BatchNormalization):
            k_running_mean = K.eval(klayer.running_mean)
            k_running_std = K.eval(klayer.running_std)
            zlayer.set_running_mean(k_running_mean)
            zlayer.set_running_std(k_running_std)
        if kmodel.get_weights():
            zmodel.set_weights(weight_converter(klayer, kmodel.get_weights()))
        zmodel.training(is_training)
        zoutput = zmodel.forward(input_data)
        self.assert_allclose(zoutput, koutput, rtol=rtol, atol=atol)

    def compare_model(self, zmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
        """
        Compare forward results for Keras model against Zoo Keras API model.
        """
        WeightLoader.load_weights_from_kmodel(zmodel, kmodel)
        zmodel.training(is_training=False)
        bigdl_output = zmodel.forward(input_data)
        keras_output = kmodel.predict(input_data)
        self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)

    def assert_forward_backward(self, model, input_data):
        """
        Test whether forward and backward can work properly.
        """
        output = model.forward(input_data)
        grad_input = model.backward(input_data, output)

    def assert_zoo_model_save_load(self, model, input_data, rtol=1e-6, atol=1e-6):
        """
        Test for ZooModel save and load.
        The loaded model should have the same class as the original model.
        The loaded model should produce the same forward and backward results as the original model.
        """
        model_class = model.__class__
        tmp_path = create_tmp_path() + ".bigdl"
        model.save_model(tmp_path)
        loaded_model = model_class.load_model(tmp_path)
        assert isinstance(loaded_model, model_class)
        self.compare_output_and_grad_input(model, loaded_model, input_data, rtol, atol)
        os.remove(tmp_path)

    def compare_output_and_grad_input(self, model1, model2, input_data, rtol=1e-6, atol=1e-6):
        # Set seed in case of random factors such as dropout.
        rng = RNG()
        rng.set_seed(1000)
        output1 = model1.forward(input_data)
        rng.set_seed(1000)
        output2 = model2.forward(input_data)
        self.assert_allclose(output1, output2, rtol, atol)
        rng.set_seed(1000)
        grad_input1 = model1.backward(input_data, output1)
        rng.set_seed(1000)
        grad_input2 = model2.backward(input_data, output1)
        self.assert_allclose(grad_input1, grad_input2, rtol, atol)

    def compare_output_and_grad_input_set_weights(self, model1, model2, input_data,
                                                  rtol=1e-6, atol=1e-6):
        if model1.get_weights():
            model2.set_weights(model1.get_weights())
        self.compare_output_and_grad_input(model1, model2, input_data, rtol, atol)
<commit_msg>Load onnx model (#604)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

import logging
import shutil
from unittest import TestCase

import keras.backend as K
import numpy as np
from bigdl.keras.converter import WeightLoader

from zoo.common.nncontext import *

np.random.seed(1337)  # for reproducibility

class ZooTestCase(TestCase):
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('py4j').setLevel(logging.INFO)

    def setup_method(self, method):
        """
        Setup any state tied to the execution of the given method in a class.
        It is invoked for every test method of a class.
        """
        K.set_image_dim_ordering("th")
        sparkConf = init_spark_conf().setMaster("local[4]").setAppName("zoo test case")
        assert str(sparkConf.get("spark.shuffle.reduceLocality.enabled")) == "false"
        assert str(sparkConf.get("spark.serializer")) \
            == "org.apache.spark.serializer.JavaSerializer"
        assert SparkContext._active_spark_context is None
        self.sc = init_nncontext(sparkConf)
        self.sc.setLogLevel("ERROR")
        self.sqlContext = SQLContext(self.sc)
        self.tmp_dirs = []

    def teardown_method(self, method):
        """
        Teardown any state that was previously setup with a setup_method call.
        """
        K.set_image_dim_ordering("th")
        self.sc.stop()
        if hasattr(self, "tmp_dirs"):
            for d in self.tmp_dirs:
                shutil.rmtree(d)

    def create_temp_dir(self):
        tmp_dir = tempfile.mkdtemp()
        self.tmp_dirs.append(tmp_dir)
        return tmp_dir

    def assert_allclose(self, a, b, rtol=1e-6, atol=1e-6, msg=None):
        # from tensorflow
        self.assertEqual(a.shape, b.shape, "Shape mismatch: expected %s, got %s." %
                         (a.shape, b.shape))
        if not np.allclose(a, b, rtol=rtol, atol=atol):
            cond = np.logical_or(
                np.abs(a - b) > atol + rtol * np.abs(b), np.isnan(a) != np.isnan(b))
            if a.ndim:
                x = a[np.where(cond)]
                y = b[np.where(cond)]
                print("not close where = ", np.where(cond))
            else:
                # np.where is broken for scalars
                x, y = a, b
            print("not close lhs = ", x)
            print("not close rhs = ", y)
            print("not close dif = ", np.abs(x - y))
            print("not close tol = ", atol + rtol * np.abs(y))
            print("dtype = %s, shape = %s" % (a.dtype, a.shape))
            np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, err_msg=msg)

    def compare_loss(self, y_a, y_b, kloss, zloss, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras loss against Zoo loss.

        # Arguments
        y_a: input/y_pred
        y_b: target/y_true
        """
        keras_output = np.mean(K.eval(kloss(K.variable(y_b), K.variable(y_a))))
        zoo_output = zloss.forward(y_a, y_b)
        np.testing.assert_allclose(zoo_output, keras_output, rtol=rtol, atol=atol)

    def compare_layer(self, klayer, zlayer, input_data, weight_converter=None,
                      is_training=False, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras layer against Zoo Keras API layer.
        """
        from keras.models import Sequential as KSequential
        from zoo.pipeline.api.keras.models import Sequential as ZSequential
        zmodel = ZSequential()
        zmodel.add(zlayer)
        kmodel = KSequential()
        kmodel.add(klayer)
        koutput = kmodel.predict(input_data)
        from zoo.pipeline.api.keras.layers import BatchNormalization
        if isinstance(zlayer, BatchNormalization):
            k_running_mean = K.eval(klayer.running_mean)
            k_running_std = K.eval(klayer.running_std)
            zlayer.set_running_mean(k_running_mean)
            zlayer.set_running_std(k_running_std)
        if kmodel.get_weights():
            zmodel.set_weights(weight_converter(klayer, kmodel.get_weights()))
        zmodel.training(is_training)
        zoutput = zmodel.forward(input_data)
        self.assert_allclose(zoutput, koutput, rtol=rtol, atol=atol)

    def compare_model(self, zmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
        """
        Compare forward results for Keras model against Zoo Keras API model.
        """
        WeightLoader.load_weights_from_kmodel(zmodel, kmodel)
        zmodel.training(is_training=False)
        bigdl_output = zmodel.forward(input_data)
        keras_output = kmodel.predict(input_data)
        self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)

    def assert_forward_backward(self, model, input_data):
        """
        Test whether forward and backward can work properly.
        """
        output = model.forward(input_data)
        grad_input = model.backward(input_data, output)

    def assert_zoo_model_save_load(self, model, input_data, rtol=1e-6, atol=1e-6):
        """
        Test for ZooModel save and load.
        The loaded model should have the same class as the original model.
        The loaded model should produce the same forward and backward results as the original model.
        """
        model_class = model.__class__
        tmp_path = create_tmp_path() + ".bigdl"
        model.save_model(tmp_path)
        loaded_model = model_class.load_model(tmp_path)
        assert isinstance(loaded_model, model_class)
        self.compare_output_and_grad_input(model, loaded_model, input_data, rtol, atol)
        os.remove(tmp_path)

    def compare_output_and_grad_input(self, model1, model2, input_data, rtol=1e-6, atol=1e-6):
        # Set seed in case of random factors such as dropout.
        rng = RNG()
        rng.set_seed(1000)
        output1 = model1.forward(input_data)
        rng.set_seed(1000)
        output2 = model2.forward(input_data)
        self.assert_allclose(output1, output2, rtol, atol)
        rng.set_seed(1000)
        grad_input1 = model1.backward(input_data, output1)
        rng.set_seed(1000)
        grad_input2 = model2.backward(input_data, output1)
        self.assert_allclose(grad_input1, grad_input2, rtol, atol)

    def compare_output_and_grad_input_set_weights(self, model1, model2, input_data,
                                                  rtol=1e-6, atol=1e-6):
        if model1.get_weights():
            model2.set_weights(model1.get_weights())
        self.compare_output_and_grad_input(model1, model2, input_data, rtol, atol)
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

from unittest import TestCase

import keras.backend as K
from bigdl.keras.converter import WeightLoader
from bigdl.util.common import *

import logging

np.random.seed(1337)  # for reproducibility

class ZooTestCase(TestCase):
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('py4j').setLevel(logging.INFO)

    def setup_method(self, method):
        """
        Setup any state tied to the execution of the given method in a class.
        It is invoked for every test method of a class.
        """
        K.set_image_dim_ordering("th")
        sparkConf = create_spark_conf().setMaster("local[4]").setAppName("zoo test case")
        self.sc = get_spark_context(sparkConf)
        self.sc.setLogLevel("ERROR")

        self.sqlContext = SQLContext(self.sc)
        init_engine()

    def teardown_method(self, method):
        """
        Teardown any state that was previously setup with a setup_method call.
        """
        K.set_image_dim_ordering("th")
        self.sc.stop()

    def assert_allclose(self, a, b, rtol=1e-6, atol=1e-6, msg=None):
        # from tensorflow
        self.assertEqual(a.shape, b.shape, "Shape mismatch: expected %s, got %s." %
                         (a.shape, b.shape))
        if not np.allclose(a, b, rtol=rtol, atol=atol):
            cond = np.logical_or(
                np.abs(a - b) > atol + rtol * np.abs(b), np.isnan(a) != np.isnan(b))
            if a.ndim:
                x = a[np.where(cond)]
                y = b[np.where(cond)]
                print("not close where = ", np.where(cond))
            else:
                # np.where is broken for scalars
                x, y = a, b
            print("not close lhs = ", x)
            print("not close rhs = ", y)
            print("not close dif = ", np.abs(x - y))
            print("not close tol = ", atol + rtol * np.abs(y))
            print("dtype = %s, shape = %s" % (a.dtype, a.shape))
            np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, err_msg=msg)

    def compare_loss(self, y_a, y_b, kloss, zloss, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras loss against Zoo loss.

        # Arguments
        y_a: input/y_pred
        y_b: target/y_true
        """
        keras_output = np.mean(K.eval(kloss(K.variable(y_b), K.variable(y_a))))
        zoo_output = zloss.forward(y_a, y_b)
        np.testing.assert_allclose(zoo_output, keras_output, rtol=rtol, atol=atol)

    def compare_layer(self, klayer, zlayer, input_data, weight_converter=None,
                      is_training=False, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras layer against Zoo Keras API layer.
        """
        from keras.models import Sequential as KSequential
        from zoo.pipeline.api.keras.models import Sequential as ZSequential
        zmodel = ZSequential()
        zmodel.add(zlayer)
        kmodel = KSequential()
        kmodel.add(klayer)
        koutput = kmodel.predict(input_data)
        from zoo.pipeline.api.keras.layers import BatchNormalization
        if isinstance(zlayer, BatchNormalization):
            k_running_mean = K.eval(klayer.running_mean)
            k_running_std = K.eval(klayer.running_std)
            zlayer.set_running_mean(k_running_mean)
            zlayer.set_running_std(k_running_std)
        if kmodel.get_weights():
            zmodel.set_weights(weight_converter(klayer, kmodel.get_weights()))
        zmodel.training(is_training)
        zoutput = zmodel.forward(input_data)
        self.assert_allclose(zoutput, koutput, rtol=rtol, atol=atol)

    def compare_model(self, zmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
        """
        Compare forward results for Keras model against Zoo Keras API model.
        """
        WeightLoader.load_weights_from_kmodel(zmodel, kmodel)
        zmodel.training(is_training=False)
        bigdl_output = zmodel.forward(input_data)
        keras_output = kmodel.predict(input_data)
        self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)

    def assert_forward_backward(self, model, input):
        """
        Test whether forward and backward can work properly.
        """
        output = model.forward(input)
        grad_input = model.backward(input, output)

    def assert_save_load(self, model, input, rtol=1e-6, atol=1e-6):
        """
        Test for model save and load.
        The loaded model should have the same class as the original model.
        The loaded model should produce the same forward and backward results as the original model.
        """
        model_class = model.__class__
        tmp_path = create_tmp_path() + ".bigdl"
        model.save_model(tmp_path)
        loaded_model = model_class.load_model(tmp_path)
        assert isinstance(loaded_model, model_class)
        rng = RNG()
        rng.set_seed(1000)
        output1 = model.forward(input)
        rng.set_seed(1000)
        output2 = loaded_model.forward(input)
        self.assert_allclose(output1, output2, rtol, atol)
        rng.set_seed(1000)
        grad_input1 = model.backward(input, output1)
        rng.set_seed(1000)
        grad_input2 = loaded_model.backward(input, output1)
        self.assert_allclose(grad_input1, grad_input2, rtol, atol)
        os.remove(tmp_path)
<commit_msg>Enable Python style check (#133)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

from unittest import TestCase

import keras.backend as K
from bigdl.keras.converter import WeightLoader
from bigdl.util.common import *

import logging

np.random.seed(1337)  # for reproducibility

class ZooTestCase(TestCase):
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('py4j').setLevel(logging.INFO)

    def setup_method(self, method):
        """
        Setup any state tied to the execution of the given method in a class.
        It is invoked for every test method of a class.
        """
        K.set_image_dim_ordering("th")
        sparkConf = create_spark_conf().setMaster("local[4]").setAppName("zoo test case")
        self.sc = get_spark_context(sparkConf)
        self.sc.setLogLevel("ERROR")

        self.sqlContext = SQLContext(self.sc)
        init_engine()

    def teardown_method(self, method):
        """
        Teardown any state that was previously setup with a setup_method call.
        """
        K.set_image_dim_ordering("th")
        self.sc.stop()

    def assert_allclose(self, a, b, rtol=1e-6, atol=1e-6, msg=None):
        # from tensorflow
        self.assertEqual(a.shape, b.shape, "Shape mismatch: expected %s, got %s." %
                         (a.shape, b.shape))
        if not np.allclose(a, b, rtol=rtol, atol=atol):
            cond = np.logical_or(
                np.abs(a - b) > atol + rtol * np.abs(b), np.isnan(a) != np.isnan(b))
            if a.ndim:
                x = a[np.where(cond)]
                y = b[np.where(cond)]
                print("not close where = ", np.where(cond))
            else:
                # np.where is broken for scalars
                x, y = a, b
            print("not close lhs = ", x)
            print("not close rhs = ", y)
            print("not close dif = ", np.abs(x - y))
            print("not close tol = ", atol + rtol * np.abs(y))
            print("dtype = %s, shape = %s" % (a.dtype, a.shape))
            np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, err_msg=msg)

    def compare_loss(self, y_a, y_b, kloss, zloss, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras loss against Zoo loss.

        # Arguments
        y_a: input/y_pred
        y_b: target/y_true
        """
        keras_output = np.mean(K.eval(kloss(K.variable(y_b), K.variable(y_a))))
        zoo_output = zloss.forward(y_a, y_b)
        np.testing.assert_allclose(zoo_output, keras_output, rtol=rtol, atol=atol)

    def compare_layer(self, klayer, zlayer, input_data, weight_converter=None,
                      is_training=False, rtol=1e-6, atol=1e-6):
        """
        Compare forward results for Keras layer against Zoo Keras API layer.
        """
        from keras.models import Sequential as KSequential
        from zoo.pipeline.api.keras.models import Sequential as ZSequential
        zmodel = ZSequential()
        zmodel.add(zlayer)
        kmodel = KSequential()
        kmodel.add(klayer)
        koutput = kmodel.predict(input_data)
        from zoo.pipeline.api.keras.layers import BatchNormalization
        if isinstance(zlayer, BatchNormalization):
            k_running_mean = K.eval(klayer.running_mean)
            k_running_std = K.eval(klayer.running_std)
            zlayer.set_running_mean(k_running_mean)
            zlayer.set_running_std(k_running_std)
        if kmodel.get_weights():
            zmodel.set_weights(weight_converter(klayer, kmodel.get_weights()))
        zmodel.training(is_training)
        zoutput = zmodel.forward(input_data)
        self.assert_allclose(zoutput, koutput, rtol=rtol, atol=atol)

    def compare_model(self, zmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
        """
        Compare forward results for Keras model against Zoo Keras API model.
        """
        WeightLoader.load_weights_from_kmodel(zmodel, kmodel)
        zmodel.training(is_training=False)
        bigdl_output = zmodel.forward(input_data)
        keras_output = kmodel.predict(input_data)
        self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)

    def assert_forward_backward(self, model, input_data):
        """
        Test whether forward and backward can work properly.
        """
        output = model.forward(input_data)
        grad_input = model.backward(input_data, output)

    def assert_zoo_model_save_load(self, model, input_data, rtol=1e-6, atol=1e-6):
        """
        Test for ZooModel save and load.
        The loaded model should have the same class as the original model.
        The loaded model should produce the same forward and backward results as the original model.
        """
        model_class = model.__class__
        tmp_path = create_tmp_path() + ".bigdl"
        model.save_model(tmp_path)
        loaded_model = model_class.load_model(tmp_path)
        assert isinstance(loaded_model, model_class)
        self.compare_output_and_grad_input(model, loaded_model, input_data, rtol, atol)
        os.remove(tmp_path)

    def compare_output_and_grad_input(self, model1, model2, input_data, rtol=1e-6, atol=1e-6):
        # Set seed in case of random factors such as dropout.
        rng = RNG()
        rng.set_seed(1000)
        output1 = model1.forward(input_data)
        rng.set_seed(1000)
        output2 = model2.forward(input_data)
        self.assert_allclose(output1, output2, rtol, atol)
        rng.set_seed(1000)
        grad_input1 = model1.backward(input_data, output1)
        rng.set_seed(1000)
        grad_input2 = model2.backward(input_data, output1)
        self.assert_allclose(grad_input1, grad_input2, rtol, atol)

    def compare_output_and_grad_input_set_weights(self, model1, model2, input_data,
                                                  rtol=1e-6, atol=1e-6):
        if model1.get_weights():
            model2.set_weights(model1.get_weights())
        self.compare_output_and_grad_input(model1, model2, input_data, rtol, atol)
<|endoftext|>


<commit_before>import signal
import sys
import pytest
import ray

# Import psutil after ray so the packaged version is used.
import psutil

from ray.test_utils import wait_for_condition

def get_pid(name):
    pids = psutil.process_iter()
    for pid in pids:
        if name in pid.name():
            return pid.pid

    return -1

def check_result(filename, num_signal, check_key):
    ray.init(num_cpus=1)
    session_dir = ray.worker._global_node.get_session_dir_path()
    raylet_out_path = filename.format(session_dir)
    pid = get_pid("raylet")
    assert pid > 0
    p = psutil.Process(pid)
    p.send_signal(num_signal)
    p.wait(timeout=15)

    def check_file():
        with open(raylet_out_path) as f:
            s = f.read()
            return check_key in s

    wait_for_condition(check_file)

@pytest.mark.skipif(sys.platform == "win32", reason="Not support on Windows.")
def test_kill_raylet_signal_log(shutdown_only):
    check_result("{}/logs/raylet.err", signal.SIGABRT, "SIGABRT")

@pytest.mark.skipif(sys.platform != "win32", reason="Only run on Windows.")
def test_kill_raylet_signal_log_win(shutdown_only):
    check_result("{}/logs/raylet.out", signal.CTRL_BREAK_EVENT, "SIGTERM")

if __name__ == "__main__":
    sys.exit(pytest.main(["-v", __file__]))
<commit_msg>Skip raylet kill windows #17682 (#17683)<commit_after>import signal
import sys
import pytest
import ray

# Import psutil after ray so the packaged version is used.
import psutil

from ray.test_utils import wait_for_condition

def get_pid(name):
    pids = psutil.process_iter()
    for pid in pids:
        if name in pid.name():
            return pid.pid

    return -1

def check_result(filename, num_signal, check_key):
    ray.init(num_cpus=1)
    session_dir = ray.worker._global_node.get_session_dir_path()
    raylet_out_path = filename.format(session_dir)
    pid = get_pid("raylet")
    assert pid > 0
    p = psutil.Process(pid)
    p.send_signal(num_signal)
    p.wait(timeout=15)

    def check_file():
        with open(raylet_out_path) as f:
            s = f.read()
            return check_key in s

    wait_for_condition(check_file)

@pytest.mark.skipif(sys.platform == "win32", reason="Not support on Windows.")
def test_kill_raylet_signal_log(shutdown_only):
    check_result("{}/logs/raylet.err", signal.SIGABRT, "SIGABRT")

@pytest.mark.skipif(sys.platform != "win32", reason="Only run on Windows.")
@pytest.mark.skip(reason="Flaky on Windows")
def test_kill_raylet_signal_log_win(shutdown_only):
    check_result("{}/logs/raylet.out", signal.CTRL_BREAK_EVENT, "SIGTERM")

if __name__ == "__main__":
    sys.exit(pytest.main(["-v", __file__]))
<|endoftext|>


<commit_before>import torch
import torch.nn as nn
from .seq2seq import Seq2Seq
from .recurrent import RecurrentAttention, bridge_bidirectional_hidden

class GNMT(Seq2Seq):

    def __init__(self, vocab_size, hidden_size=512,
                 num_layers=8, bias=True,
                 dropout=0.2, device_assignment=None, transfer_hidden=True):
        super(GNMT, self).__init__()
        self.device_assignment = device_assignment
        self.transfer_hidden = transfer_hidden
        self.encoder = ResidualRecurrentEncoder(vocab_size, hidden_size,
                                                num_layers, bias,
                                                dropout)
        if self.device_assignment is not None:
            self.encoder.cuda(self.device_assignment.get('encoder', 0))
        self.decoder = ResidualRecurrentDecoder(vocab_size, hidden_size=hidden_size,
                                                num_layers=num_layers,
                                                bias=bias,
                                                dropout=dropout)
        if self.device_assignment is not None:
            self.decoder.cuda(self.device_assignment.get('decoder', 0))

    def bridge(self, context):
        context, hidden = context
        if not self.transfer_hidden:
            new_hidden = [tuple([bridge_bidirectional_hidden(h)
                                 for h in hidden[0]])]
            for i in range(1, len(hidden)):
                new_hidden.append(hidden[i])
            return (context, tuple(new_hidden))
        else:
            return (context, None)

    def forward(self, input_encoder, input_decoder, encoder_state=None, device_ids=None):
        def cast_device(x, device):
            if device is None or x is None:
                return x
            if isinstance(x, tuple):
                return tuple([cast_device(sub_x, device) for sub_x in x])
            else:
                return x.cuda(device)
        if self.device_assignment is not None:
            input_encoder = input_encoder.cuda(
                self.device_assignment.get('encoder', 0))
            if encoder_state is not None:
                input_encoder = input_encoder.cuda(
                    self.device_assignment.get('encoder', 0))

        context = self.encode(input_encoder, encoder_state)
        context = self.bridge(context)

        if self.device_assignment is not None:
            input_decoder = input_decoder.cuda(
                self.device_assignment.get('decoder', 0))
            context = cast_device(
                context, self.device_assignment.get('decoder', 0))

        output, _ = self.decode(input_decoder, context)
        if self.device_assignment is not None:
            output = output.cuda(self.device_assignment.get('output', 0))
        return output

class ResidualRecurrentEncoder(nn.Module):

    def __init__(self, vocab_size, hidden_size=128,
                 num_layers=8, bias=True,
                 dropout=0):
        super(ResidualRecurrentEncoder, self).__init__()
        assert (hidden_size % 2 == 0)
        self.rnn_layers = nn.ModuleList()
        self.rnn_layers.append(nn.LSTM(hidden_size, hidden_size // 2, num_layers=1, bias=bias, dropout=dropout,
                                       bidirectional=True))
        for n in range(num_layers - 1):
            self.rnn_layers.append(nn.LSTM(hidden_size, hidden_size, num_layers=1, bias=bias, dropout=dropout,
                                           bidirectional=False))
        self.embedder = nn.Embedding(vocab_size, hidden_size)
        self.vocab_size = vocab_size

    def forward(self, inputs, hidden=None):
        hidden = []
        x = self.embedder(inputs)
        x, h = self.rnn_layers[0](x)
        hidden.append(h)
        for i in range(1, len(self.rnn_layers)):
            residual = x
            x, h = self.rnn_layers[i](x)
            hidden.append(h)
            x = x + residual
        return x, tuple(hidden)

class ResidualRecurrentDecoder(nn.Module):

    def __init__(self, vocab_size, hidden_size=128,
                 num_layers=8, bias=True,
                 dropout=0):
        super(ResidualRecurrentDecoder, self).__init__()

        self.rnn_layers = nn.ModuleList()
        self.rnn_layers.append(RecurrentAttention(
            hidden_size, hidden_size, num_layers=1))
        self.rnn_layers.append(
            nn.LSTM(hidden_size, hidden_size, num_layers=1, bias=bias, dropout=dropout))
        for _ in range(num_layers - 2):
            self.rnn_layers.append(
                nn.LSTM(2 * hidden_size, hidden_size, num_layers=1, bias=bias, dropout=dropout))
        self.embedder = nn.Embedding(vocab_size, hidden_size)
        self.classifier = nn.Linear(hidden_size, vocab_size)
        self.vocab_size = vocab_size

    def forward(self, inputs, context, get_attention=False):
        enc_context, hidden = context
        hidden = hidden or [None] * len(self.rnn_layers)
        next_hidden = []
        x = self.embedder(inputs)
        if get_attention:
            x, h, attentions = self.rnn_layers[0](x, hidden[0], enc_context,
                                                  get_attention=True)
        else:
            x, h = self.rnn_layers[0](x, hidden[1], enc_context)
        next_hidden.append(h)
        dec_context = x
        x, h = self.rnn_layers[1](x)
        next_hidden.append(h)
        for i in range(2, len(self.rnn_layers)):
            residual = x
            x = torch.cat([x, dec_context], 2)
            x, h = self.rnn_layers[i](x, hidden[i])
            next_hidden.append(h)
            x = x + residual
        x = x.view(-1, x.size(2))
        x = self.classifier(x)
        x = x.view(inputs.size(0), inputs.size(1), -1)
        if get_attention:
            return x, (enc_context, tuple(next_hidden)), attentions
        else:
            return x, (enc_context, tuple(next_hidden))

# # #test:
# model = GNMT(32000, device_assignment=None)
# x1 = torch.autograd.Variable(torch.rand(16, 32).long().fill_(2))
# x2 = torch.autograd.Variable(torch.rand(7, 32).long().fill_(2))
# y = model(x1, x2)
<commit_msg>gnmt bug fix<commit_after>import torch
import torch.nn as nn
from .seq2seq import Seq2Seq
from .recurrent import RecurrentAttention, bridge_bidirectional_hidden

class GNMT(Seq2Seq):

    def __init__(self, vocab_size, hidden_size=512,
                 num_layers=8, bias=True,
                 dropout=0.2, device_assignment=None, transfer_hidden=True):
        super(GNMT, self).__init__()
        self.device_assignment = device_assignment
        self.transfer_hidden = transfer_hidden
        self.encoder = ResidualRecurrentEncoder(vocab_size, hidden_size,
                                                num_layers, bias,
                                                dropout)
        if self.device_assignment is not None:
            self.encoder.cuda(self.device_assignment.get('encoder', 0))
        self.decoder = ResidualRecurrentDecoder(vocab_size, hidden_size=hidden_size,
                                                num_layers=num_layers,
                                                bias=bias,
                                                dropout=dropout)
        if self.device_assignment is not None:
            self.decoder.cuda(self.device_assignment.get('decoder', 0))

    def bridge(self, context):
        context, hidden = context
        if self.transfer_hidden:
            new_hidden = [tuple([bridge_bidirectional_hidden(h)
                                 for h in hidden[0]])]
            for i in range(1, len(hidden)):
                new_hidden.append(hidden[i])
            return (context, tuple(new_hidden))
        else:
            return (context, None)

    def forward(self, input_encoder, input_decoder, encoder_state=None, device_ids=None):
        def cast_device(x, device):
            if device is None or x is None:
                return x
            if isinstance(x, tuple):
                return tuple([cast_device(sub_x, device) for sub_x in x])
            else:
                return x.cuda(device)
        if self.device_assignment is not None:
            input_encoder = input_encoder.cuda(
                self.device_assignment.get('encoder', 0))
            if encoder_state is not None:
                input_encoder = input_encoder.cuda(
                    self.device_assignment.get('encoder', 0))

        context = self.encode(input_encoder, encoder_state)
        context = self.bridge(context)

        if self.device_assignment is not None:
            input_decoder = input_decoder.cuda(
                self.device_assignment.get('decoder', 0))
            context = cast_device(
                context, self.device_assignment.get('decoder', 0))

        output, _ = self.decode(input_decoder, context)
        if self.device_assignment is not None:
            output = output.cuda(self.device_assignment.get('output', 0))
        return output

class ResidualRecurrentEncoder(nn.Module):

    def __init__(self, vocab_size, hidden_size=128,
                 num_layers=8, bias=True,
                 dropout=0):
        super(ResidualRecurrentEncoder, self).__init__()
        assert (hidden_size % 2 == 0)
        self.rnn_layers = nn.ModuleList()
        self.rnn_layers.append(nn.LSTM(hidden_size, hidden_size // 2, num_layers=1, bias=bias, dropout=dropout,
                                       bidirectional=True))
        for n in range(num_layers - 1):
            self.rnn_layers.append(nn.LSTM(hidden_size, hidden_size, num_layers=1, bias=bias, dropout=dropout,
                                           bidirectional=False))
        self.embedder = nn.Embedding(vocab_size, hidden_size)
        self.vocab_size = vocab_size

    def forward(self, inputs, hidden=None):
        hidden = []
        x = self.embedder(inputs)
        x, h = self.rnn_layers[0](x)
        hidden.append(h)
        for i in range(1, len(self.rnn_layers)):
            residual = x
            x, h = self.rnn_layers[i](x)
            hidden.append(h)
            x = x + residual
        return x, tuple(hidden)

class ResidualRecurrentDecoder(nn.Module):

    def __init__(self, vocab_size, hidden_size=128,
                 num_layers=8, bias=True,
                 dropout=0):
        super(ResidualRecurrentDecoder, self).__init__()

        self.rnn_layers = nn.ModuleList()
        self.rnn_layers.append(RecurrentAttention(
            hidden_size, hidden_size, num_layers=1))
        self.rnn_layers.append(
            nn.LSTM(hidden_size, hidden_size, num_layers=1, bias=bias, dropout=dropout))
        for _ in range(num_layers - 2):
            self.rnn_layers.append(
                nn.LSTM(2 * hidden_size, hidden_size, num_layers=1, bias=bias, dropout=dropout))
        self.embedder = nn.Embedding(vocab_size, hidden_size)
        self.classifier = nn.Linear(hidden_size, vocab_size)
        self.vocab_size = vocab_size

    def forward(self, inputs, context, get_attention=False):
        enc_context, hidden = context
        hidden = hidden or [None] * len(self.rnn_layers)
        next_hidden = []
        x = self.embedder(inputs)
        if get_attention:
            x, h, attentions = self.rnn_layers[0](x, hidden[0], enc_context,
                                                  get_attention=True)
        else:
            x, h = self.rnn_layers[0](x, hidden[0], enc_context)
        next_hidden.append(h)
        dec_context = x
        x, h = self.rnn_layers[1](x, hidden[1])
        next_hidden.append(h)
        for i in range(2, len(self.rnn_layers)):
            residual = x
            x = torch.cat([x, dec_context], 2)
            x, h = self.rnn_layers[i](x, hidden[i])
            next_hidden.append(h)
            x = x + residual
        x = x.view(-1, x.size(2))
        x = self.classifier(x)
        x = x.view(inputs.size(0), inputs.size(1), -1)
        if get_attention:
            return x, (enc_context, tuple(next_hidden)), attentions
        else:
            return x, (enc_context, tuple(next_hidden))

# # #test:
# model = GNMT(32000, device_assignment=None)
# x1 = torch.autograd.Variable(torch.rand(16, 32).long().fill_(2))
# x2 = torch.autograd.Variable(torch.rand(7, 32).long().fill_(2))
# y = model(x1, x2)
<|endoftext|>


<commit_before><commit_msg>added models/mail.py<commit_after>from sqlalchemy import *

class Mail:

	def __init__(self, metadata):
		self.metadata = metadata
		self.table = Table('mail', metadata,
				Column('id',Integer, primary_key=True)
				Column('content', String),
				Column('date_sent', Date),
				Column('sender', String),
				Column('recipient', String),
				Column('subject', String),
				)
<|endoftext|>


<commit_before>import time
import datetime
from collections import deque

import lxml.html
import pytz
import icalendar
import requests

from .base import LegistarScraper, LegistarAPIScraper

class LegistarEventsScraper(LegistarScraper):
    def eventPages(self, since):
        # Directly use the requests library here, so that we do not
        # use a cached page, which may have expired .NET state values,
        # even in fastmode (which uses the cache).
        response = requests.get(self.EVENTSPAGE, verify=False)
        entry = response.text
        page = lxml.html.fromstring(entry)
        page.make_links_absolute(self.EVENTSPAGE)

        for page in self.eventSearch(page, since):
            yield page

    def eventSearch(self, page, since):
        payload = self.sessionSecrets(page)

        payload['ctl00_ContentPlaceHolder1_lstYears_ClientState'] = '{"value":"%s"}' % since

        payload['__EVENTTARGET'] = 'ctl00$ContentPlaceHolder1$lstYears'

        return self.pages(self.EVENTSPAGE, payload)

    def events(self, follow_links=True, since=None):
        # If an event is added to the the legistar system while we
        # are scraping, it will shift the list of events down and
        # we might revisit the same event. So, we keep track of
        # the last few events we've visited in order to
        # make sure we are not revisiting
        scraped_events = deque([], maxlen=10)

        current_year = self.now().year

        if since:
            if since > current_year:
                raise ValueError(
                    'Value of :since cannot exceed {}'.format(current_year))
            else:
                since_year = since - 1

        else:
            since_year = 0

        # Anticipate events will be scheduled for the following year to avoid
        # missing upcoming events during scrapes near the end of the current
        # year.
        for year in range(current_year + 1, since_year, -1):
            no_events_in_year = True

            for page in self.eventPages(year):
                no_events_in_year = False
                events_table = page.xpath("//table[@class='rgMasterTable']")[0]
                for event, _, _ in self.parseDataTable(events_table):
                    if follow_links and type(event["Meeting Details"]) == dict:
                        detail_url = event["Meeting Details"]['url']
                        if detail_url in scraped_events:
                            continue
                        else:
                            scraped_events.append(detail_url)

                        agenda = self.agenda(detail_url)

                    else:
                        agenda = None

                    yield event, agenda

            if no_events_in_year:  # Bail from scrape if no results returned from year
                break

    def agenda(self, detail_url):
        page = self.lxmlize(detail_url)

        payload = self.sessionSecrets(page)

        payload.update({"__EVENTARGUMENT": "3:1",
                        "__EVENTTARGET": "ctl00$ContentPlaceHolder1$menuMain"})

        for page in self.pages(detail_url, payload):
            agenda_table = page.xpath(
                "//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
            agenda = self.parseDataTable(agenda_table)
            yield from agenda

    def addDocs(self, e, events, doc_type):
        try:
            if events[doc_type] != 'Not\xa0available':
                e.add_document(note=events[doc_type]['label'],
                               url=events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError:
            pass

    def extractRollCall(self, action_detail_url):
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            rollcall_table = action_detail_page.xpath(
                "//table[@id='ctl00_ContentPlaceHolder1_gridRollCall_ctl00']")[0]
        except IndexError:
            self.warning("No rollcall found in table")
            return []
        roll_call = list(self.parseDataTable(rollcall_table))
        call_list = []
        for call, _, _ in roll_call:
            option = call['Attendance']
            call_list.append((option,
                              call['Person Name']['label']))

        return call_list

    def ical(self, ical_text):
        value = icalendar.Calendar.from_ical(ical_text)
        return value

class LegistarAPIEventScraper(LegistarAPIScraper):

    def events(self, since_datetime=None):
        # Set attribute equal to an instance of our generator yielding events
        # scraped from the Legistar web interface. This allows us to pause
        # and resume iteration as needed.
        self._events = self._scrapeWebCalendar()

        # Instantiate dictionary where events from generator are stored as they
        # are scraped.
        self._scraped_events = {}

        for api_event in self.api_events(since_datetime):

            time_str = api_event['EventTime']
            if not time_str:  # If we don't have an event time, skip it
                continue

            start_time = time.strptime(time_str, '%I:%M %p')

            start = self.toTime(api_event['EventDate'])
            api_event['start'] = start.replace(hour=start_time.tm_hour,
                                               minute=start_time.tm_min)

            api_event['status'] = self._event_status(api_event)

            if self._not_in_web_interface(api_event):
                continue

            else:
                # None if entire web calendar scraped but API event not found
                web_event = self.web_results(api_event)

                if web_event:
                    yield api_event, web_event

                else:
                    event_url = '{0}/events/{1}'.format(self.BASE_URL, api_event['EventId'])
                    self.warning('API event could not be found in web interface: {0}'.format(event_url))
                    continue

    def api_events(self, since_datetime=None):
        # scrape from oldest to newest. This makes resuming big
        # scraping jobs easier because upon a scrape failure we can
        # import everything scraped and then scrape everything newer
        # then the last event we scraped
        params = {'$orderby': 'EventLastModifiedUtc'}

        if since_datetime:
            # Minutes are often published after an event occurs – without a
            # corresponding event modification. Query all update fields so later
            # changes are always caught by our scraper, particularly when
            # scraping narrower windows of time.
            update_fields = ('EventLastModifiedUtc',
                             'EventAgendaLastPublishedUTC',
                             'EventMinutesLastPublishedUTC')

            since_fmt = " gt datetime'{}'".format(since_datetime.isoformat())
            since_filter = ' or '.join(field + since_fmt for field in update_fields)

            params['$filter'] = since_filter

        events_url = self.BASE_URL + '/events/'

        yield from self.pages(events_url,
                              params=params,
                              item_key="EventId")

    def agenda(self, event):
        agenda_url = (self.BASE_URL +
                      '/events/{}/eventitems'.format(event['EventId']))

        response = self.get(agenda_url)

        # If an event item does not have a value for
        # EventItemAgendaSequence, it is not on the agenda
        filtered_items = (item for item in response.json()
                          if (item['EventItemTitle'] and
                              item['EventItemAgendaSequence']))
        sorted_items = sorted(filtered_items,
                              key=lambda item: item['EventItemAgendaSequence'])

        for item in sorted_items:
            self._suppress_item_matter(item, agenda_url)
            yield item

    def minutes(self, event):
        minutes_url = (self.BASE_URL +
                       '/events/{}/eventitems'.format(event['EventId']))

        response = self.get(minutes_url)

        # If an event item does not have a value for
        # EventItemMinutesSequence, it is not in the minutes
        filtered_items = (item for item in response.json()
                          if (item['EventItemTitle'] and
                              item['EventItemMinutesSequence']))
        sorted_items = sorted(filtered_items,
                              key=lambda item: item['EventItemMinutesSequence'])

        for item in sorted_items:
            self._suppress_item_matter(item, minutes_url)
            yield item

    def _suppress_item_matter(self, item, agenda_url):
        '''
        Agenda items in Legistar do not always display links to
        associated matter files even if the same agenda item
        in the API references a Matter File. The agenda items
        we scrape should honor the suppression on the Legistar
        agendas.

        This is also practical because matter files that are hidden
        in the Legistar Agenda do not seem to available for scraping
        on Legistar or through the API

        Since we are not completely sure that the same suppression
        logic should be used for all Legislative Bodies, this method
        is currently just a hook for being overridden in particular
        scrapers. As of now, at least LA Metro uses this hook.
        '''
        pass

    def rollcalls(self, event):
        for item in self.agenda(event):
            if item['EventItemRollCallFlag']:
                rollcall_url = self.BASE_URL + \
                    '/eventitems/{}/rollcalls'.format(item['EventItemId'])

                response = self.get(rollcall_url)

                for item in response.json():
                    yield item

    def web_results(self, event):
        api_key = (event['EventBodyName'].strip(),
                   event['start'])

        # Check the cache of events we've already scraped from the web interface
        # for the API event at hand.
        if api_key in self._scraped_events:
            return self._scraped_events[api_key]

        else:
            # If API event not in web scrape cache, continue scraping the web
            # interface.
            for web_key, event in self._events:
                self._scraped_events[web_key] = event
                # When we find the API event, stop scraping.
                if web_key == api_key:
                    return event

    def _scrapeWebCalendar(self):
        '''Generator yielding events from Legistar in roughly reverse
        chronological order.
        '''
        web_scraper = LegistarEventsScraper(
            requests_per_minute=self.requests_per_minute)

        if self.cache_storage:
            web_scraper.cache_storage = self.cache_storage

        if self.requests_per_minute == 0:
            web_scraper.cache_write_only = False

        web_scraper.EVENTSPAGE = self.EVENTSPAGE
        web_scraper.BASE_URL = self.WEB_URL
        web_scraper.TIMEZONE = self.TIMEZONE
        web_scraper.date_format = '%m/%d/%Y'

        for event, _ in web_scraper.events(follow_links=False):
            event_key = self._event_key(event, web_scraper)
            yield event_key, event

    def _event_key(self, event, web_scraper):
        '''Since Legistar InSite contains more information about events than
        are available in the API, we need to scrape both. Then, we have
        to line them up. This method makes a key that should be
        uniquely identify every event and will allow us to link
        events from the two data sources.
        '''
        response = web_scraper.get(event['iCalendar']['url'], verify=False)
        event_time = web_scraper.ical(response.text).subcomponents[0]['DTSTART'].dt
        event_time = pytz.timezone(self.TIMEZONE).localize(event_time)

        key = (event['Name']['label'],
               event_time)

        return key

    def addDocs(self, e, events, doc_type):
        try:
            if events[doc_type] != 'Not\xa0available':
                e.add_document(note=events[doc_type]['label'],
                               url=events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError:
            pass

    def _event_status(self, event):
        '''Events can have a status of tentative, confirmed, cancelled, or
        passed (http://docs.opencivicdata.org/en/latest/data/event.html). By
        default, set status to passed if the current date and time exceeds the
        event date and time, or confirmed otherwise. Available for override in
        jurisdictional scrapers.
        '''
        if datetime.datetime.utcnow().replace(tzinfo=pytz.utc) > event['start']:
            status = 'passed'
        else:
            status = 'confirmed'

        return status

    def _not_in_web_interface(self, event):
        '''Occasionally, an event will appear in the API, but not in the web
        interface. This method checks attributes of the API event that tell us
        whether the given event is one of those cases, returning True if so, and
        False otherwise. Available for override in jurisdictional scrapers.
        '''
        return False
<commit_msg>use scrapelib's facility to turn off caching for the top level events page<commit_after>import time
import datetime
from collections import deque

import lxml.html
import pytz
import icalendar

from .base import LegistarScraper, LegistarAPIScraper

class LegistarEventsScraper(LegistarScraper):
    def eventPages(self, since):
        response = self.get(self.EVENTSPAGE, verify=False)
        entry = response.text
        page = lxml.html.fromstring(entry)
        page.make_links_absolute(self.EVENTSPAGE)

        for page in self.eventSearch(page, since):
            yield page

    def should_cache_response(self, response):
        # Never cache the top level events page, because that may result in
        # expired .NET state values.
        return (super().should_cache_response(response) and
                response.url != self.EVENTSPAGE)

    def eventSearch(self, page, since):
        payload = self.sessionSecrets(page)

        payload['ctl00_ContentPlaceHolder1_lstYears_ClientState'] = '{"value":"%s"}' % since

        payload['__EVENTTARGET'] = 'ctl00$ContentPlaceHolder1$lstYears'

        return self.pages(self.EVENTSPAGE, payload)

    def events(self, follow_links=True, since=None):
        # If an event is added to the the legistar system while we
        # are scraping, it will shift the list of events down and
        # we might revisit the same event. So, we keep track of
        # the last few events we've visited in order to
        # make sure we are not revisiting
        scraped_events = deque([], maxlen=10)

        current_year = self.now().year

        if since:
            if since > current_year:
                raise ValueError(
                    'Value of :since cannot exceed {}'.format(current_year))
            else:
                since_year = since - 1

        else:
            since_year = 0

        # Anticipate events will be scheduled for the following year to avoid
        # missing upcoming events during scrapes near the end of the current
        # year.
        for year in range(current_year + 1, since_year, -1):
            no_events_in_year = True

            for page in self.eventPages(year):
                no_events_in_year = False
                events_table = page.xpath("//table[@class='rgMasterTable']")[0]
                for event, _, _ in self.parseDataTable(events_table):
                    if follow_links and type(event["Meeting Details"]) == dict:
                        detail_url = event["Meeting Details"]['url']
                        if detail_url in scraped_events:
                            continue
                        else:
                            scraped_events.append(detail_url)

                        agenda = self.agenda(detail_url)

                    else:
                        agenda = None

                    yield event, agenda

            if no_events_in_year:  # Bail from scrape if no results returned from year
                break

    def agenda(self, detail_url):
        page = self.lxmlize(detail_url)

        payload = self.sessionSecrets(page)

        payload.update({"__EVENTARGUMENT": "3:1",
                        "__EVENTTARGET": "ctl00$ContentPlaceHolder1$menuMain"})

        for page in self.pages(detail_url, payload):
            agenda_table = page.xpath(
                "//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
            agenda = self.parseDataTable(agenda_table)
            yield from agenda

    def addDocs(self, e, events, doc_type):
        try:
            if events[doc_type] != 'Not\xa0available':
                e.add_document(note=events[doc_type]['label'],
                               url=events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError:
            pass

    def extractRollCall(self, action_detail_url):
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            rollcall_table = action_detail_page.xpath(
                "//table[@id='ctl00_ContentPlaceHolder1_gridRollCall_ctl00']")[0]
        except IndexError:
            self.warning("No rollcall found in table")
            return []
        roll_call = list(self.parseDataTable(rollcall_table))
        call_list = []
        for call, _, _ in roll_call:
            option = call['Attendance']
            call_list.append((option,
                              call['Person Name']['label']))

        return call_list

    def ical(self, ical_text):
        value = icalendar.Calendar.from_ical(ical_text)
        return value

class LegistarAPIEventScraper(LegistarAPIScraper):

    def events(self, since_datetime=None):
        # Set attribute equal to an instance of our generator yielding events
        # scraped from the Legistar web interface. This allows us to pause
        # and resume iteration as needed.
        self._events = self._scrapeWebCalendar()

        # Instantiate dictionary where events from generator are stored as they
        # are scraped.
        self._scraped_events = {}

        for api_event in self.api_events(since_datetime):

            time_str = api_event['EventTime']
            if not time_str:  # If we don't have an event time, skip it
                continue

            start_time = time.strptime(time_str, '%I:%M %p')

            start = self.toTime(api_event['EventDate'])
            api_event['start'] = start.replace(hour=start_time.tm_hour,
                                               minute=start_time.tm_min)

            api_event['status'] = self._event_status(api_event)

            if self._not_in_web_interface(api_event):
                continue

            else:
                # None if entire web calendar scraped but API event not found
                web_event = self.web_results(api_event)

                if web_event:
                    yield api_event, web_event

                else:
                    event_url = '{0}/events/{1}'.format(self.BASE_URL, api_event['EventId'])
                    self.warning('API event could not be found in web interface: {0}'.format(event_url))
                    continue

    def api_events(self, since_datetime=None):
        # scrape from oldest to newest. This makes resuming big
        # scraping jobs easier because upon a scrape failure we can
        # import everything scraped and then scrape everything newer
        # then the last event we scraped
        params = {'$orderby': 'EventLastModifiedUtc'}

        if since_datetime:
            # Minutes are often published after an event occurs – without a
            # corresponding event modification. Query all update fields so later
            # changes are always caught by our scraper, particularly when
            # scraping narrower windows of time.
            update_fields = ('EventLastModifiedUtc',
                             'EventAgendaLastPublishedUTC',
                             'EventMinutesLastPublishedUTC')

            since_fmt = " gt datetime'{}'".format(since_datetime.isoformat())
            since_filter = ' or '.join(field + since_fmt for field in update_fields)

            params['$filter'] = since_filter

        events_url = self.BASE_URL + '/events/'

        yield from self.pages(events_url,
                              params=params,
                              item_key="EventId")

    def agenda(self, event):
        agenda_url = (self.BASE_URL +
                      '/events/{}/eventitems'.format(event['EventId']))

        response = self.get(agenda_url)

        # If an event item does not have a value for
        # EventItemAgendaSequence, it is not on the agenda
        filtered_items = (item for item in response.json()
                          if (item['EventItemTitle'] and
                              item['EventItemAgendaSequence']))
        sorted_items = sorted(filtered_items,
                              key=lambda item: item['EventItemAgendaSequence'])

        for item in sorted_items:
            self._suppress_item_matter(item, agenda_url)
            yield item

    def minutes(self, event):
        minutes_url = (self.BASE_URL +
                       '/events/{}/eventitems'.format(event['EventId']))

        response = self.get(minutes_url)

        # If an event item does not have a value for
        # EventItemMinutesSequence, it is not in the minutes
        filtered_items = (item for item in response.json()
                          if (item['EventItemTitle'] and
                              item['EventItemMinutesSequence']))
        sorted_items = sorted(filtered_items,
                              key=lambda item: item['EventItemMinutesSequence'])

        for item in sorted_items:
            self._suppress_item_matter(item, minutes_url)
            yield item

    def _suppress_item_matter(self, item, agenda_url):
        '''
        Agenda items in Legistar do not always display links to
        associated matter files even if the same agenda item
        in the API references a Matter File. The agenda items
        we scrape should honor the suppression on the Legistar
        agendas.

        This is also practical because matter files that are hidden
        in the Legistar Agenda do not seem to available for scraping
        on Legistar or through the API

        Since we are not completely sure that the same suppression
        logic should be used for all Legislative Bodies, this method
        is currently just a hook for being overridden in particular
        scrapers. As of now, at least LA Metro uses this hook.
        '''
        pass

    def rollcalls(self, event):
        for item in self.agenda(event):
            if item['EventItemRollCallFlag']:
                rollcall_url = self.BASE_URL + \
                    '/eventitems/{}/rollcalls'.format(item['EventItemId'])

                response = self.get(rollcall_url)

                for item in response.json():
                    yield item

    def web_results(self, event):
        api_key = (event['EventBodyName'].strip(),
                   event['start'])

        # Check the cache of events we've already scraped from the web interface
        # for the API event at hand.
        if api_key in self._scraped_events:
            return self._scraped_events[api_key]

        else:
            # If API event not in web scrape cache, continue scraping the web
            # interface.
            for web_key, event in self._events:
                self._scraped_events[web_key] = event
                # When we find the API event, stop scraping.
                if web_key == api_key:
                    return event

    def _scrapeWebCalendar(self):
        '''Generator yielding events from Legistar in roughly reverse
        chronological order.
        '''
        web_scraper = LegistarEventsScraper(
            requests_per_minute=self.requests_per_minute)

        if self.cache_storage:
            web_scraper.cache_storage = self.cache_storage

        if self.requests_per_minute == 0:
            web_scraper.cache_write_only = False

        web_scraper.EVENTSPAGE = self.EVENTSPAGE
        web_scraper.BASE_URL = self.WEB_URL
        web_scraper.TIMEZONE = self.TIMEZONE
        web_scraper.date_format = '%m/%d/%Y'

        for event, _ in web_scraper.events(follow_links=False):
            event_key = self._event_key(event, web_scraper)
            yield event_key, event

    def _event_key(self, event, web_scraper):
        '''Since Legistar InSite contains more information about events than
        are available in the API, we need to scrape both. Then, we have
        to line them up. This method makes a key that should be
        uniquely identify every event and will allow us to link
        events from the two data sources.
        '''
        response = web_scraper.get(event['iCalendar']['url'], verify=False)
        event_time = web_scraper.ical(response.text).subcomponents[0]['DTSTART'].dt
        event_time = pytz.timezone(self.TIMEZONE).localize(event_time)

        key = (event['Name']['label'],
               event_time)

        return key

    def addDocs(self, e, events, doc_type):
        try:
            if events[doc_type] != 'Not\xa0available':
                e.add_document(note=events[doc_type]['label'],
                               url=events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError:
            pass

    def _event_status(self, event):
        '''Events can have a status of tentative, confirmed, cancelled, or
        passed (http://docs.opencivicdata.org/en/latest/data/event.html). By
        default, set status to passed if the current date and time exceeds the
        event date and time, or confirmed otherwise. Available for override in
        jurisdictional scrapers.
        '''
        if datetime.datetime.utcnow().replace(tzinfo=pytz.utc) > event['start']:
            status = 'passed'
        else:
            status = 'confirmed'

        return status

    def _not_in_web_interface(self, event):
        '''Occasionally, an event will appear in the API, but not in the web
        interface. This method checks attributes of the API event that tell us
        whether the given event is one of those cases, returning True if so, and
        False otherwise. Available for override in jurisdictional scrapers.
        '''
        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = [
    'User',
    'Item',
    'HackerNews',
    'InvalidAPIVersion',
    'InvalidItemID',
    'InvalidUserID']

"""
haxor
Unofficial Python wrapper for official Hacker News API

@author avinash sajjanshetty
@email a@sajjanshetty.com
"""

import datetime
import json

import requests

from settings import supported_api_versions

class InvalidItemID(Exception):
    pass

class InvalidUserID(Exception):
    pass

class InvalidAPIVersion(Exception):
    pass

class HTTPError(Exception):
    pass

class HackerNews(object):

    def __init__(self, version='v0'):
        """
        Args:
            version (string): specifies Hacker News API version. Default is `v0`.

        Raises:
          InvalidAPIVersion: If Hacker News version is not supported.

        """
        try:
            self.base_url = supported_api_versions[version]
        except KeyError:
            raise InvalidAPIVersion

    def _get(self, url):
        """Internal method used for GET requests

        Args:
            url (string): URL to send GET.

        Returns:
            requests' response object

        Raises:
          HTTPError: If HTTP request failed.

        """
        response = requests.get(url)
        if response.status_code == requests.codes.ok:
            return response
        else:
            raise HTTPError

    def _get_page(self, page):
        return self._get('{0}{1}.json'.format(self.base_url, page))

    def _get_page_param(self, page, param):
        return self._get('{0}{1}/{2}.json'.format(self.base_url, page, param))

    def get_item(self, item_id):
        """Returns Hacker News `Item` object.

        Args:
            item_id (int or string): Unique item id of Hacker News story, comment etc.

        Returns:
            `Item` object representing Hacker News item.

        Raises:
          InvalidItemID: If corresponding Hacker News story does not exist.

        """

        response = self._get_page_param('item', item_id).json()

        if not response:
            raise InvalidItemID

        return Item(response)

    def get_user(self, user_id):
        """Returns Hacker News `User` object.

        Args:
            user_id (string): unique user id of a Hacker News user.

        Returns:
            `User` object representing a user on Hacker News.

        Raises:
          InvalidUserID: If no such user exists on Hacker News.

        """
        response = self._get_page_param('user', user_id).json()

        if not response:
            raise InvalidUserID

        return User(response)

    def top_stories(self, limit=None):
        """Returns list of item ids of current top stories

        Args:
            limit (int): specifies the number of stories to be returned.

        Returns:
            `list` object containing ids of top stories.
        """
        return self._get_page('topstories').json()[:limit]

    def get_max_item(self):
        """Returns list of item ids of current top stories

        Args:
            limit (int): specifies the number of stories to be returned.

        Returns:
            `int` if successful.
        """
        return self._get_page('maxitem').json()

class Item(object):

    """
    Represents stories, comments, jobs, Ask HNs and polls
    """

    def __init__(self, data):
        self.item_id = data.get('id')
        self.deleted = data.get('deleted')
        self.item_type = data.get('type')
        self.by = data.get('by')
        self.submission_time = datetime.datetime.fromtimestamp(
            data.get(
                'time',
                0))
        self.text = data.get('text')
        self.dead = data.get('dead')
        self.parent = data.get('parent')
        self.kids = data.get('kids')
        self.url = data.get('url')
        self.score = data.get('score')
        self.title = data.get('title')
        self.parts = data.get('parts')
        self.raw = json.dumps(data)

    def __repr__(self):
        return '{0}: {1}'.format(self.item_id, self.title)

class User(object):

    """
    Represents a hacker i.e. a user on Hacker News
    """

    def __init__(self, data):
        self.user_id = data.get('id')
        self.delay = data.get('delay')
        self.created = datetime.datetime.fromtimestamp(data.get('time', 0))
        self.karma = data.get('karma')
        self.about = data.get('about')
        self.submitted = data.get('submitted')
        self.raw = json.dumps(data)

    def __repr__(self):
        return '{0}: {1}'.format(self.user_id, self.karma)
<commit_msg>adding proper py3k support<commit_after>#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import unicode_literals

__all__ = [
    'User',
    'Item',
    'HackerNews',
    'InvalidAPIVersion',
    'InvalidItemID',
    'InvalidUserID']

"""
haxor
Unofficial Python wrapper for official Hacker News API

@author avinash sajjanshetty
@email a@sajjanshetty.com
"""

import datetime
import json

import requests

from .settings import supported_api_versions

class InvalidItemID(Exception):
    pass

class InvalidUserID(Exception):
    pass

class InvalidAPIVersion(Exception):
    pass

class HTTPError(Exception):
    pass

class HackerNews(object):

    def __init__(self, version='v0'):
        """
        Args:
            version (string): specifies Hacker News API version. Default is `v0`.

        Raises:
          InvalidAPIVersion: If Hacker News version is not supported.

        """
        try:
            self.base_url = supported_api_versions[version]
        except KeyError:
            raise InvalidAPIVersion

    def _get(self, url):
        """Internal method used for GET requests

        Args:
            url (string): URL to send GET.

        Returns:
            requests' response object

        Raises:
          HTTPError: If HTTP request failed.

        """
        response = requests.get(url)
        if response.status_code == requests.codes.ok:
            return response
        else:
            raise HTTPError

    def _get_page(self, page):
        return self._get('{0}{1}.json'.format(self.base_url, page))

    def _get_page_param(self, page, param):
        return self._get('{0}{1}/{2}.json'.format(self.base_url, page, param))

    def get_item(self, item_id):
        """Returns Hacker News `Item` object.

        Args:
            item_id (int or string): Unique item id of Hacker News story, comment etc.

        Returns:
            `Item` object representing Hacker News item.

        Raises:
          InvalidItemID: If corresponding Hacker News story does not exist.

        """

        response = self._get_page_param('item', item_id).json()

        if not response:
            raise InvalidItemID

        return Item(response)

    def get_user(self, user_id):
        """Returns Hacker News `User` object.

        Args:
            user_id (string): unique user id of a Hacker News user.

        Returns:
            `User` object representing a user on Hacker News.

        Raises:
          InvalidUserID: If no such user exists on Hacker News.

        """
        response = self._get_page_param('user', user_id).json()

        if not response:
            raise InvalidUserID

        return User(response)

    def top_stories(self, limit=None):
        """Returns list of item ids of current top stories

        Args:
            limit (int): specifies the number of stories to be returned.

        Returns:
            `list` object containing ids of top stories.
        """
        return self._get_page('topstories').json()[:limit]

    def get_max_item(self):
        """Returns list of item ids of current top stories

        Args:
            limit (int): specifies the number of stories to be returned.

        Returns:
            `int` if successful.
        """
        return self._get_page('maxitem').json()

class Item(object):

    """
    Represents stories, comments, jobs, Ask HNs and polls
    """

    def __init__(self, data):
        self.item_id = data.get('id')
        self.deleted = data.get('deleted')
        self.item_type = data.get('type')
        self.by = data.get('by')
        self.submission_time = datetime.datetime.fromtimestamp(
            data.get(
                'time',
                0))
        self.text = data.get('text')
        self.dead = data.get('dead')
        self.parent = data.get('parent')
        self.kids = data.get('kids')
        self.url = data.get('url')
        self.score = data.get('score')
        self.title = data.get('title')
        self.parts = data.get('parts')
        self.raw = json.dumps(data)

    def __repr__(self):
        return '{0}: {1}'.format(self.item_id, self.title)

class User(object):

    """
    Represents a hacker i.e. a user on Hacker News
    """

    def __init__(self, data):
        self.user_id = data.get('id')
        self.delay = data.get('delay')
        self.created = datetime.datetime.fromtimestamp(data.get('time', 0))
        self.karma = data.get('karma')
        self.about = data.get('about')
        self.submitted = data.get('submitted')
        self.raw = json.dumps(data)

    def __repr__(self):
        return '{0}: {1}'.format(self.user_id, self.karma)
<|endoftext|>