<commit_before>import json
from django.db import connection
import math
from elo.DataProviderInterface import DataProviderInterface
from elo.model import EloModel
from model.models import UserSkill, QuestionDifficulty, DatabaseDataProvider, Skill
from questions.models import Question

def dictfetchall(cursor):
    "Returns all rows from a cursor as a dict"
    desc = cursor.description
    for row in cursor.fetchall():
        yield dict(zip([col[0] for col in desc], row))

def process_answer(answer):
    """
    Update model considering this answer
    """
    provider = DatabaseDataProvider()
    elo = EloModel(provider)
    provider.set_answer(answer)
    elo.update()

def recalculate_model():
    """
    Process all answers
    """

    cursor = connection.cursor()
    cursor.execute(
        "SELECT "
            "questions_answer.id, "
            "questions_answer.question_id as question, "
            "questions_answer.user_id as user, "
            "questions_answer.solving_time, "
            "questions_answer.correctly_solved "
        "FROM questions_answer "
        "ORDER BY questions_answer.timestamp ASC")

    provider = CachingDatabaseDataProvider()
    init_difficulty(provider)
    elo = EloModel(provider)
    for answer in dictfetchall(cursor):
        provider.set_answer(answer)
        elo.update()

    QuestionDifficulty.objects.all().delete()
    UserSkill.objects.all().delete()
    provider.save_data()

def init_difficulty(data):
    for question in data.get_questions():
        question_data = data.get_questions_data(question)
        question_data = json.loads(question_data)
        if "answer" in question_data.keys():
            value = abs(float(question_data["answer"]))
            if value <= 0:
                value = 1
            value = math.log(value) - 3
            data.set_difficulty(question, value)
        else:
            print "answer not found"

class CachingDatabaseDataProvider(DataProviderInterface):
    def __init__(self):
        cursor = connection.cursor()
        cursor.execute(
            "SELECT "
                "questions_question.id as pk, "
                "questions_question.skill_id as skill, "
                "questions_question.data as data, "
                "questions_question.type "
            "FROM questions_question "
            "LEFT OUTER JOIN model_questiondifficulty ON ( questions_question.id = model_questiondifficulty.question_id )"
        )
        self.questions = {}
        for q in dictfetchall(cursor):
            q["attempts"] = 0.
            q["total_solving_time"] = 0.
            self.questions[q["pk"]] = q

        cursor.execute(
            "SELECT "
                "model_skill.id as pk, "
                "model_skill.parent_id as parent "
            "FROM model_skill "
        )
        self.skills = {}
        for q in dictfetchall(cursor):
            self.skills[q["pk"]] = q

        self.user_skills = {}
        self.attempts_count = {}

    def save_data(self):
        data = []
        for pk, q in self.questions.items():
            if "difficulty" in q.keys() and q["difficulty"] is not None:
                qd = QuestionDifficulty(question_id=pk, value=q["difficulty"])
                if "time_intensity" in q.keys() and q["time_intensity"] is not None:
                    qd.time_intensity=q["time_intensity"]
                data.append(qd)
        QuestionDifficulty.objects.bulk_create(data)

        data = []
        for user, d in self.user_skills.items():
            for skill, value in d.items():
                data.append(UserSkill(user_id=user, skill_id=skill, value=value))
        UserSkill.objects.bulk_create(data)

    def set_answer(self, answer):
        super(CachingDatabaseDataProvider, self).set_answer(answer)
        self.mark_attempt(answer)

    def mark_attempt(self, answer):
        user = self.get_user(answer)
        question = self.get_question(answer)
        self.questions[question]["attempts"] += 1
        self.questions[question]["total_solving_time"] += self.get_solving_time(answer)
        if question not in self.attempts_count.keys():
            self.attempts_count[question] = {}
        if user not in self.attempts_count[question].keys():
            self.attempts_count[question][user] = 1
        else:
            self.attempts_count[question][user] += 1

    def get_question(self, answer):
        return answer["question"]

    def get_user(self, answer):
        return answer["user"]

    def get_skill(self, question):
        return self.questions[question]["skill"]

    def get_question_type(self, question):
        return self.questions[question]["type"]

    def get_user_skill(self, user, skill):
        try:
            return self.user_skills[user][skill]
        except:
            return None

    def set_user_skill(self, user, skill, value):
        if user not in self.user_skills.keys():
            self.user_skills[user] = {}
        self.user_skills[user][skill] = value

    def get_parent_skill(self, skill):
        return self.skills[skill]["parent"]

    def get_difficulty(self, question):
        try:
            return self.questions[question]["difficulty"]
        except:
            return None

    def set_difficulty(self, question, value):
        self.questions[question]["difficulty"] = value

    def get_solving_time(self, answer):
        return answer["solving_time"]

    def get_correctness(self, answer):
        return answer["correctly_solved"]

    def is_first_attempt(self, answer):
        user = self.get_user(answer)
        question = self.get_question(answer)
        try:
            return self.attempts_count[question][user] < 2
        except:
            return True

    def get_first_attempts_count(self, question):
        count = 0
        for _, v in self.attempts_count[question].items():
            count += v
        return count

    def get_attempts_count(self, question):
        return sum(self.attempts_count[question].values())

    def get_questions(self):
        return self.questions.keys()

    def get_questions_data(self, question):
        return self.questions[question]["data"]

    def get_time_intensity(self, question):
        try:
            return self.questions[question]["time_intensity"]
        except:
            return None

    def set_time_intensity(self, question, value):
        self.questions[question]["time_intensity"] = value<commit_msg>fix recalculate script<commit_after>import json
from django.db import connection
import math
from elo.DataProviderInterface import DataProviderInterface
from elo.model import EloModel
from model.models import UserSkill, QuestionDifficulty, DatabaseDataProvider, Skill
from questions.models import Question

def dictfetchall(cursor):
    "Returns all rows from a cursor as a dict"
    desc = cursor.description
    for row in cursor.fetchall():
        yield dict(zip([col[0] for col in desc], row))

def process_answer(answer):
    """
    Update model considering this answer
    """
    provider = DatabaseDataProvider()
    elo = EloModel(provider)
    provider.set_answer(answer)
    elo.update()

def recalculate_model():
    """
    Process all answers
    """

    cursor = connection.cursor()
    cursor.execute(
        "SELECT "
            "questions_answer.id, "
            "questions_answer.question_id as question, "
            "questions_answer.user_id as user, "
            "questions_answer.solving_time, "
            "questions_answer.correctly_solved "
        "FROM questions_answer "
        "ORDER BY questions_answer.timestamp ASC")

    provider = CachingDatabaseDataProvider()
    init_difficulty(provider)
    elo = EloModel(provider)
    for answer in dictfetchall(cursor):
        provider.set_answer(answer)
        elo.update()

    QuestionDifficulty.objects.all().delete()
    UserSkill.objects.all().delete()
    provider.save_data()

def init_difficulty(data):
    for question in data.get_questions():
        question_data = data.get_questions_data(question)
        question_data = json.loads(question_data)
        if "answer" in question_data.keys():
            value = abs(float(question_data["answer"]))
            if value <= 0:
                value = 1
            value = math.log(value) - 3
            data.set_difficulty(question, value)
        else:
            print "answer not found"

class CachingDatabaseDataProvider(DataProviderInterface):
    def __init__(self):
        cursor = connection.cursor()
        cursor.execute(
            "SELECT "
                "questions_question.id as pk, "
                "questions_question.skill_id as skill, "
                "questions_question.data as data, "
                "questions_question.type "
            "FROM questions_question "
            "LEFT OUTER JOIN model_questiondifficulty ON ( questions_question.id = model_questiondifficulty.question_id )"
        )
        self.questions = {}
        for q in dictfetchall(cursor):
            q["attempts"] = 0.
            q["total_solving_time"] = 0.
            self.questions[q["pk"]] = q

        cursor.execute(
            "SELECT "
                "model_skill.id as pk, "
                "model_skill.parent_id as parent "
            "FROM model_skill "
        )
        self.skills = {}
        for q in dictfetchall(cursor):
            self.skills[q["pk"]] = q

        self.user_skills = {}
        self.attempts_count = {}

    def save_data(self):
        data = []
        for pk, q in self.questions.items():
            if "difficulty" in q.keys() and q["difficulty"] is not None:
                qd = QuestionDifficulty(question_id=pk, value=q["difficulty"])
                if "time_intensity" in q.keys() and q["time_intensity"] is not None:
                    qd.time_intensity=q["time_intensity"]
                data.append(qd)
        QuestionDifficulty.objects.bulk_create(data)

        data = []
        for user, d in self.user_skills.items():
            for skill, value in d.items():
                data.append(UserSkill(user_id=user, skill_id=skill, value=value))
        UserSkill.objects.bulk_create(data, 10000)

    def set_answer(self, answer):
        super(CachingDatabaseDataProvider, self).set_answer(answer)
        self.mark_attempt(answer)

    def mark_attempt(self, answer):
        user = self.get_user(answer)
        question = self.get_question(answer)
        self.questions[question]["attempts"] += 1
        self.questions[question]["total_solving_time"] += self.get_solving_time(answer)
        if question not in self.attempts_count.keys():
            self.attempts_count[question] = {}
        if user not in self.attempts_count[question].keys():
            self.attempts_count[question][user] = 1
        else:
            self.attempts_count[question][user] += 1

    def get_question(self, answer):
        return answer["question"]

    def get_user(self, answer):
        return answer["user"]

    def get_skill(self, question):
        return self.questions[question]["skill"]

    def get_question_type(self, question):
        return self.questions[question]["type"]

    def get_user_skill(self, user, skill):
        try:
            return self.user_skills[user][skill]
        except:
            return None

    def set_user_skill(self, user, skill, value):
        if user not in self.user_skills.keys():
            self.user_skills[user] = {}
        self.user_skills[user][skill] = value

    def get_parent_skill(self, skill):
        return self.skills[skill]["parent"]

    def get_difficulty(self, question):
        try:
            return self.questions[question]["difficulty"]
        except:
            return None

    def set_difficulty(self, question, value):
        self.questions[question]["difficulty"] = value

    def get_solving_time(self, answer):
        return answer["solving_time"]

    def get_correctness(self, answer):
        return answer["correctly_solved"]

    def is_first_attempt(self, answer):
        user = self.get_user(answer)
        question = self.get_question(answer)
        try:
            return self.attempts_count[question][user] < 2
        except:
            return True

    def get_first_attempts_count(self, question):
        count = 0
        for _, v in self.attempts_count[question].items():
            count += v
        return count

    def get_attempts_count(self, question):
        return sum(self.attempts_count[question].values())

    def get_questions(self):
        return self.questions.keys()

    def get_questions_data(self, question):
        return self.questions[question]["data"]

    def get_time_intensity(self, question):
        try:
            return self.questions[question]["time_intensity"]
        except:
            return None

    def set_time_intensity(self, question, value):
        self.questions[question]["time_intensity"] = value<|endoftext|>


<commit_before># -*- coding: utf-8 -*-777777yyy
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import print_function, unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import getdate

class StudentApplicant(Document):
	def autoname(self):
		from frappe.model.naming import set_name_by_naming_series
		if self.student_admission:
			naming_series = None
			if self.program:
				# set the naming series from the student admission if provided.
				student_admission = get_student_admission_data(self.student_admission, self.program)
				if student_admission:
					naming_series = student_admission.get("applicant_naming_series")
				else:
					naming_series = None
			else:
				frappe.throw(_("Select the program first"))

			if naming_series:
				self.naming_series = naming_series

		set_name_by_naming_series(self)

	def validate(self):
		self.title = " ".join(filter(None, [self.first_name, self.middle_name, self.last_name]))
		if self.student_admission and self.program and self.date_of_birth:
			self.validation_from_student_admission()

	def on_update_after_submit(self):
		student = frappe.get_list("Student",  filters= {"student_applicant": self.name})
		if student:
			frappe.throw(_("Cannot change status as student {0} is linked with student application {1}").format(student[0].name, self.name))

	def on_submit(self):
		if self.paid and not self.student_admission:
			frappe.throw(_("Please select Student Admission which is mandatory for the paid student applicant"))

	def validation_from_student_admission(self):
		student_admission = get_student_admission_data(self.student_admission, self.program)

		# different validation for minimum and maximum age so that either min/max can also work independently.
		if student_admission and student_admission.minimum_age and \
			getdate(student_admission.minimum_age) < getdate(self.date_of_birth):
				frappe.throw(_("Not eligible for the admission in this program as per DOB"))

		if student_admission and student_admission.maximum_age and \
			getdate(student_admission.maximum_age) > getdate(self.date_of_birth):
				frappe.throw(_("Not eligible for the admission in this program as per DOB"))

	def on_payment_authorized(self, *args, **kwargs):
		self.db_set('paid', 1)

def get_student_admission_data(student_admission, program):
	student_admission = frappe.db.sql("""select sa.admission_start_date, sa.admission_end_date,
		sap.program, sap.minimum_age, sap.maximum_age, sap.applicant_naming_series
		from `tabStudent Admission` sa, `tabStudent Admission Program` sap
		where sa.name = sap.parent and sa.name = %s and sap.program = %s""", (student_admission, program), as_dict=1)
	if student_admission:
		return student_admission[0]
	else:
		return None
<commit_msg>fix: date of birth cannot be greater than today on student application form<commit_after># -*- coding: utf-8 -*-777777yyy
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import print_function, unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import getdate

class StudentApplicant(Document):
	def autoname(self):
		from frappe.model.naming import set_name_by_naming_series
		if self.student_admission:
			naming_series = None
			if self.program:
				# set the naming series from the student admission if provided.
				student_admission = get_student_admission_data(self.student_admission, self.program)
				if student_admission:
					naming_series = student_admission.get("applicant_naming_series")
				else:
					naming_series = None
			else:
				frappe.throw(_("Select the program first"))

			if naming_series:
				self.naming_series = naming_series

		set_name_by_naming_series(self)

	def validate(self):
		self.validate_dates()
		self.title = " ".join(filter(None, [self.first_name, self.middle_name, self.last_name]))
		if self.student_admission and self.program and self.date_of_birth:
			self.validation_from_student_admission()

	def validate_dates(self):
		if self.date_of_birth and getdate(self.date_of_birth) >= getdate():
			frappe.throw(_("Date of Birth cannot be greater than today."))

	def on_update_after_submit(self):
		student = frappe.get_list("Student",  filters= {"student_applicant": self.name})
		if student:
			frappe.throw(_("Cannot change status as student {0} is linked with student application {1}").format(student[0].name, self.name))

	def on_submit(self):
		if self.paid and not self.student_admission:
			frappe.throw(_("Please select Student Admission which is mandatory for the paid student applicant"))

	def validation_from_student_admission(self):
		student_admission = get_student_admission_data(self.student_admission, self.program)

		# different validation for minimum and maximum age so that either min/max can also work independently.
		if student_admission and student_admission.minimum_age and \
			getdate(student_admission.minimum_age) < getdate(self.date_of_birth):
				frappe.throw(_("Not eligible for the admission in this program as per DOB"))

		if student_admission and student_admission.maximum_age and \
			getdate(student_admission.maximum_age) > getdate(self.date_of_birth):
				frappe.throw(_("Not eligible for the admission in this program as per DOB"))

	def on_payment_authorized(self, *args, **kwargs):
		self.db_set('paid', 1)

def get_student_admission_data(student_admission, program):
	student_admission = frappe.db.sql("""select sa.admission_start_date, sa.admission_end_date,
		sap.program, sap.minimum_age, sap.maximum_age, sap.applicant_naming_series
		from `tabStudent Admission` sa, `tabStudent Admission Program` sap
		where sa.name = sap.parent and sa.name = %s and sap.program = %s""", (student_admission, program), as_dict=1)
	if student_admission:
		return student_admission[0]
	else:
		return None
<|endoftext|>


<commit_before>import os
import unittest
import tempfile

import numpy

import chainer
from chainer import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.serializers import npz
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check

class TestLinear(unittest.TestCase):

    in_shape = (3,)
    out_size = 2

    def setUp(self):
        in_size = numpy.prod(self.in_shape)
        self.link = links.Linear(in_size, self.out_size)
        W = self.link.W.data
        W[...] = numpy.random.uniform(-1, 1, W.shape)
        b = self.link.b.data
        b[...] = numpy.random.uniform(-1, 1, b.shape)
        self.link.zerograds()

        self.W = W.copy()  # fixed on CPU
        self.b = b.copy()  # fixed on CPU

        x_shape = (4,) + self.in_shape
        self.x = numpy.random.uniform(-1, 1, x_shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (4, self.out_size)).astype(numpy.float32)
        self.y = self.x.reshape(4, -1).dot(W.T) + b

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        self.assertEqual(y.data.dtype, numpy.float32)
        gradient_check.assert_allclose(self.y, y.data)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.W, self.link.b), eps=1e-2)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

class TestLinearParameterShapePlaceholder(unittest.TestCase):

    in_size = 3
    in_shape = (in_size,)
    out_size = 2
    in_size_or_none = None

    def setUp(self):
        self.link = links.Linear(self.in_size_or_none, self.out_size)
        temp_x = numpy.random.uniform(-1, 1,
                                      (self.out_size,
                                       self.in_size)).astype(numpy.float32)
        self.link(chainer.Variable(temp_x))
        W = self.link.W.data
        W[...] = numpy.random.uniform(-1, 1, W.shape)
        b = self.link.b.data
        b[...] = numpy.random.uniform(-1, 1, b.shape)
        self.link.zerograds()

        self.W = W.copy()  # fixed on CPU
        self.b = b.copy()  # fixed on CPU

        x_shape = (4,) + self.in_shape
        self.x = numpy.random.uniform(-1, 1, x_shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (4, self.out_size)).astype(numpy.float32)
        self.y = self.x.reshape(4, -1).dot(W.T) + b

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        self.assertEqual(y.data.dtype, numpy.float32)
        gradient_check.assert_allclose(self.y, y.data)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.W, self.link.b), eps=1e-2)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    def test_serialization(self):
        lin1 = links.Linear(None, self.out_size)
        x = chainer.Variable(self.x)
        # Must call the link to initialize weights.
        lin1(x)
        w1 = lin1.W.data
        fd, temp_file_path = tempfile.mkstemp()
        os.close(fd)
        npz.save_npz(temp_file_path, lin1)
        lin2 = links.Linear(None, self.out_size)
        npz.load_npz(temp_file_path, lin2)
        w2 = lin2.W.data
        self.assertEquals((w1 == w2).all(), True)

class TestLinearWithSpatialDimensions(TestLinear):

    in_shape = (3, 2, 2)

class TestInvalidLinear(unittest.TestCase):

    def setUp(self):
        self.link = links.Linear(3, 2)
        self.x = numpy.random.uniform(-1, 1, (4, 1, 2)).astype(numpy.float32)

    def test_invalid_size(self):
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(self.x))

testing.run_module(__name__, __file__)
<commit_msg>Fixed order of imports.<commit_after>import os
import tempfile
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import gradient_check
from chainer import links
from chainer.serializers import npz
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check

class TestLinear(unittest.TestCase):

    in_shape = (3,)
    out_size = 2

    def setUp(self):
        in_size = numpy.prod(self.in_shape)
        self.link = links.Linear(in_size, self.out_size)
        W = self.link.W.data
        W[...] = numpy.random.uniform(-1, 1, W.shape)
        b = self.link.b.data
        b[...] = numpy.random.uniform(-1, 1, b.shape)
        self.link.zerograds()

        self.W = W.copy()  # fixed on CPU
        self.b = b.copy()  # fixed on CPU

        x_shape = (4,) + self.in_shape
        self.x = numpy.random.uniform(-1, 1, x_shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (4, self.out_size)).astype(numpy.float32)
        self.y = self.x.reshape(4, -1).dot(W.T) + b

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        self.assertEqual(y.data.dtype, numpy.float32)
        gradient_check.assert_allclose(self.y, y.data)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.W, self.link.b), eps=1e-2)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

class TestLinearParameterShapePlaceholder(unittest.TestCase):

    in_size = 3
    in_shape = (in_size,)
    out_size = 2
    in_size_or_none = None

    def setUp(self):
        self.link = links.Linear(self.in_size_or_none, self.out_size)
        temp_x = numpy.random.uniform(-1, 1,
                                      (self.out_size,
                                       self.in_size)).astype(numpy.float32)
        self.link(chainer.Variable(temp_x))
        W = self.link.W.data
        W[...] = numpy.random.uniform(-1, 1, W.shape)
        b = self.link.b.data
        b[...] = numpy.random.uniform(-1, 1, b.shape)
        self.link.zerograds()

        self.W = W.copy()  # fixed on CPU
        self.b = b.copy()  # fixed on CPU

        x_shape = (4,) + self.in_shape
        self.x = numpy.random.uniform(-1, 1, x_shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (4, self.out_size)).astype(numpy.float32)
        self.y = self.x.reshape(4, -1).dot(W.T) + b

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        self.assertEqual(y.data.dtype, numpy.float32)
        gradient_check.assert_allclose(self.y, y.data)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.W, self.link.b), eps=1e-2)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    def test_serialization(self):
        lin1 = links.Linear(None, self.out_size)
        x = chainer.Variable(self.x)
        # Must call the link to initialize weights.
        lin1(x)
        w1 = lin1.W.data
        fd, temp_file_path = tempfile.mkstemp()
        os.close(fd)
        npz.save_npz(temp_file_path, lin1)
        lin2 = links.Linear(None, self.out_size)
        npz.load_npz(temp_file_path, lin2)
        w2 = lin2.W.data
        self.assertEqual((w1 == w2).all(), True)

class TestLinearWithSpatialDimensions(TestLinear):

    in_shape = (3, 2, 2)

class TestInvalidLinear(unittest.TestCase):

    def setUp(self):
        self.link = links.Linear(3, 2)
        self.x = numpy.random.uniform(-1, 1, (4, 1, 2)).astype(numpy.float32)

    def test_invalid_size(self):
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(self.x))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from PyQt4.QtCore import QVariant, QPyNullVariant
from PyQt4.QtGui import QDialog, QDialogButtonBox, QToolButton
from qgis.core import QgsField, QgsMapLayerRegistry, QgsMapLayer, \
    QgsVectorJoinInfo
from qgis.gui import QgsMessageBar
from globals import DOUBLE_FIELD_TYPE_NAME, DEBUG, NUMERIC_FIELD_TYPES, \
    SUM_BASED_COMBINATIONS, MUL_BASED_COMBINATIONS, TEXTUAL_FIELD_TYPES
from process_layer import ProcessLayer
from ui.ui_calculate_iri import Ui_CalculateIRIDialog
from utils import (LayerEditingManager,
                   reload_attrib_cbx,
                   reload_layers_in_cbx,
                   tr, toggle_select_features)

class CalculateIRIDialog(QDialog, Ui_CalculateIRIDialog):

    def __init__(self, iface, current_layer, project_definition, parent=None):
        QDialog.__init__(self, parent)
        self.iface = iface
        self.parent = parent
        self.current_layer = current_layer
        self.project_definition = project_definition
        self.setupUi(self)
        self.ok_button = self.buttonBox.button(QDialogButtonBox.Ok)
        self.calculate_iri = self.calculate_iri_check.isChecked()

        reload_layers_in_cbx(self.aal_layer,
                             [QgsMapLayer.VectorLayer],
                             [self.current_layer.id()])
        reload_attrib_cbx(self.svi_id_field, self.current_layer,
                          NUMERIC_FIELD_TYPES, TEXTUAL_FIELD_TYPES)
        self.ok_button.setEnabled(True)

    def calculate(self):
        """
        add an SVI attribute to the current layer
        """

        indicators_operator = self.indicators_combination_type.currentText()
        themes_operator = self.themes_combination_type.currentText()

        themes = self.project_definition['children'][1]['children']
        svi_attr_name = 'SVI'
        svi_field = QgsField(svi_attr_name, QVariant.Double)
        svi_field.setTypeName(DOUBLE_FIELD_TYPE_NAME)
        attr_names = ProcessLayer(self.current_layer).add_attributes(
            [svi_field])

        # get the id of the new attribute
        svi_attr_id = ProcessLayer(self.current_layer).find_attribute_id(
            attr_names[svi_attr_name])

        discarded_feats_ids = []
        try:
            with LayerEditingManager(self.current_layer, 'Add SVI', DEBUG):
                for feat in self.current_layer.getFeatures():
                    # If a feature contains any NULL value, discard_feat will
                    # be set to True and the corresponding SVI will be set to
                    # NULL
                    discard_feat = False
                    feat_id = feat.id()

                    # init svi_value to the correct value depending on
                    # themes_operator
                    if themes_operator in SUM_BASED_COMBINATIONS:
                        svi_value = 0
                    elif themes_operator in MUL_BASED_COMBINATIONS:
                        svi_value = 1

                    # iterate all themes of SVI
                    for theme in themes:
                        indicators = theme['children']

                        # init theme_result to the correct value depending on
                        # indicators_operator
                        if indicators_operator in SUM_BASED_COMBINATIONS:
                            theme_result = 0
                        elif indicators_operator in MUL_BASED_COMBINATIONS:
                            theme_result = 1

                        # iterate all indicators of a theme
                        for indicator in indicators:
                            if (feat[indicator['field']] ==
                                    QPyNullVariant(float)):
                                discard_feat = True
                                discarded_feats_ids.append(feat_id)
                                break
                            indicator_weighted = (feat[indicator['field']] *
                                                  indicator['weight'])
                            if indicators_operator in \
                                    SUM_BASED_COMBINATIONS:
                                theme_result += indicator_weighted
                            elif indicators_operator in \
                                    MUL_BASED_COMBINATIONS:
                                theme_result *= indicator_weighted
                        if discard_feat:
                            break
                        if indicators_operator == 'Average':
                            theme_result /= len(indicators)

                        # combine the indicators of each theme
                        theme_weighted = theme_result * theme['weight']
                        if themes_operator in SUM_BASED_COMBINATIONS:
                            svi_value += theme_weighted
                        elif themes_operator in MUL_BASED_COMBINATIONS:
                            svi_value *= theme_weighted
                    if discard_feat:
                        svi_value = QPyNullVariant(float)
                    else:
                        if themes_operator == 'Average':
                            svi_value /= len(themes)

                    self.current_layer.changeAttributeValue(
                        feat_id, svi_attr_id, svi_value)
            msg = ('The SVI has been calculated for fields containing '
                   'non-NULL values and it was added to the layer as '
                   'a new attribute called %s') % attr_names[svi_attr_name]
            self.iface.messageBar().pushMessage(
                tr("Info"), tr(msg), level=QgsMessageBar.INFO)
            if discarded_feats_ids:

                msg = 'Invalid indicators were found in some features'

                widget = self.iface.messageBar().createMessage(
                    tr("Warning"), msg)
                button = QToolButton(widget)
                button.setCheckable(True)
                button.setText("Select invalid features")
                current_feats_ids = self.current_layer.selectedFeaturesIds()
                button.toggled.connect(
                    lambda on, layer=self.current_layer,
                    new_feature_ids=discarded_feats_ids,
                    old_feature_ids=current_feats_ids:
                    toggle_select_features(layer,
                                           on,
                                           new_feature_ids,
                                           old_feature_ids))
                widget.layout().addWidget(button)
                self.iface.messageBar().pushWidget(widget,
                                                   QgsMessageBar.WARNING)

            self.project_definition['indicators_operator'] = \
                indicators_operator
            self.project_definition['themes_operator'] = themes_operator

            if self.calculate_iri_check.isChecked():
                self._calculateIRI(svi_attr_id, discarded_feats_ids)

        except TypeError as e:
            self.current_layer.dataProvider().deleteAttributes(
                [svi_attr_id])
            msg = 'Could not calculate SVI due to data problems: %s' % e
            self.iface.messageBar().pushMessage(tr("Error"),
                                                tr(msg),
                                                level=QgsMessageBar.CRITICAL)

    def _calculateIRI(self, svi_attr_id, discarded_feats_ids):
        """
        Copy the AAL and calculate an IRI attribute to the current layer
        """

        aal_weight = self.project_definition['children'][0]['weight']
        svi_weight = self.project_definition['children'][1]['weight']

        iri_operator = self.iri_combination_type.currentText()

        iri_attr_name = 'IRI'
        iri_field = QgsField(iri_attr_name, QVariant.Double)
        iri_field.setTypeName(DOUBLE_FIELD_TYPE_NAME)
        copy_aal_attr_name = 'AAL'
        aal_field = QgsField(copy_aal_attr_name, QVariant.Double)
        aal_field.setTypeName(DOUBLE_FIELD_TYPE_NAME)

        attr_names = ProcessLayer(self.current_layer).add_attributes(
            [aal_field, iri_field])

        # get the id of the new attributes
        iri_attr_id = ProcessLayer(self.current_layer).find_attribute_id(
            attr_names[iri_attr_name])
        copy_aal_attr_id = ProcessLayer(self.current_layer).find_attribute_id(
            attr_names[copy_aal_attr_name])

        join_layer = QgsMapLayerRegistry.instance().mapLayersByName(
            self.aal_layer.currentText())[0]
        join_info = QgsVectorJoinInfo()
        join_info.joinLayerId = join_layer.id()
        join_info.joinFieldName = self.aal_id_field.currentText()
        join_info.targetFieldName = self.svi_id_field.currentText()
        self.current_layer.addJoin(join_info)

        aal_attr_name = '%s_%s' % (self.aal_layer.currentText(),
                                   self.aal_field.currentText())

        try:
            with LayerEditingManager(self.current_layer, 'Add IRI', DEBUG):
                for feat in self.current_layer.getFeatures():
                    feat_id = feat.id()
                    svi_value = feat.attributes()[svi_attr_id]
                    aal_value = feat[aal_attr_name]
                    if feat_id in discarded_feats_ids:
                        iri_value = QPyNullVariant(float)
                    elif iri_operator == 'Sum':
                        iri_value = (
                            svi_value * svi_weight + aal_value * aal_weight)
                    elif iri_operator == 'Multiplication':
                        iri_value = (
                            svi_value * svi_weight * aal_value * aal_weight)
                    elif iri_operator == 'Average':
                        iri_value = (svi_value * svi_weight +
                                     aal_value * aal_weight) / 2.0

                    # copy AAL
                    self.current_layer.changeAttributeValue(
                        feat_id, copy_aal_attr_id, aal_value)
                    # store IRI
                    self.current_layer.changeAttributeValue(
                        feat_id, iri_attr_id, iri_value)
            self.project_definition['IRI_operator'] = iri_operator
            # set the field name for the copied AAL layer
            self.project_definition['children'][0]['field'] = attr_names[
                copy_aal_attr_name]

        except TypeError:
            self.current_layer.dataProvider().deleteAttributes(
                [iri_attr_id, copy_aal_attr_id])
            msg = 'Could not calculate IRI due to data problems'
            self.iface.messageBar().pushMessage(tr("Error"),
                                                tr(msg),
                                                level=QgsMessageBar.CRITICAL)
        finally:
            self.current_layer.removeJoin(join_info.joinLayerId)

    def on_calculate_iri_check_toggled(self, on):
        self.calculate_iri = on
        if self.calculate_iri:
            self.check_iri_fields()
        else:
            self.ok_button.setEnabled(True)

    def on_aal_field_currentIndexChanged(self, index):
        self.check_iri_fields()

    def on_aal_layer_currentIndexChanged(self, index):
        selected_layer = QgsMapLayerRegistry.instance().mapLayers().values()[
            self.aal_layer.currentIndex()]
        reload_attrib_cbx(self.aal_field, selected_layer, NUMERIC_FIELD_TYPES)
        reload_attrib_cbx(self.aal_id_field, selected_layer,
                          NUMERIC_FIELD_TYPES, TEXTUAL_FIELD_TYPES)
        self.check_iri_fields()

    def check_iri_fields(self):
        valid_state = False
        if (self.aal_field.currentText() and self.aal_layer.currentText()
                and self.aal_id_field and self.svi_id_field):
            valid_state = True
        self.ok_button.setEnabled(valid_state)
<commit_msg>Make the 'Average' operator the same as a weighted sum with equal weights<commit_after># -*- coding: utf-8 -*-
from PyQt4.QtCore import QVariant, QPyNullVariant
from PyQt4.QtGui import QDialog, QDialogButtonBox, QToolButton
from qgis.core import QgsField, QgsMapLayerRegistry, QgsMapLayer, \
    QgsVectorJoinInfo
from qgis.gui import QgsMessageBar
from globals import DOUBLE_FIELD_TYPE_NAME, DEBUG, NUMERIC_FIELD_TYPES, \
    SUM_BASED_COMBINATIONS, MUL_BASED_COMBINATIONS, TEXTUAL_FIELD_TYPES
from process_layer import ProcessLayer
from ui.ui_calculate_iri import Ui_CalculateIRIDialog
from utils import (LayerEditingManager,
                   reload_attrib_cbx,
                   reload_layers_in_cbx,
                   tr, toggle_select_features)

class CalculateIRIDialog(QDialog, Ui_CalculateIRIDialog):

    def __init__(self, iface, current_layer, project_definition, parent=None):
        QDialog.__init__(self, parent)
        self.iface = iface
        self.parent = parent
        self.current_layer = current_layer
        self.project_definition = project_definition
        self.setupUi(self)
        self.ok_button = self.buttonBox.button(QDialogButtonBox.Ok)
        self.calculate_iri = self.calculate_iri_check.isChecked()

        reload_layers_in_cbx(self.aal_layer,
                             [QgsMapLayer.VectorLayer],
                             [self.current_layer.id()])
        reload_attrib_cbx(self.svi_id_field, self.current_layer,
                          NUMERIC_FIELD_TYPES, TEXTUAL_FIELD_TYPES)
        self.ok_button.setEnabled(True)

    def calculate(self):
        """
        add an SVI attribute to the current layer
        """

        indicators_operator = self.indicators_combination_type.currentText()
        themes_operator = self.themes_combination_type.currentText()

        themes = self.project_definition['children'][1]['children']
        svi_attr_name = 'SVI'
        svi_field = QgsField(svi_attr_name, QVariant.Double)
        svi_field.setTypeName(DOUBLE_FIELD_TYPE_NAME)
        attr_names = ProcessLayer(self.current_layer).add_attributes(
            [svi_field])

        # get the id of the new attribute
        svi_attr_id = ProcessLayer(self.current_layer).find_attribute_id(
            attr_names[svi_attr_name])

        discarded_feats_ids = []
        try:
            with LayerEditingManager(self.current_layer, 'Add SVI', DEBUG):
                for feat in self.current_layer.getFeatures():
                    # If a feature contains any NULL value, discard_feat will
                    # be set to True and the corresponding SVI will be set to
                    # NULL
                    discard_feat = False
                    feat_id = feat.id()

                    # init svi_value to the correct value depending on
                    # themes_operator
                    if themes_operator in SUM_BASED_COMBINATIONS:
                        svi_value = 0
                    elif themes_operator in MUL_BASED_COMBINATIONS:
                        svi_value = 1

                    # iterate all themes of SVI
                    for theme in themes:
                        indicators = theme['children']

                        # init theme_result to the correct value depending on
                        # indicators_operator
                        if indicators_operator in SUM_BASED_COMBINATIONS:
                            theme_result = 0
                        elif indicators_operator in MUL_BASED_COMBINATIONS:
                            theme_result = 1

                        # iterate all indicators of a theme
                        for indicator in indicators:
                            if (feat[indicator['field']] ==
                                    QPyNullVariant(float)):
                                discard_feat = True
                                discarded_feats_ids.append(feat_id)
                                break
                            # For "Average" it's equivalent to use equal
                            # weights, or to sum the indicators (all weights 1)
                            # and divide by the number of indicators (we use
                            # the latter solution)
                            if indicators_operator == 'Average':
                                indicator_weighted = feat[indicator['field']]
                            else:
                                indicator_weighted = (
                                    feat[indicator['field']] *
                                    indicator['weight'])
                            if indicators_operator in \
                                    SUM_BASED_COMBINATIONS:
                                theme_result += indicator_weighted
                            elif indicators_operator in \
                                    MUL_BASED_COMBINATIONS:
                                theme_result *= indicator_weighted
                        if discard_feat:
                            break
                        if indicators_operator == 'Average':
                            theme_result /= len(indicators)

                        # combine the indicators of each theme

                        # For "Average" it's equivalent to use equal
                        # weights, or to sum the themes (all weights 1)
                        # and divide by the number of themes (we use
                        # the latter solution)
                        if themes_operator == 'Average':
                            theme_weighted = theme_result
                        else:
                            theme_weighted = theme_result * theme['weight']
                        if themes_operator in SUM_BASED_COMBINATIONS:
                            svi_value += theme_weighted
                        elif themes_operator in MUL_BASED_COMBINATIONS:
                            svi_value *= theme_weighted
                    if discard_feat:
                        svi_value = QPyNullVariant(float)
                    else:
                        if themes_operator == 'Average':
                            svi_value /= len(themes)

                    self.current_layer.changeAttributeValue(
                        feat_id, svi_attr_id, svi_value)
            msg = ('The SVI has been calculated for fields containing '
                   'non-NULL values and it was added to the layer as '
                   'a new attribute called %s') % attr_names[svi_attr_name]
            self.iface.messageBar().pushMessage(
                tr("Info"), tr(msg), level=QgsMessageBar.INFO)
            if discarded_feats_ids:

                msg = 'Invalid indicators were found in some features'

                widget = self.iface.messageBar().createMessage(
                    tr("Warning"), msg)
                button = QToolButton(widget)
                button.setCheckable(True)
                button.setText("Select invalid features")
                current_feats_ids = self.current_layer.selectedFeaturesIds()
                button.toggled.connect(
                    lambda on, layer=self.current_layer,
                    new_feature_ids=discarded_feats_ids,
                    old_feature_ids=current_feats_ids:
                    toggle_select_features(layer,
                                           on,
                                           new_feature_ids,
                                           old_feature_ids))
                widget.layout().addWidget(button)
                self.iface.messageBar().pushWidget(widget,
                                                   QgsMessageBar.WARNING)

            self.project_definition['indicators_operator'] = \
                indicators_operator
            self.project_definition['themes_operator'] = themes_operator

            if self.calculate_iri_check.isChecked():
                self._calculateIRI(svi_attr_id, discarded_feats_ids)

        except TypeError as e:
            self.current_layer.dataProvider().deleteAttributes(
                [svi_attr_id])
            msg = 'Could not calculate SVI due to data problems: %s' % e
            self.iface.messageBar().pushMessage(tr("Error"),
                                                tr(msg),
                                                level=QgsMessageBar.CRITICAL)

    def _calculateIRI(self, svi_attr_id, discarded_feats_ids):
        """
        Copy the AAL and calculate an IRI attribute to the current layer
        """

        aal_weight = self.project_definition['children'][0]['weight']
        svi_weight = self.project_definition['children'][1]['weight']

        iri_operator = self.iri_combination_type.currentText()

        iri_attr_name = 'IRI'
        iri_field = QgsField(iri_attr_name, QVariant.Double)
        iri_field.setTypeName(DOUBLE_FIELD_TYPE_NAME)
        copy_aal_attr_name = 'AAL'
        aal_field = QgsField(copy_aal_attr_name, QVariant.Double)
        aal_field.setTypeName(DOUBLE_FIELD_TYPE_NAME)

        attr_names = ProcessLayer(self.current_layer).add_attributes(
            [aal_field, iri_field])

        # get the id of the new attributes
        iri_attr_id = ProcessLayer(self.current_layer).find_attribute_id(
            attr_names[iri_attr_name])
        copy_aal_attr_id = ProcessLayer(self.current_layer).find_attribute_id(
            attr_names[copy_aal_attr_name])

        join_layer = QgsMapLayerRegistry.instance().mapLayersByName(
            self.aal_layer.currentText())[0]
        join_info = QgsVectorJoinInfo()
        join_info.joinLayerId = join_layer.id()
        join_info.joinFieldName = self.aal_id_field.currentText()
        join_info.targetFieldName = self.svi_id_field.currentText()
        self.current_layer.addJoin(join_info)

        aal_attr_name = '%s_%s' % (self.aal_layer.currentText(),
                                   self.aal_field.currentText())

        try:
            with LayerEditingManager(self.current_layer, 'Add IRI', DEBUG):
                for feat in self.current_layer.getFeatures():
                    feat_id = feat.id()
                    svi_value = feat.attributes()[svi_attr_id]
                    aal_value = feat[aal_attr_name]
                    if feat_id in discarded_feats_ids:
                        iri_value = QPyNullVariant(float)
                    elif iri_operator == 'Sum':
                        iri_value = (
                            svi_value * svi_weight + aal_value * aal_weight)
                    elif iri_operator == 'Multiplication':
                        iri_value = (
                            svi_value * svi_weight * aal_value * aal_weight)
                    elif iri_operator == 'Average':
                        # For "Average" it's equivalent to use equal
                        # weights, or to sum the indices (all weights 1)
                        # and divide by the number of indices (we use
                        # the latter solution)
                        iri_value = (svi_value + aal_value) / 2.0

                    # copy AAL
                    self.current_layer.changeAttributeValue(
                        feat_id, copy_aal_attr_id, aal_value)
                    # store IRI
                    self.current_layer.changeAttributeValue(
                        feat_id, iri_attr_id, iri_value)
            self.project_definition['IRI_operator'] = iri_operator
            # set the field name for the copied AAL layer
            self.project_definition['children'][0]['field'] = attr_names[
                copy_aal_attr_name]

        except TypeError:
            self.current_layer.dataProvider().deleteAttributes(
                [iri_attr_id, copy_aal_attr_id])
            msg = 'Could not calculate IRI due to data problems'
            self.iface.messageBar().pushMessage(tr("Error"),
                                                tr(msg),
                                                level=QgsMessageBar.CRITICAL)
        finally:
            self.current_layer.removeJoin(join_info.joinLayerId)

    def on_calculate_iri_check_toggled(self, on):
        self.calculate_iri = on
        if self.calculate_iri:
            self.check_iri_fields()
        else:
            self.ok_button.setEnabled(True)

    def on_aal_field_currentIndexChanged(self, index):
        self.check_iri_fields()

    def on_aal_layer_currentIndexChanged(self, index):
        selected_layer = QgsMapLayerRegistry.instance().mapLayers().values()[
            self.aal_layer.currentIndex()]
        reload_attrib_cbx(self.aal_field, selected_layer, NUMERIC_FIELD_TYPES)
        reload_attrib_cbx(self.aal_id_field, selected_layer,
                          NUMERIC_FIELD_TYPES, TEXTUAL_FIELD_TYPES)
        self.check_iri_fields()

    def check_iri_fields(self):
        valid_state = False
        if (self.aal_field.currentText() and self.aal_layer.currentText()
                and self.aal_id_field and self.svi_id_field):
            valid_state = True
        self.ok_button.setEnabled(valid_state)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
An unofficial API for pitchfork.com reviews.

author: Michal Czaplinski
email: mmczaplinski@gmail.com
"""

import json
import re
import difflib
import sys
from bs4 import BeautifulSoup

if sys.version_info >= (3, 0):
    from urllib.parse import urljoin
    from urllib.request import urlopen
    from urllib.request import Request
    unicode = str
else:
    from urllib2 import urlopen, Request
    from urlparse import urljoin

def replace_breaks(html):
    """
    Replaces all the
 tags in the html with newlines '\n'
    """
    breakline = html.br
    while html.br is not None:
        breakline.insert_before('\n')
        html.br.decompose()
        breakline = html.br
    return html

class Review:
    """
    Class representing the fetched review.
    Includes methods for getting the score, the text of the review
    (editorial), the album cover, label, year as well as the true
    (matched) album and artist names.
    """

    def __init__(self, searched_artist, searched_album, matched_artist,
                 matched_album, query, url, soup):
        self.searched_artist = searched_artist
        self.searched_album = searched_album
        self.matched_artist = matched_artist
        self.matched_album = matched_album
        self.query = query
        self.url = url
        self.soup = soup

    def score(self):
        """ Returns the album score. """
        rating = self.soup.find(class_='score').text
        rating = float(rating.strip())
        return rating

    def editorial(self):
        """ Returns the text of the review. """
        review_html = self.soup.find(class_='editorial')
        review_html = replace_breaks(review_html).find_all('p')
        review_text = ''
        for paragraph in review_html:
            review_text += paragraph.text + '\n\n'
        return review_text

    def cover(self):
        """ Returns the link to the album cover. """
        artwork = self.soup.find(class_='artwork')
        image_link = artwork.img['src'].strip()
        return image_link

    def artist(self):
        """ Returns the artist name that pitchfork matched to our search. """
        artist = self.matched_artist.strip()
        return artist

    def album(self):
        """ Returns the album name that pitchfork matched to our search. """
        album = self.matched_album.strip()
        return album

    def label(self):
        """ Returns the name of the record label that released the album. """
        label = self.soup.find(class_='info').h3.get_text()
        label = label[:label.index(';')].strip()
        return label

    def year(self):
        """
        Returns the year the album was released.
        In case of a reissue album, the year of original release as well as
        the year of the reissue is given separated by '/'.
        """
        year = self.soup.find(class_='info').h3.get_text()
        year = year[year.index(';')+1:].strip()
        return year
    def _json_safe_dict(self):
        """
        Returns a dictionary representation of object where
        the soup key's value's special characters are escaped
        """
        d = self.__dict__.copy()
        d['soup'] = d['soup'].prettify()
        return d

    def to_json(self):
        """
        Returns the attributes of the album review formatted as json.
        """
        d = self._json_safe_dict()
        return json.dumps(d)

    def __repr__(self):
        return self.__class__.__name__+repr((self.searched_artist,
                                             self.searched_album,
                                             self.matched_artist,
                                             self.matched_album,
                                             self.query,
                                             self.url,
                                             str(self.soup.__class__)))

class MultiReview(Review):

    def __init__(self, searched_artist, searched_album, matched_artist,
                 matched_album, query, url, soup):
        self.searched_artist = searched_artist
        self.searched_album = searched_album
        self.matched_artist = matched_artist
        self.matched_album = matched_album
        self.query = query
        self.url = url
        self.soup = soup
        self.info = soup.find('h2', text=self.matched_album).parent

    def score(self):
        """ Returns the album score. """
        rating = self.info.find(class_='score').text
        rating = float(rating.strip())
        return rating

    def label(self):
        """ Returns the name of the record label that released the album. """
        label = self.info.h3.get_text()
        label = label[:label.index(';')].strip()
        return label

    def cover(self):
        """ Returns the link to the album cover. """
        artwork = self.info.parent.find(class_='artwork')
        image_link = artwork.img['src']
        return image_link

    def year(self):
        """
        Returns the year the album was released.
        In case of a reissue album, the year of original release as well as
        the year of the reissue is given separated by '/'.
        """
        year = self.info.h3.get_text()
        year = year[year.index(';')+1:].strip()
        return year

    def _json_safe_dict(self):
        """
        Returns a dictionary representation of object where
        the soup key's value's special characters are escaped
        """
        d = self.__dict__.copy()
        d['soup'] = d['soup'].prettify()
        d['info'] = d['info'].prettify()
        return d

    def to_json(self):
        d = self._json_safe_dict()
        return json.dumps(d)

def search(artist, album):
    """
    Look for the review of the specified album by the specified artist.
    Returns either a Review object or a MultiReview object depending on
    the type of review because some pitchfork reviews cover multiple albums.
    """

    # replace spaces in the url with the '%20'
    query = re.sub('\s+', '%20', artist + '%20' + album)
    # using a custom user agent header
    request = Request(url='http://pitchfork.com/search/ac/?query=' + query,
                      data=None,
                      headers={'User-Agent': 'michalczaplinski/pitchfork-v0.1'})
    response = urlopen(request)
    text = response.read().decode()

    # the server responds with json so we load it into a dictionary
    obj = json.loads(text)

    try:
        # get the nested dictionary containing url to the review and album name
        review_dict = [x for x in obj if x['label'] == 'Reviews'][0]['objects'][0]
    except IndexError:
        raise IndexError('The search returned no results! Try again with diferent parameters.')

    url = review_dict['url']
    matched_artist = review_dict['name'].split(' - ')[0]

    # fetch the review page
    full_url = urljoin('http://pitchfork.com/', url)
    request = Request(url=full_url,
                      data=None,
                      headers={'User-Agent': 'michalczaplinski/pitchfork-v0.1'})
    response_text = urlopen(request).read()
    soup = BeautifulSoup(response_text)

    # check if the review does not review multiple albums
    if soup.find(class_='review-multi') is None:
        matched_album = review_dict['name'].split(' - ')[1]

        return Review(artist, album, matched_artist, matched_album, query, url, soup)
    else:
        # get the titles of all the albums in the multi-review
        titles = [title.get_text() for title in soup.find(class_='review-meta').find_all('h2')]

        try:
            # find the album title closest matching to the one searched for
            matched_album = difflib.get_close_matches(album, titles, cutoff=0.1)[0]
        except IndexError:
            raise IndexError('The supplied album information was insufficient...')

        return MultiReview(artist, album, matched_artist, matched_album, query, url, soup)
<commit_msg>Changed url in album review request to work with new Pitchfork url structure<commit_after>#!/usr/bin/env python

"""
An unofficial API for pitchfork.com reviews.

author: Michal Czaplinski
email: mmczaplinski@gmail.com
"""

import json
import re
import difflib
import sys
from bs4 import BeautifulSoup

if sys.version_info >= (3, 0):
    from urllib.parse import urljoin
    from urllib.request import urlopen
    from urllib.request import Request
    unicode = str
else:
    from urllib2 import urlopen, Request
    from urlparse import urljoin

def replace_breaks(html):
    """
    Replaces all the
 tags in the html with newlines '\n'
    """
    breakline = html.br
    while html.br is not None:
        breakline.insert_before('\n')
        html.br.decompose()
        breakline = html.br
    return html

class Review:
    """
    Class representing the fetched review.
    Includes methods for getting the score, the text of the review
    (editorial), the album cover, label, year as well as the true
    (matched) album and artist names.
    """

    def __init__(self, searched_artist, searched_album, matched_artist,
                 matched_album, query, url, soup):
        self.searched_artist = searched_artist
        self.searched_album = searched_album
        self.matched_artist = matched_artist
        self.matched_album = matched_album
        self.query = query
        self.url = url
        self.soup = soup

    def score(self):
        """ Returns the album score. """
        rating = self.soup.find(class_='score').text
        rating = float(rating.strip())
        return rating

    def editorial(self):
        """ Returns the text of the review. """
        review_html = self.soup.find(class_='editorial')
        review_html = replace_breaks(review_html).find_all('p')
        review_text = ''
        for paragraph in review_html:
            review_text += paragraph.text + '\n\n'
        return review_text

    def cover(self):
        """ Returns the link to the album cover. """
        artwork = self.soup.find(class_='artwork')
        image_link = artwork.img['src'].strip()
        return image_link

    def artist(self):
        """ Returns the artist name that pitchfork matched to our search. """
        artist = self.matched_artist.strip()
        return artist

    def album(self):
        """ Returns the album name that pitchfork matched to our search. """
        album = self.matched_album.strip()
        return album

    def label(self):
        """ Returns the name of the record label that released the album. """
        label = self.soup.find(class_='info').h3.get_text()
        label = label[:label.index(';')].strip()
        return label

    def year(self):
        """
        Returns the year the album was released.
        In case of a reissue album, the year of original release as well as
        the year of the reissue is given separated by '/'.
        """
        year = self.soup.find(class_='info').h3.get_text()
        year = year[year.index(';')+1:].strip()
        return year
    def _json_safe_dict(self):
        """
        Returns a dictionary representation of object where
        the soup key's value's special characters are escaped
        """
        d = self.__dict__.copy()
        d['soup'] = d['soup'].prettify()
        return d

    def to_json(self):
        """
        Returns the attributes of the album review formatted as json.
        """
        d = self._json_safe_dict()
        return json.dumps(d)

    def __repr__(self):
        return self.__class__.__name__+repr((self.searched_artist,
                                             self.searched_album,
                                             self.matched_artist,
                                             self.matched_album,
                                             self.query,
                                             self.url,
                                             str(self.soup.__class__)))

class MultiReview(Review):

    def __init__(self, searched_artist, searched_album, matched_artist,
                 matched_album, query, url, soup):
        self.searched_artist = searched_artist
        self.searched_album = searched_album
        self.matched_artist = matched_artist
        self.matched_album = matched_album
        self.query = query
        self.url = url
        self.soup = soup
        self.info = soup.find('h2', text=self.matched_album).parent

    def score(self):
        """ Returns the album score. """
        rating = self.info.find(class_='score').text
        rating = float(rating.strip())
        return rating

    def label(self):
        """ Returns the name of the record label that released the album. """
        label = self.info.h3.get_text()
        label = label[:label.index(';')].strip()
        return label

    def cover(self):
        """ Returns the link to the album cover. """
        artwork = self.info.parent.find(class_='artwork')
        image_link = artwork.img['src']
        return image_link

    def year(self):
        """
        Returns the year the album was released.
        In case of a reissue album, the year of original release as well as
        the year of the reissue is given separated by '/'.
        """
        year = self.info.h3.get_text()
        year = year[year.index(';')+1:].strip()
        return year

    def _json_safe_dict(self):
        """
        Returns a dictionary representation of object where
        the soup key's value's special characters are escaped
        """
        d = self.__dict__.copy()
        d['soup'] = d['soup'].prettify()
        d['info'] = d['info'].prettify()
        return d

    def to_json(self):
        d = self._json_safe_dict()
        return json.dumps(d)

def search(artist, album):
    """
    Look for the review of the specified album by the specified artist.
    Returns either a Review object or a MultiReview object depending on
    the type of review because some pitchfork reviews cover multiple albums.
    """

    # replace spaces in the url with the '%20'
    query = re.sub('\s+', '%20', artist + '%20' + album)
    # using a custom user agent header
    request = Request(url='http://pitchfork.com/search/?query=' + query + '#result-albumreviews',
                      data=None,
                      headers={'User-Agent': 'michalczaplinski/pitchfork-v0.1'})
    response = urlopen(request)
    text = response.read().decode()

    # the server responds with json so we load it into a dictionary
    obj = json.loads(text)

    try:
        # get the nested dictionary containing url to the review and album name
        review_dict = [x for x in obj if x['label'] == 'Reviews'][0]['objects'][0]
    except IndexError:
        raise IndexError('The search returned no results! Try again with diferent parameters.')

    url = review_dict['url']
    matched_artist = review_dict['name'].split(' - ')[0]

    # fetch the review page
    full_url = urljoin('http://pitchfork.com/', url)
    request = Request(url=full_url,
                      data=None,
                      headers={'User-Agent': 'michalczaplinski/pitchfork-v0.1'})
    response_text = urlopen(request).read()
    soup = BeautifulSoup(response_text)

    # check if the review does not review multiple albums
    if soup.find(class_='review-multi') is None:
        matched_album = review_dict['name'].split(' - ')[1]

        return Review(artist, album, matched_artist, matched_album, query, url, soup)
    else:
        # get the titles of all the albums in the multi-review
        titles = [title.get_text() for title in soup.find(class_='review-meta').find_all('h2')]

        try:
            # find the album title closest matching to the one searched for
            matched_album = difflib.get_close_matches(album, titles, cutoff=0.1)[0]
        except IndexError:
            raise IndexError('The supplied album information was insufficient...')

        return MultiReview(artist, album, matched_artist, matched_album, query, url, soup)
<|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import division
from __future__ import unicode_literals

from collections import Mapping

from activedata_etl import etl2key, key2etl
from activedata_etl.transforms import TRY_AGAIN_LATER
from mo_dots import Data, listwrap, wrap, set_default
from mo_hg.hg_mozilla_org import minimize_repo
from mo_json import json2value
from mo_logs import Log, machine_metadata, strings
from mo_math import Math
from mo_times.dates import Date
from pyLibrary.env import elasticsearch, git

DEBUG = True
DISABLE_LOG_PARSING = False
MAX_THREADS = 5

seen_tasks = {}
new_seen_tc_properties = set()

def process(source_key, source, destination, resources, please_stop=None):
    output = []
    lines = list(enumerate(source.read_lines()))
    for line_number, line in lines:
        if please_stop:
            Log.error("Shutdown detected. Stopping early")
        try:
            raw_treeherder = json2value(line)
            etl_source = consume(raw_treeherder, "etl")
            if etl_source.source.source.id <= 687:
                etl_source = wrap({"source": key2etl(source_key)})
            etl_source.type = "join"
            etl_source.source.type = "join"

            normalized = Data()
            normalized.etl = set_default({
                "id": line_number,
                "timestamp": Date.now(),
                "machine": machine_metadata,
                "revision": git.get_revision(),
                "type": "join"
            }, etl_source)
            normalize(source_key, resources, raw_treeherder, normalized)
            normalized = elasticsearch.scrub(normalized)
            output.append(normalized)
        except Exception as e:
            if TRY_AGAIN_LATER in e:
                raise e
            Log.warning("Treeherder line for key {{key}} not processed: {{line|quote}}", key=source_key, line=line, cause=e)

    keys = destination.extend({"id": etl2key(t.etl), "value": t} for t in output)
    return keys

def normalize(source_key, resources, raw_treeherder, new_treeherder):
    raw_job = raw_treeherder.job

    new_treeherder.job.type.name = consume(raw_job, "job_type.name")
    new_treeherder.job.type.description = consume(raw_job, "job_type.description")
    new_treeherder.job.type.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.symbol"),
        consume(raw_job, "signature.job_type_symbol")
    )
    new_treeherder.job.type.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.name"),
        consume(raw_job, "signature.job_type_name")
    )
    new_treeherder.job.type.group.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.symbol"),
        consume(raw_job, "signature.job_group_symbol"),
        consume(raw_job, "job_group.symbol")
    )

    new_treeherder.job.type.group.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.name"),
        consume(raw_job, "job_type.job_group"),
        consume(raw_job, "signature.job_group_name"),
        consume(raw_job, "job_group.name"),
        consume(raw_job, "job_group")
    )

    new_treeherder.job.guid = consume(raw_job, "guid")
    new_treeherder.job.id = consume(raw_job, "id")
    new_treeherder.job.coalesced_to_guid = consume(raw_job, "coalesced_to_guid")

    # BUILD
    new_treeherder.build.branch = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "repository"),
        consume(raw_job, "signature.repository")
    )
    new_treeherder.build.revision = consume(raw_job, "push.revision")
    new_treeherder.build.revision12 = new_treeherder.build.revision[0:12]
    new_treeherder.build.date = consume(raw_job, "push.time")

    new_treeherder.build.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.platform"),
        consume(raw_job, "signature.build_platform")
    )
    new_treeherder.build.os = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.os_name"),
        consume(raw_job, "signature.build_os_name")
    )
    new_treeherder.build.architecture = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.architecture"),
        consume(raw_job, "signature.build_architecture")
    )
    new_treeherder.build.product = consume(raw_job, "product.name")

    new_treeherder.run.key = consume(raw_job, "signature.name")
    new_treeherder.run.reason = consume(raw_job, "reason")
    new_treeherder.run.tier = consume(raw_job, "tier")
    new_treeherder.run.result = consume(raw_job, "result")
    new_treeherder.run.state = consume(raw_job, "state")
    if raw_treeherder.job.signature.build_system_type in ["taskcluster", "buildbot", "fx-test-jenkins", "fx-test-jenkins-dev", "qa-master", "qa-preprod-master"]:
        consume(raw_job, "signature.build_system_type")
    else:
        Log.error("Know nothing about build_system_type=={{type}}", type=raw_treeherder.job.signature.build_system_type)

    # RUN MACHINE
    new_treeherder.run.machine.name = machine_name = consume(raw_job, "machine.name")
    split_name = machine_name.split("-")
    if Math.is_integer(split_name[-1]):
        new_treeherder.run.machine.pool = "-".join(split_name[:-1])
    new_treeherder.run.machine.os = consume(raw_job, "signature.machine_os_name")
    new_treeherder.run.machine.architecture = consume(raw_job, "signature.machine_architecture")
    new_treeherder.run.machine.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "machine_platform"),
        consume(raw_job, "signature.machine_platform")
    )

    # ACTION
    new_treeherder.action.start_time = consume(raw_job, "start_time")
    new_treeherder.action.end_time = consume(raw_job, "end_time")
    new_treeherder.action.request_time = consume(raw_job, "submit_time")
    new_treeherder.action.duration = new_treeherder.action.end_time - new_treeherder.action.start_time
    new_treeherder.last_modified = consume(raw_job, "last_modified")

    new_treeherder.failure.auto_classification = consume(raw_job, "autoclassify_status")
    new_treeherder.failure.classification = consume(raw_job, "failure_classification")
    new_treeherder.failure.notes = consume(raw_job, "job_note")

    new_treeherder.repo = {
        "branch": {"name": new_treeherder.build.branch},
        "changeset": {"id": new_treeherder.build.revision}
    }
    try:
        if new_treeherder.build.branch not in NON_HG_BRANCHES:
            new_treeherder.repo = minimize_repo(resources.hg.get_revision(new_treeherder.repo))
    except Exception as e:
        Log.warning("Problem with getting info changeset {{changeset}}", changeset=new_treeherder.repo, cause=e)

    new_treeherder.bugs = consume(raw_job, "bug_job_map")

    pull_job_log(source_key, consume(raw_treeherder.job, "job_log"), new_treeherder)

    consume(raw_job, "push")
    consume(raw_job, "running_eta")
    consume(raw_job, "who")
    consume(raw_job, "signature.first_submission_timestamp")
    consume(raw_job, "signature.option_collection_hash")
    consume(raw_job, "signature.signature")
    pull_details(source_key, consume(raw_treeherder.job, "job_detail"), new_treeherder)

    new_treeherder.run.taskcluster.id = coalesce_w_conflict_detection(
        source_key,
        new_treeherder.run.taskcluster.id,
        consume(raw_job, "taskcluster_metadata.task_id")
    )
    new_treeherder.run.taskcluster.retry_id = consume(raw_job, "taskcluster_metadata.retry_id")

    pull_options(source_key, raw_treeherder, new_treeherder)

    remainder = raw_treeherder.leaves()
    if remainder:
        Log.error("Did not process {{paths}} for key={{key}}", key=source_key, paths=[k for k, _ in remainder])

def pull_options(source_key, raw_treeherder, new_treeherder):
    options = listwrap(consume(raw_treeherder, "job.option_collection.option"))
    for o in options:
        if o in _option_map:
            new_treeherder.build.type += _option_map[o]
        elif o in ["e10s"]:
            new_treeherder.run.type += [o]
        else:
            Log.warning("not known option {{option|quote}} while processing {{key}}", key=source_key, option=o)

_option_map = {
    "addon": ["addon"],
    "aarch64-debug": ["aarch64", "debug"],
    "arm-debug": ["arm", "debug"],
    "all": ["all"],
    "asan": ["asan"],
    "ccov": ["ccov"],
    "cc": ["ccov"],
    "debug": ["debug"],
    "fips": ["fips"],
    "fuzz": ["fuzz"],
    "gyp": ["gyp"],
    "gyp-asan": ["gyp", "asan"],
    "jsdcov": ["jsdcov"],
    "lto": ["lto"],  # LINK TIME OPTIMIZATION
    "make": ["make"],
    "nostylo": ["stylo-disabled"],
    "opt": ["opt"],
    "pgo": ["pgo"]
}

def pull_job_log(source_key, job_log, new_treeherder):
    for d in listwrap(job_log):
        for f in listwrap(d.failure_line):
            f.message = None
            f.job_guid = None
            f.stackwalk_stdout = None
            f.stackwalk_stderr = None
            f.subtest = None
    new_treeherder.job_log = job_log

def pull_details(source_key, details, new_treeherder):
    for d in listwrap(details):
        if d.title == "Summary":
            new_treeherder.summary = d.value
        elif d.title == "buildbot_request_id":
            new_treeherder.run.buildbot.id = d.value
        elif d.title == "Inspect Task":
            if d.url.startswith("https://tools.taskcluster.net/task-inspector/#"):
                new_treeherder.run.taskcluster.id = strings.between(d.url, "https://tools.taskcluster.net/task-inspector/#", "/")
            else:
                Log.warning("Can not extract task for ket {{key}} from {{url}}", key=source_key, url=d.url)
        elif d.title == "CPU idle":
            new_treeherder.stats.cpu_idle = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU user":
            new_treeherder.stats.cpu_user = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU system":
            new_treeherder.stats.cpu_system = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU iowait":
            new_treeherder.stats.cpu_io_wait = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU usage":
            new_treeherder.stats.cpu_usage = float(d.value.strip("%").replace(",", ""))
        elif d.title == "I/O read bytes / time":
            new_treeherder.stats.io_read_bytes = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.io_read_time = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title == "I/O write bytes / time":
            new_treeherder.stats.io_write_bytes = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.io_write_time = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title == "Swap in / out":
            new_treeherder.stats.swap_in = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.swap_out = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title in ["artifact uploaded", "One Click Loaner"]:
            pass
        elif d.title.find("-chunked"):
            pass
        elif d.title == None:
            if d.value in ["auto clobber", "purged clobber", "periodic clobber", "forced clobber"]:
                pass
            elif any(map(d.value.startswith, ["linker max vsize: ", "num_ctors: "])):
                new_treeherder.stats.linker_max_vsize = int(d.value.split(":")[1].strip())
            elif any(map(d.value.startswith, KNOWN_VALUES)):
                pass
            else:
                KNOWN_VALUES.append(d.value)
                Log.warning("value has no title {{value|quote}} while processing {{key}}", key=source_key, value=d.value)
        else:
            Log.warning("can not process detail with title of {{title}}", title=d.title)
    new_treeherder.job.details = details

def coalesce_w_conflict_detection(source_key, *args):
    output = None
    for a in args:
        if a == None:
            continue
        if isinstance(a, Mapping) and not a:
            continue
        if output == None:
            output = a
        elif a != output:
            Log.warning("tried to coalesce {{values_|json}} while processing {{key}}", key=source_key, values_=args)
        else:
            pass
    return output

def consume(props, key):
    output = props[key]
    try:
        props[key] = None
    except Exception:
        pass
    return output

KNOWN_VALUES = [
    "marionette: ",
    "--",
    "The following arguments ",
    "Tests will be run from the following files:",
    "gaia_revlink: ",
    "Unknown: ",
    "\t--this-chunk=1 --total-chunks=1 -- ",
    "\t--setenv=MOZ_BROWSER_XHTML=1 -- "
]

NON_HG_BRANCHES = [
    "bmo-master",
    "go-bouncer",
    "mozillians-tests",
    "snippets-service",
    "snippets-service",
    "snippets-tests",
    "socorro",
    "stubattribution-tests"
]
<commit_msg>more types<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import division
from __future__ import unicode_literals

from collections import Mapping

from activedata_etl import etl2key, key2etl
from activedata_etl.transforms import TRY_AGAIN_LATER
from mo_dots import Data, listwrap, wrap, set_default
from mo_hg.hg_mozilla_org import minimize_repo
from mo_json import json2value
from mo_logs import Log, machine_metadata, strings
from mo_math import Math
from mo_times.dates import Date
from pyLibrary.env import elasticsearch, git

DEBUG = True
DISABLE_LOG_PARSING = False
MAX_THREADS = 5

seen_tasks = {}
new_seen_tc_properties = set()

def process(source_key, source, destination, resources, please_stop=None):
    output = []
    lines = list(enumerate(source.read_lines()))
    for line_number, line in lines:
        if please_stop:
            Log.error("Shutdown detected. Stopping early")
        try:
            raw_treeherder = json2value(line)
            etl_source = consume(raw_treeherder, "etl")
            if etl_source.source.source.id <= 687:
                etl_source = wrap({"source": key2etl(source_key)})
            etl_source.type = "join"
            etl_source.source.type = "join"

            normalized = Data()
            normalized.etl = set_default(
                {
                    "id": line_number,
                    "timestamp": Date.now(),
                    "machine": machine_metadata,
                    "revision": git.get_revision(),
                    "type": "join",
                },
                etl_source,
            )
            normalize(source_key, resources, raw_treeherder, normalized)
            normalized = elasticsearch.scrub(normalized)
            output.append(normalized)
        except Exception as e:
            if TRY_AGAIN_LATER in e:
                raise e
            Log.warning(
                "Treeherder line for key {{key}} not processed: {{line|quote}}",
                key=source_key,
                line=line,
                cause=e,
            )

    keys = destination.extend({"id": etl2key(t.etl), "value": t} for t in output)
    return keys

def normalize(source_key, resources, raw_treeherder, new_treeherder):
    raw_job = raw_treeherder.job

    new_treeherder.job.type.name = consume(raw_job, "job_type.name")
    new_treeherder.job.type.description = consume(raw_job, "job_type.description")
    new_treeherder.job.type.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.symbol"),
        consume(raw_job, "signature.job_type_symbol"),
    )
    new_treeherder.job.type.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.name"),
        consume(raw_job, "signature.job_type_name"),
    )
    new_treeherder.job.type.group.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.symbol"),
        consume(raw_job, "signature.job_group_symbol"),
        consume(raw_job, "job_group.symbol"),
    )

    new_treeherder.job.type.group.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.name"),
        consume(raw_job, "job_type.job_group"),
        consume(raw_job, "signature.job_group_name"),
        consume(raw_job, "job_group.name"),
        consume(raw_job, "job_group"),
    )

    new_treeherder.job.guid = consume(raw_job, "guid")
    new_treeherder.job.id = consume(raw_job, "id")
    new_treeherder.job.coalesced_to_guid = consume(raw_job, "coalesced_to_guid")

    # BUILD
    new_treeherder.build.branch = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "repository"),
        consume(raw_job, "signature.repository"),
    )
    new_treeherder.build.revision = consume(raw_job, "push.revision")
    new_treeherder.build.revision12 = new_treeherder.build.revision[0:12]
    new_treeherder.build.date = consume(raw_job, "push.time")

    new_treeherder.build.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.platform"),
        consume(raw_job, "signature.build_platform"),
    )
    new_treeherder.build.os = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.os_name"),
        consume(raw_job, "signature.build_os_name"),
    )
    new_treeherder.build.architecture = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.architecture"),
        consume(raw_job, "signature.build_architecture"),
    )
    new_treeherder.build.product = consume(raw_job, "product.name")

    new_treeherder.run.key = consume(raw_job, "signature.name")
    new_treeherder.run.reason = consume(raw_job, "reason")
    new_treeherder.run.tier = consume(raw_job, "tier")
    new_treeherder.run.result = consume(raw_job, "result")
    new_treeherder.run.state = consume(raw_job, "state")
    if raw_treeherder.job.signature.build_system_type in [
        "taskcluster",
        "buildbot",
        "fx-test-jenkins",
        "fx-test-jenkins-dev",
        "qa-master",
        "qa-preprod-master",
    ]:
        consume(raw_job, "signature.build_system_type")
    else:
        Log.error(
            "Know nothing about build_system_type=={{type}}",
            type=raw_treeherder.job.signature.build_system_type,
        )

    # RUN MACHINE
    new_treeherder.run.machine.name = machine_name = consume(raw_job, "machine.name")
    split_name = machine_name.split("-")
    if Math.is_integer(split_name[-1]):
        new_treeherder.run.machine.pool = "-".join(split_name[:-1])
    new_treeherder.run.machine.os = consume(raw_job, "signature.machine_os_name")
    new_treeherder.run.machine.architecture = consume(
        raw_job, "signature.machine_architecture"
    )
    new_treeherder.run.machine.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "machine_platform"),
        consume(raw_job, "signature.machine_platform"),
    )

    # ACTION
    new_treeherder.action.start_time = consume(raw_job, "start_time")
    new_treeherder.action.end_time = consume(raw_job, "end_time")
    new_treeherder.action.request_time = consume(raw_job, "submit_time")
    new_treeherder.action.duration = (
        new_treeherder.action.end_time - new_treeherder.action.start_time
    )
    new_treeherder.last_modified = consume(raw_job, "last_modified")

    new_treeherder.failure.auto_classification = consume(raw_job, "autoclassify_status")
    new_treeherder.failure.classification = consume(raw_job, "failure_classification")
    new_treeherder.failure.notes = consume(raw_job, "job_note")

    new_treeherder.repo = {
        "branch": {"name": new_treeherder.build.branch},
        "changeset": {"id": new_treeherder.build.revision},
    }
    try:
        if new_treeherder.build.branch not in NON_HG_BRANCHES:
            new_treeherder.repo = minimize_repo(
                resources.hg.get_revision(new_treeherder.repo)
            )
    except Exception as e:
        Log.warning(
            "Problem with getting info changeset {{changeset}}",
            changeset=new_treeherder.repo,
            cause=e,
        )

    new_treeherder.bugs = consume(raw_job, "bug_job_map")

    pull_job_log(source_key, consume(raw_treeherder.job, "job_log"), new_treeherder)

    consume(raw_job, "push")
    consume(raw_job, "running_eta")
    consume(raw_job, "who")
    consume(raw_job, "signature.first_submission_timestamp")
    consume(raw_job, "signature.option_collection_hash")
    consume(raw_job, "signature.signature")
    pull_details(source_key, consume(raw_treeherder.job, "job_detail"), new_treeherder)

    new_treeherder.run.taskcluster.id = coalesce_w_conflict_detection(
        source_key,
        new_treeherder.run.taskcluster.id,
        consume(raw_job, "taskcluster_metadata.task_id"),
    )
    new_treeherder.run.taskcluster.retry_id = consume(
        raw_job, "taskcluster_metadata.retry_id"
    )

    pull_options(source_key, raw_treeherder, new_treeherder)

    remainder = raw_treeherder.leaves()
    if remainder:
        Log.error(
            "Did not process {{paths}} for key={{key}}",
            key=source_key,
            paths=[k for k, _ in remainder],
        )

def pull_options(source_key, raw_treeherder, new_treeherder):
    options = listwrap(consume(raw_treeherder, "job.option_collection.option"))
    for o in options:
        if o in _option_map:
            new_treeherder.build.type += _option_map[o]
        elif o in ["e10s"]:
            new_treeherder.run.type += [o]
        else:
            Log.warning(
                "not known option {{option|quote}} while processing {{key}}",
                key=source_key,
                option=o,
            )

_option_map = {
    "addon": ["addon"],
    "aarch64-debug": ["aarch64", "debug"],
    "arm-debug": ["arm", "debug"],
    "all": ["all"],
    "asan": ["asan"],
    "ccov": ["ccov"],
    "cc": ["ccov"],
    "debug": ["debug"],
    "fips": ["fips"],
    "fuzz": ["fuzz"],
    "gyp": ["gyp"],
    "gyp-asan": ["gyp", "asan"],
    "jsdcov": ["jsdcov"],
    "lto": ["lto"],  # LINK TIME OPTIMIZATION
    "make": ["make"],
    "nostylo": ["stylo-disabled"],
    "opt": ["opt"],
    "pgo": ["pgo"],
    "x64": ["x64"],
}

def pull_job_log(source_key, job_log, new_treeherder):
    for d in listwrap(job_log):
        for f in listwrap(d.failure_line):
            f.message = None
            f.job_guid = None
            f.stackwalk_stdout = None
            f.stackwalk_stderr = None
            f.subtest = None
    new_treeherder.job_log = job_log

def pull_details(source_key, details, new_treeherder):
    for d in listwrap(details):
        if d.title == "Summary":
            new_treeherder.summary = d.value
        elif d.title == "buildbot_request_id":
            new_treeherder.run.buildbot.id = d.value
        elif d.title == "Inspect Task":
            if d.url.startswith("https://tools.taskcluster.net/task-inspector/#"):
                new_treeherder.run.taskcluster.id = strings.between(
                    d.url, "https://tools.taskcluster.net/task-inspector/#", "/"
                )
            else:
                Log.warning(
                    "Can not extract task for ket {{key}} from {{url}}",
                    key=source_key,
                    url=d.url,
                )
        elif d.title == "CPU idle":
            new_treeherder.stats.cpu_idle = float(
                d.value.split("(")[0].replace(",", "")
            )
        elif d.title == "CPU user":
            new_treeherder.stats.cpu_user = float(
                d.value.split("(")[0].replace(",", "")
            )
        elif d.title == "CPU system":
            new_treeherder.stats.cpu_system = float(
                d.value.split("(")[0].replace(",", "")
            )
        elif d.title == "CPU iowait":
            new_treeherder.stats.cpu_io_wait = float(
                d.value.split("(")[0].replace(",", "")
            )
        elif d.title == "CPU usage":
            new_treeherder.stats.cpu_usage = float(d.value.strip("%").replace(",", ""))
        elif d.title == "I/O read bytes / time":
            new_treeherder.stats.io_read_bytes = float(
                d.value.split("/")[0].strip().replace(",", "")
            )
            new_treeherder.stats.io_read_time = float(
                d.value.split("/")[1].strip().replace(",", "")
            )
        elif d.title == "I/O write bytes / time":
            new_treeherder.stats.io_write_bytes = float(
                d.value.split("/")[0].strip().replace(",", "")
            )
            new_treeherder.stats.io_write_time = float(
                d.value.split("/")[1].strip().replace(",", "")
            )
        elif d.title == "Swap in / out":
            new_treeherder.stats.swap_in = float(
                d.value.split("/")[0].strip().replace(",", "")
            )
            new_treeherder.stats.swap_out = float(
                d.value.split("/")[1].strip().replace(",", "")
            )
        elif d.title in ["artifact uploaded", "One Click Loaner"]:
            pass
        elif d.title.find("-chunked"):
            pass
        elif d.title == None:
            if d.value in [
                "auto clobber",
                "purged clobber",
                "periodic clobber",
                "forced clobber",
            ]:
                pass
            elif any(map(d.value.startswith, ["linker max vsize: ", "num_ctors: "])):
                new_treeherder.stats.linker_max_vsize = int(
                    d.value.split(":")[1].strip()
                )
            elif any(map(d.value.startswith, KNOWN_VALUES)):
                pass
            else:
                KNOWN_VALUES.append(d.value)
                Log.warning(
                    "value has no title {{value|quote}} while processing {{key}}",
                    key=source_key,
                    value=d.value,
                )
        else:
            Log.warning("can not process detail with title of {{title}}", title=d.title)
    new_treeherder.job.details = details

def coalesce_w_conflict_detection(source_key, *args):
    output = None
    for a in args:
        if a == None:
            continue
        if isinstance(a, Mapping) and not a:
            continue
        if output == None:
            output = a
        elif a != output:
            Log.warning(
                "tried to coalesce {{values_|json}} while processing {{key}}",
                key=source_key,
                values_=args,
            )
        else:
            pass
    return output

def consume(props, key):
    output = props[key]
    try:
        props[key] = None
    except Exception:
        pass
    return output

KNOWN_VALUES = [
    "marionette: ",
    "--",
    "The following arguments ",
    "Tests will be run from the following files:",
    "gaia_revlink: ",
    "Unknown: ",
    "\t--this-chunk=1 --total-chunks=1 -- ",
    "\t--setenv=MOZ_BROWSER_XHTML=1 -- ",
]

NON_HG_BRANCHES = [
    "bmo-master",
    "go-bouncer",
    "mozillians-tests",
    "servo-auto",
    "snippets-service",
    "snippets-service",
    "snippets-tests",
    "socorro",
    "stubattribution-tests",
]
<|endoftext|>


<commit_before>"""
A management command which deletes expired accounts (e.g.,
accounts which signed up but never activated) from the database.

Calls ``RegistrationProfile.objects.delete_expired_users()``, which
contains the actual logic for determining which accounts are deleted.

"""

from django.core.management.base import NoArgsCommand
from django.core.management.base import CommandError

from registration.models import RegistrationProfile

class Command(NoArgsCommand):
    help = "Delete expired user registrations from the database"

    def handle_noargs(self, **options):
        RegistrationProfile.objects.delete_expired_users()
<commit_msg>Flip a couple imports to appease my inner style nerd<commit_after>"""
A management command which deletes expired accounts (e.g.,
accounts which signed up but never activated) from the database.

Calls ``RegistrationProfile.objects.delete_expired_users()``, which
contains the actual logic for determining which accounts are deleted.

"""

from django.core.management.base import CommandError
from django.core.management.base import NoArgsCommand

from registration.models import RegistrationProfile

class Command(NoArgsCommand):
    help = "Delete expired user registrations from the database"

    def handle_noargs(self, **options):
        RegistrationProfile.objects.delete_expired_users()
<|endoftext|>