<commit_before>
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import solve_ivp
from scipy.optimize import least_squares
from engineering_notation import EngNumber

filename = "oscilloscope.csv"

PLOT_INITAL = True
PLOT_PROGRESS = False
REPORT_PROGRESS = True
assumed_rotor_resistance = 1.0 # (TODO: set to None to estimate)

class ACMotor():
    """
    Models an induction motor based on Eq 10 in [1].
    [1] https://pdfs.semanticscholar.org/4770/15e472da4c2e05e9ff8c1b921c76a938f786.pdf

    Note: This model refers all rotor quantities to the stator, i.e. the
    quantities are as if the motor had a winding ratio of k = 1.
    """

    # parameters: (name, range)
    parameter_definitions = [
        ('stator_inductance', (0, np.inf), 'H'), # aka l_s, [Henry]
        ('stator_resistance', (0, np.inf), 'ohm'), # aka r_s, [Ohm]
        ('rotor_inductance',  (0, np.inf), 'H'), # aka l_r [Henry]
        # ('rotor_resistance',  (0, np.inf), 'ohm'), # aka r_r [Ohm]
        ('mutual_inductance_factor', (0, 1.0), ''), #[unitless] = l_m**2 / (l_s * l_r)
    ]
    # parameter index lookup
    pl = {r[0]:i for i, r in enumerate(parameter_definitions)}

    # states: (name, initial_value)
    state_definitions = [
        ('stator_current', 0.0), # aka i_s, [A]
        ('rotor_flux', 0.0),     # aka Phi_r, [Wb]
    ] # complex numbers
    # state index lookup
    sl = {r[0]:i for i, r in enumerate(state_definitions)}

    def __init__(self, params):
        self.params = params

        # Assigned in run():
        # self.stator_voltage = None
        # self.omega_stator = None
        # self.omega_rotor = None

    def get_mutual_inductance(self):
        return np.sqrt(
            self.params[ACMotor.pl['mutual_inductance_factor']]
            * self.params[ACMotor.pl['stator_inductance']]
            * self.params[ACMotor.pl['rotor_inductance']]
        )

    def system_function(self, t, y):
        # local shorthand for params
        p = self.params
        pl = ACMotor.pl
        sl = ACMotor.sl

        # rotor_resistance = p[pl['rotor_resistance']]
        rotor_resistance = assumed_rotor_resistance
        mutual_inductance = self.get_mutual_inductance()

        tau_rotor = p[pl['rotor_inductance']] / rotor_resistance # [s]
        coupling_factor = mutual_inductance / p[pl['rotor_inductance']] # aka k_r [unitless]
        r_sigma = p[pl['stator_resistance']] + coupling_factor**2 * rotor_resistance # [Ohm]
        leakage_factor = 1.0 - mutual_inductance**2 / (p[pl['rotor_inductance']] * p[pl['stator_inductance']]) # aka sigma [unitless]
        tau_stator_prime = leakage_factor * p[pl['stator_inductance']] / r_sigma # [s]

        # [1] Eq 10a
        dstator_current_dt = (
                -1.0j * self.omega_stator * tau_stator_prime * y[sl['stator_current']]
                - coupling_factor / (r_sigma * tau_rotor) * (1.0j*self.omega_rotor * tau_rotor - 1.0) * y[sl['rotor_flux']]
                + 1.0 / r_sigma * self.stator_voltage
                - y[sl['stator_current']]
            ) / tau_stator_prime

        # [1] Eq 10b
        drotor_flux_dt = (
                -1.0j * (self.omega_stator - self.omega_rotor) * tau_rotor * y[sl['rotor_flux']]
                + mutual_inductance * y[sl['stator_current']]
                - y[sl['rotor_flux']]
            ) / tau_rotor

        return [dstator_current_dt, drotor_flux_dt]

    def run(self, time_series, voltage, omega_stator, omega_rotor):
        self.stator_voltage = voltage
        self.omega_stator = omega_stator
        self.omega_rotor = omega_rotor

        y0 = np.array([x[1] for x in ACMotor.state_definitions], dtype=np.complex)

        result = solve_ivp(self.system_function, (time_series[0], time_series[-1]), y0, t_eval=time_series)
        y = result.y

        # compute derived state
        rotor_inductance = self.params[ACMotor.pl['rotor_inductance']]
        rotor_current = (1/rotor_inductance) * (y[1] - self.get_mutual_inductance() * y[0])
        return np.vstack((y, rotor_current))

def plot_data(t, y, ref, title):
    fig, (ax1, ax2) = plt.subplots(2, sharex=True)
    ax1b = ax1.twinx()
    ax1.plot(t, ref, label='Measured current')
    ax1.plot(t, y[0], label='Stator current')
    ax2.plot(t, y[2], label='Rotor current')
    ax1b.plot(t, 1000*y[1], 'g', label='Rotor flux')
    ax1.set_xlabel('time [s]')
    ax1.set_ylabel('Current [A]')
    ax1b.set_ylabel('Flux [mWb]')
    ax2.set_ylabel('Current [A]')
    plt.title(title)
    fig.legend()
    plt.show()

# load test data
t = np.arange(4096)/8000.0
voltage_step = 1.0
with open(filename, 'r') as fp:
    test_response = np.array([float(x) for x in fp.readlines()])

inital_parameters = np.zeros(len(ACMotor.parameter_definitions))
inital_parameters[ACMotor.pl['stator_inductance']] = 6.205440877238289e-05
inital_parameters[ACMotor.pl['stator_resistance']] = 0.031451061367988586
inital_parameters[ACMotor.pl['rotor_inductance']] = 0.25e-0
# inital_parameters[ACMotor.pl['rotor_resistance']] = 1.0e-0
# inital_parameters[ACMotor.pl['mutual_inductance']] = 2.40e-4
inital_parameters[ACMotor.pl['mutual_inductance_factor']] = 0.8

# inital_parameters[ACMotor.pl['stator_resistance']] = 1.298
# inital_parameters[ACMotor.pl['stator_inductance']] = 0.157228647
# inital_parameters[ACMotor.pl['rotor_resistance']] = 0.975052932
# inital_parameters[ACMotor.pl['rotor_inductance']] = 0.16674423623999998
# inital_parameters[ACMotor.pl['mutual_inductance']] = 0.157221177

# Plot initial run
if(PLOT_INITAL):
    motor = ACMotor(inital_parameters)
    y = motor.run(
        time_series = t,
        voltage = voltage_step,
        omega_stator = 0,
        omega_rotor= 0)

    plot_data(t, y, test_response, 'initial')

# Fit to data
def get_residuals(params):
    if REPORT_PROGRESS: print(params)
    motor = ACMotor(params)
    y = motor.run(
        time_series = t,
        voltage = voltage_step,
        omega_stator = 0,
        omega_rotor= 0)

    residuals = test_response - np.real(y[0])
    fitness = sum(residuals**2)
    if REPORT_PROGRESS: print(fitness)

    if(PLOT_PROGRESS):
        plot_data(t, y, test_response, 'progress')

    return residuals

optiresult = least_squares(get_residuals, inital_parameters,
    bounds=list(zip(*[x[1] for x in ACMotor.parameter_definitions])),
    x_scale='jac',
    diff_step = 1e-2 * np.array([
        7.58192590e-04,
        3.07166671e-02,
        6.85207075e-02,
        # 4.66461518e+00,
        8.67540012e-01])
)
print(optiresult.message)

print()
print('Given parameters:')
print('rotor_resistance = {}ohm'.format(EngNumber(assumed_rotor_resistance)))

print()
print('Fitted parameters:')
for i, r in enumerate(ACMotor.parameter_definitions):
    print('{} = {}{}'.format(r[0], EngNumber(optiresult.x[i]), r[2]))

print()
print('Derived parameters:')
mutual_inductance = motor.get_mutual_inductance()
coupling_factor = mutual_inductance / optiresult.x[ACMotor.pl['rotor_inductance']]
torque_constant = coupling_factor * mutual_inductance
print('mutual_inductance = {}H'.format(EngNumber(mutual_inductance)))
print('coupling_factor = {}'.format(EngNumber(coupling_factor)))
print('torque_constant = {}Nm/A^2'.format(EngNumber(torque_constant)))

motor = ACMotor(optiresult.x)
y = motor.run(
    time_series = t,
    voltage = voltage_step,
    omega_stator = 0,
    omega_rotor= 0)

final_stator_current = np.real(y[0,-1])
final_rotor_flux = np.real(y[1,-1])

final_torque_per_amp = coupling_factor * final_rotor_flux

print()
print('Final values:')
print('final_rotor_flux = {}Wb'.format(EngNumber(final_rotor_flux)))
print('final_stator_current = {}A'.format(EngNumber(final_stator_current)))
print('final_torque_per_amp = {}Nm/A'.format(EngNumber(final_torque_per_amp)))

plot_data(t, y, test_response, 'final')

<commit_msg>fix use of old motor object when calculating derived parameters<commit_after>
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import solve_ivp
from scipy.optimize import least_squares
from engineering_notation import EngNumber

filename = "oscilloscope.csv"

PLOT_INITAL = True
DO_FITTING = True
PLOT_PROGRESS = False
REPORT_PROGRESS = True
assumed_rotor_resistance = 1

class ACMotor():
    """
    Models an induction motor based on Eq 10 in [1].
    [1] https://pdfs.semanticscholar.org/4770/15e472da4c2e05e9ff8c1b921c76a938f786.pdf

    Note: This model refers all rotor quantities to the stator, i.e. the
    quantities are as if the motor had a winding ratio of k = 1.
    """

    # parameters: (name, range)
    parameter_definitions = [
        ('stator_inductance', (0, np.inf), 'H'), # aka l_s, [Henry]
        ('stator_resistance', (0, np.inf), 'ohm'), # aka r_s, [Ohm]
        ('rotor_inductance',  (0, np.inf), 'H'), # aka l_r [Henry]
        # ('rotor_resistance',  (0, np.inf), 'ohm'), # aka r_r [Ohm]
        ('mutual_inductance_factor', (0, 1.0), ''), #[unitless] = l_m**2 / (l_s * l_r)
    ]
    # parameter index lookup
    pl = {r[0]:i for i, r in enumerate(parameter_definitions)}

    # states: (name, initial_value)
    state_definitions = [
        ('stator_current', 0.0), # aka i_s, [A]
        ('rotor_flux', 0.0),     # aka Phi_r, [Wb]
    ] # complex numbers
    # state index lookup
    sl = {r[0]:i for i, r in enumerate(state_definitions)}

    def __init__(self, params):
        self.params = params

        # Assigned in run():
        # self.stator_voltage = None
        # self.omega_stator = None
        # self.omega_rotor = None

    def get_mutual_inductance(self):
        return np.sqrt(
            self.params[ACMotor.pl['mutual_inductance_factor']]
            * self.params[ACMotor.pl['stator_inductance']]
            * self.params[ACMotor.pl['rotor_inductance']]
        )

    def system_function(self, t, y):
        # local shorthand for params
        p = self.params
        pl = ACMotor.pl
        sl = ACMotor.sl

        # rotor_resistance = p[pl['rotor_resistance']]
        rotor_resistance = assumed_rotor_resistance
        mutual_inductance = self.get_mutual_inductance()

        tau_rotor = p[pl['rotor_inductance']] / rotor_resistance # [s]
        coupling_factor = mutual_inductance / p[pl['rotor_inductance']] # aka k_r [unitless]
        r_sigma = p[pl['stator_resistance']] + coupling_factor**2 * rotor_resistance # [Ohm]
        leakage_factor = 1.0 - mutual_inductance**2 / (p[pl['rotor_inductance']] * p[pl['stator_inductance']]) # aka sigma [unitless]
        tau_stator_prime = leakage_factor * p[pl['stator_inductance']] / r_sigma # [s]

        # [1] Eq 10a
        dstator_current_dt = (
                -1.0j * self.omega_stator * tau_stator_prime * y[sl['stator_current']]
                - coupling_factor / (r_sigma * tau_rotor) * (1.0j*self.omega_rotor * tau_rotor - 1.0) * y[sl['rotor_flux']]
                + 1.0 / r_sigma * self.stator_voltage
                - y[sl['stator_current']]
            ) / tau_stator_prime

        # [1] Eq 10b
        drotor_flux_dt = (
                -1.0j * (self.omega_stator - self.omega_rotor) * tau_rotor * y[sl['rotor_flux']]
                + mutual_inductance * y[sl['stator_current']]
                - y[sl['rotor_flux']]
            ) / tau_rotor

        return [dstator_current_dt, drotor_flux_dt]

    def run(self, time_series, voltage, omega_stator, omega_rotor):
        self.stator_voltage = voltage
        self.omega_stator = omega_stator
        self.omega_rotor = omega_rotor

        y0 = np.array([x[1] for x in ACMotor.state_definitions], dtype=np.complex)

        result = solve_ivp(self.system_function, (time_series[0], time_series[-1]), y0, t_eval=time_series)
        y = result.y

        # compute derived state
        rotor_inductance = self.params[ACMotor.pl['rotor_inductance']]
        rotor_current = (1/rotor_inductance) * (y[1] - self.get_mutual_inductance() * y[0])
        return np.vstack((y, rotor_current))

    def print_parameter_info(self):
        print()
        print('Given parameters:')
        print('rotor_resistance = {}ohm'.format(EngNumber(assumed_rotor_resistance)))

        print()
        print('Fitted parameters:')
        for i, r in enumerate(ACMotor.parameter_definitions):
            print('{} = {}{}'.format(r[0], EngNumber(self.params[i]), r[2]))

        print()
        print('Derived parameters:')
        mutual_inductance = motor.get_mutual_inductance()
        coupling_factor = mutual_inductance / self.params[ACMotor.pl['rotor_inductance']]
        torque_constant = coupling_factor * mutual_inductance
        print('mutual_inductance = {}H'.format(EngNumber(mutual_inductance)))
        print('coupling_factor = {}'.format(EngNumber(coupling_factor)))
        print('torque_constant = {}Nm/A^2'.format(EngNumber(torque_constant)))

    def print_run_info(self, y):
        final_stator_current_d = np.real(y[0,-1])
        final_stator_current_q = np.imag(y[0,-1])
        final_rotor_flux_d = np.real(y[1,-1])

        mutual_inductance = self.get_mutual_inductance()
        coupling_factor = mutual_inductance / self.params[ACMotor.pl['rotor_inductance']]
        final_torque_per_q_amp = coupling_factor * final_rotor_flux_d

        print()
        print('Final values:')
        print('final_rotor_flux_d = {}Wb'.format(EngNumber(final_rotor_flux_d)))
        print('final_stator_current_d = {}A'.format(EngNumber(final_stator_current_d)))
        print('final_stator_current_q = {}A'.format(EngNumber(final_stator_current_q)))
        print('final_torque_per_q_amp = {}Nm/A'.format(EngNumber(final_torque_per_q_amp)))

def plot_data(t, y, ref, title):
    fig, (ax1, ax2) = plt.subplots(2, sharex=True)
    ax1b = ax1.twinx()
    ax1.plot(t, ref, label='Measured current')
    ax1.plot(t, np.real(y[0]), label='Stator current (d)')
    ax1.plot(t, np.imag(y[0]), label='Stator current (q)')
    ax2.plot(t, np.real(y[2]), label='Rotor current (d)')
    ax2.plot(t, np.imag(y[2]), label='Rotor current (q)')
    ax1b.plot(t, 1000*np.real(y[1]), 'C2', label='Rotor flux (d)')
    ax1b.plot(t, 1000*np.imag(y[1]), 'C3', label='Rotor flux (q)')
    ax1.set_xlabel('time [s]')
    ax1.set_ylabel('Current [A]')
    ax1b.set_ylabel('Flux [mWb]')
    ax2.set_ylabel('Current [A]')
    plt.title(title)
    fig.legend()
    plt.show()

# load test data
t = np.arange(4096)/8000.0
voltage_step = 1.0
with open(filename, 'r') as fp:
    test_response = np.array([float(x) for x in fp.readlines()])

inital_parameters = np.zeros(len(ACMotor.parameter_definitions))
inital_parameters[ACMotor.pl['stator_inductance']] = 7.72181086e-04
inital_parameters[ACMotor.pl['stator_resistance']] = 3.06884624e-02
inital_parameters[ACMotor.pl['rotor_inductance']] = assumed_rotor_resistance*6.82013522e-02
# inital_parameters[ACMotor.pl['rotor_resistance']] = 1.0e-0
# inital_parameters[ACMotor.pl['mutual_inductance']] = 2.40e-4
inital_parameters[ACMotor.pl['mutual_inductance_factor']] = 8.68671978e-01

# inital_parameters[ACMotor.pl['stator_resistance']] = 1.298
# inital_parameters[ACMotor.pl['stator_inductance']] = 0.157228647
# inital_parameters[ACMotor.pl['rotor_resistance']] = 0.975052932
# inital_parameters[ACMotor.pl['rotor_inductance']] = 0.16674423623999998
# inital_parameters[ACMotor.pl['mutual_inductance']] = 0.157221177

# Plot initial run
if PLOT_INITAL:
    print()
    print('Initial run:')
    motor = ACMotor(inital_parameters)
    motor.print_parameter_info()

    y = motor.run(
        time_series = t,
        voltage = voltage_step,
        omega_stator = 0,
        omega_rotor= 0)
    motor.print_run_info(y)
    plot_data(t, y, test_response, 'initial')

# Fit to data
def get_residuals(params):
    if REPORT_PROGRESS: print(params)
    motor = ACMotor(params)
    y = motor.run(
        time_series = t,
        voltage = voltage_step,
        omega_stator = 0,
        omega_rotor= 0)

    residuals = test_response - np.real(y[0])
    fitness = sum(residuals**2)
    if REPORT_PROGRESS: print(fitness)

    if PLOT_PROGRESS:
        plot_data(t, y, test_response, 'progress')

    return residuals

if DO_FITTING:
    print()
    print('Fitting parameters:')
    optiresult = least_squares(get_residuals, inital_parameters,
        bounds=list(zip(*[x[1] for x in ACMotor.parameter_definitions])),
        x_scale='jac',
        diff_step = 1e-2 * np.array([
            7.58192590e-04,
            3.07166671e-02,
            6.85207075e-02,
            # 4.66461518e+00,
            8.67540012e-01])
    )
    print(optiresult.message)

    motor = ACMotor(optiresult.x)
    motor.print_parameter_info()

    y = motor.run(
        time_series = t,
        voltage = voltage_step,
        omega_stator = 0,
        omega_rotor= 0)
    motor.print_run_info(y)
    plot_data(t, y, test_response, 'final')

<|endoftext|>


<commit_before>def process(api, data):
    """
    Breakthroughs are rewarded to the first team to solve a given problem.

    Data Required: tid, pid
    """

    submissions = api.problems.get_problem_submissions(data["pid"], True)
    problem = api.problem.get_problem(pid=data["pid"])
    return len(submissions) == 0, {
        "name": "{} Breakthrough!".format(problem["name"]),
        "description": "Your team was the first to solve {}. Congratulations!".format(problem["name"])
    }
<commit_msg>Fixed typo.<commit_after>def process(api, data):
    """
    Breakthroughs are rewarded to the first team to solve a given problem.

    Data Required: tid, pid
    """

    submissions = api.problem.get_problem_submissions(data["pid"], True)
    problem = api.problem.get_problem(pid=data["pid"])
    return len(submissions) == 0, {
        "name": "{} Breakthrough!".format(problem["name"]),
        "description": "Your team was the first to solve {}. Congratulations!".format(problem["name"])
    }
<|endoftext|>


<commit_before>__author__ = 'Robert Meyer'

import os # To allow file paths working under Windows and Linux

from pypet import Environment
from pypet.utils.explore import cartesian_product

def multiply(traj):
    """Example of a sophisticated simulation that involves multiplying two values.

    :param traj:

        Trajectory - or more precisely a SingleRun - containing
        the parameters in a particular combination,
        it also serves as a container for results.

    """
    z=traj.x*traj.y
    traj.f_add_result('z', z, comment='Result of our simulation!')

# Create an environment that handles running
filename = os.path.join('experiments', 'example_01', 'HDF5','example_01.hdf5')
log_folder = os.path.join('experiments','example_01', 'LOGS')
env = Environment(trajectory='Multiplication',
                  filename=filename,
                  file_title='Example_01_First_Steps',
                  log_folder=log_folder,
                  comment='The first example!')

# The environment has created a trajectory container for us
traj = env.v_trajectory

# Add both parameters
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
traj.f_add_parameter('y', 1, comment='I am the second dimension!')

# Explore the parameters with a cartesian product
traj.f_explore(cartesian_product({'x':[1,2,3,4], 'y':[6,7,8]}))

# Run the simulation
env.f_run(multiply)

# Now let's see how we can reload the stored data from above.
# We do not need an environment for that, just a trajectory.
from pypet.trajectory import Trajectory

# So, first let's create a new trajectory and pass it the path and name of the HDF5 file.
# Yet, to be very clear let's delete all the old stuff.
del traj
del env
traj = Trajectory(filename=filename)

# Now we want to load all stored data.
traj.f_load(index=-1, load_parameters=2, load_results=2)

# Above `index` specifies that we want to load the trajectory with that particular index
# within the HDF5 file. We could instead also specify a `name`.
# Counting works also backwards, so `-1` yields the last or newest trajectory in the file.
#
# Next we need to specify how the data is loaded.
# Therefore, we have to set the keyword arguments `load_parameters` and `load_results`,
# here we chose both to be `2`.
# `0` would mean we do not want to load anything at all.
# `1` would mean we only want to load the empty hulls or skeletons of our parameters
# or results. Accordingly, we would add parameters or results to our trajectory
# but they would not contain any data.
# Instead `2` means we want to load the parameters and results including the data they contain.

# Finally we want to print a result of a particular run.
# Let's take the second run named `run_00000001` (Note that counting starts at 0!).
print('The result of `run_00000001` is: ')
print(traj.run_00000001.z)

raise RuntimeError()<commit_msg>removed explicit failure to make everything work again<commit_after>__author__ = 'Robert Meyer'

import os # To allow file paths working under Windows and Linux

from pypet import Environment
from pypet.utils.explore import cartesian_product

def multiply(traj):
    """Example of a sophisticated simulation that involves multiplying two values.

    :param traj:

        Trajectory - or more precisely a SingleRun - containing
        the parameters in a particular combination,
        it also serves as a container for results.

    """
    z=traj.x*traj.y
    traj.f_add_result('z', z, comment='Result of our simulation!')

# Create an environment that handles running
filename = os.path.join('experiments', 'example_01', 'HDF5','example_01.hdf5')
log_folder = os.path.join('experiments','example_01', 'LOGS')
env = Environment(trajectory='Multiplication',
                  filename=filename,
                  file_title='Example_01_First_Steps',
                  log_folder=log_folder,
                  comment='The first example!')

# The environment has created a trajectory container for us
traj = env.v_trajectory

# Add both parameters
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
traj.f_add_parameter('y', 1, comment='I am the second dimension!')

# Explore the parameters with a cartesian product
traj.f_explore(cartesian_product({'x':[1,2,3,4], 'y':[6,7,8]}))

# Run the simulation
env.f_run(multiply)

# Now let's see how we can reload the stored data from above.
# We do not need an environment for that, just a trajectory.
from pypet.trajectory import Trajectory

# So, first let's create a new trajectory and pass it the path and name of the HDF5 file.
# Yet, to be very clear let's delete all the old stuff.
del traj
del env
traj = Trajectory(filename=filename)

# Now we want to load all stored data.
traj.f_load(index=-1, load_parameters=2, load_results=2)

# Above `index` specifies that we want to load the trajectory with that particular index
# within the HDF5 file. We could instead also specify a `name`.
# Counting works also backwards, so `-1` yields the last or newest trajectory in the file.
#
# Next we need to specify how the data is loaded.
# Therefore, we have to set the keyword arguments `load_parameters` and `load_results`,
# here we chose both to be `2`.
# `0` would mean we do not want to load anything at all.
# `1` would mean we only want to load the empty hulls or skeletons of our parameters
# or results. Accordingly, we would add parameters or results to our trajectory
# but they would not contain any data.
# Instead `2` means we want to load the parameters and results including the data they contain.

# Finally we want to print a result of a particular run.
# Let's take the second run named `run_00000001` (Note that counting starts at 0!).
print('The result of `run_00000001` is: ')
print(traj.run_00000001.z)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.core.urlresolvers import reverse
from django.db import models
from django.template.defaultfilters import title
from django.utils.encoding import force_text
from django.utils.timezone import get_current_timezone_name
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.contrib import admin

from cms.exceptions import LanguageError
from cms.utils import get_cms_setting
from cms.utils.compat.dj import python_2_unicode_compatible
from cms.utils.helpers import reversion_register
from cms.utils.i18n import get_language_object
from cms.utils.placeholder import PlaceholderNoAction, get_placeholder_conf

@python_2_unicode_compatible
class Placeholder(models.Model):
    slot = models.CharField(_("slot"), max_length=50, db_index=True, editable=False)
    default_width = models.PositiveSmallIntegerField(_("width"), null=True, editable=False)
    cache_placeholder = True

    class Meta:
        app_label = 'cms'

    def __str__(self):
        return self.slot

    def clear(self, language=None):
        if language:
            qs = self.cmsplugin_set.filter(language=language)
        else:
            qs = self.cmsplugin_set.all()
        qs = qs.order_by('-level').select_related()
        for plugin in qs:
            inst, cls = plugin.get_plugin_instance()
            if inst and getattr(inst, 'cmsplugin_ptr', False):
                inst.cmsplugin_ptr._no_reorder = True
                inst._no_reorder = True
                inst.delete(no_mptt=True)
            else:
                plugin._no_reorder = True
                plugin.delete(no_mptt=True)

    def get_label(self):
        name = get_placeholder_conf("name", self.slot, default=title(self.slot))
        name = _(name)
        return name

    def get_add_url(self):
        return self._get_url('add_plugin')

    def get_edit_url(self, plugin_pk):
        return self._get_url('edit_plugin', plugin_pk)

    def get_move_url(self):
        return self._get_url('move_plugin')

    def get_delete_url(self, plugin_pk):
        return self._get_url('delete_plugin', plugin_pk)

    def get_changelist_url(self):
        return self._get_url('changelist')

    def get_clear_url(self):
        return self._get_url('clear_placeholder', self.pk)

    def get_copy_url(self):
        return self._get_url('copy_plugins')

    def get_extra_menu_items(self):
        from cms.plugin_pool import plugin_pool
        return plugin_pool.get_extra_placeholder_menu_items(self)

    def get_cache_key(self, lang):
        cache_key = '%srender_placeholder:%s.%s' % (get_cms_setting("CACHE_PREFIX"), self.pk, str(lang))
        if settings.USE_TZ:
            tz_name = force_text(get_current_timezone_name(), errors='ignore')
            cache_key += '.%s' % tz_name.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
        return cache_key

    def _get_url(self, key, pk=None):
        model = self._get_attached_model()
        args = []
        if pk:
            args.append(pk)
        if not model:
            return reverse('admin:cms_page_%s' % key, args=args)
        else:
            app_label = model._meta.app_label
            model_name = model.__name__.lower()
            return reverse('admin:%s_%s_%s' % (app_label, model_name, key), args=args)

    def _get_permission(self, request, key):
        """
        Generic method to check the permissions for a request for a given key,
        the key can be: 'add', 'change' or 'delete'. For each attached object
        permission has to be granted either on attached model or on attached object.
        """
        if request.user.is_superuser:
            return True
        if self.page:
            return self._get_object_permission(self.page, request, key)
        for obj in self._get_attached_objects():
            return self._get_object_permission(obj, request, key)

    def _get_object_permission(self, obj, request, key):
        found = False
        model = obj.__class__
        opts = model._meta
        perm_accessor = getattr(opts, 'get_%s_permission' % key)
        perm_code = '%s.%s' % (opts.app_label, perm_accessor())
        # if they don't have the permission for this attached model or object, bail out
        if not (request.user.has_perm(perm_code) or request.user.has_perm(perm_code, obj)):
            return False
        else:
            found = True
        if not (request.user.has_perm(perm_code) or request.user.has_perm(perm_code, obj)):
            return False
        else:
            found = True
        return found

    def has_change_permission(self, request):
        return self._get_permission(request, 'change')

    def has_add_permission(self, request):
        return self._get_permission(request, 'add')

    def has_delete_permission(self, request):
        return self._get_permission(request, 'delete')

    def render(self, context, width, lang=None, editable=True):
        '''
        Set editable = False to disable front-end rendering for this render.
        '''
        from cms.plugin_rendering import render_placeholder
        if not 'request' in context:
            return '<!-- missing request -->'
        width = width or self.default_width
        if width:
            context.update({'width': width})
        return render_placeholder(self, context, lang=lang, editable=editable)

    def _get_attached_fields(self):
        """
        Returns an ITERATOR of all non-cmsplugin reverse foreign key related fields.
        """
        from cms.models import CMSPlugin
        if not hasattr(self, '_attached_fields_cache'):
            self._attached_fields_cache = []
            for rel in self._meta.get_all_related_objects():
                if issubclass(rel.model, CMSPlugin):
                    continue
                from cms.admin.placeholderadmin import PlaceholderAdminMixin
                if rel.model in admin.site._registry and isinstance(admin.site._registry[rel.model], PlaceholderAdminMixin):
                    field = getattr(self, rel.get_accessor_name())
                    if field.count():
                        self._attached_fields_cache.append(rel.field)
        return self._attached_fields_cache

    def _get_attached_field(self):
        from cms.models import CMSPlugin, StaticPlaceholder, Page
        if not hasattr(self, '_attached_field_cache'):
            self._attached_field_cache = None
            relations = self._meta.get_all_related_objects()

            for rel in relations:
                if rel.model == Page or rel.model == StaticPlaceholder:
                    relations.insert(0, relations.pop(relations.index(rel)))
            for rel in relations:
                if issubclass(rel.model, CMSPlugin):
                    continue
                from cms.admin.placeholderadmin import PlaceholderAdminMixin
                if rel.model in admin.site._registry and isinstance(admin.site._registry[rel.model], PlaceholderAdminMixin):
                    field = getattr(self, rel.get_accessor_name())
                    if field.count():
                        self._attached_field_cache = rel.field
                        break
        return self._attached_field_cache

    def _get_attached_field_name(self):
        field = self._get_attached_field()
        if field:
            return field.name
        return None

    def _get_attached_model(self):
        if hasattr(self, '_attached_model_cache'):
            return self._attached_model_cache
        if self.page or self.page_set.all().count():
            from cms.models import Page
            self._attached_model_cache = Page
            return Page
        field = self._get_attached_field()
        if field:
            self._attached_model_cache = field.model
            return field.model
        self._attached_model_cache = None
        return None

    def _get_attached_models(self):
        """
        Returns a list of models of attached to this placeholder.
        """
        if hasattr(self, '_attached_models_cache'):
            return self._attached_models_cache
        self._attached_models_cache = [field.model for field in self._get_attached_fields()]
        return self._attached_models_cache

    def _get_attached_objects(self):
        """
        Returns a list of objects attached to this placeholder.
        """
        return [obj for field in self._get_attached_fields()
                for obj in getattr(self, field.related.get_accessor_name()).all()]

    def page_getter(self):
        if not hasattr(self, '_page'):
            from cms.models.pagemodel import Page
            try:
                self._page = Page.objects.get(placeholders=self)
            except (Page.DoesNotExist, Page.MultipleObjectsReturned,):
                self._page = None
        return self._page

    def page_setter(self, value):
        self._page = value

    page = property(page_getter, page_setter)

    def get_plugins_list(self, language=None):
        return list(self.get_plugins(language))

    def get_plugins(self, language=None):
        if language:
            return self.cmsplugin_set.filter(language=language).order_by('tree_id', 'lft')
        else:
            return self.cmsplugin_set.all().order_by('tree_id', 'lft')

    def get_filled_languages(self):
        """
        Returns language objects for every language for which the placeholder
        has plugins.

        This is not cached as it's meant to eb used in the frontend editor.
        """
        languages = []
        for lang_code in set(self.get_plugins().values_list('language', flat=True)):
            try:
                languages.append(get_language_object(lang_code))
            except LanguageError:
                pass
        return languages

    def get_cached_plugins(self):
        return getattr(self, '_plugins_cache', [])

    @property
    def actions(self):
        if not hasattr(self, '_actions_cache'):
            field = self._get_attached_field()
            self._actions_cache = getattr(field, 'actions', PlaceholderNoAction())
        return self._actions_cache

reversion_register(Placeholder)  # follow=["cmsplugin_set"] not following plugins since they are a spechial case
<commit_msg>slot can now have up to 255 characters<commit_after># -*- coding: utf-8 -*-
from django.core.urlresolvers import reverse
from django.db import models
from django.template.defaultfilters import title
from django.utils.encoding import force_text
from django.utils.timezone import get_current_timezone_name
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.contrib import admin

from cms.exceptions import LanguageError
from cms.utils import get_cms_setting
from cms.utils.compat.dj import python_2_unicode_compatible
from cms.utils.helpers import reversion_register
from cms.utils.i18n import get_language_object
from cms.utils.placeholder import PlaceholderNoAction, get_placeholder_conf

@python_2_unicode_compatible
class Placeholder(models.Model):
    slot = models.CharField(_("slot"), max_length=255, db_index=True, editable=False)
    default_width = models.PositiveSmallIntegerField(_("width"), null=True, editable=False)
    cache_placeholder = True

    class Meta:
        app_label = 'cms'

    def __str__(self):
        return self.slot

    def clear(self, language=None):
        if language:
            qs = self.cmsplugin_set.filter(language=language)
        else:
            qs = self.cmsplugin_set.all()
        qs = qs.order_by('-level').select_related()
        for plugin in qs:
            inst, cls = plugin.get_plugin_instance()
            if inst and getattr(inst, 'cmsplugin_ptr', False):
                inst.cmsplugin_ptr._no_reorder = True
                inst._no_reorder = True
                inst.delete(no_mptt=True)
            else:
                plugin._no_reorder = True
                plugin.delete(no_mptt=True)

    def get_label(self):
        name = get_placeholder_conf("name", self.slot, default=title(self.slot))
        name = _(name)
        return name

    def get_add_url(self):
        return self._get_url('add_plugin')

    def get_edit_url(self, plugin_pk):
        return self._get_url('edit_plugin', plugin_pk)

    def get_move_url(self):
        return self._get_url('move_plugin')

    def get_delete_url(self, plugin_pk):
        return self._get_url('delete_plugin', plugin_pk)

    def get_changelist_url(self):
        return self._get_url('changelist')

    def get_clear_url(self):
        return self._get_url('clear_placeholder', self.pk)

    def get_copy_url(self):
        return self._get_url('copy_plugins')

    def get_extra_menu_items(self):
        from cms.plugin_pool import plugin_pool
        return plugin_pool.get_extra_placeholder_menu_items(self)

    def get_cache_key(self, lang):
        cache_key = '%srender_placeholder:%s.%s' % (get_cms_setting("CACHE_PREFIX"), self.pk, str(lang))
        if settings.USE_TZ:
            tz_name = force_text(get_current_timezone_name(), errors='ignore')
            cache_key += '.%s' % tz_name.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
        return cache_key

    def _get_url(self, key, pk=None):
        model = self._get_attached_model()
        args = []
        if pk:
            args.append(pk)
        if not model:
            return reverse('admin:cms_page_%s' % key, args=args)
        else:
            app_label = model._meta.app_label
            model_name = model.__name__.lower()
            return reverse('admin:%s_%s_%s' % (app_label, model_name, key), args=args)

    def _get_permission(self, request, key):
        """
        Generic method to check the permissions for a request for a given key,
        the key can be: 'add', 'change' or 'delete'. For each attached object
        permission has to be granted either on attached model or on attached object.
        """
        if request.user.is_superuser:
            return True
        if self.page:
            return self._get_object_permission(self.page, request, key)
        for obj in self._get_attached_objects():
            return self._get_object_permission(obj, request, key)

    def _get_object_permission(self, obj, request, key):
        found = False
        model = obj.__class__
        opts = model._meta
        perm_accessor = getattr(opts, 'get_%s_permission' % key)
        perm_code = '%s.%s' % (opts.app_label, perm_accessor())
        # if they don't have the permission for this attached model or object, bail out
        if not (request.user.has_perm(perm_code) or request.user.has_perm(perm_code, obj)):
            return False
        else:
            found = True
        if not (request.user.has_perm(perm_code) or request.user.has_perm(perm_code, obj)):
            return False
        else:
            found = True
        return found

    def has_change_permission(self, request):
        return self._get_permission(request, 'change')

    def has_add_permission(self, request):
        return self._get_permission(request, 'add')

    def has_delete_permission(self, request):
        return self._get_permission(request, 'delete')

    def render(self, context, width, lang=None, editable=True):
        '''
        Set editable = False to disable front-end rendering for this render.
        '''
        from cms.plugin_rendering import render_placeholder
        if not 'request' in context:
            return '<!-- missing request -->'
        width = width or self.default_width
        if width:
            context.update({'width': width})
        return render_placeholder(self, context, lang=lang, editable=editable)

    def _get_attached_fields(self):
        """
        Returns an ITERATOR of all non-cmsplugin reverse foreign key related fields.
        """
        from cms.models import CMSPlugin
        if not hasattr(self, '_attached_fields_cache'):
            self._attached_fields_cache = []
            for rel in self._meta.get_all_related_objects():
                if issubclass(rel.model, CMSPlugin):
                    continue
                from cms.admin.placeholderadmin import PlaceholderAdminMixin
                if rel.model in admin.site._registry and isinstance(admin.site._registry[rel.model], PlaceholderAdminMixin):
                    field = getattr(self, rel.get_accessor_name())
                    if field.count():
                        self._attached_fields_cache.append(rel.field)
        return self._attached_fields_cache

    def _get_attached_field(self):
        from cms.models import CMSPlugin, StaticPlaceholder, Page
        if not hasattr(self, '_attached_field_cache'):
            self._attached_field_cache = None
            relations = self._meta.get_all_related_objects()

            for rel in relations:
                if rel.model == Page or rel.model == StaticPlaceholder:
                    relations.insert(0, relations.pop(relations.index(rel)))
            for rel in relations:
                if issubclass(rel.model, CMSPlugin):
                    continue
                from cms.admin.placeholderadmin import PlaceholderAdminMixin
                if rel.model in admin.site._registry and isinstance(admin.site._registry[rel.model], PlaceholderAdminMixin):
                    field = getattr(self, rel.get_accessor_name())
                    if field.count():
                        self._attached_field_cache = rel.field
                        break
        return self._attached_field_cache

    def _get_attached_field_name(self):
        field = self._get_attached_field()
        if field:
            return field.name
        return None

    def _get_attached_model(self):
        if hasattr(self, '_attached_model_cache'):
            return self._attached_model_cache
        if self.page or self.page_set.all().count():
            from cms.models import Page
            self._attached_model_cache = Page
            return Page
        field = self._get_attached_field()
        if field:
            self._attached_model_cache = field.model
            return field.model
        self._attached_model_cache = None
        return None

    def _get_attached_models(self):
        """
        Returns a list of models of attached to this placeholder.
        """
        if hasattr(self, '_attached_models_cache'):
            return self._attached_models_cache
        self._attached_models_cache = [field.model for field in self._get_attached_fields()]
        return self._attached_models_cache

    def _get_attached_objects(self):
        """
        Returns a list of objects attached to this placeholder.
        """
        return [obj for field in self._get_attached_fields()
                for obj in getattr(self, field.related.get_accessor_name()).all()]

    def page_getter(self):
        if not hasattr(self, '_page'):
            from cms.models.pagemodel import Page
            try:
                self._page = Page.objects.get(placeholders=self)
            except (Page.DoesNotExist, Page.MultipleObjectsReturned,):
                self._page = None
        return self._page

    def page_setter(self, value):
        self._page = value

    page = property(page_getter, page_setter)

    def get_plugins_list(self, language=None):
        return list(self.get_plugins(language))

    def get_plugins(self, language=None):
        if language:
            return self.cmsplugin_set.filter(language=language).order_by('tree_id', 'lft')
        else:
            return self.cmsplugin_set.all().order_by('tree_id', 'lft')

    def get_filled_languages(self):
        """
        Returns language objects for every language for which the placeholder
        has plugins.

        This is not cached as it's meant to eb used in the frontend editor.
        """
        languages = []
        for lang_code in set(self.get_plugins().values_list('language', flat=True)):
            try:
                languages.append(get_language_object(lang_code))
            except LanguageError:
                pass
        return languages

    def get_cached_plugins(self):
        return getattr(self, '_plugins_cache', [])

    @property
    def actions(self):
        if not hasattr(self, '_actions_cache'):
            field = self._get_attached_field()
            self._actions_cache = getattr(field, 'actions', PlaceholderNoAction())
        return self._actions_cache

reversion_register(Placeholder)  # follow=["cmsplugin_set"] not following plugins since they are a spechial case
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""DenseNet models for Keras.

# Reference

- [Densely Connected Convolutional Networks](https://arxiv.org/pdf/1608.06993.pdf)

"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import warnings

from keras.models import Model
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import AveragePooling2D
from keras.layers.pooling import GlobalAveragePooling2D
from keras.layers import Input, merge
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l2
from keras.utils.layer_utils import convert_all_kernels_in_model
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.applications.imagenet_utils import _obtain_input_shape
import keras.backend as K

TH_WEIGHTS_PATH = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Theano-Backend-TH-dim-ordering.h5'
TF_WEIGHTS_PATH = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Tensorflow-Backend-TF-dim-ordering.h5'
TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Theano-Backend-TH-dim-ordering-no-top.h5.h5'
TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Tensorflow-Backend-TF-dim-ordering-no-top.h5'

def DenseNet(depth=40, nb_dense_block=3, growth_rate=12, nb_filter=16,
             bottleneck=False, reduction=0.0, dropout_rate=None, weight_decay=1E-4,
             include_top=True, weights='cifar10',
             input_tensor=None, input_shape=None,
             classes=10):
    """Instantiate the DenseNet architecture,
        optionally loading weights pre-trained
        on CIFAR-10. Note that when using TensorFlow,
        for best performance you should set
        `image_dim_ordering="tf"` in your Keras config
        at ~/.keras/keras.json.

        The model and the weights are compatible with both
        TensorFlow and Theano. The dimension ordering
        convention used by the model is the one
        specified in your Keras config file.

        # Arguments
            depth: number or layers in the DenseNet
            nb_dense_block: number of dense blocks to add to end (generally = 3)
            growth_rate: number of filters to add per dense block
            nb_filter: initial number of filters. -1 indicates initial
                number of filters is 2 * growth_rate
            bottleneck: flag to add bottleneck blocks in between dense blocks
            reduction: reduction factor of transition blocks.
                Note : reduction value is inverted to compute compression.
            dropout_rate: dropout rate
            weight_decay: weight decay factor
            include_top: whether to include the fully-connected
                layer at the top of the network.
            weights: one of `None` (random initialization) or
                "cifar10" (pre-training on CIFAR-10)..
            input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
                to use as image input for the model.
            input_shape: optional shape tuple, only to be specified
                if `include_top` is False (otherwise the input shape
                has to be `(32, 32, 3)` (with `tf` dim ordering)
                or `(3, 32, 32)` (with `th` dim ordering).
                It should have exactly 3 inputs channels,
                and width and height should be no smaller than 8.
                E.g. `(200, 200, 3)` would be one valid value.
            classes: optional number of classes to classify images
                into, only to be specified if `include_top` is True, and
                if no `weights` argument is specified.

        # Returns
            A Keras model instance.
        """

    if weights not in {'cifar10', None}:
        raise ValueError('The `weights` argument should be either '
                         '`None` (random initialization) or `cifar10` '
                         '(pre-training on CIFAR-10).')

    if weights == 'cifar10' and include_top and classes != 10:
        raise ValueError('If using `weights` as CIFAR 10 with `include_top`'
                         ' as true, `classes` should be 10')

    # Determine proper input shape
    input_shape = _obtain_input_shape(input_shape,
                                      default_size=32,
                                      min_size=8,
                                      dim_ordering=K.image_dim_ordering(),
                                      include_top=include_top)

    if input_tensor is None:
        img_input = Input(shape=input_shape)
    else:
        if not K.is_keras_tensor(input_tensor):
            img_input = Input(tensor=input_tensor, shape=input_shape)
        else:
            img_input = input_tensor

    x = __create_dense_net(classes, img_input, include_top, depth, nb_dense_block,
                           growth_rate, nb_filter, bottleneck, reduction,
                           dropout_rate, weight_decay)

    # Ensure that the model takes into account
    # any potential predecessors of `input_tensor`.
    if input_tensor is not None:
        inputs = get_source_inputs(input_tensor)
    else:
        inputs = img_input
    # Create model.
    model = Model(inputs, x, name='densenet')

    # load weights
    if weights == 'cifar10':
        if (depth == 40) and (nb_dense_block == 3) and (growth_rate == 12) and (nb_filter == 16) and \
                (bottleneck is False) and (reduction == 0.0) and (dropout_rate == 0.0) and (weight_decay == 1E-4):
            # Default parameters match. Weights for this model exist:

            if K.image_dim_ordering() == 'th':
                if include_top:
                    weights_path = get_file('densenet_40_12_th_dim_ordering_th_kernels.h5',
                                            TH_WEIGHTS_PATH,
                                            cache_subdir='models')
                else:
                    weights_path = get_file('densenet_40_12_th_dim_ordering_th_kernels_no_top.h5',
                                            TH_WEIGHTS_PATH_NO_TOP,
                                            cache_subdir='models')

                model.load_weights(weights_path)

                if K.backend() == 'tensorflow':
                    warnings.warn('You are using the TensorFlow backend, yet you '
                                  'are using the Theano '
                                  'image dimension ordering convention '
                                  '(`image_dim_ordering="th"`). '
                                  'For best performance, set '
                                  '`image_dim_ordering="tf"` in '
                                  'your Keras config '
                                  'at ~/.keras/keras.json.')
                    convert_all_kernels_in_model(model)
            else:
                if include_top:
                    weights_path = get_file('densenet_40_12_tf_dim_ordering_tf_kernels.h5',
                                            TF_WEIGHTS_PATH,
                                            cache_subdir='models')
                else:
                    weights_path = get_file('densenet_40_12_tf_dim_ordering_tf_kernels_no_top.h5',
                                            TF_WEIGHTS_PATH_NO_TOP,
                                            cache_subdir='models')

                model.load_weights(weights_path)

                if K.backend() == 'theano':
                    convert_all_kernels_in_model(model)

    return model

def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1E-4):
    ''' Apply BatchNorm, Relu 3x3, Conv2D, optional bottleneck block and dropout

    Args:
        ip: Input keras tensor
        nb_filter: number of filters
        bottleneck: add bottleneck block
        dropout_rate: dropout rate
        weight_decay: weight decay factor

    Returns: keras tensor with batch_norm, relu and convolution2d added (optional bottleneck)

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                           beta_regularizer=l2(weight_decay))(ip)
    x = Activation('relu')(x)

    if bottleneck:
        inter_channel = nb_filter * 4  # Obtained from https://github.com/liuzhuang13/DenseNet/blob/master/densenet.lua

        x = Convolution2D(inter_channel, 1, 1, init='he_uniform', border_mode='same', bias=False,
                          W_regularizer=l2(weight_decay))(x)

        if dropout_rate:
            x = Dropout(dropout_rate)(x)

        x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                               beta_regularizer=l2(weight_decay))(x)
        x = Activation('relu')(x)

    x = Convolution2D(nb_filter, 3, 3, init="he_uniform", border_mode="same", bias=False,
                      W_regularizer=l2(weight_decay))(x)
    if dropout_rate:
        x = Dropout(dropout_rate)(x)

    return x

def __transition_block(ip, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4):
    ''' Apply BatchNorm, Relu 1x1, Conv2D, optional compression, dropout and Maxpooling2D

    Args:
        ip: keras tensor
        nb_filter: number of filters
        dropout_rate: dropout rate
        weight_decay: weight decay factor

    Returns: keras tensor, after applying batch_norm, relu-conv, dropout, maxpool

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                           beta_regularizer=l2(weight_decay))(ip)
    x = Activation('relu')(x)
    x = Convolution2D(int(nb_filter * compression), 1, 1, init="he_uniform", border_mode="same", bias=False,
                      W_regularizer=l2(weight_decay))(x)
    if dropout_rate:
        x = Dropout(dropout_rate)(x)
    x = AveragePooling2D((2, 2), strides=(2, 2))(x)

    return x

def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None, weight_decay=1E-4):
    ''' Build a __dense_block where the output of each __conv_block is fed to subsequent ones

    Args:
        x: keras tensor
        nb_layers: the number of layers of __conv_block to append to the model.
        nb_filter: number of filters
        growth_rate: growth rate
        bottleneck: bottleneck block
        dropout_rate: dropout rate
        weight_decay: weight decay factor

    Returns: keras tensor with nb_layers of __conv_block appended

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    feature_list = [x]

    for i in range(nb_layers):
        x = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay)
        feature_list.append(x)
        x = merge(feature_list, mode='concat', concat_axis=concat_axis)
        nb_filter += growth_rate

    return x, nb_filter

def __create_dense_net(nb_classes, img_input, include_top, depth=40, nb_dense_block=3, growth_rate=12, nb_filter=-1,
                       bottleneck=False, reduction=0.0, dropout_rate=None, weight_decay=1E-4):
    ''' Build the __create_dense_net model

    Args:
        nb_classes: number of classes
        img_input: tuple of shape (channels, rows, columns) or (rows, columns, channels)
        include_top: flag to include the final Dense layer
        depth: number or layers
        nb_dense_block: number of dense blocks to add to end (generally = 3)
        growth_rate: number of filters to add per dense block
        nb_filter: initial number of filters. Default -1 indicates initial number of filters is 2 * growth_rate
        bottleneck: add bottleneck blocks
        reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
        dropout_rate: dropout rate
        weight_decay: weight decay

    Returns: keras tensor with nb_layers of __conv_block appended

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    assert (depth - 4) % 3 == 0, "Depth must be 3 N + 4"
    if reduction != 0.0:
        assert reduction <= 1.0 and reduction > 0.0, "reduction value must lie between 0.0 and 1.0"

    # layers in each dense block
    nb_layers = int((depth - 4) / 3)

    if bottleneck:
        nb_layers = int(nb_layers // 2)

    # compute initial nb_filter if -1, else accept users initial nb_filter
    if nb_filter <= 0:
        nb_filter = 2 * growth_rate

    # compute compression factor
    compression = 1.0 - reduction

    # Initial convolution
    x = Convolution2D(nb_filter, 3, 3, init="he_uniform", border_mode="same", name="initial_conv2D", bias=False,
                      W_regularizer=l2(weight_decay))(img_input)

    # Add dense blocks
    for block_idx in range(nb_dense_block - 1):
        x, nb_filter = __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=bottleneck,
                                     dropout_rate=dropout_rate, weight_decay=weight_decay)
        # add __transition_block
        x = __transition_block(x, nb_filter, compression=compression, dropout_rate=dropout_rate,
                               weight_decay=weight_decay)
        nb_filter = int(nb_filter * compression)

    # The last __dense_block does not have a __transition_block
    x, nb_filter = __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=bottleneck,
                                 dropout_rate=dropout_rate, weight_decay=weight_decay)

    x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                           beta_regularizer=l2(weight_decay))(x)
    x = Activation('relu')(x)
    x = GlobalAveragePooling2D()(x)

    if include_top:
        x = Dense(nb_classes, activation='softmax', W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(x)

    return x
<commit_msg>Fixed model loading path for th dim ordering no top<commit_after># -*- coding: utf-8 -*-
"""DenseNet models for Keras.

# Reference

- [Densely Connected Convolutional Networks](https://arxiv.org/pdf/1608.06993.pdf)

"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import warnings

from keras.models import Model
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import AveragePooling2D
from keras.layers.pooling import GlobalAveragePooling2D
from keras.layers import Input, merge
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l2
from keras.utils.layer_utils import convert_all_kernels_in_model
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.applications.imagenet_utils import _obtain_input_shape
import keras.backend as K

TH_WEIGHTS_PATH = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Theano-Backend-TH-dim-ordering.h5'
TF_WEIGHTS_PATH = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Tensorflow-Backend-TF-dim-ordering.h5'
TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Theano-Backend-TH-dim-ordering-no-top.h5'
TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/titu1994/DenseNet/releases/download/v2.0/DenseNet-40-12-Tensorflow-Backend-TF-dim-ordering-no-top.h5'

def DenseNet(depth=40, nb_dense_block=3, growth_rate=12, nb_filter=16,
             bottleneck=False, reduction=0.0, dropout_rate=None, weight_decay=1E-4,
             include_top=True, weights='cifar10',
             input_tensor=None, input_shape=None,
             classes=10):
    """Instantiate the DenseNet architecture,
        optionally loading weights pre-trained
        on CIFAR-10. Note that when using TensorFlow,
        for best performance you should set
        `image_dim_ordering="tf"` in your Keras config
        at ~/.keras/keras.json.

        The model and the weights are compatible with both
        TensorFlow and Theano. The dimension ordering
        convention used by the model is the one
        specified in your Keras config file.

        # Arguments
            depth: number or layers in the DenseNet
            nb_dense_block: number of dense blocks to add to end (generally = 3)
            growth_rate: number of filters to add per dense block
            nb_filter: initial number of filters. -1 indicates initial
                number of filters is 2 * growth_rate
            bottleneck: flag to add bottleneck blocks in between dense blocks
            reduction: reduction factor of transition blocks.
                Note : reduction value is inverted to compute compression.
            dropout_rate: dropout rate
            weight_decay: weight decay factor
            include_top: whether to include the fully-connected
                layer at the top of the network.
            weights: one of `None` (random initialization) or
                "cifar10" (pre-training on CIFAR-10)..
            input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
                to use as image input for the model.
            input_shape: optional shape tuple, only to be specified
                if `include_top` is False (otherwise the input shape
                has to be `(32, 32, 3)` (with `tf` dim ordering)
                or `(3, 32, 32)` (with `th` dim ordering).
                It should have exactly 3 inputs channels,
                and width and height should be no smaller than 8.
                E.g. `(200, 200, 3)` would be one valid value.
            classes: optional number of classes to classify images
                into, only to be specified if `include_top` is True, and
                if no `weights` argument is specified.

        # Returns
            A Keras model instance.
        """

    if weights not in {'cifar10', None}:
        raise ValueError('The `weights` argument should be either '
                         '`None` (random initialization) or `cifar10` '
                         '(pre-training on CIFAR-10).')

    if weights == 'cifar10' and include_top and classes != 10:
        raise ValueError('If using `weights` as CIFAR 10 with `include_top`'
                         ' as true, `classes` should be 10')

    # Determine proper input shape
    input_shape = _obtain_input_shape(input_shape,
                                      default_size=32,
                                      min_size=8,
                                      dim_ordering=K.image_dim_ordering(),
                                      include_top=include_top)

    if input_tensor is None:
        img_input = Input(shape=input_shape)
    else:
        if not K.is_keras_tensor(input_tensor):
            img_input = Input(tensor=input_tensor, shape=input_shape)
        else:
            img_input = input_tensor

    x = __create_dense_net(classes, img_input, include_top, depth, nb_dense_block,
                           growth_rate, nb_filter, bottleneck, reduction,
                           dropout_rate, weight_decay)

    # Ensure that the model takes into account
    # any potential predecessors of `input_tensor`.
    if input_tensor is not None:
        inputs = get_source_inputs(input_tensor)
    else:
        inputs = img_input
    # Create model.
    model = Model(inputs, x, name='densenet')

    # load weights
    if weights == 'cifar10':
        if (depth == 40) and (nb_dense_block == 3) and (growth_rate == 12) and (nb_filter == 16) and \
                (bottleneck is False) and (reduction == 0.0) and (dropout_rate == 0.0) and (weight_decay == 1E-4):
            # Default parameters match. Weights for this model exist:

            if K.image_dim_ordering() == 'th':
                if include_top:
                    weights_path = get_file('densenet_40_12_th_dim_ordering_th_kernels.h5',
                                            TH_WEIGHTS_PATH,
                                            cache_subdir='models')
                else:
                    weights_path = get_file('densenet_40_12_th_dim_ordering_th_kernels_no_top.h5',
                                            TH_WEIGHTS_PATH_NO_TOP,
                                            cache_subdir='models')

                model.load_weights(weights_path)

                if K.backend() == 'tensorflow':
                    warnings.warn('You are using the TensorFlow backend, yet you '
                                  'are using the Theano '
                                  'image dimension ordering convention '
                                  '(`image_dim_ordering="th"`). '
                                  'For best performance, set '
                                  '`image_dim_ordering="tf"` in '
                                  'your Keras config '
                                  'at ~/.keras/keras.json.')
                    convert_all_kernels_in_model(model)
            else:
                if include_top:
                    weights_path = get_file('densenet_40_12_tf_dim_ordering_tf_kernels.h5',
                                            TF_WEIGHTS_PATH,
                                            cache_subdir='models')
                else:
                    weights_path = get_file('densenet_40_12_tf_dim_ordering_tf_kernels_no_top.h5',
                                            TF_WEIGHTS_PATH_NO_TOP,
                                            cache_subdir='models')

                model.load_weights(weights_path)

                if K.backend() == 'theano':
                    convert_all_kernels_in_model(model)

    return model

def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1E-4):
    ''' Apply BatchNorm, Relu 3x3, Conv2D, optional bottleneck block and dropout

    Args:
        ip: Input keras tensor
        nb_filter: number of filters
        bottleneck: add bottleneck block
        dropout_rate: dropout rate
        weight_decay: weight decay factor

    Returns: keras tensor with batch_norm, relu and convolution2d added (optional bottleneck)

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                           beta_regularizer=l2(weight_decay))(ip)
    x = Activation('relu')(x)

    if bottleneck:
        inter_channel = nb_filter * 4  # Obtained from https://github.com/liuzhuang13/DenseNet/blob/master/densenet.lua

        x = Convolution2D(inter_channel, 1, 1, init='he_uniform', border_mode='same', bias=False,
                          W_regularizer=l2(weight_decay))(x)

        if dropout_rate:
            x = Dropout(dropout_rate)(x)

        x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                               beta_regularizer=l2(weight_decay))(x)
        x = Activation('relu')(x)

    x = Convolution2D(nb_filter, 3, 3, init="he_uniform", border_mode="same", bias=False,
                      W_regularizer=l2(weight_decay))(x)
    if dropout_rate:
        x = Dropout(dropout_rate)(x)

    return x

def __transition_block(ip, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4):
    ''' Apply BatchNorm, Relu 1x1, Conv2D, optional compression, dropout and Maxpooling2D

    Args:
        ip: keras tensor
        nb_filter: number of filters
        dropout_rate: dropout rate
        weight_decay: weight decay factor

    Returns: keras tensor, after applying batch_norm, relu-conv, dropout, maxpool

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                           beta_regularizer=l2(weight_decay))(ip)
    x = Activation('relu')(x)
    x = Convolution2D(int(nb_filter * compression), 1, 1, init="he_uniform", border_mode="same", bias=False,
                      W_regularizer=l2(weight_decay))(x)
    if dropout_rate:
        x = Dropout(dropout_rate)(x)
    x = AveragePooling2D((2, 2), strides=(2, 2))(x)

    return x

def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None, weight_decay=1E-4):
    ''' Build a __dense_block where the output of each __conv_block is fed to subsequent ones

    Args:
        x: keras tensor
        nb_layers: the number of layers of __conv_block to append to the model.
        nb_filter: number of filters
        growth_rate: growth rate
        bottleneck: bottleneck block
        dropout_rate: dropout rate
        weight_decay: weight decay factor

    Returns: keras tensor with nb_layers of __conv_block appended

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    feature_list = [x]

    for i in range(nb_layers):
        x = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay)
        feature_list.append(x)
        x = merge(feature_list, mode='concat', concat_axis=concat_axis)
        nb_filter += growth_rate

    return x, nb_filter

def __create_dense_net(nb_classes, img_input, include_top, depth=40, nb_dense_block=3, growth_rate=12, nb_filter=-1,
                       bottleneck=False, reduction=0.0, dropout_rate=None, weight_decay=1E-4):
    ''' Build the __create_dense_net model

    Args:
        nb_classes: number of classes
        img_input: tuple of shape (channels, rows, columns) or (rows, columns, channels)
        include_top: flag to include the final Dense layer
        depth: number or layers
        nb_dense_block: number of dense blocks to add to end (generally = 3)
        growth_rate: number of filters to add per dense block
        nb_filter: initial number of filters. Default -1 indicates initial number of filters is 2 * growth_rate
        bottleneck: add bottleneck blocks
        reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
        dropout_rate: dropout rate
        weight_decay: weight decay

    Returns: keras tensor with nb_layers of __conv_block appended

    '''

    concat_axis = 1 if K.image_dim_ordering() == "th" else -1

    assert (depth - 4) % 3 == 0, "Depth must be 3 N + 4"
    if reduction != 0.0:
        assert reduction <= 1.0 and reduction > 0.0, "reduction value must lie between 0.0 and 1.0"

    # layers in each dense block
    nb_layers = int((depth - 4) / 3)

    if bottleneck:
        nb_layers = int(nb_layers // 2)

    # compute initial nb_filter if -1, else accept users initial nb_filter
    if nb_filter <= 0:
        nb_filter = 2 * growth_rate

    # compute compression factor
    compression = 1.0 - reduction

    # Initial convolution
    x = Convolution2D(nb_filter, 3, 3, init="he_uniform", border_mode="same", name="initial_conv2D", bias=False,
                      W_regularizer=l2(weight_decay))(img_input)

    # Add dense blocks
    for block_idx in range(nb_dense_block - 1):
        x, nb_filter = __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=bottleneck,
                                     dropout_rate=dropout_rate, weight_decay=weight_decay)
        # add __transition_block
        x = __transition_block(x, nb_filter, compression=compression, dropout_rate=dropout_rate,
                               weight_decay=weight_decay)
        nb_filter = int(nb_filter * compression)

    # The last __dense_block does not have a __transition_block
    x, nb_filter = __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=bottleneck,
                                 dropout_rate=dropout_rate, weight_decay=weight_decay)

    x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay),
                           beta_regularizer=l2(weight_decay))(x)
    x = Activation('relu')(x)
    x = GlobalAveragePooling2D()(x)

    if include_top:
        x = Dense(nb_classes, activation='softmax', W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(x)

    return x
<|endoftext|>


<commit_before>import re
from ...lang import number_replace
from ...lang.tokenize import wiki_tokenize_func

ENTITY_MATCH_RE = re.compile(r'<(.+)\|(.+)>')

def parse_types_text(text, dbpedia_types, numeric=True):
    """
    :type dbpedia_types: dict
    :type dbpedia_redirects: dict
    """
    line = wiki_tokenize_func(text)
    new_line = []
    for word in line:
        match = ENTITY_MATCH_RE.search(word)
        if match:
            uri = match.group(1)
            orig_text = match.group(2)
            orig_text = orig_text.replace('_', ' ')
            stop = False
            for uri in (uri, uri.capitalize()):
                if uri in dbpedia_types:
                    new_line.append(match.expand('<dbpedia:' + dbpedia_types[uri][0]+'>'))
                    stop = True
                    break
            if not stop:
                if numeric:
                    new_line.extend([number_replace(x) for x in wiki_tokenize_func(orig_text)])
                else:
                    new_line.extend(wiki_tokenize_func(orig_text))

        elif numeric:
            new_line.append(number_replace(word))
        else:
            new_line.append(word)
    return ' '.join(new_line)
<commit_msg>update entity parser for dbm<commit_after>import re
from ...lang import number_replace
from ...lang.tokenize import wiki_tokenize_func

ENTITY_MATCH_RE = re.compile(r'<(.+)\|(.+)>')

def parse_types_text(text, dbpedia_types, numeric=True):
    """
    :type dbpedia_types: dict
    :type dbpedia_redirects: dict
    """
    line = wiki_tokenize_func(text)
    new_line = []
    for word in line:
        match = ENTITY_MATCH_RE.search(word)
        if match:
            uri = match.group(1)
            orig_text = match.group(2)
            orig_text = orig_text.replace('_', ' ')
            stop = False
            for uri in (uri, uri.capitalize()):
                if uri in dbpedia_types:
                    new_line.append(match.expand('<dbpedia:' + dbpedia_types['types'][uri][0]+'>'))
                    stop = True
                    break
            if not stop:
                if numeric:
                    new_line.extend([number_replace(x) for x in wiki_tokenize_func(orig_text)])
                else:
                    new_line.extend(wiki_tokenize_func(orig_text))

        elif numeric:
            new_line.append(number_replace(word))
        else:
            new_line.append(word)
    return ' '.join(new_line)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

from lxml import etree
import requests

from ..models.service_definition_interface import ServiceDefinitionInterface

class ServiceDefinition(ServiceDefinitionInterface):
    id = u"xml"

    def _parse_data(self, submission, fields):
        if len(fields) > 0:

            # Build xml to be parsed
            xml_doc = etree.fromstring(submission)
            tree = etree.ElementTree(xml_doc)
            matched_nodes_paths = []
            root_element = tree.getroot()
            root_path = tree.getpath(root_element)

            def remove_root_path(path_):
                return path_.replace(root_path, "")

            def is_group(node_):
                for nested_node_ in node_.iterchildren():
                    if nested_node_.iterchildren():
                        return True
                return False

            def process_node(node_, matched_nodes_paths_):
                """
                Removes node from XML tree if it's not included in subset of fields
                :param node_: lxml.etree._Element
                :param matched_nodes_paths_: tuple. Nodes to keep
                """
                # Calling process_node(xml, `matched_nodes_path`) will first loop through all children of the root element.
                # Then for each child, it will loop through its children if any etc...
                # When all children are processed, it checks whether the node should be removed or not.
                # Trivial case, it's removed. Otherwise, the parent of the current node is tagged as `do_not_delete`.
                # It let us know when the parent is processed, it has to be kept because one of its children matches
                # one of the occurrences of `subset_fields`
                # For example, with `subset_fields = ['question_2', 'question_3'] and this `xml`:
                # <root>
                #   <group>
                #       <question_1>Value1</question1>
                #       <question_2>Value2</question1>
                #   </group>
                #   <question3>Value3</question3>
                # </root>
                #
                #  - `<question_1>` is processed first and removed.
                #  - `<question_2>` is processed second and kept. `<group>` is tagged as `do_not_delete`
                #  - `<group>` is processed, is not part of `subset_field` but it's tagged as `do_not_delete`. It's kept
                #  - `<question_3>` is processed and kept
                #
                # Results:
                # <root>
                #   <group>
                #       <question_2>Value2</question1>
                #   </group>
                #   <question3>Value3</question3>
                # </root>

                for child in node_.getchildren():
                    process_node(child, matched_nodes_paths_)

                node_path = remove_root_path(tree.getpath(node_))

                # if `node_path` does not match one of the occurrences previously found,
                # it must be removed.
                if not node_path.startswith(matched_nodes_paths_) and node_.get("do_not_delete") != "true":
                    node_.getparent().remove(node_)
                elif node_path != "":
                    node_.getparent().set("do_not_delete", "true")

                if node_.attrib.get("do_not_delete"):
                    del node_.attrib["do_not_delete"]

            # Keep all paths of nodes that match the subset of fields
            for field_ in fields:
                for node in tree.iter(field_):
                    matched_node_path = remove_root_path(tree.getpath(node))
                    # If node is a group, we need to add a trailing slash for later comparison in `process_node`
                    if is_group(node):
                        matched_node_path += "/"
                    matched_nodes_paths.append(matched_node_path)

            process_node(root_element, tuple(matched_nodes_paths))

            return etree.tostring(tree, pretty_print=True)

        return submission

    def _prepare_request_kwargs(self):
        return {
            "headers": {"Content-Type": "application/xml"},
            "data": self._data
        }

<commit_msg>Added some comments<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

from lxml import etree
import requests

from ..models.service_definition_interface import ServiceDefinitionInterface

class ServiceDefinition(ServiceDefinitionInterface):
    id = u"xml"

    def _parse_data(self, submission, fields):
        if len(fields) > 0:

            # Build xml to be parsed
            xml_doc = etree.fromstring(submission)
            tree = etree.ElementTree(xml_doc)
            matched_nodes_paths = []
            root_element = tree.getroot()
            root_path = tree.getpath(root_element)

            def remove_root_path(path_):
                return path_.replace(root_path, "")

            def is_group(node_):
                """
                Checks whether `node_` has children that are also xml nodes with children.
                Not text.
                It lets us assume `node_` is a group.

                :param node_: lxml.etree._Element
                :return: bool
                """
                for nested_node_ in node_.iterchildren():
                    if nested_node_.iterchildren():
                        return True
                return False

            def process_node(node_, matched_nodes_paths_):
                """
                Removes node from XML tree if it's not included in subset of fields
                :param node_: lxml.etree._Element
                :param matched_nodes_paths_: tuple. Nodes to keep
                """
                # Calling process_node(xml, `matched_nodes_path`) will first loop through all children of the root element.
                # Then for each child, it will loop through its children if any etc...
                # When all children are processed, it checks whether the node should be removed or not.
                # Trivial case, it's removed. Otherwise, the parent of the current node is tagged as `do_not_delete`.
                # It let us know when the parent is processed, it has to be kept because one of its children matches
                # one of the occurrences of `subset_fields`
                # For example, with `subset_fields = ['question_2', 'question_3'] and this `xml`:
                # <root>
                #   <group>
                #       <question_1>Value1</question1>
                #       <question_2>Value2</question1>
                #   </group>
                #   <question3>Value3</question3>
                # </root>
                #
                #  - `<question_1>` is processed first and removed.
                #  - `<question_2>` is processed second and kept. `<group>` is tagged as `do_not_delete`
                #  - `<group>` is processed, is not part of `subset_field` but it's tagged as `do_not_delete`. It's kept
                #  - `<question_3>` is processed and kept
                #
                # Results:
                # <root>
                #   <group>
                #       <question_2>Value2</question1>
                #   </group>
                #   <question3>Value3</question3>
                # </root>

                for child in node_.getchildren():
                    process_node(child, matched_nodes_paths_)

                node_path = remove_root_path(tree.getpath(node_))

                # if `node_path` does not match one of the occurrences previously found,
                # it must be removed.
                if not node_path.startswith(matched_nodes_paths_) and node_.get("do_not_delete") != "true":
                    node_.getparent().remove(node_)
                elif node_path != "":
                    node_.getparent().set("do_not_delete", "true")

                if node_.attrib.get("do_not_delete"):
                    del node_.attrib["do_not_delete"]

            # Keep all paths of nodes that match the subset of fields
            for field_ in fields:
                for node in tree.iter(field_):
                    matched_node_path = remove_root_path(tree.getpath(node))
                    # If node is a group, we need to add a trailing slash for later comparison in `process_node`
                    if is_group(node):
                        matched_node_path += "/"
                    matched_nodes_paths.append(matched_node_path)

            process_node(root_element, tuple(matched_nodes_paths))

            return etree.tostring(tree, pretty_print=True)

        return submission

    def _prepare_request_kwargs(self):
        return {
            "headers": {"Content-Type": "application/xml"},
            "data": self._data
        }

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Tests for the LinksSource class (Links from a page)."""
from __future__ import unicode_literals
from datetime import datetime

from kuma.scrape.sources import LinksSource
from kuma.wiki.models import Revision
from . import mock_requester, mock_storage

def test_init_blank():
    source = LinksSource()
    assert source.path == '/en-US/'
    assert source.locale == 'en-US'

def test_init_slash():
    source = LinksSource('/')
    assert source.path == '/en-US/'

def test_init_full_url():
    url = 'https://developer.mozilla.org/en-US/docs/Web/CSS'
    source = LinksSource(url)
    assert source.path == '/en-US/docs/Web/CSS'
    assert source.locale == 'en-US'

def test_init_non_english():
    url = '/fr/docs/Web/CSS'
    source = LinksSource(url)
    assert source.path == '/fr/docs/Web/CSS'
    assert source.locale == 'fr'

def test_gather_homepage(client, db):
    source = LinksSource('/')
    html = client.get('/en-US/').content
    requester = mock_requester(content=html)
    storage = mock_storage()
    resources = source.gather(requester, storage)
    assert source.state == source.STATE_DONE
    assert source.freshness == source.FRESH_YES

    # The exact list of resources will change as the homepage changes
    expected_paths = [
        '/en-US/docs/Learn',
        '/en-US/docs/Mozilla/Firefox',
        '/en-US/docs/Web/JavaScript',
    ]
    for path in expected_paths:
        spec = ('document', path, {})
        assert spec in resources

    # These appear on the homepage, but shouldn't in the paths to scrape
    unexpected_paths = [
        '/en-US/search',
    ]
    for path in unexpected_paths:
        spec = ('document', path, {})
        assert spec not in resources

def test_gather_ignores_links(client, root_doc, simple_user):
    user_profile_url = simple_user.get_absolute_url()
    content = """

Absolute Link

Relative Link

Later in this page.

Profile Link

""" % user_profile_url
    new_rev = Revision(
        document=root_doc,
        creator=root_doc.current_revision.creator,
        content=content,
        created=datetime(2017, 6, 5))
    new_rev.save()
    base_path = root_doc.get_absolute_url()
    html = client.get(base_path).content

    source = LinksSource(base_path)
    requester = mock_requester(content=html)
    storage = mock_storage()
    resources = source.gather(requester, storage)
    assert source.state == source.STATE_DONE
    assert source.freshness == source.FRESH_YES

    expected = ('document', '/en-US/docs/Absolute/Link', {})
    assert expected in resources

    for doc, path, options in resources:
        assert base_path not in path
        assert "Relative/Link" not in path
        assert "#later" not in path
        assert user_profile_url not in path
<commit_msg>bug 1370594: Don't expect Firefox link on homepage<commit_after># -*- coding: utf-8 -*-
"""Tests for the LinksSource class (Links from a page)."""
from __future__ import unicode_literals
from datetime import datetime

from kuma.scrape.sources import LinksSource
from kuma.wiki.models import Revision
from . import mock_requester, mock_storage

def test_init_blank():
    source = LinksSource()
    assert source.path == '/en-US/'
    assert source.locale == 'en-US'

def test_init_slash():
    source = LinksSource('/')
    assert source.path == '/en-US/'

def test_init_full_url():
    url = 'https://developer.mozilla.org/en-US/docs/Web/CSS'
    source = LinksSource(url)
    assert source.path == '/en-US/docs/Web/CSS'
    assert source.locale == 'en-US'

def test_init_non_english():
    url = '/fr/docs/Web/CSS'
    source = LinksSource(url)
    assert source.path == '/fr/docs/Web/CSS'
    assert source.locale == 'fr'

def test_gather_homepage(client, db):
    source = LinksSource('/')
    html = client.get('/en-US/').content
    requester = mock_requester(content=html)
    storage = mock_storage()
    resources = source.gather(requester, storage)
    assert source.state == source.STATE_DONE
    assert source.freshness == source.FRESH_YES

    # The exact list of resources will change as the homepage changes
    expected_paths = [
        '/en-US/docs/Learn',
        '/en-US/docs/Tools',
        '/en-US/docs/Web/JavaScript',
    ]
    for path in expected_paths:
        spec = ('document', path, {})
        assert spec in resources

    # These appear on the homepage, but shouldn't be in the paths to scrape
    unexpected_paths = [
        '/en-US/search',
    ]
    for path in unexpected_paths:
        spec = ('document', path, {})
        assert spec not in resources

def test_gather_ignores_links(client, root_doc, simple_user):
    user_profile_url = simple_user.get_absolute_url()
    content = """

Absolute Link

Relative Link

Later in this page.

Profile Link

""" % user_profile_url
    new_rev = Revision(
        document=root_doc,
        creator=root_doc.current_revision.creator,
        content=content,
        created=datetime(2017, 6, 5))
    new_rev.save()
    base_path = root_doc.get_absolute_url()
    html = client.get(base_path).content

    source = LinksSource(base_path)
    requester = mock_requester(content=html)
    storage = mock_storage()
    resources = source.gather(requester, storage)
    assert source.state == source.STATE_DONE
    assert source.freshness == source.FRESH_YES

    expected = ('document', '/en-US/docs/Absolute/Link', {})
    assert expected in resources

    for doc, path, options in resources:
        assert base_path not in path
        assert "Relative/Link" not in path
        assert "#later" not in path
        assert user_profile_url not in path
<|endoftext|>