<commit_before>from codepage import OrdinalLookup, Codepage, rCommand
from string import ascii_lowercase, ascii_uppercase, digits
import re
import lzma
try:
    import brotli
except:
    print("Please install the 'brotli' module: 'sudo -H pip3 install brotli'")
    __import__("sys").exit()

symbols = ".!\"#$%&'()*+,-/:;<=>?@[\]^_`{|}~\r"
whitespace = "\n "
default_order = "wslnu"
charset_fragment_lookup = {
    "w": whitespace, "s": symbols, "l": ascii_lowercase, "n": digits,
    "u": ascii_uppercase
}
default_charset = (
    whitespace + symbols + ascii_lowercase + digits + ascii_uppercase
)
Codepage.remove("”")
gap = OrdinalLookup["”"]
RAW_ENCODING = 120
DICTIONARY_ENCODING = 121
CHARSET_ENCODING = 122
RLE_ENCODING = 123
BROTLI_ENCODING = 124
LZMA_ENCODING = 125

def Compressed(string, escape=False):
    """
    Compressed(string) -> str
    Returns the shortest Charcoal compressed literal of the given string.

    """
    if not string:
        return "””"
    if not all(
        character == "¶" or character == "⸿" or
        character >= " " and character <= "~"
        for character in string
    ):
        if not escape:
            return string
        if len(re.findall("[^ -~¶⸿]", string)) > 3:
            return "”" + Codepage[RAW_ENCODING] + string + "”"
        return (
            "´" * (string[0] in "+X*|-\\/<>^KLTVY7¬") +
            rCommand.sub("´\1", string)
        )
    original_string, string = string, re.sub(
        "¶", "\n", re.sub("⸿", "\r", string)
    )
    compressed_charset = CompressCharset(string)
    compressed_rle = CompressRLE(string)
    compressed_brotli = CompressBrotli(string)
    compressed_lzma = CompressLZMA(string)
    compressed_permuted = CompressPermutations(string)
    compressed = CompressString(string)
    string_length = len(original_string) - 2
    minimum_length = min(
        len(compressed_charset), len(compressed_rle), len(compressed_brotli),
        len(compressed_lzma), len(compressed_permuted), len(compressed),
        string_length
    )
    if string_length == minimum_length:
        if not escape:
            return original_string
        if len(re.findall("[^ -~¶⸿]", original_string)) > 3:
            return "”" + Codepage[RAW_ENCODING] + original_string + "”"
        return (
            "´" * (original_string[0] in "+X*|-\\/<>^KLTVY7¬") +
            rCommand.sub("´\1", original_string)
        )
    if len(compressed) == minimum_length:
        return "“" + compressed + "”"
    if len(compressed_permuted) == minimum_length:
        return "”" + compressed_permuted + "”"
    if len(compressed_charset) == minimum_length:
        return "”" + compressed_charset + "”"
    if len(compressed_rle) == minimum_length:
        return "”" + compressed_rle + "”"
    if len(compressed_brotli) == minimum_length:
        return "”" + compressed_brotli + "”"
    return "”" + compressed_lzma + "”"

def CompressCharset(string):
    """
    CompressPermutations(string) -> str
    Returns without delimiters the given string compressed \
using a character set of only the characters in the string.

    """
    occurrences = [0] * 97
    for c in string:
        occurrences[0 if c == "\n" else 1 if c == "\r" else ord(c) - 30] += 1
    items = sorted([i for i, n in enumerate(occurrences) if n])[::-1]
    charset = "".join([chr(n + 30) if n > 1 else "\n\r"[n] for n in items])
    base = len(charset)
    result = ""
    number = 1
    if base == 1:
        number = len(string)
    else:
        for character in string:
            number = number * base + charset.index(character)
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    number = 1
    length = 0
    for character in charset:
        number = number * 97 + default_charset.index(character)
    while number:
        result = Codepage[number % 255] + result
        length += 1
        number //= 255
    return Codepage[CHARSET_ENCODING] + Codepage[length] + result

def CompressRLE(string):
    """
    CompressRLE(string) -> str
    Returns without delimiters the given string compressed \
using run-length encoding.

    """
    number, previous, count = 1, string[0], -1
    for character in string:
        if character != previous or count == 32:
            number = number * 97 + default_charset.index(previous)
            number = number * 32 + count
            count = 0
            previous = character
        else:
            count += 1
    number = number * 97 + default_charset.index(string[-1])
    number = number * 32 + count
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[RLE_ENCODING] + result

def CompressBrotli(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using Google's brotli compression method.

    """
    compressed = brotli.compress(string)
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[BROTLI_ENCODING] + result

def CompressLZMA(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using the lzstring compression method.

    """
    compressed = lzma.compress(
        string.encode("ascii"),
        format=lzma.FORMAT_RAW,
        filters=[{'id': lzma.FILTER_LZMA2, 'preset': 9 | lzma.PRESET_EXTREME}]
    )
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[LZMA_ENCODING] + result

def CompressPermutations(string):
    """
    CompressPermutations(string) -> str
    Returns without delimiters the given string compressed \
using a permuted codepage.

    """
    numeric = lowercase = uppercase = whitespace = symbol = 0
    for character in string:
        if character >= "0" and character <= "9":
            numeric -= .1
        elif character >= "a" and character <= "z":
            lowercase -= 0.03846
        elif character >= "A" and character <= "Z":
            uppercase -= 0.03846
        elif character == "\n" or character == " ":
            whitespace -= .5
        else:
            symbol -= .03125
    result = "".join(map(lambda t: t[1], sorted([
        (whitespace, "w"), (symbol, "s"), (lowercase, "l"), (numeric, "n"),
        (uppercase, "u")
    ])))
    index, base = 0, 5
    charset = "".join(
        charset_fragment_lookup[character] for character in result
    )
    for character in default_order[:-1]:
        index = index * base + result.index(character)
        result = result.replace(character, "", 1)
        base -= 1
    return Codepage[index] + Compress([
        charset.index(character) for character in string
    ])

def CompressString(string):
    """
    CompressString(string) -> str
    Returns without delimiters the given string compressed.

    """
    return Compress([
        default_charset.index(character) for character in string
    ])

def Compress(ordinals):
    """
    Compress(ordinals) -> str
    Returns without delimiters the given string compressed.

    """
    base, result, number = max(ordinals) + 1, "", 1
    if base == 1:
        number = len(ordinals)
    else:
        for ordinal in ordinals:
            number = number * base + ordinal
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[base - 1] + result

def Decompressed(string):
    """
    Decompressed(string) -> str
    Returns the decompressed form of the given Charcoal string.

    """
    if string == "””":
        return ""
    if string[-1] != "”":
        return string
    if string[0] == "”":
        ordinal = OrdinalLookup.get(string[1], ord(string[1]))
        alphabet_id = ordinal - (ordinal > gap)
        if alphabet_id < 120:
            return DecompressPermutations(string[1:-1])
        return [
            lambda string: string,
            lambda string: "", # TODO
            lambda string: DecompressCharset(string),
            lambda string: DecompressRLE(string),
            lambda string: DecompressBrotli(string),
            lambda string: DecompressLZMA(string)
        ][alphabet_id - 120](string[2:-1])
    elif string[0] == "“":
        return DecompressString(string[1:-1])

def DecompressCharset(string):
    """
    DecompressCharset(string) -> str
    Returns the original form of the given string compressed \
using a given character set, passed without delimiters.

    """
    length = OrdinalLookup.get(string[0], ord(string[0]))
    length += length <= gap
    number = 0
    for character in string[1:length]:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    charset = ""
    while number > 1:
        charset = default_charset[number % 97] + charset
        number //= 97
    number = 0
    for character in string[length:]:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    base = len(charset)
    if base == 1:
        return charset[0] * number
    result = ""
    while number > 1:
        result = charset[number % base] + result
        number //= base
    return result

def DecompressRLE(string):
    """
    DecompressRLE(string) -> str
    Returns the original form of the given string compressed \
using run-length encoding, passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    result = ""
    while number > 1:
        count = number % 32
        number //= 32
        result = default_charset[number % 97] * (count + 1) + result
        number //= 97
    return result

def DecompressBrotli(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return brotli.decompress(bytes(compressed)).decode("ascii")

def DecompressLZMA(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return lzma.decompress(bytes(compressed)).decode("ascii")

def DecompressPermutations(string):
    """
    DecompressPermutations(string) -> str
    Returns the original form of the given string compressed \
using a permuted codepage, passed without delimiters.

    """
    index = OrdinalLookup.get(string[0], ord(string[0]))
    if index > gap:
        index -= 1
    base, letters = 2, ["u"]
    while index:
        letters.insert(index % base, default_order[5 - base])
        index //= base
        base += 1
    while base <= 5:
        letters.insert(0, default_order[5 - base])
        base += 1
    charset = "".join(charset_fragment_lookup[c] for c in letters)
    return "".join([charset[n] for n in Decompress(string[1:])])

def DecompressString(string):
    """
    DecompressString(string) -> str
    Returns the original form of the given string compressed, \
passed without delimiters.

    """
    return "".join([default_charset[n] for n in Decompress(string)])

def Decompress(string):
    """
    Decompress(string) -> list
    Returns the ordinals in the original form of the given string compressed.

    """
    number, result = 0, []
    base = OrdinalLookup.get(string[0], ord(string[0])) + 1
    if base > gap:
        base -= 1
    for character in string[1:]:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = (number * 255) + ordinal - (ordinal > gap)
    if base == 1:
        return [0] * number
    while number > 1:
        remainder = number % base
        result = [remainder] + result
        number //= base
    return result
<commit_msg>Fix call to brotli.compress<commit_after>from codepage import OrdinalLookup, Codepage, rCommand
from string import ascii_lowercase, ascii_uppercase, digits
import re
import lzma
try:
    import brotli
except:
    print("Please install the 'brotli' module: 'sudo -H pip3 install brotli'")
    __import__("sys").exit()

symbols = ".!\"#$%&'()*+,-/:;<=>?@[\]^_`{|}~\r"
whitespace = "\n "
default_order = "wslnu"
charset_fragment_lookup = {
    "w": whitespace, "s": symbols, "l": ascii_lowercase, "n": digits,
    "u": ascii_uppercase
}
default_charset = (
    whitespace + symbols + ascii_lowercase + digits + ascii_uppercase
)
Codepage.remove("”")
gap = OrdinalLookup["”"]
RAW_ENCODING = 120
DICTIONARY_ENCODING = 121
CHARSET_ENCODING = 122
RLE_ENCODING = 123
BROTLI_ENCODING = 124
LZMA_ENCODING = 125

def Compressed(string, escape=False):
    """
    Compressed(string) -> str
    Returns the shortest Charcoal compressed literal of the given string.

    """
    if not string:
        return "””"
    if not all(
        character == "¶" or character == "⸿" or
        character >= " " and character <= "~"
        for character in string
    ):
        if not escape:
            return string
        if len(re.findall("[^ -~¶⸿]", string)) > 3:
            return "”" + Codepage[RAW_ENCODING] + string + "”"
        return (
            "´" * (string[0] in "+X*|-\\/<>^KLTVY7¬") +
            rCommand.sub("´\1", string)
        )
    original_string, string = string, re.sub(
        "¶", "\n", re.sub("⸿", "\r", string)
    )
    compressed_charset = CompressCharset(string)
    compressed_rle = CompressRLE(string)
    compressed_brotli = CompressBrotli(string)
    compressed_lzma = CompressLZMA(string)
    compressed_permuted = CompressPermutations(string)
    compressed = CompressString(string)
    string_length = len(original_string) - 2
    minimum_length = min(
        len(compressed_charset), len(compressed_rle), len(compressed_brotli),
        len(compressed_lzma), len(compressed_permuted), len(compressed),
        string_length
    )
    if string_length == minimum_length:
        if not escape:
            return original_string
        if len(re.findall("[^ -~¶⸿]", original_string)) > 3:
            return "”" + Codepage[RAW_ENCODING] + original_string + "”"
        return (
            "´" * (original_string[0] in "+X*|-\\/<>^KLTVY7¬") +
            rCommand.sub("´\1", original_string)
        )
    if len(compressed) == minimum_length:
        return "“" + compressed + "”"
    if len(compressed_permuted) == minimum_length:
        return "”" + compressed_permuted + "”"
    if len(compressed_charset) == minimum_length:
        return "”" + compressed_charset + "”"
    if len(compressed_rle) == minimum_length:
        return "”" + compressed_rle + "”"
    if len(compressed_brotli) == minimum_length:
        return "”" + compressed_brotli + "”"
    return "”" + compressed_lzma + "”"

def CompressCharset(string):
    """
    CompressPermutations(string) -> str
    Returns without delimiters the given string compressed \
using a character set of only the characters in the string.

    """
    occurrences = [0] * 97
    for c in string:
        occurrences[0 if c == "\n" else 1 if c == "\r" else ord(c) - 30] += 1
    items = sorted([i for i, n in enumerate(occurrences) if n])[::-1]
    charset = "".join([chr(n + 30) if n > 1 else "\n\r"[n] for n in items])
    base = len(charset)
    result = ""
    number = 1
    if base == 1:
        number = len(string)
    else:
        for character in string:
            number = number * base + charset.index(character)
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    number = 1
    length = 0
    for character in charset:
        number = number * 97 + default_charset.index(character)
    while number:
        result = Codepage[number % 255] + result
        length += 1
        number //= 255
    return Codepage[CHARSET_ENCODING] + Codepage[length] + result

def CompressRLE(string):
    """
    CompressRLE(string) -> str
    Returns without delimiters the given string compressed \
using run-length encoding.

    """
    number, previous, count = 1, string[0], -1
    for character in string:
        if character != previous or count == 32:
            number = number * 97 + default_charset.index(previous)
            number = number * 32 + count
            count = 0
            previous = character
        else:
            count += 1
    number = number * 97 + default_charset.index(string[-1])
    number = number * 32 + count
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[RLE_ENCODING] + result

def CompressBrotli(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using Google's brotli compression method.

    """
    compressed = brotli.compress(string.encode("ascii"))
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[BROTLI_ENCODING] + result

def CompressLZMA(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using the lzstring compression method.

    """
    compressed = lzma.compress(
        string.encode("ascii"),
        format=lzma.FORMAT_RAW,
        filters=[{'id': lzma.FILTER_LZMA2, 'preset': 9 | lzma.PRESET_EXTREME}]
    )
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[LZMA_ENCODING] + result

def CompressPermutations(string):
    """
    CompressPermutations(string) -> str
    Returns without delimiters the given string compressed \
using a permuted codepage.

    """
    numeric = lowercase = uppercase = whitespace = symbol = 0
    for character in string:
        if character >= "0" and character <= "9":
            numeric -= .1
        elif character >= "a" and character <= "z":
            lowercase -= 0.03846
        elif character >= "A" and character <= "Z":
            uppercase -= 0.03846
        elif character == "\n" or character == " ":
            whitespace -= .5
        else:
            symbol -= .03125
    result = "".join(map(lambda t: t[1], sorted([
        (whitespace, "w"), (symbol, "s"), (lowercase, "l"), (numeric, "n"),
        (uppercase, "u")
    ])))
    index, base = 0, 5
    charset = "".join(
        charset_fragment_lookup[character] for character in result
    )
    for character in default_order[:-1]:
        index = index * base + result.index(character)
        result = result.replace(character, "", 1)
        base -= 1
    return Codepage[index] + Compress([
        charset.index(character) for character in string
    ])

def CompressString(string):
    """
    CompressString(string) -> str
    Returns without delimiters the given string compressed.

    """
    return Compress([
        default_charset.index(character) for character in string
    ])

def Compress(ordinals):
    """
    Compress(ordinals) -> str
    Returns without delimiters the given string compressed.

    """
    base, result, number = max(ordinals) + 1, "", 1
    if base == 1:
        number = len(ordinals)
    else:
        for ordinal in ordinals:
            number = number * base + ordinal
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[base - 1] + result

def Decompressed(string):
    """
    Decompressed(string) -> str
    Returns the decompressed form of the given Charcoal string.

    """
    if string == "””":
        return ""
    if string[-1] != "”":
        return string
    if string[0] == "”":
        ordinal = OrdinalLookup.get(string[1], ord(string[1]))
        alphabet_id = ordinal - (ordinal > gap)
        if alphabet_id < 120:
            return DecompressPermutations(string[1:-1])
        return [
            lambda string: string,
            lambda string: "", # TODO
            lambda string: DecompressCharset(string),
            lambda string: DecompressRLE(string),
            lambda string: DecompressBrotli(string),
            lambda string: DecompressLZMA(string)
        ][alphabet_id - 120](string[2:-1])
    elif string[0] == "“":
        return DecompressString(string[1:-1])

def DecompressCharset(string):
    """
    DecompressCharset(string) -> str
    Returns the original form of the given string compressed \
using a given character set, passed without delimiters.

    """
    length = OrdinalLookup.get(string[0], ord(string[0]))
    length += length <= gap
    number = 0
    for character in string[1:length]:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    charset = ""
    while number > 1:
        charset = default_charset[number % 97] + charset
        number //= 97
    number = 0
    for character in string[length:]:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    base = len(charset)
    if base == 1:
        return charset[0] * number
    result = ""
    while number > 1:
        result = charset[number % base] + result
        number //= base
    return result

def DecompressRLE(string):
    """
    DecompressRLE(string) -> str
    Returns the original form of the given string compressed \
using run-length encoding, passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    result = ""
    while number > 1:
        count = number % 32
        number //= 32
        result = default_charset[number % 97] * (count + 1) + result
        number //= 97
    return result

def DecompressBrotli(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return brotli.decompress(bytes(compressed)).decode("ascii")

def DecompressLZMA(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return lzma.decompress(bytes(compressed)).decode("ascii")

def DecompressPermutations(string):
    """
    DecompressPermutations(string) -> str
    Returns the original form of the given string compressed \
using a permuted codepage, passed without delimiters.

    """
    index = OrdinalLookup.get(string[0], ord(string[0]))
    if index > gap:
        index -= 1
    base, letters = 2, ["u"]
    while index:
        letters.insert(index % base, default_order[5 - base])
        index //= base
        base += 1
    while base <= 5:
        letters.insert(0, default_order[5 - base])
        base += 1
    charset = "".join(charset_fragment_lookup[c] for c in letters)
    return "".join([charset[n] for n in Decompress(string[1:])])

def DecompressString(string):
    """
    DecompressString(string) -> str
    Returns the original form of the given string compressed, \
passed without delimiters.

    """
    return "".join([default_charset[n] for n in Decompress(string)])

def Decompress(string):
    """
    Decompress(string) -> list
    Returns the ordinals in the original form of the given string compressed.

    """
    number, result = 0, []
    base = OrdinalLookup.get(string[0], ord(string[0])) + 1
    if base > gap:
        base -= 1
    for character in string[1:]:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = (number * 255) + ordinal - (ordinal > gap)
    if base == 1:
        return [0] * number
    while number > 1:
        remainder = number % base
        result = [remainder] + result
        number //= base
    return result
<|endoftext|>


<commit_before>import functools

from pylearn2.models.mlp import MLP, CompositeLayer
from pylearn2.space import CompositeSpace, VectorSpace
from theano import tensor as T

from adversarial import AdversaryPair, AdversaryCost2, Generator

class ConditionalAdversaryPair(AdversaryPair):
    def __init__(self, generator, discriminator, data_space, condition_space,
                 inferer=None,
                 inference_monitoring_batch_size=128,
                 monitor_generator=True,
                 monitor_discriminator=True,
                 monitor_inference=True,
                 shrink_d=0.):
        super(ConditionalAdversaryPair, self).__init__(generator, discriminator, inferer,
                                                       inference_monitoring_batch_size, monitor_generator, monitor_discriminator,
                                                       monitor_inference, shrink_d)

        self.data_space = data_space
        self.condition_space = condition_space

        self.input_source = self.discriminator.get_input_source()
        self.output_space = self.discriminator.get_output_space()

class ConditionalGenerator(Generator):
    def __init__(self, mlp, input_condition_space, noise_dim=100, *args, **kwargs):
        super(ConditionalGenerator, self).__init__(mlp, *args, **kwargs)

        self.noise_dim = noise_dim
        self.noise_space = VectorSpace(dim=self.noise_dim)

        self.condition_space = input_condition_space

        self.input_space = CompositeSpace([self.noise_space, self.condition_space])
        self.mlp.set_input_space(self.input_space)

    def sample_and_noise(self, conditional_data, default_input_include_prob=1., default_input_scale=1.,
                         all_g_layers=False):
        """
        Retrieve a sample (and the noise used to generate the sample)
        conditioned on some input data.

        Parameters
        ----------
        conditional_data: member of self.condition_space
            A minibatch of conditional data to feedforward.

        default_input_include_prob: float
            WRITEME

        default_input_scale: float
            WRITEME

        all_g_layers: boolean
            If true, return all generator layers in `other_layers` slot
            of this method's return value. (Otherwise returns `None` in
            this slot.)

        Returns
        -------
        net_output: 3-tuple
            Tuple of the form `(sample, noise, other_layers)`.
        """

        num_samples = conditional_data.shape[0]

        noise = self.get_noise((num_samples, self.noise_dim))
        # TODO necessary?
        formatted_noise = self.noise_space.format_as(noise, self.noise_space)

        # Build inputs: concatenate noise with conditional data
        inputs = (formatted_noise, conditional_data)

        # Feedforward
        # if all_g_layers:
        #     rval = self.mlp.dropout_fprop(inputs, default_input_include_prob=default_input_include_prob,
        #                                   default_input_scale=default_input_scale, return_all=all_g_layers)
        #     other_layers, rval = rval[:-1], rval[-1]
        # else:
        rval = self.mlp.dropout_fprop(inputs, default_input_include_prob=default_input_include_prob,
                                      default_input_scale=default_input_scale)
            # other_layers = None

        return rval, formatted_noise, None# , other_layers

    def sample(self, conditional_data, **kwargs):
        sample, _, _ = self.sample_and_noise(conditional_data, **kwargs)
        return sample

    def get_monitoring_channels(self, data):
        # TODO
        return {}

class CompositeMLPLayer(CompositeLayer):
    """A CompositeLayer where each of the components are MLPs.

    Supports forwarding dropout parameters to each MLP independently."""

    def __init__(self, layers, *args, **kwargs):
        for layer in layers:
            assert isinstance(layer, MLP), "CompositeMLPLayer only supports MLP component layers"

        super(CompositeMLPLayer, self).__init__(layers=layers, *args, **kwargs)

    def validate_layer_names(self, req_names):
        all_names = []
        for mlp_layer in self.layers:
            all_names.extend(mlp_layer.layers)

        if any(req_name not in all_names for req_name in req_names):
            unknown_names = [req_name for req_name in req_names
                             if req_name not in all_names]
            raise ValueError("No MLPs in this CompositeMLPLayer have layer(s) named %s" %
                             ", ".join(unknown_names))

    def dropout_fprop(self, state_below, *args, **kwargs):
        """Extension of Layer#fprop which forwards on dropout parameters
        to MLP sub-layers."""

        rvals = []
        for i, mlp in enumerate(self.layers):
            if self.routing_needed and i in self.layers_to_inputs:
                cur_state_below = [state_below[j]
                                   for j in self.layers_to_inputs[i]]
                # This is to mimic the behavior of CompositeSpace's restrict
                # method, which only returns a CompositeSpace when the number
                # of components is greater than 1
                if len(cur_state_below) == 1:
                    cur_state_below, = cur_state_below
            else:
                cur_state_below = state_below

            rvals.append(mlp.dropout_fprop(cur_state_below, *args, **kwargs))

        return tuple(rvals)

class ConditionalDiscriminator(MLP):
    def __init__(self, data_mlp, condition_mlp, joint_mlp,
                 input_data_space, input_condition_space, input_source=('data', 'condition'),
                 *args, **kwargs):
        """
        A discriminator acting within a cGAN which may "condition" on
        extra information.

        Parameters
        ----------
        data_mlp: pylearn2.models.mlp.MLP
            MLP which processes the data-space information. Must output
            a `VectorSpace` of some sort.

        condition_mlp: pylearn2.models.mlp.MLP
            MLP which processes the condition-space information. Must
            output a `VectorSpace` of some sort.

        joint_mlp: pylearn2.models.mlp.MLP
            MLP which processes the combination of the outputs of the
            data MLP and the condition MLP.

        input_data_space : pylearn2.space.CompositeSpace
            Space which contains the empirical / model-generated data

        input_condition_space : pylearn2.space.CompositeSpace
            Space which contains the extra data being conditioned on

        kwargs : dict
            Passed on to MLP superclass.
        """

        # Make sure user isn't trying to override any fixed keys
        for illegal_key in ['input_source', 'input_space', 'layers']:
            assert illegal_key not in kwargs

        # First feed forward in parallel along the data and condition
        # MLPs; then feed the composite output to the joint MLP
        layers = [
            CompositeMLPLayer(layer_name='discriminator_composite',
                              layers=[data_mlp, condition_mlp],
                              inputs_to_layers={0: [0], 1: [1]}),
            joint_mlp
        ]

        super(ConditionalDiscriminator, self).__init__(
            layers=layers,
            input_space=CompositeSpace([input_data_space, input_condition_space]),
            input_source=input_source,
            *args, **kwargs)

    @functools.wraps(MLP.dropout_fprop)
    def dropout_fprop(self, state_below, default_input_include_prob=0.5,
                      input_include_probs=None, default_input_scale=2.,
                      input_scales=None, per_example=True):
        """Extended version of MLP#dropout_fprop which supports passing
        on dropout parameters to nested MLPs within this MLP.

        Coupled with `CompositeMLPLayer`, which is a core part of the
        ConditionalDiscriminator setup.
        """

        if input_include_probs is None:
            input_include_probs = {}
        if input_scales is None:
            input_scales = {}

        layer_name_set = set(input_include_probs.keys())
        layer_name_set.update(input_scales.keys())

        # Remove layers from the outer net
        layer_name_set.remove(set(layer.layer_name for layer in self.layers))

        # Make sure remaining layers are contained within sub-MLPs
        # NOTE: Assumes composite layer is only at position zero
        self.layers[0].validate_layer_names(list(input_include_probs.keys()))
        self.layers[0].validate_layer_names(list(input_scales.keys()))

        theano_rng = MRG_RandomStreams(max(self.rng.randint(2 ** 15), 1))

        for layer in self.layrers:
            layer_name = layer.layer_name

            if layer_name in input_include_probs:
                include_prob = input_include_probs[layer_name]
            else:
                include_prob = default_input_include_prob

            if layer_name in input_scales:
                scale = input_scales[layer_name]
            else:
                scale = default_input_scale

            # Forward propagate
            if isinstance(layer, CompositeMLPLayer):
                # This is a composite MLP layer -- forward on the
                # dropout parameters
                state_below = layer.dropout_fprop(state_below,
                                                  default_input_include_prob=default_input_include_prob,
                                                  input_include_probs=input_include_probs,
                                                  default_input_scale=default_input_scale,
                                                  input_scales=input_scales,
                                                  per_example=per_example)
            else:
                state_below = self.apply_dropout(
                    state=state_below,
                    include_prob=include_prob,
                    theano_rng=theano_rng,
                    scale=scale,
                    mask_value=layer.dropout_input_mask_value,
                    input_space=layer.get_input_space(),
                    per_example=per_example
                )

                state_below = layer.fprop(state_below)

        return state_below

class ConditionalAdversaryCost(AdversaryCost2):
    """
    Defines the cost expression for a cGAN.
    """

    # We need to see labels for the real-world examples, so that we can
    # condition the generator + discriminator on them
    supervised = True

    def __init__(self, **kwargs):
        super(ConditionalAdversaryCost, self).__init__(**kwargs)

    def get_samples_and_objectives(self, model, data):
        space, sources = self.get_data_specs(model)
        space.validate(data)
        assert isinstance(model, ConditionalAdversaryPair)

        G, D = model.generator, model.discriminator

        # X_data: empirical data to be sent to the discriminator. We'll
        # make an equal amount of generated data and send this to the
        # discriminator as well.
        #
        # X_condition: Conditional data for each empirical sample.
        #
        # TODO something wrong here -- where does X_condition come from
        # on this invocation?
        (X_data, X_condition), y = data
        m = X_data.shape[space.get_batch_axis()]

        # Expected discriminator output: 1 for real data, 0 for
        # generated samples
        y1 = T.alloc(1, m, 1)
        y0 = T.alloc(0, m, 1)

        S, z, other_layers = G.sample_and_noise(X_condition,
                                                default_input_include_prob=self.generator_default_input_include_prob,
                                                default_input_scale=self.generator_default_input_scale,
                                                all_g_layers=(self.infer_layer is not None))

        if self.noise_both != 0.:
            rng = MRG_RandomStreams(2014 / 6 + 2)
            S = S + rng.normal(size=S.shape, dtype=S.dtype) * self.noise_both
            X_data = X_data + rng.normal(size=X_data.shape, dtype=X_data.dtype) * self.noise_both

        fprop_args = [self.discriminator_default_input_include_prob,
                      self.discriminator_input_include_probs,
                      self.discriminator_default_input_scale,
                      self.discriminator_input_scales]

        # Run discriminator on empirical data (1 expected)
        y_hat1 = D.dropout_fprop(X_data, *fprop_args)

        # Run discriminator on generated data (0 expected)
        y_hat0 = D.dropout_fprop(S, *fprop_args)

        # Compute discriminator objective
        d_obj = 0.5 * (D.layers[-1].cost(y1, y_hat1) + d.layers[-1].cost(y0, y_hat0))

        # Compute generator objective
        if self.no_drop_in_d_for_g:
            y_hat0_no_drop = D.dropout_fprop(S)
            g_obj = d.layers[-1].cost(y1, y_hat0_no_drop)
        else:
            g_obj = d.layers[-1].cost(y1, y_hat0)

        if self.blend_obj:
            g_obj = (self.zurich_coeff * g_obj - self.minimax_coeff * d_obj) / (self.zurich_coeff + self.minimax_coeff)

        if model.inferer is not None:
            # Change this if we ever switch to using dropout in the
            # construction of S.
            S_nograd = block_gradient(S)  # Redundant as long as we have custom get_gradients
            pred = model.inferer.dropout_fprop(S_nograd, *fprop_args)

            if self.infer_layer is None:
                target = z
            else:
                target = other_layers[self.infer_layer]

            i_obj = model.inferer.layers[-1].cost(target, pred)
        else:
            i_obj = 0

        return S, d_obj, g_obj, i_obj
<commit_msg>Fix layer name manipulation.. twice<commit_after>import functools

from pylearn2.models.mlp import MLP, CompositeLayer
from pylearn2.space import CompositeSpace, VectorSpace
from theano import tensor as T

from adversarial import AdversaryPair, AdversaryCost2, Generator

class ConditionalAdversaryPair(AdversaryPair):
    def __init__(self, generator, discriminator, data_space, condition_space,
                 inferer=None,
                 inference_monitoring_batch_size=128,
                 monitor_generator=True,
                 monitor_discriminator=True,
                 monitor_inference=True,
                 shrink_d=0.):
        super(ConditionalAdversaryPair, self).__init__(generator, discriminator, inferer,
                                                       inference_monitoring_batch_size, monitor_generator, monitor_discriminator,
                                                       monitor_inference, shrink_d)

        self.data_space = data_space
        self.condition_space = condition_space

        self.input_source = self.discriminator.get_input_source()
        self.output_space = self.discriminator.get_output_space()

class ConditionalGenerator(Generator):
    def __init__(self, mlp, input_condition_space, noise_dim=100, *args, **kwargs):
        super(ConditionalGenerator, self).__init__(mlp, *args, **kwargs)

        self.noise_dim = noise_dim
        self.noise_space = VectorSpace(dim=self.noise_dim)

        self.condition_space = input_condition_space

        self.input_space = CompositeSpace([self.noise_space, self.condition_space])
        self.mlp.set_input_space(self.input_space)

    def sample_and_noise(self, conditional_data, default_input_include_prob=1., default_input_scale=1.,
                         all_g_layers=False):
        """
        Retrieve a sample (and the noise used to generate the sample)
        conditioned on some input data.

        Parameters
        ----------
        conditional_data: member of self.condition_space
            A minibatch of conditional data to feedforward.

        default_input_include_prob: float
            WRITEME

        default_input_scale: float
            WRITEME

        all_g_layers: boolean
            If true, return all generator layers in `other_layers` slot
            of this method's return value. (Otherwise returns `None` in
            this slot.)

        Returns
        -------
        net_output: 3-tuple
            Tuple of the form `(sample, noise, other_layers)`.
        """

        num_samples = conditional_data.shape[0]

        noise = self.get_noise((num_samples, self.noise_dim))
        # TODO necessary?
        formatted_noise = self.noise_space.format_as(noise, self.noise_space)

        # Build inputs: concatenate noise with conditional data
        inputs = (formatted_noise, conditional_data)

        # Feedforward
        # if all_g_layers:
        #     rval = self.mlp.dropout_fprop(inputs, default_input_include_prob=default_input_include_prob,
        #                                   default_input_scale=default_input_scale, return_all=all_g_layers)
        #     other_layers, rval = rval[:-1], rval[-1]
        # else:
        rval = self.mlp.dropout_fprop(inputs, default_input_include_prob=default_input_include_prob,
                                      default_input_scale=default_input_scale)
            # other_layers = None

        return rval, formatted_noise, None# , other_layers

    def sample(self, conditional_data, **kwargs):
        sample, _, _ = self.sample_and_noise(conditional_data, **kwargs)
        return sample

    def get_monitoring_channels(self, data):
        # TODO
        return {}

class CompositeMLPLayer(CompositeLayer):
    """A CompositeLayer where each of the components are MLPs.

    Supports forwarding dropout parameters to each MLP independently."""

    def __init__(self, layers, *args, **kwargs):
        for layer in layers:
            assert isinstance(layer, MLP), "CompositeMLPLayer only supports MLP component layers"

        super(CompositeMLPLayer, self).__init__(layers=layers, *args, **kwargs)

    def validate_layer_names(self, req_names):
        all_names = []
        for mlp_layer in self.layers:
            all_names.extend([sub_layer.layer_name for sub_layer in mlp_layer.layers])

        if any(req_name not in all_names for req_name in req_names):
            unknown_names = [req_name for req_name in req_names
                             if req_name not in all_names]
            raise ValueError("No MLPs in this CompositeMLPLayer have layer(s) named %s" %
                             ", ".join(unknown_names))

    def dropout_fprop(self, state_below, *args, **kwargs):
        """Extension of Layer#fprop which forwards on dropout parameters
        to MLP sub-layers."""

        rvals = []
        for i, mlp in enumerate(self.layers):
            if self.routing_needed and i in self.layers_to_inputs:
                cur_state_below = [state_below[j]
                                   for j in self.layers_to_inputs[i]]
                # This is to mimic the behavior of CompositeSpace's restrict
                # method, which only returns a CompositeSpace when the number
                # of components is greater than 1
                if len(cur_state_below) == 1:
                    cur_state_below, = cur_state_below
            else:
                cur_state_below = state_below

            rvals.append(mlp.dropout_fprop(cur_state_below, *args, **kwargs))

        return tuple(rvals)

class ConditionalDiscriminator(MLP):
    def __init__(self, data_mlp, condition_mlp, joint_mlp,
                 input_data_space, input_condition_space, input_source=('data', 'condition'),
                 *args, **kwargs):
        """
        A discriminator acting within a cGAN which may "condition" on
        extra information.

        Parameters
        ----------
        data_mlp: pylearn2.models.mlp.MLP
            MLP which processes the data-space information. Must output
            a `VectorSpace` of some sort.

        condition_mlp: pylearn2.models.mlp.MLP
            MLP which processes the condition-space information. Must
            output a `VectorSpace` of some sort.

        joint_mlp: pylearn2.models.mlp.MLP
            MLP which processes the combination of the outputs of the
            data MLP and the condition MLP.

        input_data_space : pylearn2.space.CompositeSpace
            Space which contains the empirical / model-generated data

        input_condition_space : pylearn2.space.CompositeSpace
            Space which contains the extra data being conditioned on

        kwargs : dict
            Passed on to MLP superclass.
        """

        # Make sure user isn't trying to override any fixed keys
        for illegal_key in ['input_source', 'input_space', 'layers']:
            assert illegal_key not in kwargs

        # First feed forward in parallel along the data and condition
        # MLPs; then feed the composite output to the joint MLP
        layers = [
            CompositeMLPLayer(layer_name='discriminator_composite',
                              layers=[data_mlp, condition_mlp],
                              inputs_to_layers={0: [0], 1: [1]}),
            joint_mlp
        ]

        super(ConditionalDiscriminator, self).__init__(
            layers=layers,
            input_space=CompositeSpace([input_data_space, input_condition_space]),
            input_source=input_source,
            *args, **kwargs)

    @functools.wraps(MLP.dropout_fprop)
    def dropout_fprop(self, state_below, default_input_include_prob=0.5,
                      input_include_probs=None, default_input_scale=2.,
                      input_scales=None, per_example=True):
        """Extended version of MLP#dropout_fprop which supports passing
        on dropout parameters to nested MLPs within this MLP.

        Coupled with `CompositeMLPLayer`, which is a core part of the
        ConditionalDiscriminator setup.
        """

        if input_include_probs is None:
            input_include_probs = {}
        if input_scales is None:
            input_scales = {}

        layer_name_set = set(input_include_probs.keys())
        layer_name_set.update(input_scales.keys())

        # Remove layers from the outer net
        layer_name_set.difference_update(set(layer.layer_name for layer in self.layers))

        # Make sure remaining layers are contained within sub-MLPs
        # NOTE: Assumes composite layer is only at position zero
        self.layers[0].validate_layer_names(list(input_include_probs.keys()))
        self.layers[0].validate_layer_names(list(input_scales.keys()))

        theano_rng = MRG_RandomStreams(max(self.rng.randint(2 ** 15), 1))

        for layer in self.layrers:
            layer_name = layer.layer_name

            if layer_name in input_include_probs:
                include_prob = input_include_probs[layer_name]
            else:
                include_prob = default_input_include_prob

            if layer_name in input_scales:
                scale = input_scales[layer_name]
            else:
                scale = default_input_scale

            # Forward propagate
            if isinstance(layer, CompositeMLPLayer):
                # This is a composite MLP layer -- forward on the
                # dropout parameters
                state_below = layer.dropout_fprop(state_below,
                                                  default_input_include_prob=default_input_include_prob,
                                                  input_include_probs=input_include_probs,
                                                  default_input_scale=default_input_scale,
                                                  input_scales=input_scales,
                                                  per_example=per_example)
            else:
                state_below = self.apply_dropout(
                    state=state_below,
                    include_prob=include_prob,
                    theano_rng=theano_rng,
                    scale=scale,
                    mask_value=layer.dropout_input_mask_value,
                    input_space=layer.get_input_space(),
                    per_example=per_example
                )

                state_below = layer.fprop(state_below)

        return state_below

class ConditionalAdversaryCost(AdversaryCost2):
    """
    Defines the cost expression for a cGAN.
    """

    # We need to see labels for the real-world examples, so that we can
    # condition the generator + discriminator on them
    supervised = True

    def __init__(self, **kwargs):
        super(ConditionalAdversaryCost, self).__init__(**kwargs)

    def get_samples_and_objectives(self, model, data):
        space, sources = self.get_data_specs(model)
        space.validate(data)
        assert isinstance(model, ConditionalAdversaryPair)

        G, D = model.generator, model.discriminator

        # X_data: empirical data to be sent to the discriminator. We'll
        # make an equal amount of generated data and send this to the
        # discriminator as well.
        #
        # X_condition: Conditional data for each empirical sample.
        #
        # TODO something wrong here -- where does X_condition come from
        # on this invocation?
        (X_data, X_condition), y = data
        m = X_data.shape[space.get_batch_axis()]

        # Expected discriminator output: 1 for real data, 0 for
        # generated samples
        y1 = T.alloc(1, m, 1)
        y0 = T.alloc(0, m, 1)

        S, z, other_layers = G.sample_and_noise(X_condition,
                                                default_input_include_prob=self.generator_default_input_include_prob,
                                                default_input_scale=self.generator_default_input_scale,
                                                all_g_layers=(self.infer_layer is not None))

        if self.noise_both != 0.:
            rng = MRG_RandomStreams(2014 / 6 + 2)
            S = S + rng.normal(size=S.shape, dtype=S.dtype) * self.noise_both
            X_data = X_data + rng.normal(size=X_data.shape, dtype=X_data.dtype) * self.noise_both

        fprop_args = [self.discriminator_default_input_include_prob,
                      self.discriminator_input_include_probs,
                      self.discriminator_default_input_scale,
                      self.discriminator_input_scales]

        # Run discriminator on empirical data (1 expected)
        y_hat1 = D.dropout_fprop(X_data, *fprop_args)

        # Run discriminator on generated data (0 expected)
        y_hat0 = D.dropout_fprop(S, *fprop_args)

        # Compute discriminator objective
        d_obj = 0.5 * (D.layers[-1].cost(y1, y_hat1) + d.layers[-1].cost(y0, y_hat0))

        # Compute generator objective
        if self.no_drop_in_d_for_g:
            y_hat0_no_drop = D.dropout_fprop(S)
            g_obj = d.layers[-1].cost(y1, y_hat0_no_drop)
        else:
            g_obj = d.layers[-1].cost(y1, y_hat0)

        if self.blend_obj:
            g_obj = (self.zurich_coeff * g_obj - self.minimax_coeff * d_obj) / (self.zurich_coeff + self.minimax_coeff)

        if model.inferer is not None:
            # Change this if we ever switch to using dropout in the
            # construction of S.
            S_nograd = block_gradient(S)  # Redundant as long as we have custom get_gradients
            pred = model.inferer.dropout_fprop(S_nograd, *fprop_args)

            if self.infer_layer is None:
                target = z
            else:
                target = other_layers[self.infer_layer]

            i_obj = model.inferer.layers[-1].cost(target, pred)
        else:
            i_obj = 0

        return S, d_obj, g_obj, i_obj
<|endoftext|>


<commit_before>from django.contrib.gis.geos import Point
from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E08000029"
    addresses_name = "local.2019-05-02/Version 1/Democracy_Club__02May2019soli.tsv"
    stations_name = "local.2019-05-02/Version 1/Democracy_Club__02May2019soli.tsv"
    elections = ["europarl.2019-05-23"]
    csv_delimiter = "\t"
    allow_station_point_from_postcode = False

    def station_record_to_dict(self, record):

        # 3x station changes for EU Parl elections
        if record.polling_place_id == "6775":
            record = record._replace(polling_place_name="Hampton in Arden Library")
            record = record._replace(polling_place_address_1="39 Fentham Road")
            record = record._replace(polling_place_address_2="Hampton in Arden")
            record = record._replace(polling_place_address_3="")
            record = record._replace(polling_place_address_4="")
            record = record._replace(polling_place_postcode="B92 0AY")
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")
            record = record._replace(polling_place_uprn="200003829766")

        if record.polling_place_id == "7075":
            record = record._replace(
                polling_place_name="Monkspath Junior and Infant School"
            )
            record = record._replace(polling_place_address_1="Farmhouse Way")
            record = record._replace(polling_place_address_2="")
            record = record._replace(polling_place_address_3="")
            record = record._replace(polling_place_address_4="")
            record = record._replace(polling_place_postcode="B90 4EH")
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")
            record = record._replace(polling_place_uprn="100071489020")

        if record.polling_place_id == "7049":
            record = record._replace(polling_place_name="Sharmans Cross Junior School")
            record = record._replace(polling_place_address_1="Sharmans Cross Road")
            record = record._replace(polling_place_address_2="")
            record = record._replace(polling_place_address_3="")
            record = record._replace(polling_place_address_4="")
            record = record._replace(polling_place_postcode="B91 1PH")
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")
            record = record._replace(polling_place_uprn="100071401403")

        # The Loft Above Asda
        if record.polling_place_id == "6826":
            record = record._replace(polling_place_uprn="010023647341")

        # Tudor Grange Leisure Centre
        if record.polling_place_id == "7027":
            rec = super().station_record_to_dict(record)
            rec["location"] = Point(-1.7881577, 52.4124167, srid=4326)
            return rec

        # Three Trees Community Centre
        if record.polling_place_id == "6824":
            record = record._replace(polling_place_uprn="100071461342")

        # Elmwood Place
        if record.polling_place_id in ["7041", "7011"]:
            record = record._replace(polling_place_uprn="10090946409")

        # Auckland Hall
        if record.polling_place_id == "7004":
            record = record._replace(polling_place_uprn="200003829755")

        # Dorridge Methodist Church
        if record.polling_place_id == "7081":
            record = record._replace(polling_place_uprn="100071001475")

        # Catherine de Barnes Village Hall
        if record.polling_place_id == "6777":
            rec = super().station_record_to_dict(record)
            rec["location"] = Point(-1.7382134, 52.4203089, srid=4326)
            return rec

        return super().station_record_to_dict(record)

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn == "10090949380":
            rec["postcode"] = "B93 0FH"

        if uprn in [
            "10090945527",  # B377RN -> B376RL : 3C Woodlands Way, Chelmsley Wood
            "10090945525",  # B377RN -> B376RL : 3A Woodlands Way, Chelmsley Wood
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "100071001341",  # B911DA -> B911JW : 90 Grange Road, Solihull
            "10090946742",  # B901FT -> B930EJ : Apartment 16, Leasowes House, 3 Main Street, Dickens Heath, Solihull
            "10090948318",  # B901GL -> B913AB : Apartment 5, Market Court, 61 Old Dickens Heath Road, Shirley, Solihull
            "200003823593",  # B904EA -> B946QT : The Flat Cheswick Green Inn, Tanworth Lane, Shirley, Solihull
            "10090947460",  # B904LG -> B377NZ : 18 Archer Drive, Solihull
            "10090947804",  # CV49BN -> B901FT : 12 Eagle Drive, Solihull
            "200003834455",  # B927AW -> B927AH : St Michaels Residential Home, 251 Warwick Road, Solihull
            "10090946771",  # B920JP -> B930FD : Caravan Firs Farm, Barston Lane, Solihull
            "10090946731",  # B377WA -> B930EJ : Apartment 23, Fillingham Court, 66 Fillingham Close, Chelmsley Wood
            "10090946121",  # B913EP -> B920LD : Apartment 10, Scarlet Oak, 911-913 Warwick Road, Solihull
            "10090948319",  # B912AW -> B913AB : Flat 2, 58 Lode Lane, Solihull
            "100070965323",  # B376ES -> B376EU : 77 Overgreen Drive, Kingshurst
            "100070965320",  # B376ES -> B376EU : 77A Overgreen Drive, Kingshurst
            "100070965321",  # B376ES -> B376EU : 77B Overgreen Drive, Kingshurst
            "100070965322",  # B376ES -> B376EU : 77C Overgreen Drive, Kingshurst
            "10023648240",  # B946DQ -> B946DE : 66B Salter Street, Hockley Heath
            "200003831389",  # B360UD -> B360UF : 275 Windward Way, Smith`s Wood
        ]:
            rec["accept_suggestion"] = False

        return rec
<commit_msg>Import Solihull addresses<commit_after>from django.contrib.gis.geos import Point
from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E08000029"
    addresses_name = "parl.2019-12-12/Version 1/Democracy_Club__12December2019.CSV"
    stations_name = "parl.2019-12-12/Version 1/Democracy_Club__12December2019.CSV"
    elections = ["europarl.2019-05-23"]
    allow_station_point_from_postcode = False

    def station_record_to_dict(self, record):

        # 3x station changes for EU Parl elections
        if record.polling_place_id == "6775":
            record = record._replace(polling_place_name="Hampton in Arden Library")
            record = record._replace(polling_place_address_1="39 Fentham Road")
            record = record._replace(polling_place_address_2="Hampton in Arden")
            record = record._replace(polling_place_address_3="")
            record = record._replace(polling_place_address_4="")
            record = record._replace(polling_place_postcode="B92 0AY")
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")
            record = record._replace(polling_place_uprn="200003829766")

        if record.polling_place_id == "7075":
            record = record._replace(
                polling_place_name="Monkspath Junior and Infant School"
            )
            record = record._replace(polling_place_address_1="Farmhouse Way")
            record = record._replace(polling_place_address_2="")
            record = record._replace(polling_place_address_3="")
            record = record._replace(polling_place_address_4="")
            record = record._replace(polling_place_postcode="B90 4EH")
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")
            record = record._replace(polling_place_uprn="100071489020")

        if record.polling_place_id == "7049":
            record = record._replace(polling_place_name="Sharmans Cross Junior School")
            record = record._replace(polling_place_address_1="Sharmans Cross Road")
            record = record._replace(polling_place_address_2="")
            record = record._replace(polling_place_address_3="")
            record = record._replace(polling_place_address_4="")
            record = record._replace(polling_place_postcode="B91 1PH")
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")
            record = record._replace(polling_place_uprn="100071401403")

        # The Loft Above Asda
        if record.polling_place_id == "6826":
            record = record._replace(polling_place_uprn="010023647341")

        # Tudor Grange Leisure Centre
        if record.polling_place_id == "7027":
            rec = super().station_record_to_dict(record)
            rec["location"] = Point(-1.7881577, 52.4124167, srid=4326)
            return rec

        # Three Trees Community Centre
        if record.polling_place_id == "6824":
            record = record._replace(polling_place_uprn="100071461342")

        # Elmwood Place
        if record.polling_place_id in ["7041", "7011"]:
            record = record._replace(polling_place_uprn="10090946409")

        # Auckland Hall
        if record.polling_place_id == "7004":
            record = record._replace(polling_place_uprn="200003829755")

        # Dorridge Methodist Church
        if record.polling_place_id == "7081":
            record = record._replace(polling_place_uprn="100071001475")

        # Catherine de Barnes Village Hall
        if record.polling_place_id == "6777":
            rec = super().station_record_to_dict(record)
            rec["location"] = Point(-1.7382134, 52.4203089, srid=4326)
            return rec

        return super().station_record_to_dict(record)

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn == "10090949380":
            rec["postcode"] = "B93 0FH"

        if (record.addressline1, record.addressline2) == (
            "101 Noble Way",
            "Cheswick Green",
        ):
            rec["uprn"] = "10090950327"
            rec["accept_suggestion"] = True

        if uprn in [
            "10090945527",  # B377RN -> B376RL : 3C Woodlands Way, Chelmsley Wood
            "10090945525",  # B377RN -> B376RL : 3A Woodlands Way, Chelmsley Wood
        ]:
            rec["accept_suggestion"] = True

        if record.addressline6 in [
            "B90 4AY",  # stray odd-looking property
            "CV7 7HL",  # single property with spurious-looking station
        ]:
            return None

        if uprn in [
            "100071001341",  # B911DA -> B911JW : 90 Grange Road, Solihull
            "10090946742",  # B901FT -> B930EJ : Apartment 16, Leasowes House, 3 Main Street, Dickens Heath, Solihull
            "10090948318",  # B901GL -> B913AB : Apartment 5, Market Court, 61 Old Dickens Heath Road, Shirley, Solihull
            "10090947804",  # CV49BN -> B901FT : 12 Eagle Drive, Solihull
            "200003834455",  # B927AW -> B927AH : St Michaels Residential Home, 251 Warwick Road, Solihull
            "10090946771",  # B920JP -> B930FD : Caravan Firs Farm, Barston Lane, Solihull
            "10090948319",  # B912AW -> B913AB : Flat 2, 58 Lode Lane, Solihull
            "100070965323",  # B376ES -> B376EU : 77 Overgreen Drive, Kingshurst
            "100070965320",  # B376ES -> B376EU : 77A Overgreen Drive, Kingshurst
            "100070965321",  # B376ES -> B376EU : 77B Overgreen Drive, Kingshurst
            "100070965322",  # B376ES -> B376EU : 77C Overgreen Drive, Kingshurst
        ]:
            rec["accept_suggestion"] = False

        return rec
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000076"
    addresses_name = "local.2019-05-02/Version 3/Democracy_Club__02May2019Tendring.tsv"
    stations_name = "local.2019-05-02/Version 3/Democracy_Club__02May2019Tendring.tsv"
    elections = ["local.2019-05-02"]
    csv_delimiter = "\t"

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "10007940599",  # CO168TA -> CO148TA : 106 Kirby Road, Walton-on-the-Naze, Essex
            "10007941701",  # CO169DT -> CO169BX : The Annexe, 5 Bentley Road, Weeley, Clacton-on-Sea, Essex
            "10007944266",  # CO151PT -> CO151QA : Flat 1 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "10007944355",  # CO151PT -> CO151QA : Flat 2 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "100090634962",  # CO123PJ -> CO123PF : 302A Main Road, Harwich, Essex
            "100090635741",  # CO154LJ -> CO123UB : 11 Nightingale Way, Clacton-on-Sea, Essex
            "100091272552",  # CO130AG -> CO130DT : 30 Walton Road, Frinton-on-Sea, Essex
            "100091460461",  # CO112UJ -> CO112TA : Spinnels Farm, Spinnels Lane, Wix, Manningtree, Essex
            "100091462057",  # CO130PA -> CO130PB : Melrose, Frinton Road, Kirby Cross, Frinton-on-Sea, Essex
            "100091463517",  # CO168ET -> CO151UG : 2 Beach Road, Lee-Over-Sands, Clacton-on-Sea, Essex
            "100091464541",  # CO153AZ -> CO153AT : 135, Old Road, Clacton-on-Sea, Essex
            "10024056898",  # CO78RF -> CO78RE : 2 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056899",  # CO78RF -> CO78RE : 9 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056900",  # CO78RF -> CO78RE : 15 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056901",  # CO78RF -> CO78RE : 19 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056902",  # CO78RF -> CO78RE : 10 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056903",  # CO78RF -> CO78RE : 24 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059148",  # CO78RF -> CO78RE : 17 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059227",  # CO78RF -> CO78RE : 7 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059229",  # CO78RF -> CO78RE : 4 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060309",  # CO78RF -> CO78RE : 1 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060310",  # CO78RF -> CO78RE : 2 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060311",  # CO78RF -> CO78RE : 15 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060312",  # CO78RF -> CO78RE : 21 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060345",  # CO78RF -> CO78RE : 19 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060445",  # CO78RF -> CO78RE : 7 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090655999",  # CO78RF -> CO78RE : 1 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657509",  # CO78RF -> CO78RE : 21 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657800",  # CO78RF -> CO78RE : 3 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10094246358",  # CO168SG -> CO168TA : 422 Seawick Holiday Park, Beach Road, St Osyth, Clacton-on-Sea, Essex
            "10094246534",  # CO156LY -> CO154BE : 406 Valley Farm Holiday Park, Valley Road, Clacton-on-Sea, Essex
            "200001487728",  # CO77DW -> CO77DJ : Brendon, Colchester Road, Frating, Colchester, Essex
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "200001737111",  # CO168TA -> CO168SG : 101A Priory Close, Bel Air Chalet Estate, St Osyth, Clacton-on-Sea, Essex
            "100091274872",  # CO153DL -> CO111BU : 57 Empire Court, Warwick Road, Clacton-on-Sea, Essex
            "10090656718",  # CO125JE -> CO148BG : 52A Harwich Road, Little Oakley, Harwich, Essex
            "100091463343",  # CO123LD -> CO151PP : 18 Orwell Road, Harwich, Essex
            "100090623245",  # CO70AG -> CO111AH : 25A High Street, Brightlingsea, Colchester, Essex
        ]:
            rec["accept_suggestion"] = False

        return rec
<commit_msg>import Tendring (closes #1949)<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000076"
    addresses_name = (
        "europarl.2019-05-23/Version 1/Democracy_Club__23May2019tendring.tsv"
    )
    stations_name = (
        "europarl.2019-05-23/Version 1/Democracy_Club__23May2019tendring.tsv"
    )
    elections = ["europarl.2019-05-23"]
    csv_delimiter = "\t"

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "10007940599",  # CO168TA -> CO148TA : 106 Kirby Road, Walton-on-the-Naze, Essex
            "10007941701",  # CO169DT -> CO169BX : The Annexe, 5 Bentley Road, Weeley, Clacton-on-Sea, Essex
            "10007944266",  # CO151PT -> CO151QA : Flat 1 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "10007944355",  # CO151PT -> CO151QA : Flat 2 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "100090634962",  # CO123PJ -> CO123PF : 302A Main Road, Harwich, Essex
            "100090635741",  # CO154LJ -> CO123UB : 11 Nightingale Way, Clacton-on-Sea, Essex
            "100091272552",  # CO130AG -> CO130DT : 30 Walton Road, Frinton-on-Sea, Essex
            "100091460461",  # CO112UJ -> CO112TA : Spinnels Farm, Spinnels Lane, Wix, Manningtree, Essex
            "100091462057",  # CO130PA -> CO130PB : Melrose, Frinton Road, Kirby Cross, Frinton-on-Sea, Essex
            "100091463517",  # CO168ET -> CO151UG : 2 Beach Road, Lee-Over-Sands, Clacton-on-Sea, Essex
            "100091464541",  # CO153AZ -> CO153AT : 135, Old Road, Clacton-on-Sea, Essex
            "10024056898",  # CO78RF -> CO78RE : 2 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056899",  # CO78RF -> CO78RE : 9 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056900",  # CO78RF -> CO78RE : 15 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056901",  # CO78RF -> CO78RE : 19 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056902",  # CO78RF -> CO78RE : 10 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056903",  # CO78RF -> CO78RE : 24 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059148",  # CO78RF -> CO78RE : 17 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059227",  # CO78RF -> CO78RE : 7 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059229",  # CO78RF -> CO78RE : 4 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060309",  # CO78RF -> CO78RE : 1 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060310",  # CO78RF -> CO78RE : 2 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060311",  # CO78RF -> CO78RE : 15 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060312",  # CO78RF -> CO78RE : 21 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060345",  # CO78RF -> CO78RE : 19 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060445",  # CO78RF -> CO78RE : 7 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090655999",  # CO78RF -> CO78RE : 1 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657509",  # CO78RF -> CO78RE : 21 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657800",  # CO78RF -> CO78RE : 3 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10094246358",  # CO168SG -> CO168TA : 422 Seawick Holiday Park, Beach Road, St Osyth, Clacton-on-Sea, Essex
            "10094246534",  # CO156LY -> CO154BE : 406 Valley Farm Holiday Park, Valley Road, Clacton-on-Sea, Essex
            "200001487728",  # CO77DW -> CO77DJ : Brendon, Colchester Road, Frating, Colchester, Essex
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "200001737111",  # CO168TA -> CO168SG : 101A Priory Close, Bel Air Chalet Estate, St Osyth, Clacton-on-Sea, Essex
            "100091463343",  # CO123LD -> CO151PP : 18 Orwell Road, Harwich, Essex
            "100090623245",  # CO70AG -> CO111AH : 25A High Street, Brightlingsea, Colchester, Essex
        ]:
            rec["accept_suggestion"] = False

        return rec
<|endoftext|>


<commit_before>from data_importers.management.commands import (
    BaseXpressDCCsvInconsistentPostcodesImporter,
)

class Command(BaseXpressDCCsvInconsistentPostcodesImporter):
    council_id = "E06000016"
    stations_name = "parl.2019-12-12/Version 1/Democracy_Club__12December2019leices.tsv"
    addresses_name = (
        "parl.2019-12-12/Version 1/Democracy_Club__12December2019leices.tsv"
    )
    elections = ["parl.2019-12-12"]
    csv_delimiter = "\t"
    csv_encoding = "windows-1252"
    allow_station_point_from_postcode = False

    station_postcode_search_fields = [
        "polling_place_postcode",
        "polling_place_address_2",
    ]
    station_address_fields = ["polling_place_name", "polling_place_address_1"]

    def station_record_to_dict(self, record):

        if record.polling_place_id == "7560":
            record = record._replace(
                polling_place_address_1="Entrance from Kedleston Road"
            )

        return super().station_record_to_dict(record)

    def address_record_to_dict(self, record):
        uprn = record.property_urn.strip().lstrip("0")
        rec = super().address_record_to_dict(record)

        if len(record.addressline6) <= 6:
            return None

        if record.addressline6 == "LE3 3AT":
            rec["postcode"] = "LE1 3AT"

        if uprn in ["2465203014", "2465203012", "2465203013"]:
            rec["postcode"] = "LE1 5TE"

        if uprn in [
            "2465188649",  # LE16NB -> LE16EY : First Floor Flat 108 Y.M.C.A., 7 East Street, Leicester
            "2465188810",  # LE16NB -> LE16EY : Third Floor Cluster Flat 3, Room 311-314 Y.M.C.A., 7 East Street, Leicester
            "2465188643",  # LE16NB -> LE16EY : First Floor Flat 102 Y.M.C.A., 7 East Street, Leicester
            "2465188809",  # LE16NB -> LE16EY : Third Floor cluster Flat 2, Room 307-310 Y.M.C.A., 7 East Street, Leicester
            "2465188807",  # LE16NB -> LE16EY : Second Floor Flat 217 Y.M.C.A., 7 East Street, Leicester
            "2465188808",  # LE16NB -> LE16EY : Third Floor Cluster Flat 1, Room 301-306 Y.M.C.A., 7 East Street, Leicester
            "2465188806",  # LE16NB -> LE16EY : Second Floor Flat 216 Y.M.C.A., 7 East Street, Leicester
            "2465188645",  # LE16NB -> LE16EY : First Floor Flat 104 Y.M.C.A., 7 East Street, Leicester
            "2465188646",  # LE16NB -> LE16EY : First Floor Flat 105 Y.M.C.A., 7 East Street, Leicester
            "2465188642",  # LE16NB -> LE16EY : First Floor Flat 101 Y.M.C.A., 7 East Street, Leicester
            "2465188647",  # LE16NB -> LE16EY : First Floor Flat 106 Y.M.C.A., 7 East Street, Leicester
            "2465188813",  # LE16NB -> LE16EY : Third Floor Flat 316 Y.M.C.A., 7 East Street, Leicester
            "2465188648",  # LE16NB -> LE16EY : First Floor Flat 107 Y.M.C.A., 7 East Street, Leicester
            "2465188803",  # LE16NB -> LE16EY : Second Floor Cluster Flat 6 Room 207-210 Y.M.C.A., 7 East Street, Leicester
            "2465188644",  # LE16NB -> LE16EY : First Floor Flat 103 Y.M.C.A., 7 East Street, Leicester
            "2465188802",  # LE16NB -> LE16EY : Second Floor Cluster Flat 5 Room 201-206 Y.M.C.A., 7 East Street, Leicester
            "2465188812",  # LE16NB -> LE16EY : Third Floor Flat 315 Y.M.C.A., 7 East Street, Leicester
            "2465188811",  # LE16NB -> LE16EY : Third Floor Cluster Flat 4, Rooms 317-320 Y.M.C.A., 7 East Street, Leicester
            "2465156194",  # LE21DB -> LE21DA : The Flat, 34 St Peters Road, Leicester
            "2465184336",  # LE21DF -> LE21DB : Second Floor Flat 3, 38 St Peters Road, Leicester
            "2465184335",  # LE21DF -> LE21DB : First Floor Flat 2, 38 St Peters Road, Leicester
            "2465184334",  # LE21DF -> LE21DB : Ground Floor Flat 1, 38 St Peters Road, Leicester
            "2465179600",  # LE21DF -> LE21DB : 40 St Peters Road, Leicester
            "2465159692",  # LE31BH -> LE31BL : Flat 1, 10 Fullhurst Avenue, Leicester
            "2465156924",  # LE31BH -> LE31BL : Flat Over Shop, 6 Fullhurst Avenue, Leicester
            "2465192335",  # LE20PF -> LE17GZ : Flat 1, 79A London Road, Leicester
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "2465161850",  # LE54HH -> LE54HG : 337 St Saviours Road, Leicester
            "2465100401",  # LE20GU -> LE20GH : 63 Melbourne Road, Leicester
            "2465167645",  # LE46PN -> LE46QJ : The Flat, 43 Melton Road, Leicester
            "2465199613",  # LE35HU -> LE45BD : Cluster Room Second Floor 327 Tudor Studios, 164 Tudor Road, Leicester
            "2465199614",  # LE35HU -> LE45BD : Cluster Room Second Floor 328 Tudor Studios, 164 Tudor Road, Leicester
            "2465199616",  # LE35HU -> LE51WU : Cluster Room Second Floor 330 Tudor Studios, 164 Tudor Road, Leicester
        ]:
            rec["accept_suggestion"] = False

        if (
            record.addressline1
            in [
                "Second Floor Cluster Flat 7 Room 212-215 Y.M.C.A.",
                "Second Floor Cluster Flat 8, Room 218-221 Y.M.C.A.",
            ]
            and record.addressline6 == "LE1 6NB"
        ):
            rec["postcode"] = "LE1 6EY"

        return rec
<commit_msg>import script for leicester (closes #3256)<commit_after>from data_importers.management.commands import (
    BaseXpressDCCsvInconsistentPostcodesImporter,
)

class Command(BaseXpressDCCsvInconsistentPostcodesImporter):
    council_id = "LCE"
    addresses_name = (
        "2021-03-19T11:45:34.451844/Democracy Club Polling Stations List 6 May 2021.csv"
    )
    stations_name = (
        "2021-03-19T11:45:34.451844/Democracy Club Polling Stations List 6 May 2021.csv"
    )
    elections = ["2021-05-06"]
    csv_delimiter = ","
    csv_encoding = "windows-1252"

    station_postcode_search_fields = [
        "polling_place_postcode",
        "polling_place_address_2",
    ]
    station_address_fields = ["polling_place_name", "polling_place_address_1"]

    def station_record_to_dict(self, record):

        # Evington Community Centre Kedleston Road
        if record.polling_place_id == "8567":
            record = record._replace(
                polling_place_address_1="Entrance from Kedleston Road"
            )

        # Splitting polling station addresses up accordingly
        if record.polling_place_id in [
            "8459",  # Mobile Polling Station, The Linwood Centre
            "8582",  # Goodwood Bowling & Social Club, Entrances From Uppingham Rd & Crofters Dr
            "8509",  # Shree Shakti Mandir, Moira Street
        ]:
            line2, postcode = record.polling_place_address_2.split(",")
            record = record._replace(
                polling_place_address_2=line2, polling_place_postcode=postcode
            )

        return super().station_record_to_dict(record)

    def address_record_to_dict(self, record):
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "2465131722",  # HUMBERSTONE FARM, THURMASTON LANE, LEICESTER
        ]:
            return None

        if record.addressline6 in ["LE1 7GT", "LE5 4GA"]:
            return None

        if (
            "Thurcaston House" in record.addressline1
            or "Thurcaston House" in record.addressline2
        ):
            return None

        return super().address_record_to_dict(record)
<|endoftext|>


<commit_before>import os
import sys
import argparse
import candle
import p3b5 as bmk

import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
from torch.utils.data import DataLoader

file_path = os.path.dirname(os.path.realpath(__file__))
lib_path = os.path.abspath(os.path.join(file_path, '..'))
sys.path.append(lib_path)
lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
sys.path.append(lib_path2)

import darts

def train(trainloader, validloader, model, architecture, criterion, optimizer, lr, args, tasks, device, meter):

    valid_iter = iter(trainloader)

    for step, (data, target) in enumerate(trainloader):

        batch_size = data.size(0)
        model.train()

        data = data.to(device)

        for task, label in target.items():
            target[task] = target[task].to(device)

        x_search, target_search = next(valid_iter)
        x_search = x_search.to(device)

        for task, label in target_search.items():
            target_search[task] = target_search[task].to(device)

        # 1. update alpha
        architecture.step(
            data,
            target,
            x_search,
            target_search,
            lr,
            optimizer,
            unrolled=args.unrolled
        )

        logits = model(data)
        loss = darts.multitask_loss(target, logits, criterion, reduce='mean')

        # 2. update weight
        optimizer.zero_grad()
        loss.backward()
        nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
        optimizer.step()

        prec1 = darts.multitask_accuracy_topk(logits, target)
        meter.update_batch_loss(loss.item(), batch_size)
        meter.update_batch_accuracy(prec1, batch_size)

        if step % args.log_interval == 0:
            print(f'Step: {step} loss: {meter.loss_meter.avg:.4}')

    meter.update_epoch()
    meter.save(args.savepath)

def infer(validloader, model, criterion, args, tasks, device, meter):
    model.eval()

    with torch.no_grad():
        for step, (data, target) in enumerate(validloader):

            data = data.to(device)
            for task, label in target.items():
                target[task] = target[task].to(device)

            batch_size = data.size(0)

            logits = model(data)
            loss = darts.multitask_loss(target, logits, criterion, reduce='mean')

            prec1 = darts.multitask_accuracy_topk(logits, target)
            meter.update_batch_loss(loss.item(), batch_size)
            meter.update_batch_accuracy(prec1, batch_size)

            if step % args.log_interval == 0:
                print(f'>> Validation: {step} loss: {meter.loss_meter.avg:.4}')

    meter.update_epoch()
    meter.save(args.savepath)

    return meter.loss_meter.avg

if __name__=='__main__':
    main()
<commit_msg>Fixed save_path keyword.<commit_after>import os
import sys
import argparse
import candle
import p3b5 as bmk

import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
from torch.utils.data import DataLoader

file_path = os.path.dirname(os.path.realpath(__file__))
lib_path = os.path.abspath(os.path.join(file_path, '..'))
sys.path.append(lib_path)
lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
sys.path.append(lib_path2)

import darts

def train(trainloader, validloader, model, architecture, criterion, optimizer, lr, args, tasks, device, meter):

    valid_iter = iter(trainloader)

    for step, (data, target) in enumerate(trainloader):

        batch_size = data.size(0)
        model.train()

        data = data.to(device)

        for task, label in target.items():
            target[task] = target[task].to(device)

        x_search, target_search = next(valid_iter)
        x_search = x_search.to(device)

        for task, label in target_search.items():
            target_search[task] = target_search[task].to(device)

        # 1. update alpha
        architecture.step(
            data,
            target,
            x_search,
            target_search,
            lr,
            optimizer,
            unrolled=args.unrolled
        )

        logits = model(data)
        loss = darts.multitask_loss(target, logits, criterion, reduce='mean')

        # 2. update weight
        optimizer.zero_grad()
        loss.backward()
        nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
        optimizer.step()

        prec1 = darts.multitask_accuracy_topk(logits, target)
        meter.update_batch_loss(loss.item(), batch_size)
        meter.update_batch_accuracy(prec1, batch_size)

        if step % args.log_interval == 0:
            print(f'Step: {step} loss: {meter.loss_meter.avg:.4}')

    meter.update_epoch()
    meter.save(args.save_path)

def infer(validloader, model, criterion, args, tasks, device, meter):
    model.eval()

    with torch.no_grad():
        for step, (data, target) in enumerate(validloader):

            data = data.to(device)
            for task, label in target.items():
                target[task] = target[task].to(device)

            batch_size = data.size(0)

            logits = model(data)
            loss = darts.multitask_loss(target, logits, criterion, reduce='mean')

            prec1 = darts.multitask_accuracy_topk(logits, target)
            meter.update_batch_loss(loss.item(), batch_size)
            meter.update_batch_accuracy(prec1, batch_size)

            if step % args.log_interval == 0:
                print(f'>> Validation: {step} loss: {meter.loss_meter.avg:.4}')

    meter.update_epoch()
    meter.save(args.save_path)

    return meter.loss_meter.avg

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>import unittest
from rudix import *

class RudixTest(unittest.TestCase):
    def test_version_compare(self):
        self.assertEqual(version_compare('1.0', '2.0'), -1)
        self.assertEqual(version_compare('1.0', '1.0.1'), -1)
        self.assertEqual(version_compare('1.0.2', '1.0.10'), -1)
        self.assertEqual(version_compare('1.0.1-2', '1.0.1-10'), -1)
        self.assertEqual(version_compare('1.11-1', '1.11.1-0'), -1)
        self.assertEqual(version_compare('2.1.0b1-0', '2.1.2-0'), -1)
        self.assertEqual(version_compare('2.2.1-10', '3.0-0'), -1)
        self.assertEqual(version_compare('1.0-1', '1.0.1'), -1)
        self.assertEqual(version_compare('1.0-3', '1.0.2'), -1)
        self.assertEqual(version_compare('1.0', '1.0-1'), -1)
        self.assertEqual(version_compare('1.0.2', '1.0.10'), -1)
        self.assertEqual(version_compare('1.0.1-2', '1.0.1-10'), -1)
        self.assertEqual(version_compare('R13B', 'R14B'), -1)
        # -0 is the same as an empty release number)
        self.assertEqual(version_compare('1.0.1', '1.0.1-0'), 0)

    def test_sort_version(self):
        l = ['1.0', '1.0-2', '1.0.1', '1.2', '1.7', '1.7.1']
        l2 = sorted(l, cmp=version_compare)
        for i in zip(l,l2):
            self.assertEqual(i[0], i[1])

    def test_communicate(self):
        self.assertEqual(communicate(['echo', 'rudix']), ['rudix'])

    def test_normalization(self):
        self.assertEqual( normalize('rudix'), 'org.rudix.pkg.rudix' )
        self.assertEqual( normalize('org.rudix.pkg.rudix'),
                          'org.rudix.pkg.rudix' )
        self.assertEqual( denormalize('org.rudix.pkg.rudix'), 'rudix' )
        self.assertEqual( denormalize('rudix'), 'rudix' )

    def test_process(self):
        self.assertEqual( process(['-h']), 0 )
        self.assertEqual( process(['-v']), 0 )
        self.assertEqual( process(['-a']), 2 ) # option -a not used
        self.assertEqual( process(['help']), 0 )
        self.assertEqual( process(['version']), 0 )
        self.assertEqual( process(['foo']), 2 ) # command  foo doesn't exists

if __name__ == '__main__':
    unittest.main()
<commit_msg>rudix.py - added test cases: test_is_package_installed, test_get_packages, test_get_package_info, test_get_package_content.<commit_after>import unittest
from rudix import *

# NOTICE: we will assume org.rudix.pkg.rudix is installed.

class RudixTest(unittest.TestCase):
    def test_version_compare(self):
        self.assertEqual(version_compare('1.0', '2.0'), -1)
        self.assertEqual(version_compare('1.0', '1.0.1'), -1)
        self.assertEqual(version_compare('1.0.2', '1.0.10'), -1)
        self.assertEqual(version_compare('1.0.1-2', '1.0.1-10'), -1)
        self.assertEqual(version_compare('1.11-1', '1.11.1-0'), -1)
        self.assertEqual(version_compare('2.1.0b1-0', '2.1.2-0'), -1)
        self.assertEqual(version_compare('2.2.1-10', '3.0-0'), -1)
        self.assertEqual(version_compare('1.0-1', '1.0.1'), -1)
        self.assertEqual(version_compare('1.0-3', '1.0.2'), -1)
        self.assertEqual(version_compare('1.0', '1.0-1'), -1)
        self.assertEqual(version_compare('1.0.2', '1.0.10'), -1)
        self.assertEqual(version_compare('1.0.1-2', '1.0.1-10'), -1)
        self.assertEqual(version_compare('R13B', 'R14B'), -1)
        # -0 is the same as an empty release number)
        self.assertEqual(version_compare('1.0.1', '1.0.1-0'), 0)

    def test_sort_version(self):
        l = ['1.0', '1.0-2', '1.0.1', '1.2', '1.7', '1.7.1']
        l2 = sorted(l, cmp=version_compare)
        for i in zip(l,l2):
            self.assertEqual(i[0], i[1])

    def test_communicate(self):
        self.assertEqual(communicate(['echo', 'rudix']), ['rudix'])

    def test_is_package_installed(self):
        self.assertEqual( is_package_installed('rudix'), True )
        self.assertEqual( is_package_installed('rrrrr'), False )

    def test_get_packages(self):
        self.assertEqual( 'org.rudix.pkg.rudix' in get_packages(), True )
        self.assertEqual( 'org.rudix.pkg.rrrrr' in get_packages(), False )

    def test_get_package_info(self):
        self.assertEqual( len(get_package_info('rudix')), 2 )

    def test_get_package_content(self):
        self.assertEqual(
            '/usr/local/bin/rudix' in get_package_content('rudix'), True )

    def test_normalization(self):
        self.assertEqual( normalize('rudix'), 'org.rudix.pkg.rudix' )
        self.assertEqual( normalize('org.rudix.pkg.rudix'),
                          'org.rudix.pkg.rudix' )
        self.assertEqual( denormalize('org.rudix.pkg.rudix'), 'rudix' )
        self.assertEqual( denormalize('rudix'), 'rudix' )

    def test_process(self):
        self.assertEqual( process(['-h']), 0 )
        self.assertEqual( process(['-v']), 0 )
        self.assertEqual( process(['-a']), 2 ) # option -a not used
        self.assertEqual( process(['help']), 0 )
        self.assertEqual( process(['version']), 0 )
        self.assertEqual( process(['foo']), 2 ) # command  foo doesn't exists

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright (C) 2014 Rémi Bèges
# For conditions of distribution and use, see copyright notice in the LICENSE file

import tkinter as Tk
import ttk as ttk
from pubsub import pub
from threading import Thread, Lock, Event, Timer

"""
To handle which variables the MCU should send or not
"""

# TODO : Count the attempts to read a variable, and output error after 10 trials

class VariableManager(Thread):
    def __init__(self,model,**kwargs):
        Thread.__init__(self)
        self.model = model
        self.variables = dict()
        self.running = True
        self.event = Event()
        self.event.clear()
        self.last_update_time = 0
        self.timer = None
        self.l = Lock()

        # Subscription
        pub.subscribe(self.add_follower,'using_var')
        pub.subscribe(self.remove_follower,'stop_using_var')
        pub.subscribe(self.variable_state_check,'var_value_update')

    def add_follower(self,varid):
        self.l.acquire()
        if not varid in self.variables:
            self.variables[varid] = dict()
            self.variables[varid]['amount'] = 1
            self.variables[varid]['read_status'] = "off"
        else:
            self.variables[varid]['amount'] += 1
        self.l.release()
        # Update 500 ms later
        #self.timer = Timer(0.5, self.event.set)
        #self.timer.start()

    def remove_follower(self,varid):
        self.l.acquire()
        if not varid in self.variables:
            print("VariableManager error : Trying to remove unregistered ID")
            return

        if self.variables[varid]['amount'] == 0:
            print("VariableManager error : Trying to reduced null follower amount")
            return

        self.variables[varid]['amount'] -= 1
        self.l.release()

        # Update 500 ms later
        #self.timer = Timer(0.5, self.event.set)
        #self.timer.start()

    def variable_state_check(self,varid,data):
        self.l.acquire()
        if varid in self.variables:
            self.variables[varid]['read_status'] = "on"
        else:
            # To fix variables that are received but not needed
            self.variables[varid] = dict()
            self.variables[varid]['read_status'] = "on"
            self.variables[varid]['amount'] = 0
        self.l.release()

    def stop(self):
        self.l.acquire()
        self.running = False
        self.variables = dict()
        self.l.release()
        self.event.set()

    def run(self):
        while(self.running):
            #Update every 500ms
            self.event.wait(0.5)
            self.l.acquire()
            # Ask MCU to start sending all needed variables variable
            for key, item in self.variables.items():
                if item['read_status'] == "off" and item['amount'] > 0:
                    #print("Start :",key)
                    self.model.read_var(key)

                if item['read_status'] == "on" and item['amount'] == 0:
                    self.model.stop_read_var(key)
                    #print("Stop : ",key)
                    item['read_status'] = "off"
            self.l.release()
            self.event.clear()
<commit_msg>VariableManager bugfix<commit_after># Copyright (C) 2014 Rémi Bèges
# For conditions of distribution and use, see copyright notice in the LICENSE file

import tkinter as Tk
import ttk as ttk
from pubsub import pub
from threading import Thread, Lock, Event, Timer

"""
To handle which variables the MCU should send or not
"""

# TODO : Count the attempts to read a variable, and output error after 10 trials

class VariableManager(Thread):
    def __init__(self,model,**kwargs):
        Thread.__init__(self)
        self.model = model
        self.variables = dict()
        self.running = True
        self.event = Event()
        self.event.clear()
        self.l = Lock()

        # Subscription
        pub.subscribe(self.add_follower,'using_var')
        pub.subscribe(self.remove_follower,'stop_using_var')
        pub.subscribe(self.variable_state_check,'var_value_update')
        pub.subscribe(self.table_received_listener,'logtable_update')

    def add_follower(self,varid):
        self.l.acquire()
        if not varid in self.variables:
            self.variables[varid] = dict()
            self.variables[varid]['amount'] = 1
            self.variables[varid]['read_status'] = "off"
        else:
            self.variables[varid]['amount'] += 1
        self.l.release()

    def remove_follower(self,varid):
        self.l.acquire()
        if not varid in self.variables:
            self.l.release()
            return

        if self.variables[varid]['amount'] == 0:
            print("VariableManager error : Trying to reduced null follower amount")
            self.l.release()
            return

        self.variables[varid]['amount'] -= 1
        self.l.release()

    def table_received_listener(self,varlist):
        self.reset()

    def reset(self):
        self.variables = dict()

    def variable_state_check(self,varid,data):
        self.l.acquire()
        if varid in self.variables:
            self.variables[varid]['read_status'] = "on"
        else:
            # To fix variables that are received but not needed
            self.variables[varid] = dict()
            self.variables[varid]['read_status'] = "on"
            self.variables[varid]['amount'] = 0
        self.l.release()

    def stop(self):
        self.l.acquire()
        self.running = False
        self.variables = dict()
        self.l.release()
        self.event.set()

    def run(self):
        while(self.running):
            #Update every 500ms
            self.event.wait(0.5)
            self.l.acquire()
            # Ask MCU to start sending all needed variables variable
            for key, item in self.variables.items():
                if item['read_status'] == "off" and item['amount'] > 0:
                    #print("Start :",key)
                    self.model.read_var(key)

                if item['read_status'] == "on" and item['amount'] == 0:
                    self.model.stop_read_var(key)
                    #print("Stop : ",key)
                    item['read_status'] = "off"
            self.l.release()
            self.event.clear()
<|endoftext|>