<commit_before># Copyright (c) 2015 Damir Jelić, Denis Kasak
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import unittest

from mario.core import arg_rewrite_func, Kind
from mario.parser import make_parser, parse_rule_string

# PARSER TESTS

simple_rule = '''[test]
arg matches {data} regex_string
plumb run firefox'''

simple_res = [
    ['test', (
        [
            ['arg', 'matches', '{data}', ['regex_string']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

multiple_margs_rule = '''[test]
arg matches {data} regex_string
                   regex_inbetween
plumb run firefox'''

multiple_margs_res = [
    ['test', (
        [
            ['arg', 'matches', '{data}', ['regex_string', 'regex_inbetween']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

multiple_rules = '''[test]
arg matches {data} regex_string
                   regex_inbetween
plumb run firefox
[test2]
arg is {data} something
plumb run echo {data}'''

multiple_res = [
    ['test', (
        [
            ['arg', 'matches', '{data}', ['regex_string', 'regex_inbetween']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )],
    ['test2', (
        [
            ['arg', 'is', '{data}', ['something']]
        ],
        [
            ['plumb', 'run', 'echo {data}']
        ]
    )]
]

rule_with_comment = '''# this is a comment
[test] # even here?
# another one

arg matches {data} regex_string # commenting is fun

   # maybe here with some whitespace?

plumb run firefox # inline commenting wherever I want

#even here ?'''

rule_complex_variable = '''[test]
arg matches /bla/{data}/bla.py regex_string
plumb run firefox'''

complex_var_res = [
    ['test', (
        [
            ['arg', 'matches', '/bla/{data}/bla.py', ['regex_string']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

rule_utf8_names = '''[čest]
arg matches /bla/{data}/ćla.py regex_stringić # comments ¹²³
plumb run firefȭx'''

res_utf8_names = [
    ['čest', (
        [
            ['arg', 'matches', '/bla/{data}/ćla.py', ['regex_stringić']]
        ],
        [
            ['plumb', 'run', 'firefȭx']
        ]
    )]
]

multiple_variables_rule = '''[test]
arg matches /bla/{data}/{another} regex_string
plumb run firefox'''

multiple_variables_res = [
    ['test', (
        [
            ['arg', 'matches', '/bla/{data}/{another}', ['regex_string']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

liberal_whitespace = '''[test] # co
arg     matches         {data}         regex_string      #   white      space
plumb     run        firefox'''

data_object_rule = '''[test]
data matches regex_string
plumb run firefox'''

data_multiple_margs_rule = '''[test]
data matches regex_string
             regex_inbetween
plumb run firefox'''

class ParserTest(unittest.TestCase):
    def parser_test_helper(self, rule, result):
        parser = make_parser()
        res = parse_rule_string(parser, rule)
        self.assertEqual(result, res)

    def test_validate_parser(self):
        parser = make_parser()
        parser.validate()

    def test_simple_rule(self):
        self.parser_test_helper(simple_rule, simple_res)

    def test_multiple_match_args(self):
        self.parser_test_helper(multiple_margs_rule, multiple_margs_res)

    def test_multiple_rules(self):
        self.parser_test_helper(multiple_rules, multiple_res)

    def test_rule_with_comment(self):
        self.parser_test_helper(rule_with_comment, simple_res)

    def test_complex_var(self):
        self.parser_test_helper(rule_complex_variable, complex_var_res)

    def test_utf8(self):
        self.parser_test_helper(rule_utf8_names, res_utf8_names)

    def test_multiple_variables(self):
        self.parser_test_helper(multiple_variables_rule, multiple_variables_res)

    def test_whitespace(self):
        self.parser_test_helper(liberal_whitespace, simple_res)

    def test_data_object(self):
        self.parser_test_helper(data_object_rule, simple_res)

    def test_data_multiple_marg(self):
        self.parser_test_helper(data_multiple_margs_rule, multiple_margs_res)

# CORE TESTS

class CoreTest(unittest.TestCase):
    def test_arg_rewrite_simple(self):
        self.assertEqual(
            arg_rewrite_func({'data' : 'oolong',
                              'kind' : Kind['raw']
                             },
                             ['{data}', ['oo,', 'g,g jing']],
                             (), {}),
            (True, {'data': 'long jing', 'kind': Kind['raw']}, (), {})
        )

    def test_get_var_references_basic(self):
        self.assertListEqual(
            list(get_var_references('{0}')),
            ['{0}']
        )

    def test_get_var_references_basic_spaces(self):
        self.assertListEqual(
            list(get_var_references('   {0} \t  ')),
            ['{0}']
        )

    def test_get_var_references_multiple_vars(self):
        self.assertListEqual(
            list(get_var_references("{0}www{1}foo {abc}")),
            ['{0}', '{1}', '{abc}']
        )

if __name__ == '__main__':
        unittest.main()
<commit_msg>tests: Fix test failure due to not importing get_var_references.<commit_after># Copyright (c) 2015 Damir Jelić, Denis Kasak
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import unittest

from mario.core import get_var_references, arg_rewrite_func, Kind
from mario.parser import make_parser, parse_rule_string

# PARSER TESTS

simple_rule = '''[test]
arg matches {data} regex_string
plumb run firefox'''

simple_res = [
    ['test', (
        [
            ['arg', 'matches', '{data}', ['regex_string']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

multiple_margs_rule = '''[test]
arg matches {data} regex_string
                   regex_inbetween
plumb run firefox'''

multiple_margs_res = [
    ['test', (
        [
            ['arg', 'matches', '{data}', ['regex_string', 'regex_inbetween']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

multiple_rules = '''[test]
arg matches {data} regex_string
                   regex_inbetween
plumb run firefox
[test2]
arg is {data} something
plumb run echo {data}'''

multiple_res = [
    ['test', (
        [
            ['arg', 'matches', '{data}', ['regex_string', 'regex_inbetween']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )],
    ['test2', (
        [
            ['arg', 'is', '{data}', ['something']]
        ],
        [
            ['plumb', 'run', 'echo {data}']
        ]
    )]
]

rule_with_comment = '''# this is a comment
[test] # even here?
# another one

arg matches {data} regex_string # commenting is fun

   # maybe here with some whitespace?

plumb run firefox # inline commenting wherever I want

#even here ?'''

rule_complex_variable = '''[test]
arg matches /bla/{data}/bla.py regex_string
plumb run firefox'''

complex_var_res = [
    ['test', (
        [
            ['arg', 'matches', '/bla/{data}/bla.py', ['regex_string']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

rule_utf8_names = '''[čest]
arg matches /bla/{data}/ćla.py regex_stringić # comments ¹²³
plumb run firefȭx'''

res_utf8_names = [
    ['čest', (
        [
            ['arg', 'matches', '/bla/{data}/ćla.py', ['regex_stringić']]
        ],
        [
            ['plumb', 'run', 'firefȭx']
        ]
    )]
]

multiple_variables_rule = '''[test]
arg matches /bla/{data}/{another} regex_string
plumb run firefox'''

multiple_variables_res = [
    ['test', (
        [
            ['arg', 'matches', '/bla/{data}/{another}', ['regex_string']]
        ],
        [
            ['plumb', 'run', 'firefox']
        ]
    )]
]

liberal_whitespace = '''[test] # co
arg     matches         {data}         regex_string      #   white      space
plumb     run        firefox'''

data_object_rule = '''[test]
data matches regex_string
plumb run firefox'''

data_multiple_margs_rule = '''[test]
data matches regex_string
             regex_inbetween
plumb run firefox'''

class ParserTest(unittest.TestCase):
    def parser_test_helper(self, rule, result):
        parser = make_parser()
        res = parse_rule_string(parser, rule)
        self.assertEqual(result, res)

    def test_validate_parser(self):
        parser = make_parser()
        parser.validate()

    def test_simple_rule(self):
        self.parser_test_helper(simple_rule, simple_res)

    def test_multiple_match_args(self):
        self.parser_test_helper(multiple_margs_rule, multiple_margs_res)

    def test_multiple_rules(self):
        self.parser_test_helper(multiple_rules, multiple_res)

    def test_rule_with_comment(self):
        self.parser_test_helper(rule_with_comment, simple_res)

    def test_complex_var(self):
        self.parser_test_helper(rule_complex_variable, complex_var_res)

    def test_utf8(self):
        self.parser_test_helper(rule_utf8_names, res_utf8_names)

    def test_multiple_variables(self):
        self.parser_test_helper(multiple_variables_rule, multiple_variables_res)

    def test_whitespace(self):
        self.parser_test_helper(liberal_whitespace, simple_res)

    def test_data_object(self):
        self.parser_test_helper(data_object_rule, simple_res)

    def test_data_multiple_marg(self):
        self.parser_test_helper(data_multiple_margs_rule, multiple_margs_res)

# CORE TESTS

class CoreTest(unittest.TestCase):
    def test_arg_rewrite_simple(self):
        self.assertEqual(
            arg_rewrite_func({'data' : 'oolong',
                              'kind' : Kind['raw']
                             },
                             ['{data}', ['oo,', 'g,g jing']],
                             (), {}),
            (True, {'data': 'long jing', 'kind': Kind['raw']}, (), {})
        )

    def test_get_var_references_basic(self):
        self.assertListEqual(
            list(get_var_references('{0}')),
            ['{0}']
        )

    def test_get_var_references_basic_spaces(self):
        self.assertListEqual(
            list(get_var_references('   {0} \t  ')),
            ['{0}']
        )

    def test_get_var_references_multiple_vars(self):
        self.assertListEqual(
            list(get_var_references("{0}www{1}foo {abc}")),
            ['{0}', '{1}', '{abc}']
        )

if __name__ == '__main__':
        unittest.main()
<|endoftext|>


<commit_before>__version_tuple__ = (0, 1, 5, None, None)
__version__ = '0.1.5'
<commit_msg>Release 0.1.6<commit_after>__version_tuple__ = (0, 1, 6)
__version__ = '0.1.6'
<|endoftext|>


<commit_before>config.bind('<Ctrl-Shift-y>', 'hint links spawn --detach mpv --force-window yes {hint-url}')
config.bind('<Ctrl-Shift-p>', 'set content.proxy http://bop.ps.gameop.net:8080"')

c.content.headers.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
c.tabs.padding = {"top": 5, "bottom": 5, "left": 0, "right": 0}
c.fonts.monospace = 'Hack'
c.fonts.tabs = '10pt monospace'
config.source('nord-qutebrowser.py')
c.colors.tabs.bar.bg = "#2E3440"
c.colors.tabs.even.bg = "#2E3440"
c.colors.tabs.even.fg = "#ffffff"
c.colors.tabs.odd.bg = "#2E3440"
c.colors.tabs.odd.fg = "#ffffff"
c.colors.tabs.selected.even.bg = "#4c566a"
c.colors.tabs.selected.even.fg = "#ffffff"
c.colors.tabs.selected.odd.bg = "#4c566a"
c.colors.tabs.selected.odd.fg = "#ffffff"
c.hints.chars = "asdfkl"
<commit_msg>[qutebrowser] hint colors<commit_after>config.bind('<Ctrl-Shift-y>', 'hint links spawn --detach mpv --force-window yes {hint-url}')
config.bind('<Ctrl-Shift-p>', 'set content.proxy http://bop.ps.gameop.net:8080"')

c.content.headers.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
c.tabs.padding = {"top": 5, "bottom": 5, "left": 0, "right": 0}
c.fonts.monospace = 'Hack'
c.fonts.tabs = '10pt monospace'
config.source('nord-qutebrowser.py')
c.colors.tabs.bar.bg = "#2E3440"
c.colors.tabs.even.bg = "#2E3440"
c.colors.tabs.even.fg = "#ffffff"
c.colors.tabs.odd.bg = "#2E3440"
c.colors.tabs.odd.fg = "#ffffff"
c.colors.tabs.selected.even.bg = "#4c566a"
c.colors.tabs.selected.even.fg = "#ffffff"
c.colors.tabs.selected.odd.bg = "#4c566a"
c.colors.tabs.selected.odd.fg = "#ffffff"
c.hints.chars = "asdfkl"
c.hints.border = "1px solid #2e3440"
c.colors.hints.bg = "#2e3440"
c.colors.hints.fg = "#ffffff"
c.colors.hints.match.fg = "#bf616a"

<|endoftext|>


<commit_before># This script is used to import Northern Ireland postcode information from the
# National Statistics Postcode Database.
# http://www.ons.gov.uk/about-statistics/geography/products/geog-products-postcode/nspd/
#
# The fields of NSPD Open CSV file are:
#   Postcode (7), Postcode (8), Postcode (sp), Start date, End date, County,
#   council, ward, usertype, Easting, Northing, quality, SHA, IT cluster,
#   country, GOR, Stats region, Parliamentary constituency, Euro region,
#   TEC/LEC, Travel to Work area, Primary Care Org, NUTS, 1991 census ED,
#   1991 census ED, ED indicator, Pre-July 2006 SHA, LEA, Pre 2002 Health
#   Authority, 1991 ward code, 1991 ward code, 1998 ward code, 2005 stats ward,
#   OA code, OA indicator, CAS ward, National Park, SOA (Lower), Datazone, SOA
#   (Middle), Urban/rural, Urban/rural, Urban/rural, Intermediate, SOA (NI), OA
#   classification, Pre October 2006 PCO

import sys
import csv
from django.contrib.gis.geos import Point
from django.core.management.base import BaseCommand
from mapit.postcodes.models import Postcode
from mapit.areas.models import Area, Generation

class Command(BaseCommand):
    help = 'Imports Northern Ireland postcodes from the NSPD, creates the areas if need be'

    def handle(self, *args, **options):
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        euro_area, created = Area.objects.get_or_create(country='N', type='EUR',
            generation_low__lte=current_generation, generation_high__gte=current_generation,
            defaults = { 'generation_low': new_generation, 'generation_high': new_generation }
        )
        euro_area.generation_high = new_generation
        euro_area.save()
        euro_area.names.get_or_create(type='S', name='Northern Ireland')

        # Read in ward name -> electoral area name/area
        ni_eas = csv.reader(open('../../data/ni-electoral-areas.csv'))
        ward_to_electoral_area = {}
        e = {}
        for district, electoral_area, ward in ni_eas:
            if not district:
                district = last_district
            if not electoral_area:
                electoral_area = last_electoral_area
            last_district = district
            last_electoral_area = electoral_area
            if electoral_area not in e:
                ea = Area.objects.get_or_create_with_name(
                    country='N', type='LGE', name_type='M', name=electoral_area,
                )
                e[electoral_area] = ea
            ward_to_electoral_area.setdefault(district, {})[ward] = e[electoral_area]

        # Read in new ONS code to names
        snac = csv.reader(open('../../data/snac-2009-ni-cons2ward.csv'))
        code_to_area = {}
        ward_to_parl = {}
        for parl_code, parl_name, ward_code, ward_name, district_code, district_name in snac:
            if district_name not in ward_to_electoral_area:
                raise Exception, "District %s is missing" % district_name
            if ward_name not in ward_to_electoral_area[district_name]:
                raise Exception, "Ward %s, district %s is missing" % (ward_name, district_name)

            ward_code = ward_code.replace(' ', '')

            if district_code not in code_to_area:
                district_area = Area.objects.get_or_create_with_code(
                    country='N', type='LGD', code_type='ons', code=district_code,
                )
                district_area.names.get_or_create(type='S', name=district_name)
                code_to_area[district_code] = district_area

            if ward_code not in code_to_area:
                ward_area = Area.objects.get_or_create_with_code(
                    country='N', type='LGW', code_type='ons', code=ward_code
                )
                ward.names.get_or_create(type='S', name=ward_name)
                ward_area.parent_area = ward_to_electoral_area[district_name][ward_name]
                ward_area.parent_area.parent_area = code_to_area[district_code]
                code_to_area[ward_code] = ward_area

            if parl_code not in code_to_area:
                parl_area = Area.objects.get_or_create_with_code(
                    country='N', type='WMC', code_type='ons', code=parl_code,
                )
                parl_area.names.get_or_create(type='S', name=parl_name)
                code_to_area[parl_code] = parl_area
                name_to_area[parl_name] = parl_area

            ward_to_parl[ward_code] = code_to_area[parl_code]

        # Read in old SNAC for NI Assembly boundaries, still the same until 2011
        snac = csv.reader(open('../../data/snac-2003-ni-cons2ward.csv'))
        ward_to_assembly = {}
        for parl_code, parl_name, ward_code, ward_name, district_code, district_name in snac:
            if 'NIE' + parl_code not in code_to_area:
                nia_area = Area.objects.get_or_create_with_name(
                    country='N', type='NIE', name_type='S', name=parl_name,
                )
                code_to_area['NIE' + parl_code] = nia_area
            ward_to_assembly[ward_code] = code_to_area['NIE' + parl_code]

        # The manual fix for the split ward in the 2010 boundaries
        derryaghy = csv.reader(open('../../data/Derryaghy-postcodes.csv'))
        derryaghy_fix = {}
        for postcode, parl_name in derryaghy:
            derryaghy_fix[postcode] = name_to_area[parl_name]

        count = 0
        for row in csv.reader(sys.stdin):
            if row[4]: continue # Terminated postcode
            if row[11] == '9': continue # PO Box etc.

            postcode = row[0].strip().replace(' ', '')
            if postcode[0:2] != 'BT': continue # Only importing NI from NSPD

            # Create/update the postcode
            location = Point(map(float, row[9:11]), srid=27700)
            try:
                pc = Postcode.objects.get(postcode=postcode)
                if pc.location != location:
                    pc.location = location
                    pc.save()
            except Postcode.DoesNotExist:
                pc = Postcode.objects.create(postcode=postcode, location=location)

            # Create/update the areas
            ons_code = ''.join(row[5:8])
            ward = Area.objects.get(codes__type='ons', codes__code=ons_code)
            electoral_area = ward.parent_area
            council = electoral_area.parent_area
            nia_area = ward_to_assembly[ons_code]
            parl_area = ward_to_parl[ons_code]
            if postcode in derryaghy_fix:
                parl_area = derryaghy_fix[postcode]
            pc.areas.add(ward, electoral_area, council, nia_area, parl_area, euro_area)

            count += 1
            if count % 10000 == 0:
                print "Imported %d" % count
<commit_msg>CSV has blank column at end.<commit_after># This script is used to import Northern Ireland postcode information from the
# National Statistics Postcode Database.
# http://www.ons.gov.uk/about-statistics/geography/products/geog-products-postcode/nspd/
#
# The fields of NSPD Open CSV file are:
#   Postcode (7), Postcode (8), Postcode (sp), Start date, End date, County,
#   council, ward, usertype, Easting, Northing, quality, SHA, IT cluster,
#   country, GOR, Stats region, Parliamentary constituency, Euro region,
#   TEC/LEC, Travel to Work area, Primary Care Org, NUTS, 1991 census ED,
#   1991 census ED, ED indicator, Pre-July 2006 SHA, LEA, Pre 2002 Health
#   Authority, 1991 ward code, 1991 ward code, 1998 ward code, 2005 stats ward,
#   OA code, OA indicator, CAS ward, National Park, SOA (Lower), Datazone, SOA
#   (Middle), Urban/rural, Urban/rural, Urban/rural, Intermediate, SOA (NI), OA
#   classification, Pre October 2006 PCO

import sys
import csv
from django.contrib.gis.geos import Point
from django.core.management.base import BaseCommand
from mapit.postcodes.models import Postcode
from mapit.areas.models import Area, Generation

class Command(BaseCommand):
    help = 'Imports Northern Ireland postcodes from the NSPD, creates the areas if need be'

    def handle(self, *args, **options):
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        euro_area, created = Area.objects.get_or_create(country='N', type='EUR',
            generation_low__lte=current_generation, generation_high__gte=current_generation,
            defaults = { 'generation_low': new_generation, 'generation_high': new_generation }
        )
        euro_area.generation_high = new_generation
        euro_area.save()
        euro_area.names.get_or_create(type='S', name='Northern Ireland')

        # Read in ward name -> electoral area name/area
        ni_eas = csv.reader(open('../../data/ni-electoral-areas.csv'))
        ward_to_electoral_area = {}
        e = {}
        for district, electoral_area, ward, dummy in ni_eas:
            if not district:
                district = last_district
            if not electoral_area:
                electoral_area = last_electoral_area
            last_district = district
            last_electoral_area = electoral_area
            if electoral_area not in e:
                ea = Area.objects.get_or_create_with_name(
                    country='N', type='LGE', name_type='M', name=electoral_area,
                )
                e[electoral_area] = ea
            ward_to_electoral_area.setdefault(district, {})[ward] = e[electoral_area]

        # Read in new ONS code to names
        snac = csv.reader(open('../../data/snac-2009-ni-cons2ward.csv'))
        code_to_area = {}
        ward_to_parl = {}
        for parl_code, parl_name, ward_code, ward_name, district_code, district_name in snac:
            if district_name not in ward_to_electoral_area:
                raise Exception, "District %s is missing" % district_name
            if ward_name not in ward_to_electoral_area[district_name]:
                raise Exception, "Ward %s, district %s is missing" % (ward_name, district_name)

            ward_code = ward_code.replace(' ', '')

            if district_code not in code_to_area:
                district_area = Area.objects.get_or_create_with_code(
                    country='N', type='LGD', code_type='ons', code=district_code,
                )
                district_area.names.get_or_create(type='S', name=district_name)
                code_to_area[district_code] = district_area

            if ward_code not in code_to_area:
                ward_area = Area.objects.get_or_create_with_code(
                    country='N', type='LGW', code_type='ons', code=ward_code
                )
                ward.names.get_or_create(type='S', name=ward_name)
                ward_area.parent_area = ward_to_electoral_area[district_name][ward_name]
                ward_area.parent_area.parent_area = code_to_area[district_code]
                code_to_area[ward_code] = ward_area

            if parl_code not in code_to_area:
                parl_area = Area.objects.get_or_create_with_code(
                    country='N', type='WMC', code_type='ons', code=parl_code,
                )
                parl_area.names.get_or_create(type='S', name=parl_name)
                code_to_area[parl_code] = parl_area
                name_to_area[parl_name] = parl_area

            ward_to_parl[ward_code] = code_to_area[parl_code]

        # Read in old SNAC for NI Assembly boundaries, still the same until 2011
        snac = csv.reader(open('../../data/snac-2003-ni-cons2ward.csv'))
        ward_to_assembly = {}
        for parl_code, parl_name, ward_code, ward_name, district_code, district_name in snac:
            if 'NIE' + parl_code not in code_to_area:
                nia_area = Area.objects.get_or_create_with_name(
                    country='N', type='NIE', name_type='S', name=parl_name,
                )
                code_to_area['NIE' + parl_code] = nia_area
            ward_to_assembly[ward_code] = code_to_area['NIE' + parl_code]

        # The manual fix for the split ward in the 2010 boundaries
        derryaghy = csv.reader(open('../../data/Derryaghy-postcodes.csv'))
        derryaghy_fix = {}
        for postcode, parl_name in derryaghy:
            derryaghy_fix[postcode] = name_to_area[parl_name]

        count = 0
        for row in csv.reader(sys.stdin):
            if row[4]: continue # Terminated postcode
            if row[11] == '9': continue # PO Box etc.

            postcode = row[0].strip().replace(' ', '')
            if postcode[0:2] != 'BT': continue # Only importing NI from NSPD

            # Create/update the postcode
            location = Point(map(float, row[9:11]), srid=27700)
            try:
                pc = Postcode.objects.get(postcode=postcode)
                if pc.location != location:
                    pc.location = location
                    pc.save()
            except Postcode.DoesNotExist:
                pc = Postcode.objects.create(postcode=postcode, location=location)

            # Create/update the areas
            ons_code = ''.join(row[5:8])
            ward = Area.objects.get(codes__type='ons', codes__code=ons_code)
            electoral_area = ward.parent_area
            council = electoral_area.parent_area
            nia_area = ward_to_assembly[ons_code]
            parl_area = ward_to_parl[ons_code]
            if postcode in derryaghy_fix:
                parl_area = derryaghy_fix[postcode]
            pc.areas.add(ward, electoral_area, council, nia_area, parl_area, euro_area)

            count += 1
            if count % 10000 == 0:
                print "Imported %d" % count
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from collections import defaultdict
from odoo import api, fields, models, _
from odoo.osv import expression
from odoo.exceptions import ValidationError

class AccountAnalyticDistribution(models.Model):
    _name = 'account.analytic.distribution'
    _description = 'Analytic Account Distribution'
    _rec_name = 'account_id'

    account_id = fields.Many2one('account.analytic.account', string='Analytic Account', required=True)
    percentage = fields.Float(string='Percentage', required=True, default=100.0)
    name = fields.Char(string='Name', related='account_id.name', readonly=False)
    tag_id = fields.Many2one('account.analytic.tag', string="Parent tag", required=True)

    _sql_constraints = [
        ('check_percentage', 'CHECK(percentage >= 0 AND percentage <= 100)',
         'The percentage of an analytic distribution should be between 0 and 100.')
    ]

class AccountAnalyticTag(models.Model):
    _name = 'account.analytic.tag'
    _description = 'Analytic Tags'
    name = fields.Char(string='Analytic Tag', index=True, required=True)
    color = fields.Integer('Color Index')
    active = fields.Boolean(default=True, help="Set active to false to hide the Analytic Tag without removing it.")
    active_analytic_distribution = fields.Boolean('Analytic Distribution')
    analytic_distribution_ids = fields.One2many('account.analytic.distribution', 'tag_id', string="Analytic Accounts")
    company_id = fields.Many2one('res.company', string='Company')

class AccountAnalyticGroup(models.Model):
    _name = 'account.analytic.group'
    _description = 'Analytic Categories'
    _parent_store = True
    _rec_name = 'complete_name'

    name = fields.Char(required=True)
    description = fields.Text(string='Description')
    parent_id = fields.Many2one('account.analytic.group', string="Parent", ondelete='cascade', domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    parent_path = fields.Char(index=True)
    children_ids = fields.One2many('account.analytic.group', 'parent_id', string="Childrens")
    complete_name = fields.Char('Complete Name', compute='_compute_complete_name', store=True)
    company_id = fields.Many2one('res.company', string='Company', default=lambda self: self.env.company)

    @api.depends('name', 'parent_id.complete_name')
    def _compute_complete_name(self):
        for group in self:
            if group.parent_id:
                group.complete_name = '%s / %s' % (group.parent_id.complete_name, group.name)
            else:
                group.complete_name = group.name

class AccountAnalyticAccount(models.Model):
    _name = 'account.analytic.account'
    _inherit = ['mail.thread']
    _description = 'Analytic Account'
    _order = 'code, name asc'

    @api.model
    def read_group(self, domain, fields, groupby, offset=0, limit=None, orderby=False, lazy=True):
        """
            Override read_group to calculate the sum of the non-stored fields that depend on the user context
        """
        res = super(AccountAnalyticAccount, self).read_group(domain, fields, groupby, offset=offset, limit=limit, orderby=orderby, lazy=lazy)
        accounts = self.env['account.analytic.account']
        for line in res:
            if '__domain' in line:
                accounts = self.search(line['__domain'])
            if 'balance' in fields:
                line['balance'] = sum(accounts.mapped('balance'))
            if 'debit' in fields:
                line['debit'] = sum(accounts.mapped('debit'))
            if 'credit' in fields:
                line['credit'] = sum(accounts.mapped('credit'))
        return res

    @api.depends('line_ids.amount')
    def _compute_debit_credit_balance(self):
        Curr = self.env['res.currency']
        analytic_line_obj = self.env['account.analytic.line']
        domain = [
            ('account_id', 'in', self.ids),
            ('company_id', 'in', [False] + self.env.companies.ids)
        ]
        if self._context.get('from_date', False):
            domain.append(('date', '>=', self._context['from_date']))
        if self._context.get('to_date', False):
            domain.append(('date', '<=', self._context['to_date']))
        if self._context.get('tag_ids'):
            tag_domain = expression.OR([[('tag_ids', 'in', [tag])] for tag in self._context['tag_ids']])
            domain = expression.AND([domain, tag_domain])

        user_currency = self.env.company.currency_id
        credit_groups = analytic_line_obj.read_group(
            domain=domain + [('amount', '>=', 0.0)],
            fields=['account_id', 'currency_id', 'amount'],
            groupby=['account_id', 'currency_id'],
            lazy=False,
        )
        data_credit = defaultdict(float)
        for l in credit_groups:
            data_credit[l['account_id'][0]] += Curr.browse(l['currency_id'][0])._convert(
                l['amount'], user_currency, self.env.company, fields.Date.today())

        debit_groups = analytic_line_obj.read_group(
            domain=domain + [('amount', '<', 0.0)],
            fields=['account_id', 'currency_id', 'amount'],
            groupby=['account_id', 'currency_id'],
            lazy=False,
        )
        data_debit = defaultdict(float)
        for l in debit_groups:
            data_debit[l['account_id'][0]] += Curr.browse(l['currency_id'][0])._convert(
                l['amount'], user_currency, self.env.company, fields.Date.today())

        for account in self:
            account.debit = abs(data_debit.get(account.id, 0.0))
            account.credit = data_credit.get(account.id, 0.0)
            account.balance = account.credit - account.debit

    name = fields.Char(string='Analytic Account', index=True, required=True, tracking=True)
    code = fields.Char(string='Reference', index=True, tracking=True)
    active = fields.Boolean('Active', help="If the active field is set to False, it will allow you to hide the account without removing it.", default=True)

    group_id = fields.Many2one('account.analytic.group', string='Group', domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")

    line_ids = fields.One2many('account.analytic.line', 'account_id', string="Analytic Lines")

    company_id = fields.Many2one('res.company', string='Company', default=lambda self: self.env.company)

    # use auto_join to speed up name_search call
    partner_id = fields.Many2one('res.partner', string='Customer', auto_join=True, tracking=True, domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")

    balance = fields.Monetary(compute='_compute_debit_credit_balance', string='Balance')
    debit = fields.Monetary(compute='_compute_debit_credit_balance', string='Debit')
    credit = fields.Monetary(compute='_compute_debit_credit_balance', string='Credit')

    currency_id = fields.Many2one(related="company_id.currency_id", string="Currency", readonly=True)

    def name_get(self):
        res = []
        for analytic in self:
            name = analytic.name
            if analytic.code:
                name = '[' + analytic.code + '] ' + name
            if analytic.partner_id:
                name = name + ' - ' + analytic.partner_id.commercial_partner_id.name
            res.append((analytic.id, name))
        return res

    @api.model
    def _name_search(self, name, args=None, operator='ilike', limit=100, name_get_uid=None):
        if operator not in ('ilike', 'like', '=', '=like', '=ilike'):
            return super(AccountAnalyticAccount, self)._name_search(name, args, operator, limit, name_get_uid=name_get_uid)
        args = args or []
        if operator == 'ilike' and not (name or '').strip():
            domain = []
        else:
            # `partner_id` is in auto_join and the searches using ORs with auto_join fields doesn't work
            # we have to cut the search in two searches ... https://github.com/odoo/odoo/issues/25175
            partner_ids = self.env['res.partner']._search([('name', operator, name)], limit=limit, access_rights_uid=name_get_uid)
            domain = ['|', '|', ('code', operator, name), ('name', operator, name), ('partner_id', 'in', partner_ids)]
        analytic_account_ids = self._search(expression.AND([domain, args]), limit=limit, access_rights_uid=name_get_uid)
        return models.lazy_name_get(self.browse(analytic_account_ids).with_user(name_get_uid))

class AccountAnalyticLine(models.Model):
    _name = 'account.analytic.line'
    _description = 'Analytic Line'
    _order = 'date desc, id desc'

    @api.model
    def _default_user(self):
        return self.env.context.get('user_id', self.env.user.id)

    name = fields.Char('Description', required=True)
    date = fields.Date('Date', required=True, index=True, default=fields.Date.context_today)
    amount = fields.Monetary('Amount', required=True, default=0.0)
    unit_amount = fields.Float('Quantity', default=0.0)
    product_uom_id = fields.Many2one('uom.uom', string='Unit of Measure', domain="[('category_id', '=', product_uom_category_id)]")
    product_uom_category_id = fields.Many2one(related='product_uom_id.category_id', readonly=True)
    account_id = fields.Many2one('account.analytic.account', 'Analytic Account', required=True, ondelete='restrict', index=True, domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    partner_id = fields.Many2one('res.partner', string='Partner', domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    user_id = fields.Many2one('res.users', string='User', default=_default_user)
    tag_ids = fields.Many2many('account.analytic.tag', 'account_analytic_line_tag_rel', 'line_id', 'tag_id', string='Tags', copy=True, domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    company_id = fields.Many2one('res.company', string='Company', required=True, readonly=True, default=lambda self: self.env.company)
    currency_id = fields.Many2one(related="company_id.currency_id", string="Currency", readonly=True, store=True, compute_sudo=True)
    group_id = fields.Many2one('account.analytic.group', related='account_id.group_id', store=True, readonly=True, compute_sudo=True)

    @api.constrains('company_id', 'account_id')
    def _check_company_id(self):
        for line in self:
            if line.account_id.company_id and line.company_id.id != line.account_id.company_id.id:
                raise ValidationError(_('The selected account belongs to another company that the one you\'re trying to create an analytic item for'))
<commit_msg>[FIX] analytic: partner without name<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from collections import defaultdict
from odoo import api, fields, models, _
from odoo.osv import expression
from odoo.exceptions import ValidationError

class AccountAnalyticDistribution(models.Model):
    _name = 'account.analytic.distribution'
    _description = 'Analytic Account Distribution'
    _rec_name = 'account_id'

    account_id = fields.Many2one('account.analytic.account', string='Analytic Account', required=True)
    percentage = fields.Float(string='Percentage', required=True, default=100.0)
    name = fields.Char(string='Name', related='account_id.name', readonly=False)
    tag_id = fields.Many2one('account.analytic.tag', string="Parent tag", required=True)

    _sql_constraints = [
        ('check_percentage', 'CHECK(percentage >= 0 AND percentage <= 100)',
         'The percentage of an analytic distribution should be between 0 and 100.')
    ]

class AccountAnalyticTag(models.Model):
    _name = 'account.analytic.tag'
    _description = 'Analytic Tags'
    name = fields.Char(string='Analytic Tag', index=True, required=True)
    color = fields.Integer('Color Index')
    active = fields.Boolean(default=True, help="Set active to false to hide the Analytic Tag without removing it.")
    active_analytic_distribution = fields.Boolean('Analytic Distribution')
    analytic_distribution_ids = fields.One2many('account.analytic.distribution', 'tag_id', string="Analytic Accounts")
    company_id = fields.Many2one('res.company', string='Company')

class AccountAnalyticGroup(models.Model):
    _name = 'account.analytic.group'
    _description = 'Analytic Categories'
    _parent_store = True
    _rec_name = 'complete_name'

    name = fields.Char(required=True)
    description = fields.Text(string='Description')
    parent_id = fields.Many2one('account.analytic.group', string="Parent", ondelete='cascade', domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    parent_path = fields.Char(index=True)
    children_ids = fields.One2many('account.analytic.group', 'parent_id', string="Childrens")
    complete_name = fields.Char('Complete Name', compute='_compute_complete_name', store=True)
    company_id = fields.Many2one('res.company', string='Company', default=lambda self: self.env.company)

    @api.depends('name', 'parent_id.complete_name')
    def _compute_complete_name(self):
        for group in self:
            if group.parent_id:
                group.complete_name = '%s / %s' % (group.parent_id.complete_name, group.name)
            else:
                group.complete_name = group.name

class AccountAnalyticAccount(models.Model):
    _name = 'account.analytic.account'
    _inherit = ['mail.thread']
    _description = 'Analytic Account'
    _order = 'code, name asc'

    @api.model
    def read_group(self, domain, fields, groupby, offset=0, limit=None, orderby=False, lazy=True):
        """
            Override read_group to calculate the sum of the non-stored fields that depend on the user context
        """
        res = super(AccountAnalyticAccount, self).read_group(domain, fields, groupby, offset=offset, limit=limit, orderby=orderby, lazy=lazy)
        accounts = self.env['account.analytic.account']
        for line in res:
            if '__domain' in line:
                accounts = self.search(line['__domain'])
            if 'balance' in fields:
                line['balance'] = sum(accounts.mapped('balance'))
            if 'debit' in fields:
                line['debit'] = sum(accounts.mapped('debit'))
            if 'credit' in fields:
                line['credit'] = sum(accounts.mapped('credit'))
        return res

    @api.depends('line_ids.amount')
    def _compute_debit_credit_balance(self):
        Curr = self.env['res.currency']
        analytic_line_obj = self.env['account.analytic.line']
        domain = [
            ('account_id', 'in', self.ids),
            ('company_id', 'in', [False] + self.env.companies.ids)
        ]
        if self._context.get('from_date', False):
            domain.append(('date', '>=', self._context['from_date']))
        if self._context.get('to_date', False):
            domain.append(('date', '<=', self._context['to_date']))
        if self._context.get('tag_ids'):
            tag_domain = expression.OR([[('tag_ids', 'in', [tag])] for tag in self._context['tag_ids']])
            domain = expression.AND([domain, tag_domain])

        user_currency = self.env.company.currency_id
        credit_groups = analytic_line_obj.read_group(
            domain=domain + [('amount', '>=', 0.0)],
            fields=['account_id', 'currency_id', 'amount'],
            groupby=['account_id', 'currency_id'],
            lazy=False,
        )
        data_credit = defaultdict(float)
        for l in credit_groups:
            data_credit[l['account_id'][0]] += Curr.browse(l['currency_id'][0])._convert(
                l['amount'], user_currency, self.env.company, fields.Date.today())

        debit_groups = analytic_line_obj.read_group(
            domain=domain + [('amount', '<', 0.0)],
            fields=['account_id', 'currency_id', 'amount'],
            groupby=['account_id', 'currency_id'],
            lazy=False,
        )
        data_debit = defaultdict(float)
        for l in debit_groups:
            data_debit[l['account_id'][0]] += Curr.browse(l['currency_id'][0])._convert(
                l['amount'], user_currency, self.env.company, fields.Date.today())

        for account in self:
            account.debit = abs(data_debit.get(account.id, 0.0))
            account.credit = data_credit.get(account.id, 0.0)
            account.balance = account.credit - account.debit

    name = fields.Char(string='Analytic Account', index=True, required=True, tracking=True)
    code = fields.Char(string='Reference', index=True, tracking=True)
    active = fields.Boolean('Active', help="If the active field is set to False, it will allow you to hide the account without removing it.", default=True)

    group_id = fields.Many2one('account.analytic.group', string='Group', domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")

    line_ids = fields.One2many('account.analytic.line', 'account_id', string="Analytic Lines")

    company_id = fields.Many2one('res.company', string='Company', default=lambda self: self.env.company)

    # use auto_join to speed up name_search call
    partner_id = fields.Many2one('res.partner', string='Customer', auto_join=True, tracking=True, domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")

    balance = fields.Monetary(compute='_compute_debit_credit_balance', string='Balance')
    debit = fields.Monetary(compute='_compute_debit_credit_balance', string='Debit')
    credit = fields.Monetary(compute='_compute_debit_credit_balance', string='Credit')

    currency_id = fields.Many2one(related="company_id.currency_id", string="Currency", readonly=True)

    def name_get(self):
        res = []
        for analytic in self:
            name = analytic.name
            if analytic.code:
                name = '[' + analytic.code + '] ' + name
            if analytic.partner_id.commercial_partner_id.name:
                name = name + ' - ' + analytic.partner_id.commercial_partner_id.name
            res.append((analytic.id, name))
        return res

    @api.model
    def _name_search(self, name, args=None, operator='ilike', limit=100, name_get_uid=None):
        if operator not in ('ilike', 'like', '=', '=like', '=ilike'):
            return super(AccountAnalyticAccount, self)._name_search(name, args, operator, limit, name_get_uid=name_get_uid)
        args = args or []
        if operator == 'ilike' and not (name or '').strip():
            domain = []
        else:
            # `partner_id` is in auto_join and the searches using ORs with auto_join fields doesn't work
            # we have to cut the search in two searches ... https://github.com/odoo/odoo/issues/25175
            partner_ids = self.env['res.partner']._search([('name', operator, name)], limit=limit, access_rights_uid=name_get_uid)
            domain = ['|', '|', ('code', operator, name), ('name', operator, name), ('partner_id', 'in', partner_ids)]
        analytic_account_ids = self._search(expression.AND([domain, args]), limit=limit, access_rights_uid=name_get_uid)
        return models.lazy_name_get(self.browse(analytic_account_ids).with_user(name_get_uid))

class AccountAnalyticLine(models.Model):
    _name = 'account.analytic.line'
    _description = 'Analytic Line'
    _order = 'date desc, id desc'

    @api.model
    def _default_user(self):
        return self.env.context.get('user_id', self.env.user.id)

    name = fields.Char('Description', required=True)
    date = fields.Date('Date', required=True, index=True, default=fields.Date.context_today)
    amount = fields.Monetary('Amount', required=True, default=0.0)
    unit_amount = fields.Float('Quantity', default=0.0)
    product_uom_id = fields.Many2one('uom.uom', string='Unit of Measure', domain="[('category_id', '=', product_uom_category_id)]")
    product_uom_category_id = fields.Many2one(related='product_uom_id.category_id', readonly=True)
    account_id = fields.Many2one('account.analytic.account', 'Analytic Account', required=True, ondelete='restrict', index=True, domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    partner_id = fields.Many2one('res.partner', string='Partner', domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    user_id = fields.Many2one('res.users', string='User', default=_default_user)
    tag_ids = fields.Many2many('account.analytic.tag', 'account_analytic_line_tag_rel', 'line_id', 'tag_id', string='Tags', copy=True, domain="['|', ('company_id', '=', False), ('company_id', '=', company_id)]")
    company_id = fields.Many2one('res.company', string='Company', required=True, readonly=True, default=lambda self: self.env.company)
    currency_id = fields.Many2one(related="company_id.currency_id", string="Currency", readonly=True, store=True, compute_sudo=True)
    group_id = fields.Many2one('account.analytic.group', related='account_id.group_id', store=True, readonly=True, compute_sudo=True)

    @api.constrains('company_id', 'account_id')
    def _check_company_id(self):
        for line in self:
            if line.account_id.company_id and line.company_id.id != line.account_id.company_id.id:
                raise ValidationError(_('The selected account belongs to another company that the one you\'re trying to create an analytic item for'))
<|endoftext|>


<commit_before>"""Lens training + nearest neighbors classification pipeline."""

import os

import sys
sys.path.insert(1, 'google_research/')

import flax
from flax import nn
from flax.training import checkpoints

import jax
import jax.numpy as jnp
from jax.config import config
config.enable_omnistaging()

import numpy as np

import pandas as pd

import json

import copy

from pkg_resources import resource_filename

from fs_gcsfs import GCSFS

from google_research.protein_lm import domains, models

from contextual_lenses.contextual_lenses import reduce_fn_name_to_fn

from contextual_lenses.train_utils import create_optimizer, train, \
create_representation_model, create_transformer_representation_model, \
architecture_to_layers

from contextual_lenses.encoders import encoder_fn_name_to_fn

from contextual_lenses.loss_fns import cross_entropy_loss

from contextual_lenses.pfam_utils import family_ids, pfam_num_categories, \
pfam_evaluate, create_pfam_batches, pfam_nearest_neighbors_classification

from contextual_lenses.load_transformer import load_transformer_params

from absl import app, flags

# Define flags
FLAGS = flags.FLAGS

flags.DEFINE_string('encoder_fn_name', None, 'Name of encoder_fn to use. None if using Transformer.')
flags.DEFINE_string('encoder_fn_kwargs_path', 'medium_transformer_kwargs.json', 'Path to encoder_fn_kwargs.')
flags.DEFINE_string('reduce_fn_name', 'linear_max_pool', 'Name of reduce_fn to use.')
flags.DEFINE_string('reduce_fn_kwargs_path', 'linear_pool_256.json', 'Path to reduce_fn_kwargs.')

flags.DEFINE_integer('epochs', 10, 'Number of epochs for lens training.')
flags.DEFINE_integer('batch_size', 64, 'Batch size for training.')
flags.DEFINE_list('learning_rate', [0.0, 1e-3, 1e-3], 'Learning rates for encoder, lens, and predictor.')
flags.DEFINE_list('weight_decay', [0.0, 0.0, 0.0], 'Weight decays for encoder, lens, and predictor.')
flags.DEFINE_integer('train_families', 1000, 'Number of famlies used to train lens.')
flags.DEFINE_integer('lens_train_samples', 50, 'Number of samples used to train lens.')
flags.DEFINE_integer('knn_train_samples', 5, 'Number of samples used to train nearest neighbors algorithm.')

flags.DEFINE_boolean('use_transformer', True, 'Whether or not to use transformer encoder')
flags.DEFINE_boolean('use_bert', True, 'Whether or not to use bidirectional transformer.')
flags.DEFINE_string('restore_transformer_dir', None, 'Directory to load pretrained transformer from.')

# Train lens and measure performance of lens and nearest neighbors classifier.
def main(_):

	gcsfs = GCSFS('sequin-public')

	num_families = len(family_ids)
	loss_fn_kwargs = {
	  	'num_classes': num_families
	}

	lens_knn_train_family_accessions = []
	for _ in range(1, FLAGS.train_families+1):
  		family_name = 'PF%05d' % _
  		lens_knn_train_family_accessions.append(family_name)

	knn_test_family_accessions = []
	for _ in range(15001, 16001):
		family_name = 'PF%05d' % _
		knn_test_family_accessions.append(family_name)

	train_batches, train_indexes = create_pfam_batches(family_accessions=lens_knn_train_family_accessions,
													   batch_size=FLAGS.batch_size,
													   samples=FLAGS.lens_train_samples,
													   epochs=FLAGS.epochs,
													   drop_remainder=True)

	encoder_fn = encoder_fn_name_to_fn(FLAGS.encoder_fn_name)
	encoder_fn_kwargs = json.load(open(resource_filename('contextual_lenses.resources', os.path.join('encoder_fn_kwargs_resources', FLAGS.encoder_fn_kwargs_path))))

	reduce_fn = reduce_fn_name_to_fn(FLAGS.reduce_fn_name)
	reduce_fn_kwargs = json.load(open(resource_filename('contextual_lenses.resources', os.path.join('reduce_fn_kwargs_resources', FLAGS.reduce_fn_kwargs_path))))

	if FLAGS.use_transformer:

		if FLAGS.use_bert:
			model_cls = models.FlaxBERT
		else:
			model_cls = models.FlaxLM

		if FLAGS.restore_transformer_dir is not None:
			pretrained_transformer_params = load_transformer_params(restore_transformer_dir, model_cls)
		else:
			pretrained_transformer_params = None

		model = create_transformer_representation_model(transformer_kwargs=encoder_fn_kwargs,
	                                                    reduce_fn=reduce_fn,
	                                                    reduce_fn_kwargs=reduce_fn_kwargs,
	                                                    num_categories=pfam_num_categories,
	                                                    output_features=num_families,
	                                                    output='prediction',
	                                                    bidirectional=FLAGS.use_bert,
	                                                    encoder_fn_params=pretrained_transformer_params)

		embedding_model = create_transformer_representation_model(transformer_kwargs=encoder_fn_kwargs,
	                                                    		  reduce_fn=reduce_fn,
				                                                  reduce_fn_kwargs=reduce_fn_kwargs,
				                                                  num_categories=pfam_num_categories,
				                                                  output_features=num_families,
				                                                  output='embedding',
				                                                  bidirectional=FLAGS.use_bert,
				                                                  encoder_fn_params=pretrained_transformer_params)
	else:
		model = create_representation_model(encoder_fn=encoder_fn,
		                                    encoder_fn_kwargs=encoder_fn_kwargs,
		                                    reduce_fn=reduce_fn,
		                                    reduce_fn_kwargs=reduce_fn_kwargs,
		                                    num_categories=pfam_num_categories,
		                                    output_features=num_families,
		                                  	output='prediction')

		embedding_model = create_representation_model(encoder_fn=encoder_fn,
				                                      encoder_fn_kwargs=encoder_fn_kwargs,
				                                      reduce_fn=reduce_fn,
				                                      reduce_fn_kwargs=reduce_fn_kwargs,
				                                      num_categories=pfam_num_categories,
				                                      output_features=num_families,
				                                  	  output='embedding')

	layers = architecture_to_layers(FLAGS.encoder_fn_name, FLAGS.reduce_fn_name)

	with gcsfs.open('test_model.txt', 'w') as f:
		f.write('CREATED MODEL!')

	optimizer = train(model=model,
                      train_data=train_batches,
                      loss_fn=cross_entropy_loss,
                      loss_fn_kwargs=loss_fn_kwargs,
                      learning_rate=FLAGS.learning_rate,
                      weight_decay=FLAGS.weight_decay,
                      layers=layers)
	trained_params = copy.deepcopy(optimizer.target.params)

	with gcsfs.open('test_learning.txt', 'w') as f:
		f.write('MODEL TRAINED!')

	results, preds = pfam_evaluate(predict_fn=optimizer.target,
                                   test_family_accessions=lens_knn_train_family_accessions,
                                   title=None,
                                   loss_fn_kwargs=loss_fn_kwargs,
                                   batch_size=FLAGS.batch_size)

	lens_accuracy = results['accuracy']
	lens_cross_entropy = float(results['cross_entropy'])

	embedding_optimizer = create_optimizer(embedding_model,
										   learning_rate=FLAGS.learning_rate,
										   weight_decay=FLAGS.weight_decay,
										   layers=layers)

	assert(embedding_optimizer.target.params.keys()==trained_params.keys()), 'Optimizer parameters do not match!'
	for layer in embedding_optimizer.target.params.keys():
		embedding_optimizer.target.params[layer] = trained_params[layer]

	train_knn_results = pfam_nearest_neighbors_classification(encoder=embedding_optimizer.target,
                                                              train_family_accessions=lens_knn_train_family_accessions,
                                                              test_family_accessions=lens_knn_train_family_accessions,
                                                              batch_size=FLAGS.batch_size,
                                                              train_samples=FLAGS.knn_train_samples)[0]
	train_knn_accuracy = train_knn_results['1-nn accuracy']

	test_knn_results = pfam_nearest_neighbors_classification(encoder=embedding_optimizer.target,
                                                             train_family_accessions=knn_test_family_accessions,
                                                             test_family_accessions=knn_test_family_accessions,
                                                             batch_size=FLAGS.batch_size,
                                                             train_samples=FLAGS.knn_train_samples)[0]
	test_knn_accuracy = test_knn_results['1-nn accuracy']

	datum = {
				'encoder_fn_name': FLAGS.encoder_fn_name,
				'encoder_fn_kwargs_path': FLAGS.encoder_fn_kwargs_path,
				'reduce_fn_name': FLAGS.reduce_fn_name,
				'reduce_fn_kwargs_path': FLAGS.reduce_fn_kwargs_path,
				'epochs': FLAGS.epochs,
				'batch_size': FLAGS.batch_size,
				'encoder_lr': FLAGS.learning_rate[0],
				'lens_lr': FLAGS.learning_rate[1],
				'predictor_lr': FLAGS.learning_rate[2],
				'encoder_wd': FLAGS.weight_decay[0],
				'lens_wd': FLAGS.weight_decay[1],
				'predictor_wd': FLAGS.weight_decay[2],
				'train_families': FLAGS.train_families,
				'use_transformer': FLAGS.use_transformer,
				'use_bert': FLAGS.use_bert,
				'restore_transformer_dir': FLAGS.restore_transformer_dir,
				'knn_train_samples': FLAGS.knn_train_samples,
				'lens_cross_entropy': lens_cross_entropy,
				'lens_accuracy': lens_accuracy,
				'train_knn_accuracy': train_knn_accuracy,
				'test_knn_accuracy': test_knn_accuracy
			}

	print(datum)
	df = pd.DataFrame([datum])

	with gcsfs.open(os.path.join('sweep_data', 'experiment' + '.csv'), 'w') as gcs_file:
		df.to_csv(gcs_file)

if __name__ == '__main__':
	app.run(main)

<commit_msg>updatig kwargs path variables<commit_after>"""Lens training + nearest neighbors classification pipeline."""

import os

import sys
sys.path.insert(1, 'google_research/')

import flax
from flax import nn
from flax.training import checkpoints

import jax
import jax.numpy as jnp
from jax.config import config
config.enable_omnistaging()

import numpy as np

import pandas as pd

import json

import copy

from pkg_resources import resource_filename

from fs_gcsfs import GCSFS

from google_research.protein_lm import domains, models

from contextual_lenses.contextual_lenses import reduce_fn_name_to_fn

from contextual_lenses.train_utils import create_optimizer, train, \
create_representation_model, create_transformer_representation_model, \
architecture_to_layers

from contextual_lenses.encoders import encoder_fn_name_to_fn

from contextual_lenses.loss_fns import cross_entropy_loss

from contextual_lenses.pfam_utils import family_ids, pfam_num_categories, \
pfam_evaluate, create_pfam_batches, pfam_nearest_neighbors_classification

from contextual_lenses.load_transformer import load_transformer_params

from absl import app, flags

# Define flags
FLAGS = flags.FLAGS

flags.DEFINE_string('encoder_fn_name', None, 'Name of encoder_fn to use. None if using Transformer.')
flags.DEFINE_string('encoder_fn_kwargs_path', 'medium_transformer_kwargs', 'Path to encoder_fn_kwargs.')
flags.DEFINE_string('reduce_fn_name', 'linear_max_pool', 'Name of reduce_fn to use.')
flags.DEFINE_string('reduce_fn_kwargs_path', 'linear_pool_256', 'Path to reduce_fn_kwargs.')

flags.DEFINE_integer('epochs', 10, 'Number of epochs for lens training.')
flags.DEFINE_integer('batch_size', 64, 'Batch size for training.')
flags.DEFINE_list('learning_rate', [0.0, 1e-3, 1e-3], 'Learning rates for encoder, lens, and predictor.')
flags.DEFINE_list('weight_decay', [0.0, 0.0, 0.0], 'Weight decays for encoder, lens, and predictor.')
flags.DEFINE_integer('train_families', 1000, 'Number of famlies used to train lens.')
flags.DEFINE_integer('lens_train_samples', 50, 'Number of samples used to train lens.')
flags.DEFINE_integer('knn_train_samples', 5, 'Number of samples used to train nearest neighbors algorithm.')

flags.DEFINE_boolean('use_transformer', True, 'Whether or not to use transformer encoder')
flags.DEFINE_boolean('use_bert', True, 'Whether or not to use bidirectional transformer.')
flags.DEFINE_string('restore_transformer_dir', None, 'Directory to load pretrained transformer from.')

# Train lens and measure performance of lens and nearest neighbors classifier.
def main(_):

	gcsfs = GCSFS('sequin-public')

	num_families = len(family_ids)
	loss_fn_kwargs = {
	  	'num_classes': num_families
	}

	lens_knn_train_family_accessions = []
	for _ in range(1, FLAGS.train_families+1):
  		family_name = 'PF%05d' % _
  		lens_knn_train_family_accessions.append(family_name)

	knn_test_family_accessions = []
	for _ in range(15001, 16001):
		family_name = 'PF%05d' % _
		knn_test_family_accessions.append(family_name)

	train_batches, train_indexes = create_pfam_batches(family_accessions=lens_knn_train_family_accessions,
													   batch_size=FLAGS.batch_size,
													   samples=FLAGS.lens_train_samples,
													   epochs=FLAGS.epochs,
													   drop_remainder=True)

	encoder_fn = encoder_fn_name_to_fn(FLAGS.encoder_fn_name)
	encoder_fn_kwargs = json.load(open(resource_filename('contextual_lenses.resources', os.path.join('encoder_fn_kwargs_resources', FLAGS.encoder_fn_kwargs_path + '.json'))))

	reduce_fn = reduce_fn_name_to_fn(FLAGS.reduce_fn_name)
	reduce_fn_kwargs = json.load(open(resource_filename('contextual_lenses.resources', os.path.join('reduce_fn_kwargs_resources', FLAGS.reduce_fn_kwargs_path + '.json'))))

	if FLAGS.use_transformer:

		if FLAGS.use_bert:
			model_cls = models.FlaxBERT
		else:
			model_cls = models.FlaxLM

		if FLAGS.restore_transformer_dir is not None:
			pretrained_transformer_params = load_transformer_params(restore_transformer_dir, model_cls)
		else:
			pretrained_transformer_params = None

		model = create_transformer_representation_model(transformer_kwargs=encoder_fn_kwargs,
	                                                    reduce_fn=reduce_fn,
	                                                    reduce_fn_kwargs=reduce_fn_kwargs,
	                                                    num_categories=pfam_num_categories,
	                                                    output_features=num_families,
	                                                    output='prediction',
	                                                    bidirectional=FLAGS.use_bert,
	                                                    encoder_fn_params=pretrained_transformer_params)

		embedding_model = create_transformer_representation_model(transformer_kwargs=encoder_fn_kwargs,
	                                                    		  reduce_fn=reduce_fn,
				                                                  reduce_fn_kwargs=reduce_fn_kwargs,
				                                                  num_categories=pfam_num_categories,
				                                                  output_features=num_families,
				                                                  output='embedding',
				                                                  bidirectional=FLAGS.use_bert,
				                                                  encoder_fn_params=pretrained_transformer_params)
	else:
		model = create_representation_model(encoder_fn=encoder_fn,
		                                    encoder_fn_kwargs=encoder_fn_kwargs,
		                                    reduce_fn=reduce_fn,
		                                    reduce_fn_kwargs=reduce_fn_kwargs,
		                                    num_categories=pfam_num_categories,
		                                    output_features=num_families,
		                                  	output='prediction')

		embedding_model = create_representation_model(encoder_fn=encoder_fn,
				                                      encoder_fn_kwargs=encoder_fn_kwargs,
				                                      reduce_fn=reduce_fn,
				                                      reduce_fn_kwargs=reduce_fn_kwargs,
				                                      num_categories=pfam_num_categories,
				                                      output_features=num_families,
				                                  	  output='embedding')

	layers = architecture_to_layers(FLAGS.encoder_fn_name, FLAGS.reduce_fn_name)

	with gcsfs.open('test_model.txt', 'w') as f:
		f.write('CREATED MODEL!')

	optimizer = train(model=model,
                      train_data=train_batches,
                      loss_fn=cross_entropy_loss,
                      loss_fn_kwargs=loss_fn_kwargs,
                      learning_rate=FLAGS.learning_rate,
                      weight_decay=FLAGS.weight_decay,
                      layers=layers)
	trained_params = copy.deepcopy(optimizer.target.params)

	with gcsfs.open('test_learning.txt', 'w') as f:
		f.write('MODEL TRAINED!')

	results, preds = pfam_evaluate(predict_fn=optimizer.target,
                                   test_family_accessions=lens_knn_train_family_accessions,
                                   title=None,
                                   loss_fn_kwargs=loss_fn_kwargs,
                                   batch_size=FLAGS.batch_size)

	lens_accuracy = results['accuracy']
	lens_cross_entropy = float(results['cross_entropy'])

	embedding_optimizer = create_optimizer(embedding_model,
										   learning_rate=FLAGS.learning_rate,
										   weight_decay=FLAGS.weight_decay,
										   layers=layers)

	assert(embedding_optimizer.target.params.keys()==trained_params.keys()), 'Optimizer parameters do not match!'
	for layer in embedding_optimizer.target.params.keys():
		embedding_optimizer.target.params[layer] = trained_params[layer]

	train_knn_results = pfam_nearest_neighbors_classification(encoder=embedding_optimizer.target,
                                                              train_family_accessions=lens_knn_train_family_accessions,
                                                              test_family_accessions=lens_knn_train_family_accessions,
                                                              batch_size=FLAGS.batch_size,
                                                              train_samples=FLAGS.knn_train_samples)[0]
	train_knn_accuracy = train_knn_results['1-nn accuracy']

	test_knn_results = pfam_nearest_neighbors_classification(encoder=embedding_optimizer.target,
                                                             train_family_accessions=knn_test_family_accessions,
                                                             test_family_accessions=knn_test_family_accessions,
                                                             batch_size=FLAGS.batch_size,
                                                             train_samples=FLAGS.knn_train_samples)[0]
	test_knn_accuracy = test_knn_results['1-nn accuracy']

	datum = {
				'encoder_fn_name': FLAGS.encoder_fn_name,
				'encoder_fn_kwargs_path': FLAGS.encoder_fn_kwargs_path,
				'reduce_fn_name': FLAGS.reduce_fn_name,
				'reduce_fn_kwargs_path': FLAGS.reduce_fn_kwargs_path,
				'epochs': FLAGS.epochs,
				'batch_size': FLAGS.batch_size,
				'encoder_lr': FLAGS.learning_rate[0],
				'lens_lr': FLAGS.learning_rate[1],
				'predictor_lr': FLAGS.learning_rate[2],
				'encoder_wd': FLAGS.weight_decay[0],
				'lens_wd': FLAGS.weight_decay[1],
				'predictor_wd': FLAGS.weight_decay[2],
				'train_families': FLAGS.train_families,
				'use_transformer': FLAGS.use_transformer,
				'use_bert': FLAGS.use_bert,
				'restore_transformer_dir': FLAGS.restore_transformer_dir,
				'knn_train_samples': FLAGS.knn_train_samples,
				'lens_cross_entropy': lens_cross_entropy,
				'lens_accuracy': lens_accuracy,
				'train_knn_accuracy': train_knn_accuracy,
				'test_knn_accuracy': test_knn_accuracy
			}

	print(datum)
	df = pd.DataFrame([datum])

	with gcsfs.open(os.path.join('sweep_data', 'experiment' + '.csv'), 'w') as gcs_file:
		df.to_csv(gcs_file)

if __name__ == '__main__':
	app.run(main)

<|endoftext|>


<commit_before>import collections
import os
import re

from jira import JIRA, JIRAError, utils
from jira.resources import Attachment
from requests import Request

PADDING = 3
CHARS_LIMIT = 255

def _get_filename(path):
    limit = CHARS_LIMIT - PADDING
    fname = os.path.basename(path)
    filename = fname.split('.')[0]
    filename_extension = fname.split('.')[1:]
    count = (
        len('.'.join(filename_extension).encode('utf-8')) + 1
        if filename_extension
        else 0
    )
    char_limit = 0

    for char in filename:
        count += len(char.encode('utf-8'))
        if count > limit:
            break
        else:
            char_limit += 1

    if not char_limit:
        raise JIRAError('Attachment filename is very long.')

    tmp = [filename[:char_limit]]
    tmp.extend(filename_extension)
    filename = '.'.join(tmp)
    return filename

def _upload_file(manager, issue, upload_file, filename):
    # This method will fix original method jira.JIRA.add_attachment (jira/client.py line 591)
    url = manager._get_url('issue/' + str(issue) + '/attachments')
    files = {
        'file': (filename, upload_file),
    }
    headers = {
        'X-Atlassian-Token': 'nocheck',
    }
    req = Request('POST', url, headers=headers, files=files, auth=manager._session.auth)
    prepped = req.prepare()
    prepped.body = re.sub(
        b'filename=.*', b'filename="%s"\r' % filename.encode('utf-8'), prepped.body
    )
    r = manager._session.send(prepped)

    js = utils.json_loads(r)

    if not js or not isinstance(js, collections.Iterable):
        raise JIRAError("Unable to parse JSON: %s" % js)

    attachment = Attachment(manager._options, manager._session, js[0])

    if attachment.size == 0:
        raise JIRAError(
            "Added empty attachment?!: r: %s\nattachment: %s" % (r, attachment)
        )

    return attachment

def add_attachment(manager, issue, path):
    """
    Replace jira's method 'add_attachment' while don't well fixed this issue
    https://github.com/shazow/urllib3/issues/303
    And we need to set filename limit equaled 252 chars.
    :param manager: [jira.JIRA instance]
    :param issue: [jira.JIRA.resources.Issue instance]
    :param path: [string]
    :return: [jira.JIRA.resources.Attachment instance]
    """
    filename = _get_filename(path)

    with open(path, 'rb') as f:
        return _upload_file(manager, issue, f, filename)

def service_desk(manager, id):
    """In Jira v8.7.1 / SD 4.7.1 a service desk id must is an integer. So it needs to use this workaround."""
    try:
        return manager.service_desk(id)
    except JIRAError as e:
        if 'java.lang.NumberFormatException' in e.text:
            service_desks = [sd for sd in manager.service_desks() if sd.projectKey == 'WAL']
            if len(service_desks):
                return service_desks[0]
            else:
                msg = 'The Service Desk with ID {id} does not exist.'.format(id=id)
                raise JIRAError(text=msg, status_code=404)
        else:
            raise e

JIRA.waldur_add_attachment = add_attachment
JIRA.waldur_service_desk = service_desk

<commit_msg>[WAL-2876] Fix a linter error.<commit_after>import collections
import os
import re

from jira import JIRA, JIRAError, utils
from jira.resources import Attachment
from requests import Request

PADDING = 3
CHARS_LIMIT = 255

def _get_filename(path):
    limit = CHARS_LIMIT - PADDING
    fname = os.path.basename(path)
    filename = fname.split('.')[0]
    filename_extension = fname.split('.')[1:]
    count = (
        len('.'.join(filename_extension).encode('utf-8')) + 1
        if filename_extension
        else 0
    )
    char_limit = 0

    for char in filename:
        count += len(char.encode('utf-8'))
        if count > limit:
            break
        else:
            char_limit += 1

    if not char_limit:
        raise JIRAError('Attachment filename is very long.')

    tmp = [filename[:char_limit]]
    tmp.extend(filename_extension)
    filename = '.'.join(tmp)
    return filename

def _upload_file(manager, issue, upload_file, filename):
    # This method will fix original method jira.JIRA.add_attachment (jira/client.py line 591)
    url = manager._get_url('issue/' + str(issue) + '/attachments')
    files = {
        'file': (filename, upload_file),
    }
    headers = {
        'X-Atlassian-Token': 'nocheck',
    }
    req = Request('POST', url, headers=headers, files=files, auth=manager._session.auth)
    prepped = req.prepare()
    prepped.body = re.sub(
        b'filename=.*', b'filename="%s"\r' % filename.encode('utf-8'), prepped.body
    )
    r = manager._session.send(prepped)

    js = utils.json_loads(r)

    if not js or not isinstance(js, collections.Iterable):
        raise JIRAError("Unable to parse JSON: %s" % js)

    attachment = Attachment(manager._options, manager._session, js[0])

    if attachment.size == 0:
        raise JIRAError(
            "Added empty attachment?!: r: %s\nattachment: %s" % (r, attachment)
        )

    return attachment

def add_attachment(manager, issue, path):
    """
    Replace jira's method 'add_attachment' while don't well fixed this issue
    https://github.com/shazow/urllib3/issues/303
    And we need to set filename limit equaled 252 chars.
    :param manager: [jira.JIRA instance]
    :param issue: [jira.JIRA.resources.Issue instance]
    :param path: [string]
    :return: [jira.JIRA.resources.Attachment instance]
    """
    filename = _get_filename(path)

    with open(path, 'rb') as f:
        return _upload_file(manager, issue, f, filename)

def service_desk(manager, id):
    """In Jira v8.7.1 / SD 4.7.1 a service desk id must is an integer. So it needs to use this workaround."""
    try:
        return manager.service_desk(id)
    except JIRAError as e:
        if 'java.lang.NumberFormatException' in e.text:
            service_desks = [sd for sd in manager.service_desks() if sd.projectKey == 'WAL']
            if len(service_desks):
                return service_desks[0]
            else:
                msg = 'The Service Desk with ID {id} does not exist.'.format(id=id)
                raise JIRAError(text=msg, status_code=404)
        else:
            raise e

JIRA.waldur_add_attachment = add_attachment
JIRA.waldur_service_desk = service_desk
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2006 Christopher Lenz
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.

"""Core classes for markup processing."""

import htmlentitydefs
import re
from StringIO import StringIO

__all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Namespace', 'QName']

class StreamEventKind(object):
    """A kind of event on an XML stream."""

    __slots__ = ['name']

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return self.name

class Stream(object):
    """Represents a stream of markup events.

    This class is basically an iterator over the events.

    Also provided are ways to serialize the stream to text. The `serialize()`
    method will return an iterator over generated strings, while `render()`
    returns the complete generated text at once. Both accept various parameters
    that impact the way the stream is serialized.

    Stream events are tuples of the form:

      (kind, data, position)

    where `kind` is the event kind (such as `START`, `END`, `TEXT`, etc), `data`
    depends on the kind of event, and `position` is a `(line, offset)` tuple
    that contains the location of the original element or text in the input.
    """
    __slots__ = ['events']

    START = StreamEventKind('start') # a start tag
    END = StreamEventKind('end') # an end tag
    TEXT = StreamEventKind('text') # literal text
    EXPR = StreamEventKind('expr') # an expression
    SUB = StreamEventKind('sub') # a "subprogram"
    PROLOG = StreamEventKind('prolog') # XML prolog
    DOCTYPE = StreamEventKind('doctype') # doctype declaration
    START_NS = StreamEventKind('start-ns') # start namespace mapping
    END_NS = StreamEventKind('end-ns') # end namespace mapping
    PI = StreamEventKind('pi') # processing instruction
    COMMENT = StreamEventKind('comment') # comment

    def __init__(self, events):
        """Initialize the stream with a sequence of markup events.

        @oaram events: a sequence or iterable providing the events
        """
        self.events = events

    def __iter__(self):
        return iter(self.events)

    def render(self, method='xml', encoding='utf-8', **kwargs):
        """Return a string representation of the stream.

        @param method: determines how the stream is serialized; can be either
                       'xml' or 'html', or a custom `Serializer` subclass
        @param encoding: how the output string should be encoded; if set to
                         `None`, this method returns a `unicode` object

        Any additional keyword arguments are passed to the serializer, and thus
        depend on the `method` parameter value.
        """
        output = u''.join(list(self.serialize(method=method, **kwargs)))
        if encoding is not None:
            return output.encode('utf-8')
        return output

    def select(self, path):
        """Return a new stream that contains the events matching the given
        XPath expression.

        @param path: a string containing the XPath expression
        """
        from markup.path import Path
        path = Path(path)
        return path.select(self)

    def serialize(self, method='xml', **kwargs):
        """Generate strings corresponding to a specific serialization of the
        stream.

        Unlike the `render()` method, this method is a generator this returns
        the serialized output incrementally, as opposed to returning a single
        string.

        @param method: determines how the stream is serialized; can be either
                       'xml' or 'html', or a custom `Serializer` subclass
        """
        from markup import output
        cls = method
        if isinstance(method, basestring):
            cls = {'xml': output.XMLSerializer,
                   'html': output.HTMLSerializer}[method]
        else:
            assert issubclass(cls, serializers.Serializer)
        serializer = cls(**kwargs)
        return serializer.serialize(self)

    def __str__(self):
        return self.render()

    def __unicode__(self):
        return self.render(encoding=None)

class Attributes(list):

    def __init__(self, attrib=None):
        list.__init__(self, map(lambda (k, v): (QName(k), v), attrib or []))

    def __contains__(self, name):
        return name in [attr for attr, value in self]

    def get(self, name, default=None):
        for attr, value in self:
            if attr == name:
                return value
        return default

    def remove(self, name):
        for idx, (attr, _) in enumerate(self):
            if attr == name:
                del self[idx]
                break

    def set(self, name, value):
        for idx, (attr, _) in enumerate(self):
            if attr == name:
                self[idx] = (attr, value)
                break
        else:
            self.append((QName(name), value))

class Markup(unicode):
    """Marks a string as being safe for inclusion in HTML/XML output without
    needing to be escaped.
    """
    def __new__(self, text='', *args):
        if args:
            text %= tuple([escape(arg) for arg in args])
        return unicode.__new__(self, text)

    def __add__(self, other):
        return Markup(unicode(self) + Markup.escape(other))

    def __mod__(self, args):
        if not isinstance(args, (list, tuple)):
            args = [args]
        return Markup(unicode.__mod__(self,
                                      tuple([escape(arg) for arg in args])))

    def __mul__(self, num):
        return Markup(unicode(self) * num)

    def join(self, seq):
        return Markup(unicode(self).join([Markup.escape(item) for item in seq]))

    def stripentities(self, keepxmlentities=False):
        """Return a copy of the text with any character or numeric entities
        replaced by the equivalent UTF-8 characters.

        If the `keepxmlentities` parameter is provided and evaluates to `True`,
        the core XML entities (&, ', >, < and ").
        """
        def _replace_entity(match):
            if match.group(1): # numeric entity
                ref = match.group(1)
                if ref.startswith('x'):
                    ref = int(ref[1:], 16)
                else:
                    ref = int(ref, 10)
                return unichr(ref)
            else: # character entity
                ref = match.group(2)
                if keepxmlentities and ref in ('amp', 'apos', 'gt', 'lt', 'quot'):
                    return '&%s;' % ref
                try:
                    codepoint = htmlentitydefs.name2codepoint[ref]
                    return unichr(codepoint)
                except KeyError:
                    if keepxmlentities:
                        return '&%s;' % ref
                    else:
                        return ref
        return Markup(re.sub(r'&(?:#((?:\d+)|(?:[xX][0-9a-fA-F]+));?|(\w+);)',
                             _replace_entity, self))

    def striptags(self):
        """Return a copy of the text with all XML/HTML tags removed."""
        return Markup(re.sub(r'<[^>]*?>', '', self))

    def escape(cls, text, quotes=True):
        """Create a Markup instance from a string and escape special characters
        it may contain (<, >, & and \").

        If the `quotes` parameter is set to `False`, the \" character is left
        as is. Escaping quotes is generally only required for strings that are
        to be used in attribute values.
        """
        if isinstance(text, cls):
            return text
        text = unicode(text)
        if not text:
            return cls()
        text = text.replace('&', '&') \
                   .replace('<', '<') \
                   .replace('>', '>')
        if quotes:
            text = text.replace('"', '"')
        return cls(text)
    escape = classmethod(escape)

    def unescape(self):
        """Reverse-escapes &, <, > and \" and returns a `unicode` object."""
        if not self:
            return ''
        return unicode(self).replace('"', '"') \
                            .replace('>', '>') \
                            .replace('<', '<') \
                            .replace('&', '&')

    def plaintext(self, keeplinebreaks=True):
        """Returns the text as a `unicode` string with all entities and tags
        removed.
        """
        text = unicode(self.striptags().stripentities())
        if not keeplinebreaks:
            text = text.replace('\n', ' ')
        return text

    def sanitize(self):
        from markup.filters import HTMLSanitizer
        from markup.input import HTMLParser
        sanitize = HTMLSanitizer()
        text = self.stripentities(keepxmlentities=True)
        return Stream(sanitize(HTMLParser(StringIO(text)), None))

escape = Markup.escape

def unescape(text):
    """Reverse-escapes &, <, > and \" and returns a `unicode` object."""
    if not isinstance(text, Markup):
        return text
    return text.unescape()

class Namespace(object):

    def __init__(self, uri):
        self.uri = uri

    def __getitem__(self, name):
        return QName(self.uri + '}' + name)

    __getattr__ = __getitem__

    def __repr__(self):
        return '<Namespace "%s">' % self.uri

    def __str__(self):
        return self.uri

    def __unicode__(self):
        return unicode(self.uri)

class QName(unicode):
    """A qualified element or attribute name.

    The unicode value of instances of this class contains the qualified name of
    the element or attribute, in the form `{namespace}localname`. The namespace
    URI can be obtained through the additional `namespace` attribute, while the
    local name can be accessed through the `localname` attribute.
    """
    __slots__ = ['namespace', 'localname']

    def __new__(cls, qname):
        if isinstance(qname, QName):
            return qname

        parts = qname.split('}', 1)
        if qname.find('}') > 0:
            self = unicode.__new__(cls, '{' + qname)
            self.namespace = parts[0]
            self.localname = parts[1]
        else:
            self = unicode.__new__(cls, qname)
            self.namespace = None
            self.localname = qname
        return self
<commit_msg>Actually use the specified encoding in `Stream.render()`.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2006 Christopher Lenz
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.

"""Core classes for markup processing."""

import htmlentitydefs
import re
from StringIO import StringIO

__all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Namespace', 'QName']

class StreamEventKind(object):
    """A kind of event on an XML stream."""

    __slots__ = ['name']

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return self.name

class Stream(object):
    """Represents a stream of markup events.

    This class is basically an iterator over the events.

    Also provided are ways to serialize the stream to text. The `serialize()`
    method will return an iterator over generated strings, while `render()`
    returns the complete generated text at once. Both accept various parameters
    that impact the way the stream is serialized.

    Stream events are tuples of the form:

      (kind, data, position)

    where `kind` is the event kind (such as `START`, `END`, `TEXT`, etc), `data`
    depends on the kind of event, and `position` is a `(line, offset)` tuple
    that contains the location of the original element or text in the input.
    """
    __slots__ = ['events']

    START = StreamEventKind('start') # a start tag
    END = StreamEventKind('end') # an end tag
    TEXT = StreamEventKind('text') # literal text
    EXPR = StreamEventKind('expr') # an expression
    SUB = StreamEventKind('sub') # a "subprogram"
    PROLOG = StreamEventKind('prolog') # XML prolog
    DOCTYPE = StreamEventKind('doctype') # doctype declaration
    START_NS = StreamEventKind('start-ns') # start namespace mapping
    END_NS = StreamEventKind('end-ns') # end namespace mapping
    PI = StreamEventKind('pi') # processing instruction
    COMMENT = StreamEventKind('comment') # comment

    def __init__(self, events):
        """Initialize the stream with a sequence of markup events.

        @oaram events: a sequence or iterable providing the events
        """
        self.events = events

    def __iter__(self):
        return iter(self.events)

    def render(self, method='xml', encoding='utf-8', **kwargs):
        """Return a string representation of the stream.

        @param method: determines how the stream is serialized; can be either
                       'xml' or 'html', or a custom `Serializer` subclass
        @param encoding: how the output string should be encoded; if set to
                         `None`, this method returns a `unicode` object

        Any additional keyword arguments are passed to the serializer, and thus
        depend on the `method` parameter value.
        """
        output = u''.join(list(self.serialize(method=method, **kwargs)))
        if encoding is not None:
            return output.encode(encoding)
        return output

    def select(self, path):
        """Return a new stream that contains the events matching the given
        XPath expression.

        @param path: a string containing the XPath expression
        """
        from markup.path import Path
        path = Path(path)
        return path.select(self)

    def serialize(self, method='xml', **kwargs):
        """Generate strings corresponding to a specific serialization of the
        stream.

        Unlike the `render()` method, this method is a generator this returns
        the serialized output incrementally, as opposed to returning a single
        string.

        @param method: determines how the stream is serialized; can be either
                       'xml' or 'html', or a custom `Serializer` subclass
        """
        from markup import output
        cls = method
        if isinstance(method, basestring):
            cls = {'xml': output.XMLSerializer,
                   'html': output.HTMLSerializer}[method]
        else:
            assert issubclass(cls, serializers.Serializer)
        serializer = cls(**kwargs)
        return serializer.serialize(self)

    def __str__(self):
        return self.render()

    def __unicode__(self):
        return self.render(encoding=None)

class Attributes(list):

    def __init__(self, attrib=None):
        list.__init__(self, map(lambda (k, v): (QName(k), v), attrib or []))

    def __contains__(self, name):
        return name in [attr for attr, value in self]

    def get(self, name, default=None):
        for attr, value in self:
            if attr == name:
                return value
        return default

    def remove(self, name):
        for idx, (attr, _) in enumerate(self):
            if attr == name:
                del self[idx]
                break

    def set(self, name, value):
        for idx, (attr, _) in enumerate(self):
            if attr == name:
                self[idx] = (attr, value)
                break
        else:
            self.append((QName(name), value))

class Markup(unicode):
    """Marks a string as being safe for inclusion in HTML/XML output without
    needing to be escaped.
    """
    def __new__(self, text='', *args):
        if args:
            text %= tuple([escape(arg) for arg in args])
        return unicode.__new__(self, text)

    def __add__(self, other):
        return Markup(unicode(self) + Markup.escape(other))

    def __mod__(self, args):
        if not isinstance(args, (list, tuple)):
            args = [args]
        return Markup(unicode.__mod__(self,
                                      tuple([escape(arg) for arg in args])))

    def __mul__(self, num):
        return Markup(unicode(self) * num)

    def join(self, seq):
        return Markup(unicode(self).join([Markup.escape(item) for item in seq]))

    def stripentities(self, keepxmlentities=False):
        """Return a copy of the text with any character or numeric entities
        replaced by the equivalent UTF-8 characters.

        If the `keepxmlentities` parameter is provided and evaluates to `True`,
        the core XML entities (&, ', >, < and ").
        """
        def _replace_entity(match):
            if match.group(1): # numeric entity
                ref = match.group(1)
                if ref.startswith('x'):
                    ref = int(ref[1:], 16)
                else:
                    ref = int(ref, 10)
                return unichr(ref)
            else: # character entity
                ref = match.group(2)
                if keepxmlentities and ref in ('amp', 'apos', 'gt', 'lt', 'quot'):
                    return '&%s;' % ref
                try:
                    codepoint = htmlentitydefs.name2codepoint[ref]
                    return unichr(codepoint)
                except KeyError:
                    if keepxmlentities:
                        return '&%s;' % ref
                    else:
                        return ref
        return Markup(re.sub(r'&(?:#((?:\d+)|(?:[xX][0-9a-fA-F]+));?|(\w+);)',
                             _replace_entity, self))

    def striptags(self):
        """Return a copy of the text with all XML/HTML tags removed."""
        return Markup(re.sub(r'<[^>]*?>', '', self))

    def escape(cls, text, quotes=True):
        """Create a Markup instance from a string and escape special characters
        it may contain (<, >, & and \").

        If the `quotes` parameter is set to `False`, the \" character is left
        as is. Escaping quotes is generally only required for strings that are
        to be used in attribute values.
        """
        if isinstance(text, cls):
            return text
        text = unicode(text)
        if not text:
            return cls()
        text = text.replace('&', '&') \
                   .replace('<', '<') \
                   .replace('>', '>')
        if quotes:
            text = text.replace('"', '"')
        return cls(text)
    escape = classmethod(escape)

    def unescape(self):
        """Reverse-escapes &, <, > and \" and returns a `unicode` object."""
        if not self:
            return ''
        return unicode(self).replace('"', '"') \
                            .replace('>', '>') \
                            .replace('<', '<') \
                            .replace('&', '&')

    def plaintext(self, keeplinebreaks=True):
        """Returns the text as a `unicode` string with all entities and tags
        removed.
        """
        text = unicode(self.striptags().stripentities())
        if not keeplinebreaks:
            text = text.replace('\n', ' ')
        return text

    def sanitize(self):
        from markup.filters import HTMLSanitizer
        from markup.input import HTMLParser
        sanitize = HTMLSanitizer()
        text = self.stripentities(keepxmlentities=True)
        return Stream(sanitize(HTMLParser(StringIO(text)), None))

escape = Markup.escape

def unescape(text):
    """Reverse-escapes &, <, > and \" and returns a `unicode` object."""
    if not isinstance(text, Markup):
        return text
    return text.unescape()

class Namespace(object):

    def __init__(self, uri):
        self.uri = uri

    def __getitem__(self, name):
        return QName(self.uri + '}' + name)

    __getattr__ = __getitem__

    def __repr__(self):
        return '<Namespace "%s">' % self.uri

    def __str__(self):
        return self.uri

    def __unicode__(self):
        return unicode(self.uri)

class QName(unicode):
    """A qualified element or attribute name.

    The unicode value of instances of this class contains the qualified name of
    the element or attribute, in the form `{namespace}localname`. The namespace
    URI can be obtained through the additional `namespace` attribute, while the
    local name can be accessed through the `localname` attribute.
    """
    __slots__ = ['namespace', 'localname']

    def __new__(cls, qname):
        if isinstance(qname, QName):
            return qname

        parts = qname.split('}', 1)
        if qname.find('}') > 0:
            self = unicode.__new__(cls, '{' + qname)
            self.namespace = parts[0]
            self.localname = parts[1]
        else:
            self = unicode.__new__(cls, qname)
            self.namespace = None
            self.localname = qname
        return self
<|endoftext|>