<commit_before>#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Specification record for future flags.

A source reference also implies a specific set of future flags in use by the
parser at that location. Can be different inside a module due to e.g. the
in-lining of "exec" statements with their own future imports, or in-lining of
code from other modules.
"""

from nuitka.PythonVersions import python_version
from nuitka.utils.InstanceCounters import counted_del, counted_init

# These defaults have changed with Python versions.
_future_division_default = python_version >= 300
_future_absolute_import_default = python_version >= 300
_future_generator_stop_default = python_version >= 370

class FutureSpec(object):
    @counted_init
    def __init__(self):
        self.future_division  = _future_division_default
        self.unicode_literals = False
        self.absolute_import  = _future_absolute_import_default
        self.future_print     = False
        self.barry_bdfl       = False
        self.generator_stop   = _future_generator_stop_default

    __del__ = counted_del()

    def __repr__(self):
        return "<FutureSpec %s>" % ','.join(self.asFlags())

    def clone(self):
        result = FutureSpec()

        result.future_division   = self.future_division
        result.unicode_literals  = self.unicode_literals
        result.absolute_import   = self.absolute_import
        result.future_print      = self.future_print
        result.barry_bdfl        = self.barry_bdfl

        return result

    def isFutureDivision(self):
        return self.future_division

    def enableFutureDivision(self):
        self.future_division = True

    def isFuturePrint(self):
        return self.future_print

    def enableFuturePrint(self):
        self.future_print = True

    def enableUnicodeLiterals(self):
        self.unicode_literals = True

    def enableAbsoluteImport(self):
        self.absolute_import = True

    def enableBarry(self):
        self.barry_bdfl = True

    def enableGeneratorStop(self):
        self.generator_stop = True

    def isAbsoluteImport(self):
        return self.absolute_import

    def isGeneratorStop(self):
        return self.generator_stop

    def asFlags(self):
        """ Create a list of C identifiers to represent the flag values.

            This is for use in code generation and to restore from
            saved modules.
        """

        result = []

        if python_version < 300 and self.future_division:
            result.append("CO_FUTURE_DIVISION")

        if self.unicode_literals:
            result.append("CO_FUTURE_UNICODE_LITERALS")

        if python_version < 300 and self.absolute_import:
            result.append("CO_FUTURE_ABSOLUTE_IMPORT")

        if python_version < 300 and self.future_print:
            result.append("CO_FUTURE_PRINT_FUNCTION")

        if python_version >= 300 and self.barry_bdfl:
            result.append("CO_FUTURE_BARRY_AS_BDFL")

        if python_version >= 350 and self.generator_stop:
            result.append("CO_FUTURE_GENERATOR_STOP")

        return tuple(result)

def fromFlags(flags):
    flags = flags.split(',')

    result = FutureSpec()

    if "CO_FUTURE_DIVISION" in flags:
        result.enableFutureDivision()

    if "CO_FUTURE_UNICODE_LITERALS" in flags:
        result.enableUnicodeLiterals()

    if "CO_FUTURE_ABSOLUTE_IMPORT" in flags:
        result.enableAbsoluteImport()

    if "CO_FUTURE_PRINT_FUNCTION" in flags:
        result.enableFuturePrint()

    if "CO_FUTURE_BARRY_AS_BDFL" in flags:
        result.enableBarry()

    if "CO_FUTURE_GENERATOR_STOP" in flags:
        result.enableGeneratorStop()

    # Check if we are going to give similar results than what we got.
    assert tuple(result.asFlags()) == tuple(flags), (result, result.asFlags(), flags)

    return result
<commit_msg>Python3.7: Do list code flag that is default.<commit_after>#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Specification record for future flags.

A source reference also implies a specific set of future flags in use by the
parser at that location. Can be different inside a module due to e.g. the
in-lining of "exec" statements with their own future imports, or in-lining of
code from other modules.
"""

from nuitka.PythonVersions import python_version
from nuitka.utils.InstanceCounters import counted_del, counted_init

# These defaults have changed with Python versions.
_future_division_default = python_version >= 300
_future_absolute_import_default = python_version >= 300
_future_generator_stop_default = python_version >= 370

class FutureSpec(object):
    @counted_init
    def __init__(self):
        self.future_division  = _future_division_default
        self.unicode_literals = False
        self.absolute_import  = _future_absolute_import_default
        self.future_print     = False
        self.barry_bdfl       = False
        self.generator_stop   = _future_generator_stop_default

    __del__ = counted_del()

    def __repr__(self):
        return "<FutureSpec %s>" % ','.join(self.asFlags())

    def clone(self):
        result = FutureSpec()

        result.future_division   = self.future_division
        result.unicode_literals  = self.unicode_literals
        result.absolute_import   = self.absolute_import
        result.future_print      = self.future_print
        result.barry_bdfl        = self.barry_bdfl

        return result

    def isFutureDivision(self):
        return self.future_division

    def enableFutureDivision(self):
        self.future_division = True

    def isFuturePrint(self):
        return self.future_print

    def enableFuturePrint(self):
        self.future_print = True

    def enableUnicodeLiterals(self):
        self.unicode_literals = True

    def enableAbsoluteImport(self):
        self.absolute_import = True

    def enableBarry(self):
        self.barry_bdfl = True

    def enableGeneratorStop(self):
        self.generator_stop = True

    def isAbsoluteImport(self):
        return self.absolute_import

    def isGeneratorStop(self):
        return self.generator_stop

    def asFlags(self):
        """ Create a list of C identifiers to represent the flag values.

            This is for use in code generation and to restore from
            saved modules.
        """

        result = []

        if python_version < 300 and self.future_division:
            result.append("CO_FUTURE_DIVISION")

        if self.unicode_literals:
            result.append("CO_FUTURE_UNICODE_LITERALS")

        if python_version < 300 and self.absolute_import:
            result.append("CO_FUTURE_ABSOLUTE_IMPORT")

        if python_version < 300 and self.future_print:
            result.append("CO_FUTURE_PRINT_FUNCTION")

        if python_version >= 300 and self.barry_bdfl:
            result.append("CO_FUTURE_BARRY_AS_BDFL")

        if python_version >= 350 and python_version < 370 and self.generator_stop:
            result.append("CO_FUTURE_GENERATOR_STOP")

        return tuple(result)

def fromFlags(flags):
    flags = flags.split(',')

    result = FutureSpec()

    if "CO_FUTURE_DIVISION" in flags:
        result.enableFutureDivision()

    if "CO_FUTURE_UNICODE_LITERALS" in flags:
        result.enableUnicodeLiterals()

    if "CO_FUTURE_ABSOLUTE_IMPORT" in flags:
        result.enableAbsoluteImport()

    if "CO_FUTURE_PRINT_FUNCTION" in flags:
        result.enableFuturePrint()

    if "CO_FUTURE_BARRY_AS_BDFL" in flags:
        result.enableBarry()

    if "CO_FUTURE_GENERATOR_STOP" in flags:
        result.enableGeneratorStop()

    # Check if we are going to give similar results than what we got.
    assert tuple(result.asFlags()) == tuple(flags), (result, result.asFlags(), flags)

    return result
<|endoftext|>


<commit_before>import collections, re, pprint
import schema
from models import *
import json

class Converter:
    """
    Takes two DataFiles and converts the formatted data in one and
    converts it to another data format.
    """

    def __init__(self, item_model):
        self.item_model = item_model
        self.items = list()

    def import_data(self, source, item_map):
        pass

    def export_data(self, target, item_mapper):
        pass

def prepare_left_to_sell_file():

    try:
        f = open('data/NewLeftToSell.csv', 'r')
    except IOError:
        return

    # read all lines of the csv into a 2D list, rows as the first level, columns 2nd.
    rows_list = [list(line) for line in csv.reader(f)]

    # the size labels
    sizes = rows_list[6][3:12]

    # the tags
    tag_names = rows_list[6][12:]

    pattern = re.compile("(^\d{6})\s/\s/\s(\w{3})\s{2}(.+)")

    variants = list()

    # the dress inventories
    for row in rows_list[8:]:
        if row[0] == '':
            break

        match = pattern.match(row[0])
        style = match.group(1)
        color_code = match.group(2)
        color_name = match.group(3)

        # fix color name abbreviations
        color_abbreviations = {"blk/watermelon": "black/watermelon",
                               "blush/mid": "blush/midnight",
                               "prb  - prussian blue": "prussian blue",
                               "blk/midnight": "black/midnight",
                               "blk pewter": "black pewter",
                               "blk/teal": "black/teal"}
        if color_name in color_abbreviations:
            color_name = color_abbreviations[color_name]

        price = int_from_str(row[1].strip())

        quantities = row[3:12]

        tag_matrix = row[12:]
        row_tags = []
        for index, name in enumerate(tag_names):
            if tag_matrix[index] != '':
                row_tags.append(tag_names[index])

        # create rows for each of the sizes
        for index, size in enumerate(sizes):
            item = collections.OrderedDict()
            item["Style"] = style
            item["Color"] = color_code
            item["Color Desc"] = color_name
            item["Size Desc"] = size
            item["Left to Sell"] = quantities[index]
            item["price"] = price
            item["tags"] = row_tags

            variants.append(item)

    # initialize the output csv file.
    target = DataFile(filename='data/LeftToSell.csv', schema=schema.left_to_sell_items)

    target.data = variants

    target.save()

def main():

    # this prepares the LTS report from the weird table layout to the old LTS report format that's
    # specified in the schema.
    prepare_left_to_sell_file()

    # open the data files and unserialze the data according to their schema
    source = DataFile(filename='data/TheiaDresses.csv', schema=schema.js_group_dresses)
    source.load()

    # load in the available inventory from the LTS report
    left_to_sell = DataFile(filename='data/LeftToSell.csv', schema=schema.left_to_sell_items)
    left_to_sell.load()

    # check for styles in the LTS data that are missing from the main dress list
    for lts_item in left_to_sell.data:
        lts_item_missing_in_main_data = True
        for main_item in source.data:
            if main_item["Style Number"] == lts_item["Style"]:
                lts_item_missing_in_main_data = False
                break
        if lts_item_missing_in_main_data:
            print lts_item["Style"], lts_item['Color Desc']

    inventory = Inventory(left_to_sell.data)
    left_to_sell_data = LeftToSellData(left_to_sell.data)

    # initialize the output csv file.
    target = DataFile(filename='data/shopify_products.csv', schema=schema.shopify_product)

    # missing image list
    missing_image_list = DataFile(filename='data/prods_missing_images.csv', schema=schema.dresses_missing_images)

    # load the images from the gallery
    # **todo** modify to use boto to list all the files in the s3 bucket directly.
    # that way we don't need to keep the local directory and bucket in sync.
    gallery = ImageGallery(directory='data/images')

    # using the source data create a set of Product instances
    products = list()
    for item in source.data:
        product = Product(
            title=item['Dress Name'],
            body=item['Dress Description'],
            collection=item['Collection'],
            price=item['Price (USD)'],
            sale_price=item['Sale Price'],
            on_sale=item['On Sale'],
            style_number=item['Style Number'],
            oversell=item['Oversell'],
            waitlist=item['Waitlist'],
            fulfillment=item['Fulfillment'],
            is_published=True
        )
        product.add_option('Size', item['Sizes'])
        product.add_option('Color', item['Colors'])
        if item['Waitlist']:
            product.add_tag('waitlist')

        lts_tags = left_to_sell_data.get_tags(item["Style Number"])
        if lts_tags != None and len(lts_tags) > 0:
            for tag in lts_tags:
                product.add_tag(tag)

        if product.is_on_sale():
            product.add_tag("On Sale")

        product.populate_variants(inventory)

        products.append(product)

    # check inventory levels for each product
    # mark product as in stock if any variants have any stock
    for product in products:
        product.in_stock = False
        for variant in product.variants:
            if variant.quantity > 0:
                product.in_stock = True

    # using these Product instances create the target data matching the target schema
    for product in products:

        # we are not selling the out of stock items on the site so we don't add them to the export
        # we are only preselling the spring 2015 collection
        # if not product.in_stock and product.collection != 'Spring 2015':
        #     # print('continue')
        #     continue

        if product.in_stock or product.waitlist:
            # add to 'Shop' Collection
            product.product_type = 'Theia Shop'
        else:
            product.product_type = 'Theia Collection'

        # get the product images
        images = gallery.get_product_images(product.style_number)

        # don't add the product if it's missing an image.
        if len(images) == 0:
            product_missing_images = collections.OrderedDict()
            product_missing_images["Style Number"] = product.style_number
            product_missing_images["Dress Name"] = product.title
            product_missing_images["Collection"] = product.collection

            missing_image_list.data.append(product_missing_images)

            continue

        # default is None
        featured_image = None
        if len(images) > 0:
            # check for a front view image
            for i, img in enumerate(images):
                if 'front' in img.description:
                    featured_image = img
            # if nothing found then just set to the first image
            if featured_image == None:
                featured_image = img

        product_rows = list()

        # we will need x number of rows where x is equal to the number of
        # variants plus the number of extra product images (not associated
        # with a specific variant.

        first = True

        for variant in product.variants:

            row = collections.OrderedDict()

            # check for an image to use for this variant
            variant_color = variant.get_option_value_by_term('Color')
            variant_image = None
            for image in images:
                if image.color == variant_color and 'front' in image.description:
                    variant_image = image
                    break

            # if no variant_image skip this row and go to the next variant
            if variant_image == None:
                continue

            row["Handle"] = product.handle

            if first:
                first = False

                # 1st row is the general product data
                row["Title"] = product.title
                row["Body (HTML)"] = product.body
                row["Vendor"] = product.vendor
                row["Type"] = product.product_type
                row["Tags"] = product.tags
                row["Published"] = product.is_published
                row["Collection"] = product.collection

                if variant_image == None:
                     row["Image Src"] = ''
                     row["Image Alt Text"] = ''
                else:
                    row["Image Src"] = variant_image.get_url()
                    row["Image Alt Text"] = variant_image.get_img_alt_data_string()

            # following rows are data for the variants

            # add each option
            for i, option in enumerate(variant.option_combo, start=1):
                option_name_header = "Option%s Name" % i
                option_value_header = "Option%s Value" % i

                # unpack the single item dictionary
                for k, v in option.iteritems():
                    option_name = k
                    option_value = v

                row[option_name_header] = option_name
                row[option_value_header] = option_value

            row["Variant SKU"] = variant.sku
            row["Variant Inventory Tracker"] = "shopify"
            row["Variant Inventory Qty"] = variant.quantity
            row["Variant Inventory Policy"] = "continue" if product.oversell else "deny"
            row["Variant Fulfillment Service"] = product.fulfillment
            row["Variant Requires Shipping"] = True
            row["Variant Price"] = product.get_price()
            if product.is_on_sale():
                row["Variant Compare At Price"] = product.get_regular_price()

            # Weight unit and Variant Grams are independent of eachother
            row["Variant Weight Unit"] = 'lb' # used for displaying the weight
            row["Variant Grams"] = 900 # used inside Shopify, must always be in grams
            row["Variant Image"] = variant_image.get_url()

            product_rows.append(row)

        # additional rows are needed for extra images
        for img in images:
            row = collections.OrderedDict()
            row["Handle"] = product.handle
            row["Image Src"] = img.get_url()
            row["Image Alt Text"] = img.get_img_alt_data_string()
            product_rows.append(row)

        # add row(s) to the data file
        target.data += product_rows

    # save the target file
    target.save()

    # save the missing images.
    missing_image_list.save()

if __name__ == '__main__':
    main()
<commit_msg>Changes "On Sale" tag to just "Sale".<commit_after>import collections, re, pprint
import schema
from models import *
import json

class Converter:
    """
    Takes two DataFiles and converts the formatted data in one and
    converts it to another data format.
    """

    def __init__(self, item_model):
        self.item_model = item_model
        self.items = list()

    def import_data(self, source, item_map):
        pass

    def export_data(self, target, item_mapper):
        pass

def prepare_left_to_sell_file():

    try:
        f = open('data/NewLeftToSell.csv', 'r')
    except IOError:
        return

    # read all lines of the csv into a 2D list, rows as the first level, columns 2nd.
    rows_list = [list(line) for line in csv.reader(f)]

    # the size labels
    sizes = rows_list[6][3:12]

    # the tags
    tag_names = rows_list[6][12:]

    pattern = re.compile("(^\d{6})\s/\s/\s(\w{3})\s{2}(.+)")

    variants = list()

    # the dress inventories
    for row in rows_list[8:]:
        if row[0] == '':
            break

        match = pattern.match(row[0])
        style = match.group(1)
        color_code = match.group(2)
        color_name = match.group(3)

        # fix color name abbreviations
        color_abbreviations = {"blk/watermelon": "black/watermelon",
                               "blush/mid": "blush/midnight",
                               "prb  - prussian blue": "prussian blue",
                               "blk/midnight": "black/midnight",
                               "blk pewter": "black pewter",
                               "blk/teal": "black/teal"}
        if color_name in color_abbreviations:
            color_name = color_abbreviations[color_name]

        price = int_from_str(row[1].strip())

        quantities = row[3:12]

        tag_matrix = row[12:]
        row_tags = []
        for index, name in enumerate(tag_names):
            if tag_matrix[index] != '':
                row_tags.append(tag_names[index])

        # create rows for each of the sizes
        for index, size in enumerate(sizes):
            item = collections.OrderedDict()
            item["Style"] = style
            item["Color"] = color_code
            item["Color Desc"] = color_name
            item["Size Desc"] = size
            item["Left to Sell"] = quantities[index]
            item["price"] = price
            item["tags"] = row_tags

            variants.append(item)

    # initialize the output csv file.
    target = DataFile(filename='data/LeftToSell.csv', schema=schema.left_to_sell_items)

    target.data = variants

    target.save()

def main():

    # this prepares the LTS report from the weird table layout to the old LTS report format that's
    # specified in the schema.
    prepare_left_to_sell_file()

    # open the data files and unserialze the data according to their schema
    source = DataFile(filename='data/TheiaDresses.csv', schema=schema.js_group_dresses)
    source.load()

    # load in the available inventory from the LTS report
    left_to_sell = DataFile(filename='data/LeftToSell.csv', schema=schema.left_to_sell_items)
    left_to_sell.load()

    # check for styles in the LTS data that are missing from the main dress list
    for lts_item in left_to_sell.data:
        lts_item_missing_in_main_data = True
        for main_item in source.data:
            if main_item["Style Number"] == lts_item["Style"]:
                lts_item_missing_in_main_data = False
                break
        if lts_item_missing_in_main_data:
            print lts_item["Style"], lts_item['Color Desc']

    inventory = Inventory(left_to_sell.data)
    left_to_sell_data = LeftToSellData(left_to_sell.data)

    # initialize the output csv file.
    target = DataFile(filename='data/shopify_products.csv', schema=schema.shopify_product)

    # missing image list
    missing_image_list = DataFile(filename='data/prods_missing_images.csv', schema=schema.dresses_missing_images)

    # load the images from the gallery
    # **todo** modify to use boto to list all the files in the s3 bucket directly.
    # that way we don't need to keep the local directory and bucket in sync.
    gallery = ImageGallery(directory='data/images')

    # using the source data create a set of Product instances
    products = list()
    for item in source.data:
        product = Product(
            title=item['Dress Name'],
            body=item['Dress Description'],
            collection=item['Collection'],
            price=item['Price (USD)'],
            sale_price=item['Sale Price'],
            on_sale=item['On Sale'],
            style_number=item['Style Number'],
            oversell=item['Oversell'],
            waitlist=item['Waitlist'],
            fulfillment=item['Fulfillment'],
            is_published=True
        )
        product.add_option('Size', item['Sizes'])
        product.add_option('Color', item['Colors'])
        if item['Waitlist']:
            product.add_tag('waitlist')

        lts_tags = left_to_sell_data.get_tags(item["Style Number"])
        if lts_tags != None and len(lts_tags) > 0:
            for tag in lts_tags:
                product.add_tag(tag)

        if product.is_on_sale():
            product.add_tag("Sale")

        product.populate_variants(inventory)

        products.append(product)

    # check inventory levels for each product
    # mark product as in stock if any variants have any stock
    for product in products:
        product.in_stock = False
        for variant in product.variants:
            if variant.quantity > 0:
                product.in_stock = True

    # using these Product instances create the target data matching the target schema
    for product in products:

        # we are not selling the out of stock items on the site so we don't add them to the export
        # we are only preselling the spring 2015 collection
        # if not product.in_stock and product.collection != 'Spring 2015':
        #     # print('continue')
        #     continue

        if product.in_stock or product.waitlist:
            # add to 'Shop' Collection
            product.product_type = 'Theia Shop'
        else:
            product.product_type = 'Theia Collection'

        # get the product images
        images = gallery.get_product_images(product.style_number)

        # don't add the product if it's missing an image.
        if len(images) == 0:
            product_missing_images = collections.OrderedDict()
            product_missing_images["Style Number"] = product.style_number
            product_missing_images["Dress Name"] = product.title
            product_missing_images["Collection"] = product.collection

            missing_image_list.data.append(product_missing_images)

            continue

        # default is None
        featured_image = None
        if len(images) > 0:
            # check for a front view image
            for i, img in enumerate(images):
                if 'front' in img.description:
                    featured_image = img
            # if nothing found then just set to the first image
            if featured_image == None:
                featured_image = img

        product_rows = list()

        # we will need x number of rows where x is equal to the number of
        # variants plus the number of extra product images (not associated
        # with a specific variant.

        first = True

        for variant in product.variants:

            row = collections.OrderedDict()

            # check for an image to use for this variant
            variant_color = variant.get_option_value_by_term('Color')
            variant_image = None
            for image in images:
                if image.color == variant_color and 'front' in image.description:
                    variant_image = image
                    break

            # if no variant_image skip this row and go to the next variant
            if variant_image == None:
                continue

            row["Handle"] = product.handle

            if first:
                first = False

                # 1st row is the general product data
                row["Title"] = product.title
                row["Body (HTML)"] = product.body
                row["Vendor"] = product.vendor
                row["Type"] = product.product_type
                row["Tags"] = product.tags
                row["Published"] = product.is_published
                row["Collection"] = product.collection

                if variant_image == None:
                     row["Image Src"] = ''
                     row["Image Alt Text"] = ''
                else:
                    row["Image Src"] = variant_image.get_url()
                    row["Image Alt Text"] = variant_image.get_img_alt_data_string()

            # following rows are data for the variants

            # add each option
            for i, option in enumerate(variant.option_combo, start=1):
                option_name_header = "Option%s Name" % i
                option_value_header = "Option%s Value" % i

                # unpack the single item dictionary
                for k, v in option.iteritems():
                    option_name = k
                    option_value = v

                row[option_name_header] = option_name
                row[option_value_header] = option_value

            row["Variant SKU"] = variant.sku
            row["Variant Inventory Tracker"] = "shopify"
            row["Variant Inventory Qty"] = variant.quantity
            row["Variant Inventory Policy"] = "continue" if product.oversell else "deny"
            row["Variant Fulfillment Service"] = product.fulfillment
            row["Variant Requires Shipping"] = True
            row["Variant Price"] = product.get_price()
            if product.is_on_sale():
                row["Variant Compare At Price"] = product.get_regular_price()

            # Weight unit and Variant Grams are independent of eachother
            row["Variant Weight Unit"] = 'lb' # used for displaying the weight
            row["Variant Grams"] = 900 # used inside Shopify, must always be in grams
            row["Variant Image"] = variant_image.get_url()

            product_rows.append(row)

        # additional rows are needed for extra images
        for img in images:
            row = collections.OrderedDict()
            row["Handle"] = product.handle
            row["Image Src"] = img.get_url()
            row["Image Alt Text"] = img.get_img_alt_data_string()
            product_rows.append(row)

        # add row(s) to the data file
        target.data += product_rows

    # save the target file
    target.save()

    # save the missing images.
    missing_image_list.save()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Anomaly-related algorithms."""

import numpy

from nupic.algorithms.anomaly_likelihood import AnomalyLikelihood
from nupic.utils import MovingAverage

def computeRawAnomalyScore(activeColumns, prevPredictedColumns):
  """Computes the raw anomaly score.

  The raw anomaly score is the fraction of active columns not predicted.

  @param activeColumns: array of active column indices
  @param prevPredictedColumns: array of columns indices predicted in prev step
  @return anomaly score 0..1 (float)
  """
  nActiveColumns = len(activeColumns)
  if nActiveColumns > 0:
    # Test whether each element of a 1-D array is also present in a second
    # array. Sum to get the total # of columns that are active and were
    # predicted.
    score = numpy.in1d(activeColumns, prevPredictedColumns).sum()
    # Get the percent of active columns that were NOT predicted, that is
    # our anomaly score.
    score = (nActiveColumns - score) / float(nActiveColumns)
  elif len(prevPredictedColumns) > 0:
    # There were predicted columns but none active.
    score = 1.0
  else:
    # There were no predicted or active columns.
    score = 0.0

  return score

class Anomaly(object):
  """Utility class for generating anomaly scores in different ways.

  Supported modes:
    MODE_PURE - the raw anomaly score as computed by computeRawAnomalyScore
    MODE_LIKELIHOOD - uses the AnomalyLikelihood class on top of the raw
        anomaly scores
    MODE_WEIGHTED - multiplies the likelihood result with the raw anomaly score
        that was used to generate the likelihood
  """

  # anomaly modes supported
  MODE_PURE = "pure"
  MODE_LIKELIHOOD = "likelihood"
  MODE_WEIGHTED = "weighted"
  _supportedModes = (MODE_PURE, MODE_LIKELIHOOD, MODE_WEIGHTED)

  def __init__(self, slidingWindowSize = None, mode=MODE_PURE):
    """
    @param slidingWindowSize (optional) - how many elements are summed up;
        enables moving average on final anomaly score; int >= 0
    @param mode (optional) - (string) how to compute anomaly;
        possible values are:
          - "pure" - the default, how much anomal the value is;
              float 0..1 where 1=totally unexpected
          - "likelihood" - uses the anomaly_likelihood code;
              models probability of receiving this value and anomalyScore
          - "weighted" - "pure" anomaly weighted by "likelihood"
              (anomaly * likelihood)
    """
    self._mode = mode
    if slidingWindowSize is not None:
      self._movingAverage = MovingAverage(windowSize=slidingWindowSize)
    else:
      self._movingAverage = None

    if self._mode == Anomaly.MODE_LIKELIHOOD:
      self._likelihood = AnomalyLikelihood() # probabilistic anomaly
    if not self._mode in Anomaly._supportedModes:
      raise ValueError("Invalid anomaly mode; only supported modes are: "
                       "Anomaly.MODE_PURE, Anomaly.MODE_LIKELIHOOD, "
                       "Anomaly.MODE_WEIGHTED; you used: %r" % self._mode)

  def compute(self, activeColumns, predictedColumns,
				inputValue=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    @param activeColumns: array of active column indices
    @param predictedColumns: array of columns indices predicted in this step
                             (used for anomaly in step T+1)
    @param inputValue: (optional) value of current input to encoders
				(eg "cat" for category encoder)
                              	(used in anomaly-likelihood)
    @param timestamp: (optional) date timestamp when the sample occured
                              	(used in anomaly-likelihood)
    @return the computed anomaly score; float 0..1
    """
    # Start by computing the raw anomaly score.
    anomalyScore = computeRawAnomalyScore(activeColumns, predictedColumns)

    # Compute final anomaly based on selected mode.
    if self._mode == Anomaly.MODE_PURE:
      score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      if inputValue is None:
        raise ValueError("Selected anomaly mode 'Anomaly.MODE_LIKELIHOOD' "
                 "requires 'inputValue' as parameter to compute() method. ")

      probability = self._likelihood.anomalyProbability(
          inputValue, anomalyScore, timestamp)
      # low likelihood -> hi anomaly
      score = 1 - probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(
          inputValue, anomalyScore, timestamp)
      score = anomalyScore * (1 - probability)

    # Last, do moving-average if windowSize was specified.
    if self._movingAverage is not None:
      score = self._movingAverage.next(score)

    return score

  def __str__(self):
    windowSize = 0
    if self._movingAverage is not None:
      windowSize = self._movingAverage.windowSize
    return "Anomaly:\tmode=%s\twindowSize=%r" % (self._mode, windowSize)

<commit_msg>anomaly: make score an object variable + show it in print()<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Anomaly-related algorithms."""

import numpy

from nupic.algorithms.anomaly_likelihood import AnomalyLikelihood
from nupic.utils import MovingAverage

def computeRawAnomalyScore(activeColumns, prevPredictedColumns):
  """Computes the raw anomaly score.

  The raw anomaly score is the fraction of active columns not predicted.

  @param activeColumns: array of active column indices
  @param prevPredictedColumns: array of columns indices predicted in prev step
  @return anomaly score 0..1 (float)
  """
  nActiveColumns = len(activeColumns)
  if nActiveColumns > 0:
    # Test whether each element of a 1-D array is also present in a second
    # array. Sum to get the total # of columns that are active and were
    # predicted.
    score = numpy.in1d(activeColumns, prevPredictedColumns).sum()
    # Get the percent of active columns that were NOT predicted, that is
    # our anomaly score.
    score = (nActiveColumns - score) / float(nActiveColumns)
  elif len(prevPredictedColumns) > 0:
    # There were predicted columns but none active.
    score = 1.0
  else:
    # There were no predicted or active columns.
    score = 0.0

  return score

class Anomaly(object):
  """Utility class for generating anomaly scores in different ways.

  Supported modes:
    MODE_PURE - the raw anomaly score as computed by computeRawAnomalyScore
    MODE_LIKELIHOOD - uses the AnomalyLikelihood class on top of the raw
        anomaly scores
    MODE_WEIGHTED - multiplies the likelihood result with the raw anomaly score
        that was used to generate the likelihood
  """

  # anomaly modes supported
  MODE_PURE = "pure"
  MODE_LIKELIHOOD = "likelihood"
  MODE_WEIGHTED = "weighted"
  _supportedModes = (MODE_PURE, MODE_LIKELIHOOD, MODE_WEIGHTED)

  def __init__(self, slidingWindowSize = None, mode=MODE_PURE):
    """
    @param slidingWindowSize (optional) - how many elements are summed up;
        enables moving average on final anomaly score; int >= 0
    @param mode (optional) - (string) how to compute anomaly;
        possible values are:
          - "pure" - the default, how much anomal the value is;
              float 0..1 where 1=totally unexpected
          - "likelihood" - uses the anomaly_likelihood code;
              models probability of receiving this value and anomalyScore
          - "weighted" - "pure" anomaly weighted by "likelihood"
              (anomaly * likelihood)
    """
    self._mode = mode
    if slidingWindowSize is not None:
      self._movingAverage = MovingAverage(windowSize=slidingWindowSize)
    else:
      self._movingAverage = None

    if self._mode == Anomaly.MODE_LIKELIHOOD:
      self._likelihood = AnomalyLikelihood() # probabilistic anomaly
    if not self._mode in Anomaly._supportedModes:
      raise ValueError("Invalid anomaly mode; only supported modes are: "
                       "Anomaly.MODE_PURE, Anomaly.MODE_LIKELIHOOD, "
                       "Anomaly.MODE_WEIGHTED; you used: %r" % self._mode)
    self.score = 1 # current anomaly score (starts at 1, anything is unexpected)

  def compute(self, activeColumns, predictedColumns,
				inputValue=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    @param activeColumns: array of active column indices
    @param predictedColumns: array of columns indices predicted in this step
                             (used for anomaly in step T+1)
    @param inputValue: (optional) value of current input to encoders
				(eg "cat" for category encoder)
                              	(used in anomaly-likelihood)
    @param timestamp: (optional) date timestamp when the sample occured
                              	(used in anomaly-likelihood)
    @return the computed anomaly score; float 0..1
    """
    # Start by computing the raw anomaly score.
    anomalyScore = computeRawAnomalyScore(activeColumns, predictedColumns)

    # Compute final anomaly based on selected mode.
    if self._mode == Anomaly.MODE_PURE:
      self.score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      if inputValue is None:
        raise ValueError("Selected anomaly mode 'Anomaly.MODE_LIKELIHOOD' "
                 "requires 'inputValue' as parameter to compute() method. ")

      probability = self._likelihood.anomalyProbability(
          inputValue, anomalyScore, timestamp)
      # low likelihood -> hi anomaly
      self.score = 1 - probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(
          inputValue, anomalyScore, timestamp)
      self.score = anomalyScore * (1 - probability)

    # Last, do moving-average if windowSize was specified.
    if self._movingAverage is not None:
      self.score = self._movingAverage.next(self.score)

    return self.score

  def __str__(self):
    windowSize = 0
    if self._movingAverage is not None:
      windowSize = self._movingAverage.windowSize
    msg = "Anomaly:\tmode=%s\twindowSize=%r\tscore=%r" % (self._mode, windowSize, self.score)
    return msg

<|endoftext|>


<commit_before>import torch
from torch import nn
from torch.utils.data import DataLoader
import numpy as np
from .. import networks
from .enums import *
import os
import shutil
from typing import Optional
from itertools import chain
import sys
import time
import time
import logging

try:
    from tensorboardX import SummaryWriter
except:
    SummaryWriter = None

class Trainer():
    def __init__(
        self,
        output_folder,
        train_data,
        model,
        options,
        validation_data=None,
        use_tensorboard=True,
        experiment=None,
        cache_populated=False,
    ):
        self.options = options.copy()
        self.use_tensorboard = (
            use_tensorboard if SummaryWriter is not None else False
        )
        self.experiment = experiment
        self.prepare_directories(output_folder)
        self.model = self.build_model(model)

        freeze_layers = self.options.pop('freeze_layers', [])
        self.freeze_layers(freeze_layers)

        self.device = torch.device(
            'cpu'
            if options['device'] == 'cpu' or not torch.cuda.is_available()
            else 'cuda'
        )
        self.model = self.model.to(self.device)

        self.writer = (
            SummaryWriter(log_dir=self.output_folder)
            if use_tensorboard else None
        )

        self.setup_loss()
        self.model_tag = self.options.pop('model_tag', None)
        self.num_epoch = 0

        self.optimizer, self.scheduler = self.create_optimizer_and_scheduler(
            self.model,
            self.options
        )

        self.module = self.model
        resume = self.options.pop('resume', False)
        resume_location = self.options.pop('resume_location', None)
        resume_prefix = self.options.pop('resume_prefix', ('latest',))
        load_only_model = self.options.pop('load_only_model', False)
        if resume:
            self.resume(resume_location, load_only_model=load_only_model, prefixes=resume_prefix)
        if options['data_parallel'] and options['device'] == 'cuda':
            self.model = nn.DataParallel(self.module)
            self.module = self.model.module
        self.model.train()

        self.dataloaders = {
            'training': self.create_dataloader(train_data),
        }
        if validation_data:
            self.dataloaders['validation'] = self.create_dataloader(
                validation_data
            )
        self.cache_populated = cache_populated

    def setup_loss(self):
        self.output_target_map = {
            'estimates': ['source_spectrograms'],
            'embedding': ['assignments', 'weights']
        }
        self.loss_dictionary = {
            target: (LossFunctions[fn.upper()].value(), float(weight))
            for (fn, target, weight)
            in options['loss_function']
        }
        self.loss_keys = sorted(list(self.loss_dictionary))

    @staticmethod
    def build_model(model):
        return (
            networks.SeparationModel(model)
            if (type(model) is str and '.json' in model) or type(model) is dict
            else model
        )

    def freeze_layers(self, freeze_layers):
        for name, parameter in self.model.named_parameters():
            if name in freeze_layers:
                logging.info(f'Freezing {name} layer in model.')
                parameter.requires_grad = False

    def prepare_directories(self, output_folder):
        self.output_folder = output_folder
        self.checkpoint_folder = os.path.join(output_folder, 'checkpoints')
        self.config_folder = os.path.join(output_folder, 'config')

        os.makedirs(self.output_folder, exist_ok=True)
        os.makedirs(self.checkpoint_folder, exist_ok=True)
        os.makedirs(self.config_folder, exist_ok=True)

    def create_dataloader(self, dataset):
        if not dataset:
            return None

        input_keys = [[connection[0]] + connection[1] for connection in self.module.connections]
        input_keys = list(chain.from_iterable(input_keys))
        input_keys += self.module.output_keys

        output_keys = [self.output_target_map[k] for k in input_keys if k in self.output_target_map]
        output_keys = list(chain.from_iterable(output_keys))

        dataset.data_keys_for_training = input_keys + output_keys

        return DataLoader(
            dataset,
            batch_size=self.options['batch_size'],
            num_workers=self.options['num_workers'],
            sampler=Samplers[
                self.options['sample_strategy'].upper()
            ].value(dataset),
            pin_memory=True,
        )

    def create_optimizer_and_scheduler(self, model, options):
        parameters = filter(lambda p: p.requires_grad, model.parameters())

        optimizer = Optimizers[options['optimizer'].upper()].value(
            parameters,
            lr=options['learning_rate'],
            weight_decay=options['weight_decay']
        )
        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
            optimizer,
            'min',
            factor=options['learning_rate_decay'],
            patience=options['patience'],
        )
        return optimizer, scheduler

    def calculate_loss(self, outputs, targets):
        if self.module.layers['mel_projection'].num_mels > 0:
            if 'assignments' in targets:
                targets['assignments'] = self.module.project_data(
                    targets['assignments'],
                    clamp=True
                )
                targets['assignments'] = (targets['assignments'] > 0).float()
            if 'weights' in targets:
                targets['weights'] = self.module.project_data(
                    targets['weights'],
                    clamp=False
                )
                if 'threshold' in self.dataloaders['training'].dataset.options['weight_type']:
                    targets['weights'] = (targets['weights'] > 0).float()
        loss = {}
        for key in self.loss_keys:
            loss_function = self.loss_dictionary[key][0]
            weight = self.loss_dictionary[key][1]
            target_keys = self.output_target_map[key]
            arguments = [outputs[key]] + [targets[t] for t in target_keys]
            _loss = weight * loss_function(*arguments)
            self.check_loss(_loss)
            loss[key] = _loss
        return loss

    def check_loss(self, loss):
        if np.isnan(loss.item()):
            raise ValueError("Loss went to nan - aborting training.")

    def prepare_data(self, data):
        """Moves all the data to the requested device (cuda or cpu).
        """
        for key in data:
            data[key] = data[key].float().to(self.device)
            if key not in self.loss_keys and self.model.training:
                data[key] = data[key].requires_grad_()
        return data

    def forward(self, data):
        """Performs a forward pass on the data using the model.
        """
        if self.model.training:
            output = self.model(data)
        else:
            output = self.module(data)
        return output

    def run_epoch(self, key):
        epoch_loss = 0
        num_batches = len(self.dataloaders[key])
        logging.info(f"Starting {key} epoch")
        for step, data in enumerate(self.dataloaders[key]):
            #TODO factor this out into a loop that can be overridden by other classes.
            data = self.prepare_data(data)
            output = self.forward(data)
            loss = self.calculate_loss(output, data)
            loss['total_loss'] = sum(list(loss.values()))
            if step % 100 == 0:
                logging.info(f"Step {step}/{num_batches}, loss: {epoch_loss/(step + 1)}")
            epoch_loss += loss['total_loss'].item()

            if self.model.training:
                self.optimizer.zero_grad()
                loss['total_loss'].backward()
                self.optimizer.step()
            step += 1
        return {'loss': epoch_loss / float(num_batches)}

    def before_epoch(self):
        """
        Do whatever needs to be done before each epoch (e.g. lengthen sequences).
        Can use this to implement curriculum learning.
        """
        if self.options['curriculum_learning']:
            settings = self.options['curriculum_learning']
            for setting in settings:
                if setting['num_epoch'] == self.num_epoch:
                    logging.info(
                        f"Curriculum learning - running @ epoch: {setting['num_epoch']}, "
                        f"command: {setting['command']}, "
                        f"args: {setting['args']}"
                    )
                    if setting['command'] == 'set_current_length':
                        for key, dataloader in self.dataloaders.items():
                            dataloader.dataset.set_current_length(*setting['args'])
                    if setting['command'] == 'save':
                        self.save(is_best=False, prefix=f'epoch{self.num_epoch}')

    def log_metrics(self, data, step, scope):
        if self.use_tensorboard:
            prefix = 'training' if self.model.training else 'validation'
            for key in data:
                label = os.path.join(prefix, key)
                self.writer.add_scalar(label, data[key], step)
        if self.experiment is not None:
            context = (
                self.experiment.train
                if self.model.training
                else self.experiment.validate
            )
            with context():
                for key in data:
                    self.experiment.log_metric(key, data[key], step)

    def populate_cache(self):
        for key, dataloader in self.dataloaders.items():
            num_batches = len(dataloader)
            if dataloader.dataset.overwrite_cache:
                logging.info(f'Populating cache for {key} w/ {num_batches} batches')
                for i, data in enumerate(dataloader):
                    if i % 50 == 0:
                        logging.info(f'{i}/{num_batches} batches completed')
                    continue
                logging.info(f'Done populating cache for {key}')

    def switch_to_cache(self):
        for key, dataloader in self.dataloaders.items():
            dataloader.dataset.switch_to_cache()

    def fit(self):
        lowest_validation_loss = np.inf

        self.populate_cache()
        self.switch_to_cache()

        logging.info(f"Training data for {self.options['num_epochs'] - self.num_epoch} epochs")
        fit_start_time = time.time()

        for self.num_epoch in range(self.num_epoch, self.options['num_epochs']):
            self.before_epoch()
            start_time = time.time()
            epoch_loss = self.run_epoch('training')
            self.log_metrics(epoch_loss, self.num_epoch, 'epoch')
            validation_loss = self.validate('validation')
            self.save(validation_loss['loss'] < lowest_validation_loss)
            lowest_validation_loss = (
                validation_loss['loss']
                if validation_loss['loss'] < lowest_validation_loss
                else lowest_validation_loss
            )

            epoch_elapsed_time = time.time() - start_time
            epoch_elapsed_time = time.strftime("%H:%M:%S", time.gmtime(epoch_elapsed_time))
            full_elapsed_time = time.time() - fit_start_time
            full_elapsed_time = time.strftime("%H:%M:%S", time.gmtime(full_elapsed_time))

            logging.info(
                f"Epoch summary: {self.num_epoch}\t"
                f"Training loss: {epoch_loss['loss']}\t"
                f"Validation loss: {validation_loss['loss']}\t"
                f"Epoch took: {epoch_elapsed_time}\t"
                f"Time since start: {full_elapsed_time}"
            )

    def validate(self, key) -> float:
        """Calculate loss on validation set

        Args:
            dataloader - a dataloader yielding the validation data if there is
                any, `None` otherwise

        Returns:
            `np.inf` if there is no validation dataset (`dataloader` is `None`)
            else the loss over the given validation data
        """
        if key not in self.dataloaders:
            return {'loss': np.inf}

        self.model.eval()
        with torch.no_grad():
            validation_loss = self.run_epoch(key)
        self.log_metrics(validation_loss, self.num_epoch, 'epoch')
        self.model.train()
        self.scheduler.step(validation_loss['loss'])
        if self.scheduler.num_bad_epochs + 1 == self.options['patience']:
            logging.info(f'Exceeded patience at epoch {self.num_epoch} - adjusting learning rate')

        return validation_loss

    def save(self, is_best: bool, prefix: str = '') -> str:
        """Saves the model being trained with either `latest` or `best` prefix
        based on validation loss

        Args:
            is_best - whether or not the model is known to have improved
                accuracy from the last epoch (based on validation loss). Always
                False if no validation data was given.
            [path] - path to folder in which to save model. If not given, model
                saved in checkpoint folder

        Returns:
            path to saved model
        """
        if not prefix:
            prefix = 'latest'
        if self.model_tag is not None:
            prefix = f'{prefix}.{self.model_tag}'
        optimizer_path = os.path.join(self.checkpoint_folder, f'{prefix}.opt.pth')
        model_path = os.path.join(
             self.checkpoint_folder,
            f'{prefix}.model.pth'
        )
        dataset_options = self.dataloaders['training'].dataset.options
        metadata = {
            key: val
            for key, val in dataset_options.items()
            if key in [
                'n_fft',
                'hop_length',
                'format',
                'sample_rate',
                'num_channels'
            ]
        }

        optimizer_state = {
            'optimizer': self.optimizer.state_dict(),
            'num_epoch': self.num_epoch
        }

        torch.save(optimizer_state, optimizer_path)
        self.module.save(model_path, {'metadata': metadata})
        if is_best:
            torch.save(optimizer_state, optimizer_path.replace('latest', 'best'))
            self.module.save(model_path.replace('latest', 'best'), {'metadata': metadata})
        return model_path

    def resume(self, resume_location, load_only_model=True, prefixes=('best', 'latest')):
        for prefix in prefixes:
            checkpoint_folder = self.checkpoint_folder
            if resume_location is not None:
                checkpoint_folder = resume_location

            optimizer_path = os.path.join(checkpoint_folder, f'{prefix}.opt.pth')
            model_path = os.path.join(checkpoint_folder, f'{prefix}.model.pth')
            logging.info(f'Looking for {model_path}')

            if os.path.exists(model_path) and os.path.exists(optimizer_path):
                optimizer_state = torch.load(optimizer_path, map_location=lambda storage, loc: storage)
                model_dict = torch.load(model_path, map_location=lambda storage, loc: storage)
                self.module.load_state_dict(model_dict['state_dict'])
                if not load_only_model:
                    self.optimizer.load_state_dict(optimizer_state['optimizer'])
                    self.num_epoch = optimizer_state['num_epoch']
                logging.info(f'Resuming from {model_path}')
                return model_path
            else:
                model_path = None
        logging.info('No model found! Training a new model from scratch.')
        return model_path<commit_msg>making self.setup_loss easy to subclass and override<commit_after>import torch
from torch import nn
from torch.utils.data import DataLoader
import numpy as np
from .. import networks
from .enums import *
import os
import shutil
from typing import Optional
from itertools import chain
import sys
import time
import time
import logging

try:
    from tensorboardX import SummaryWriter
except:
    SummaryWriter = None

class Trainer():
    def __init__(
        self,
        output_folder,
        train_data,
        model,
        options,
        validation_data=None,
        use_tensorboard=True,
        experiment=None,
    ):
        self.options = options.copy()
        self.use_tensorboard = (
            use_tensorboard if SummaryWriter is not None else False
        )
        self.experiment = experiment
        self.prepare_directories(output_folder)
        self.model = self.build_model(model)

        freeze_layers = self.options.pop('freeze_layers', [])
        self.freeze_layers(freeze_layers)

        self.device = torch.device(
            'cpu'
            if options['device'] == 'cpu' or not torch.cuda.is_available()
            else 'cuda'
        )
        self.model = self.model.to(self.device)

        self.writer = (
            SummaryWriter(log_dir=self.output_folder)
            if use_tensorboard else None
        )

        output_target_map = {
            'estimates': ['source_spectrograms'],
            'embedding': ['assignments', 'weights']
        }
        self.setup_loss(LossFunctions, output_target_map)
        self.model_tag = self.options.pop('model_tag', None)
        self.num_epoch = 0

        self.optimizer, self.scheduler = self.create_optimizer_and_scheduler(
            self.model,
            self.options
        )

        self.module = self.model
        resume = self.options.pop('resume', False)
        resume_location = self.options.pop('resume_location', None)
        resume_prefix = self.options.pop('resume_prefix', ('latest',))
        load_only_model = self.options.pop('load_only_model', False)
        if resume:
            self.resume(resume_location, load_only_model=load_only_model, prefixes=resume_prefix)

        if options['data_parallel'] and options['device'] == 'cuda':
            self.model = nn.DataParallel(self.module)
            self.module = self.model.module
        self.model.train()

        self.dataloaders = {
            'training': self.create_dataloader(train_data),
        }
        if validation_data:
            self.dataloaders['validation'] = self.create_dataloader(
                validation_data
            )

    def setup_loss(self, loss_functions, output_target_map):
        self.output_target_map = output_target_map
        self.loss_dictionary = {
            target: (loss_functions[fn.upper()].value(), float(weight))
            for (fn, target, weight)
            in options['loss_function']
        }
        self.loss_keys = sorted(list(self.loss_dictionary))

    @staticmethod
    def build_model(model):
        return (
            networks.SeparationModel(model)
            if (type(model) is str and '.json' in model) or type(model) is dict
            else model
        )

    def freeze_layers(self, freeze_layers):
        for name, parameter in self.model.named_parameters():
            if name in freeze_layers:
                logging.info(f'Freezing {name} layer in model.')
                parameter.requires_grad = False

    def prepare_directories(self, output_folder):
        self.output_folder = output_folder
        self.checkpoint_folder = os.path.join(output_folder, 'checkpoints')
        self.config_folder = os.path.join(output_folder, 'config')

        os.makedirs(self.output_folder, exist_ok=True)
        os.makedirs(self.checkpoint_folder, exist_ok=True)
        os.makedirs(self.config_folder, exist_ok=True)

    def create_dataloader(self, dataset):
        if not dataset:
            return None

        input_keys = [[connection[0]] + connection[1] for connection in self.module.connections]
        input_keys = list(chain.from_iterable(input_keys))
        input_keys += self.module.output_keys

        output_keys = [self.output_target_map[k] for k in input_keys if k in self.output_target_map]
        output_keys = list(chain.from_iterable(output_keys))

        dataset.data_keys_for_training = input_keys + output_keys

        return DataLoader(
            dataset,
            batch_size=self.options['batch_size'],
            num_workers=self.options['num_workers'],
            sampler=Samplers[
                self.options['sample_strategy'].upper()
            ].value(dataset),
            pin_memory=True,
        )

    def create_optimizer_and_scheduler(self, model, options):
        parameters = filter(lambda p: p.requires_grad, model.parameters())

        optimizer = Optimizers[options['optimizer'].upper()].value(
            parameters,
            lr=options['learning_rate'],
            weight_decay=options['weight_decay']
        )
        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
            optimizer,
            'min',
            factor=options['learning_rate_decay'],
            patience=options['patience'],
        )
        return optimizer, scheduler

    def preprocess_targets(self, targets):
        if self.module.layers['mel_projection'].num_mels > 0:
            if 'assignments' in targets:
                targets['assignments'] = self.module.project_data(
                    targets['assignments'],
                    clamp=True
                )
                targets['assignments'] = (targets['assignments'] > 0).float()
            if 'weights' in targets:
                targets['weights'] = self.module.project_data(
                    targets['weights'],
                    clamp=False
                )
                if 'threshold' in self.dataloaders['training'].dataset.options['weight_type']:
                    targets['weights'] = (targets['weights'] > 0).float()
        return targets

    def calculate_loss(self, outputs, targets):
        targets = self.preprocess_targets(targets)
        loss = {}
        for key in self.loss_keys:
            loss_function = self.loss_dictionary[key][0]
            weight = self.loss_dictionary[key][1]
            target_keys = self.output_target_map[key]
            arguments = [outputs[key]] + [targets[t] for t in target_keys]
            _loss = weight * loss_function(*arguments)
            self.check_loss(_loss)
            loss[key] = _loss
        return loss

    def check_loss(self, loss):
        if np.isnan(loss.item()):
            raise ValueError("Loss went to nan - aborting training.")

    def prepare_data(self, data):
        """Moves all the data to the requested device (cuda or cpu).
        """
        for key in data:
            data[key] = data[key].float().to(self.device)
            if key not in self.loss_keys and self.model.training:
                data[key] = data[key].requires_grad_()
        return data

    def forward(self, data):
        """Performs a forward pass on the data using the model.
        """
        if self.model.training:
            output = self.model(data)
        else:
            output = self.module(data)
        return output

    def run_epoch(self, key):
        epoch_loss = 0
        num_batches = len(self.dataloaders[key])
        logging.info(f"Starting {key} epoch")
        for step, data in enumerate(self.dataloaders[key]):
            #TODO factor this out into a loop that can be overridden by other classes.
            data = self.prepare_data(data)
            output = self.forward(data)
            loss = self.calculate_loss(output, data)
            loss['total_loss'] = sum(list(loss.values()))
            if step % 100 == 0:
                logging.info(f"Step {step}/{num_batches}, loss: {epoch_loss/(step + 1)}")
            epoch_loss += loss['total_loss'].item()

            if self.model.training:
                self.optimizer.zero_grad()
                loss['total_loss'].backward()
                self.optimizer.step()
            step += 1
        return {'loss': epoch_loss / float(num_batches)}

    def before_epoch(self):
        """
        Do whatever needs to be done before each epoch (e.g. lengthen sequences).
        Can use this to implement curriculum learning.
        """
        if self.options['curriculum_learning']:
            settings = self.options['curriculum_learning']
            for setting in settings:
                if setting['num_epoch'] == self.num_epoch:
                    logging.info(
                        f"Curriculum learning - running @ epoch: {setting['num_epoch']}, "
                        f"command: {setting['command']}, "
                        f"args: {setting['args']}"
                    )
                    if setting['command'] == 'set_current_length':
                        for key, dataloader in self.dataloaders.items():
                            dataloader.dataset.set_current_length(*setting['args'])
                    if setting['command'] == 'save':
                        self.save(is_best=False, prefix=f'epoch{self.num_epoch}')

    def log_metrics(self, data, step, scope):
        if self.use_tensorboard:
            prefix = 'training' if self.model.training else 'validation'
            for key in data:
                label = os.path.join(prefix, key)
                self.writer.add_scalar(label, data[key], step)
        if self.experiment is not None:
            context = (
                self.experiment.train
                if self.model.training
                else self.experiment.validate
            )
            with context():
                for key in data:
                    self.experiment.log_metric(key, data[key], step)

    def populate_cache(self):
        for key, dataloader in self.dataloaders.items():
            num_batches = len(dataloader)
            if dataloader.dataset.overwrite_cache:
                logging.info(f'Populating cache for {key} w/ {num_batches} batches')
                for i, data in enumerate(dataloader):
                    if i % 50 == 0:
                        logging.info(f'{i}/{num_batches} batches completed')
                    continue
                logging.info(f'Done populating cache for {key}')

    def switch_to_cache(self):
        for key, dataloader in self.dataloaders.items():
            dataloader.dataset.switch_to_cache()

    def fit(self):
        lowest_validation_loss = np.inf

        self.populate_cache()
        self.switch_to_cache()

        logging.info(f"Training data for {self.options['num_epochs'] - self.num_epoch} epochs")
        fit_start_time = time.time()

        for self.num_epoch in range(self.num_epoch, self.options['num_epochs']):
            self.before_epoch()
            start_time = time.time()
            epoch_loss = self.run_epoch('training')
            self.log_metrics(epoch_loss, self.num_epoch, 'epoch')
            validation_loss = self.validate('validation')
            self.save(validation_loss['loss'] < lowest_validation_loss)
            lowest_validation_loss = (
                validation_loss['loss']
                if validation_loss['loss'] < lowest_validation_loss
                else lowest_validation_loss
            )

            epoch_elapsed_time = time.time() - start_time
            epoch_elapsed_time = time.strftime("%H:%M:%S", time.gmtime(epoch_elapsed_time))
            full_elapsed_time = time.time() - fit_start_time
            full_elapsed_time = time.strftime("%H:%M:%S", time.gmtime(full_elapsed_time))

            logging.info(
                f"Epoch summary: {self.num_epoch}\t"
                f"Training loss: {epoch_loss['loss']}\t"
                f"Validation loss: {validation_loss['loss']}\t"
                f"Epoch took: {epoch_elapsed_time}\t"
                f"Time since start: {full_elapsed_time}"
            )

    def validate(self, key) -> float:
        """Calculate loss on validation set

        Args:
            dataloader - a dataloader yielding the validation data if there is
                any, `None` otherwise

        Returns:
            `np.inf` if there is no validation dataset (`dataloader` is `None`)
            else the loss over the given validation data
        """
        if key not in self.dataloaders:
            return {'loss': np.inf}

        self.model.eval()
        with torch.no_grad():
            validation_loss = self.run_epoch(key)
        self.log_metrics(validation_loss, self.num_epoch, 'epoch')
        self.model.train()
        self.scheduler.step(validation_loss['loss'])
        if self.scheduler.num_bad_epochs + 1 == self.options['patience']:
            logging.info(f'Exceeded patience at epoch {self.num_epoch} - adjusting learning rate')

        return validation_loss

    def save(self, is_best: bool, prefix: str = '') -> str:
        """Saves the model being trained with either `latest` or `best` prefix
        based on validation loss

        Args:
            is_best - whether or not the model is known to have improved
                accuracy from the last epoch (based on validation loss). Always
                False if no validation data was given.
            [path] - path to folder in which to save model. If not given, model
                saved in checkpoint folder

        Returns:
            path to saved model
        """
        if not prefix:
            prefix = 'latest'
        if self.model_tag is not None:
            prefix = f'{prefix}.{self.model_tag}'
        optimizer_path = os.path.join(self.checkpoint_folder, f'{prefix}.opt.pth')
        model_path = os.path.join(
             self.checkpoint_folder,
            f'{prefix}.model.pth'
        )
        dataset_options = self.dataloaders['training'].dataset.options
        metadata = {
            key: val
            for key, val in dataset_options.items()
            if key in [
                'n_fft',
                'hop_length',
                'format',
                'sample_rate',
                'num_channels'
            ]
        }

        optimizer_state = {
            'optimizer': self.optimizer.state_dict(),
            'num_epoch': self.num_epoch
        }

        torch.save(optimizer_state, optimizer_path)
        self.module.save(model_path, {'metadata': metadata})
        if is_best:
            torch.save(optimizer_state, optimizer_path.replace('latest', 'best'))
            self.module.save(model_path.replace('latest', 'best'), {'metadata': metadata})
        return model_path

    def resume(self, resume_location, load_only_model=True, prefixes=('best', 'latest')):
        for prefix in prefixes:
            checkpoint_folder = self.checkpoint_folder
            if resume_location is not None:
                checkpoint_folder = resume_location

            optimizer_path = os.path.join(checkpoint_folder, f'{prefix}.opt.pth')
            model_path = os.path.join(checkpoint_folder, f'{prefix}.model.pth')
            logging.info(f'Looking for {model_path}')

            if os.path.exists(model_path) and os.path.exists(optimizer_path):
                optimizer_state = torch.load(optimizer_path, map_location=lambda storage, loc: storage)
                model_dict = torch.load(model_path, map_location=lambda storage, loc: storage)
                self.module.load_state_dict(model_dict['state_dict'])
                if not load_only_model:
                    self.optimizer.load_state_dict(optimizer_state['optimizer'])
                    self.num_epoch = optimizer_state['num_epoch']
                logging.info(f'Resuming from {model_path}')
                return model_path
            else:
                model_path = None
        logging.info('No model found! Training a new model from scratch.')
        return model_path<|endoftext|>


<commit_before>"""Collected page objects for Commotion Router web UI.
    Individual page elements are contained in xelements.py files.
"""

import objects.router.router as cro
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException

# Core identifiers for each page.
LOCATORS = {
    "common": {
        "commotion_logo": "device", # ID
        "version": "credits", # Class - requires add'l filtering
    },
    "login": {
        "password_field": "focus_password", # ID
        "error": "error", # Class
        "submit": "cbi-button-apply", # Class
        "reset": "cbi-button-reset", # Class
    },
    "home": {
        "apps-header": "appsH2", # Class
        "user-add-app": "add_app", # ID
    },
    "admin": {
        "url-stok": False,
        "logout": False,
    },
}

class CRCommonPage(object):
    """Page objects common to all Commotion Router pages"""

    def __init__(self):
        """Set page within context of router."""
        page_url = None
        __sb = None
        commotion_node_ip = None

    # This is dumb and duplicative. We only need node_ip.
    __, commotion_client_ip = cro.get_commotion_client_ip()
    commotion_node_ip = cro.get_commotion_node_ip(commotion_client_ip)

    def _verify_correct_page(self, __sb, page_url):
        """Sanity check defined page url against url in browser"""
        __sb.get(page_url)

        # Wait for known-good page element
        self.wait_for_page_load(__sb)

        try:
            # this assert may not work as expected
            assert (__sb.current_url == page_url) is True
            print __sb.current_url + " matches " + page_url
        except AssertionError:
            print "Rendered url %s does not match expected url %s" % (
                    __sb.current_url, page_url)

    def wait_for_page_load(self, __sb):
        """Tell selenium to wait for locator before proceeding"""
        print "Waiting for presence of known-good page element"
        try:
            WebDriverWait(__sb, 10).until(
                EC.presence_of_element_located((By.ID, "device")))
        except NoSuchElementException:
            message = "Page element 'device' not found!"
            print message
            raise message
        else:
            print "%s loaded successfully" % __sb.current_url

    def wait_for_element_of_type(self, __sb, etype, element):
        """
        Tell selenium to wait for a specific locator of specific type
        before proceeding.

        Valid types: ID, CLASS_NAME, CSS_SELECTOR, LINK_TEXT, NAME,
            PARTIAL_LINK_TEXT, TAG_NAME, XPATH
        """
        print "Waiting for %s, type %s" % (element, etype)
        try:
            WebDriverWait(__sb, 10).until(
                EC.presence_of_element_located((
                    (
                        getattr(By,etype)), element
                    )))
        except NoSuchElementException:
            print "Page element %s of type %s not found!" % (
                element, etype
                )
        else:
            print "Page element %s found." % element
            return True

    # thisnode url
    # Header
    # Footer
    # Body

class CRHomePage(CRCommonPage):
    """Objects found on Commotion Router's default landing page"""
    def __init__(self, browser):
        super(CRHomePage, self).__init__()
        __sb = browser
        self.page_url = ('https://' + CRCommonPage.commotion_node_ip
            + '/cgi-bin/luci')
        self._verify_correct_page(__sb, self.page_url)

    def show_current_rev(self, __sb, test_rev):
        """Check page footer for commotion version number.
        This is actually a common object but common class
        isn't written to accept tests."""
        print "Checking footer for correct Commotion Revision"
        CRCommonPage.wait_for_element_of_type(
            self, __sb, "CLASS_NAME", LOCATORS["common"]["version"]
            )
        print "Comparing versions"
        page_rev = __sb.find_element_by_class_name(LOCATORS["common"]
                                                   ["version"])
        # Could also use page_rev.text.endswith(test_rev)
        if test_rev not in page_rev.text:
            print "Footer version %s does not match test version %s", (
                page_rev.text, test_rev)
            return False
        else:
            return True

    def users_can_add_apps(self, __sb):
        """When enabled, unprivileged users can add apps from the homepage"""
        print "Checking for app add button..."
        try:
            __sb.find_element_by_id(LOCATORS["home"]["user-add-app"])
        except NoSuchElementException:
            return False
        else:
            print "Users can add applications from the homepage"
            return True

class CRLoginPageObjects(CRCommonPage):
    """Page objects specific to Commotion Router login page.
        Note that the login page triggers a DOM-less cert error
    """

    def __init__(self, browser):
        super(CRLoginPageObjects, self).__init__()
        __sb = browser
        self.page_url = ('https://' + CRCommonPage.commotion_node_ip
            + '/cgi-bin/luci/admin')
        self._verify_correct_page(__sb, self.page_url)

    def password_required(self, __sb):
        """Admin pages should require a password if stok url token is not
        present.
        """
        print "Checking for password field..."
        try:
            __sb.find_element_by_id(LOCATORS["login"]["password_field"])
        except NoSuchElementException:
            print "Login page element %s not found" % (
                LOCATORS["login"]["password_field"]
                )
            return False
        else:
            print "Login page requires a password"
            return True

    def incorrect_pass_returns_error(self, __sb, password):
        """The login form should reject incorrect passwords"""
        print "Testing user-supplied password"
        __sb.find_element_by_id(
            LOCATORS["login"]["password_field"]
            ).send_keys(password)

        if "\n" not in password:
            # Click submit if password doesn't have a newline
            __sb.find_element_by_class_name("cbi-button-apply").click()

        CRCommonPage.wait_for_element_of_type(
            self, __sb, "CLASS_NAME", LOCATORS["login"]["error"]
            )
        # Rewrite as try/except NoSuchElementException/else
        if __sb.find_element_by_class_name(
            LOCATORS["login"]["error"]).is_displayed():
            print "Login page displays error message on incorrect password"
            return True
        else:
            print "Login page does not display error message " \
                "on incorrect password"
            return False

    def correct_pass_allows_access(self, __sb, password):
        """Correct password in login form should allow access to admin
        pages"""
        pass

class CRAdminPageObjects(CRCommonPage):
    """Page objects accessible only to authenticated admin users"""
    # Side Nav
    ## Logout
    # URL stok (Note: This also allows csrf vuln)
    #page_url = 'https://' + self.netinfo.commotion_node_ip \
            #+ '/cgi-bin/luci/admin'
    pass
<commit_msg>Renamed router login page class<commit_after>"""Collected page objects for Commotion Router web UI.
    Individual page elements are contained in xelements.py files.
"""

import objects.router.router as cro
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException

# Core identifiers for each page.
LOCATORS = {
    "common": {
        "commotion_logo": "device", # ID
        "version": "credits", # Class - requires add'l filtering
    },
    "login": {
        "password_field": "focus_password", # ID
        "error": "error", # Class
        "submit": "cbi-button-apply", # Class
        "reset": "cbi-button-reset", # Class
    },
    "home": {
        "apps-header": "appsH2", # Class
        "user-add-app": "add_app", # ID
    },
    "admin": {
        "url-stok": False,
        "logout": False,
    },
}

class CRCommonPage(object):
    """Page objects common to all Commotion Router pages"""

    def __init__(self):
        """Set page within context of router."""
        page_url = None
        __sb = None
        commotion_node_ip = None

    # This is dumb and duplicative. We only need node_ip.
    __, commotion_client_ip = cro.get_commotion_client_ip()
    commotion_node_ip = cro.get_commotion_node_ip(commotion_client_ip)

    def _verify_correct_page(self, __sb, page_url):
        """Sanity check defined page url against url in browser"""
        __sb.get(page_url)

        # Wait for known-good page element
        self.wait_for_page_load(__sb)

        try:
            # this assert may not work as expected
            assert (__sb.current_url == page_url) is True
            print __sb.current_url + " matches " + page_url
        except AssertionError:
            print "Rendered url %s does not match expected url %s" % (
                    __sb.current_url, page_url)

    def wait_for_page_load(self, __sb):
        """Tell selenium to wait for locator before proceeding"""
        print "Waiting for presence of known-good page element"
        try:
            WebDriverWait(__sb, 10).until(
                EC.presence_of_element_located((By.ID, "device")))
        except NoSuchElementException:
            message = "Page element 'device' not found!"
            print message
            raise message
        else:
            print "%s loaded successfully" % __sb.current_url

    def wait_for_element_of_type(self, __sb, etype, element):
        """
        Tell selenium to wait for a specific locator of specific type
        before proceeding.

        Valid types: ID, CLASS_NAME, CSS_SELECTOR, LINK_TEXT, NAME,
            PARTIAL_LINK_TEXT, TAG_NAME, XPATH
        """
        print "Waiting for %s, type %s" % (element, etype)
        try:
            WebDriverWait(__sb, 10).until(
                EC.presence_of_element_located((
                    (
                        getattr(By,etype)), element
                    )))
        except NoSuchElementException:
            print "Page element %s of type %s not found!" % (
                element, etype
                )
        else:
            print "Page element %s found." % element
            return True

    # thisnode url
    # Header
    # Footer
    # Body

class CRHomePage(CRCommonPage):
    """Objects found on Commotion Router's default landing page"""
    def __init__(self, browser):
        super(CRHomePage, self).__init__()
        __sb = browser
        self.page_url = ('https://' + CRCommonPage.commotion_node_ip
            + '/cgi-bin/luci')
        self._verify_correct_page(__sb, self.page_url)

    def show_current_rev(self, __sb, test_rev):
        """Check page footer for commotion version number.
        This is actually a common object but common class
        isn't written to accept tests."""
        print "Checking footer for correct Commotion Revision"
        CRCommonPage.wait_for_element_of_type(
            self, __sb, "CLASS_NAME", LOCATORS["common"]["version"]
            )
        print "Comparing versions"
        page_rev = __sb.find_element_by_class_name(LOCATORS["common"]
                                                   ["version"])
        # Could also use page_rev.text.endswith(test_rev)
        if test_rev not in page_rev.text:
            print "Footer version %s does not match test version %s", (
                page_rev.text, test_rev)
            return False
        else:
            return True

    def users_can_add_apps(self, __sb):
        """When enabled, unprivileged users can add apps from the homepage"""
        print "Checking for app add button..."
        try:
            __sb.find_element_by_id(LOCATORS["home"]["user-add-app"])
        except NoSuchElementException:
            return False
        else:
            print "Users can add applications from the homepage"
            return True

class CRLoginPage(CRCommonPage):
    """Page objects specific to Commotion Router login page.
        Note that the login page triggers a DOM-less cert error
    """

    def __init__(self, browser):
        super(CRLoginPage, self).__init__()
        __sb = browser
        self.page_url = ('https://' + CRCommonPage.commotion_node_ip
            + '/cgi-bin/luci/admin')
        self._verify_correct_page(__sb, self.page_url)

    def password_required(self, __sb):
        """Admin pages should require a password if stok url token is not
        present.
        """
        print "Checking for password field..."
        try:
            __sb.find_element_by_id(LOCATORS["login"]["password_field"])
        except NoSuchElementException:
            print "Login page element %s not found" % (
                LOCATORS["login"]["password_field"]
                )
            return False
        else:
            print "Login page requires a password"
            return True

    def incorrect_pass_returns_error(self, __sb, password):
        """The login form should reject incorrect passwords"""
        print "Testing user-supplied password"
        __sb.find_element_by_id(
            LOCATORS["login"]["password_field"]
            ).send_keys(password)

        if "\n" not in password:
            # Click submit if password doesn't have a newline
            __sb.find_element_by_class_name("cbi-button-apply").click()

        CRCommonPage.wait_for_element_of_type(
            self, __sb, "CLASS_NAME", LOCATORS["login"]["error"]
            )
        # Rewrite as try/except NoSuchElementException/else
        if __sb.find_element_by_class_name(
            LOCATORS["login"]["error"]).is_displayed():
            print "Login page displays error message on incorrect password"
            return True
        else:
            print "Login page does not display error message " \
                "on incorrect password"
            return False

    def correct_pass_allows_access(self, __sb, password):
        """Correct password in login form should allow access to admin
        pages"""
        pass

class CRAdminPageObjects(CRCommonPage):
    """Page objects accessible only to authenticated admin users"""
    # Side Nav
    ## Logout
    # URL stok (Note: This also allows csrf vuln)
    #page_url = 'https://' + self.netinfo.commotion_node_ip \
            #+ '/cgi-bin/luci/admin'
    pass
<|endoftext|>