<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from jinja2 import Environment, PackageLoader

import regex
import logging
import datetime
import numpy as np
import shapely.geometry as geom

from shapely.ops import unary_union
from collections import Counter

from kraken.rpred import ocr_record
from kraken.lib.util import make_printable
from kraken.lib.segmentation import is_in_region

from typing import List, Tuple, Iterable, Optional, Sequence, Dict

logger = logging.getLogger(__name__)

__all__ = ['serialize', 'render_report']

def _rescale(val: Sequence[float], low: float, high: float) -> List[float]:
    """
    Rescales a list of confidence value between 0 and 1 to an interval [low,
    high].

    Args:
        val (float): List of values in interval (0,1)
        low (float): Lower bound of rescaling interval
        high (float): Upper bound of rescaling interval

    Returns:
        Rescaled value (float).
    """
    return [(high - low) * x + low for x in val]

def max_bbox(boxes: Iterable[Sequence[int]]) -> Tuple[int, int, int, int]:
    """
    Calculates the minimal bounding box containing all contained in an
    iterator.

    Args:
        boxes (iterator): An iterator returning tuples of the format ((x0, y0),
        (x1, y1), ... (xn, yn)).
    Returns:
        A box (x0, y0, x1, y1) covering all bounding boxes in the input
        argument.
    """
    flat_box = [point for pol in boxes for point in pol]
    flat_box = [x for point in flat_box for x in point]
    xmin, xmax = min(flat_box[::2]), max(flat_box[::2])
    ymin, ymax = min(flat_box[1::2]), max(flat_box[1::2])
    o = xmin, ymin, xmax, ymax  # type: ignore
    return o

def serialize(records: Sequence[ocr_record],
              image_name: str = None,
              image_size: Tuple[int, int] = (0, 0),
              writing_mode: str = 'horizontal-tb',
              scripts: Optional[Iterable[str]] = None,
              regions: Optional[Dict[str, List[List[Tuple[int, int]]]]] = None,
              template: str = 'hocr') -> str:
    """
    Serializes a list of ocr_records into an output document.

    Serializes a list of predictions and their corresponding positions by doing
    some hOCR-specific preprocessing and then renders them through one of
    several jinja2 templates.

    Note: Empty records are ignored for serialization purposes.

    Args:
        records (iterable): List of kraken.rpred.ocr_record
        image_name (str): Name of the source image
        image_size (tuple): Dimensions of the source image
        writing_mode (str): Sets the principal layout of lines and the
                            direction in which blocks progress. Valid values
                            are horizontal-tb, vertical-rl, and
                            vertical-lr.
        scripts (list): List of scripts contained in the OCR records
        regions (list): Dictionary mapping region types to a list of region
                        polygons.
        template (str): Selector for the serialization format. May be
                        'hocr' or 'alto'.

    Returns:
            (str) rendered template.
    """
    logger.info(f'Serialize {len(records)} records from {image_name} with template {template}.')
    page = {'entities': [],
            'size': image_size,
            'name': image_name,
            'writing_mode': writing_mode,
            'scripts': scripts,
            'date': datetime.datetime.now().isoformat(),
            'base_dir': [rec.base_dir for rec in records][0]}  # type: dict
    seg_idx = 0
    char_idx = 0
    region_map = {}
    idx = 0
    if regions is not None:
        for id, regs in regions.items():
            for reg in regs:
                region_map[idx] = (id, geom.Polygon(reg), reg)
                idx += 1

    # build region and line type dict
    page['types'] = list(set(line.script for line in records if line.script is not None))
    if regions is not None:
        page['types'].extend(list(regions.keys()))

    is_in_reg = -1
    for idx, record in enumerate(records):
        if record.type == 'baselines':
            l_obj = geom.LineString(record.baseline)
        else:
            l_obj = geom.LineString(record.line)
        reg = list(filter(lambda x: is_in_region(l_obj, x[1][1]), region_map.items()))
        if len(reg) == 0:
            cur_ent = page['entities']
        elif reg[0][0] != is_in_reg:
            reg = reg[0]
            is_in_reg = reg[0]
            region = {'index': reg[0],
                      'bbox': [int(x) for x in reg[1][1].bounds],
                      'boundary': [list(x) for x in reg[1][2]],
                      'region_type': reg[1][0],
                      'lines': [],
                      'type': 'region'
                      }
            page['entities'].append(region)
            cur_ent = region['lines']

        # set field to indicate the availability of baseline segmentation in
        # addition to bounding boxes
        if record.type == 'baselines':
            page['seg_type'] = 'baselines'
        line = {'index': idx,
                'bbox': max_bbox([record.line]),
                'cuts': record.cuts,
                'confidences': record.confidences,
                'recognition': [],
                'boundary': [list(x) for x in record.line],
                'type': 'line'
                }
        if record.script is not None:
            line['script'] = record.script
        if record.type == 'baselines':
            line['baseline'] = [list(x) for x in record.baseline]
        splits = regex.split(r'(\s+)', record.prediction)
        line_offset = 0
        logger.debug(f'Record contains {len(splits)} segments')
        for segment in splits:
            if len(segment) == 0:
                continue
            seg_bbox = max_bbox(record.cuts[line_offset:line_offset + len(segment)])
            seg_struct = {'bbox': seg_bbox,
                          'confidences': record.confidences[line_offset:line_offset + len(segment)],
                          'cuts': record.cuts[line_offset:line_offset + len(segment)],
                          'text': segment,
                          'recognition': [{'bbox': max_bbox([cut]),
                                           'boundary': cut,
                                           'confidence': conf,
                                           'text': char,
                                           'index': cid}
                                          for conf, cut, char, cid in
                                          zip(record.confidences[line_offset:line_offset + len(segment)],
                                              record.cuts[line_offset:line_offset + len(segment)],
                                              segment,
                                              range(char_idx, char_idx + len(segment)))],
                          'index': seg_idx}
            # compute convex hull of all characters in segment
            if record.type == 'baselines':
                pols = []
                for x in record.cuts[line_offset:line_offset + len(segment)]:
                    try:
                        pol = geom.Polygon(x)
                    except ValueError:
                        pol = geom.LineString(x).buffer(0.5, cap_style=2)
                    if pol.area == 0.0:
                        pol = pol.buffer(0.5)
                    # if area is still 0 it's probably a point
                    if pol.area == 0.0:
                        pol = geom.Point(x[0]).buffer(0.5)
                    pols.append(pol)
                pols = unary_union(pols)
                coords = np.array(pols.convex_hull.exterior.coords, dtype=np.uint).tolist()
                seg_struct['boundary'] = coords
            line['recognition'].append(seg_struct)
            char_idx += len(segment)
            seg_idx += 1
            line_offset += len(segment)
        cur_ent.append(line)
    logger.debug('Initializing jinja environment.')
    env = Environment(loader=PackageLoader('kraken', 'templates'),
                      trim_blocks=True,
                      lstrip_blocks=False,
                      autoescape=True)
    env.tests['whitespace'] = str.isspace
    env.filters['rescale'] = _rescale
    logger.debug('Retrieving template.')
    tmpl = env.get_template(template)
    logger.debug('Rendering data.')
    return tmpl.render(page=page)

def render_report(model: str,
                  chars: int,
                  errors: int,
                  char_confusions: Counter,
                  scripts: Counter,
                  insertions: Counter,
                  deletions: int,
                  substitutions: Counter) -> str:
    """
    Renders an accuracy report.

    Args:
        model (str): Model name.
        errors (int): Number of errors on test set.
        char_confusions (dict): Dictionary mapping a tuple (gt, pred) to a
                                number of occurrences.
        scripts (dict): Dictionary counting character per script.
        insertions (dict): Dictionary counting insertion operations per Unicode
                           script
        deletions (int): Number of deletions
        substitutions (dict): Dictionary counting substitution operations per
                              Unicode script.

    Returns:
        A string containing the rendered report.
    """
    logger.info(f'Serializing report for {model}.')

    report = {'model': model,
              'chars': chars,
              'errors': errors,
              'accuracy': (chars-errors)/chars * 100,
              'insertions': sum(insertions.values()),
              'deletions': deletions,
              'substitutions': sum(substitutions.values()),
              'scripts': sorted([{'script': k,
                                  'count': v,
                                  'errors': insertions[k] + substitutions[k],
                                  'accuracy': 100 * (v-(insertions[k] + substitutions[k]))/v} for k, v in scripts.items()],
                                key=lambda x: x['accuracy'],
                                reverse=True),
              'counts': sorted([{'correct': make_printable(k[0]),
                                 'generated': make_printable(k[1]),
                                 'errors': v} for k, v in char_confusions.items() if k[0] != k[1]],
                               key=lambda x: x['errors'],
                               reverse=True)}
    logger.debug('Initializing jinja environment.')
    env = Environment(loader=PackageLoader('kraken', 'templates'),
                      trim_blocks=True,
                      lstrip_blocks=True,
                      autoescape=True)
    logger.debug('Retrieving template.')
    tmpl = env.get_template('report')
    logger.debug('Rendering data.')
    return tmpl.render(report=report)
<commit_msg>Fix crash during serialization of pages without lines<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from jinja2 import Environment, PackageLoader

import regex
import logging
import datetime
import numpy as np
import shapely.geometry as geom

from shapely.ops import unary_union
from collections import Counter

from kraken.rpred import ocr_record
from kraken.lib.util import make_printable
from kraken.lib.segmentation import is_in_region

from typing import List, Tuple, Iterable, Optional, Sequence, Dict

logger = logging.getLogger(__name__)

__all__ = ['serialize', 'render_report']

def _rescale(val: Sequence[float], low: float, high: float) -> List[float]:
    """
    Rescales a list of confidence value between 0 and 1 to an interval [low,
    high].

    Args:
        val (float): List of values in interval (0,1)
        low (float): Lower bound of rescaling interval
        high (float): Upper bound of rescaling interval

    Returns:
        Rescaled value (float).
    """
    return [(high - low) * x + low for x in val]

def max_bbox(boxes: Iterable[Sequence[int]]) -> Tuple[int, int, int, int]:
    """
    Calculates the minimal bounding box containing all contained in an
    iterator.

    Args:
        boxes (iterator): An iterator returning tuples of the format ((x0, y0),
        (x1, y1), ... (xn, yn)).
    Returns:
        A box (x0, y0, x1, y1) covering all bounding boxes in the input
        argument.
    """
    flat_box = [point for pol in boxes for point in pol]
    flat_box = [x for point in flat_box for x in point]
    xmin, xmax = min(flat_box[::2]), max(flat_box[::2])
    ymin, ymax = min(flat_box[1::2]), max(flat_box[1::2])
    o = xmin, ymin, xmax, ymax  # type: ignore
    return o

def serialize(records: Sequence[ocr_record],
              image_name: str = None,
              image_size: Tuple[int, int] = (0, 0),
              writing_mode: str = 'horizontal-tb',
              scripts: Optional[Iterable[str]] = None,
              regions: Optional[Dict[str, List[List[Tuple[int, int]]]]] = None,
              template: str = 'hocr') -> str:
    """
    Serializes a list of ocr_records into an output document.

    Serializes a list of predictions and their corresponding positions by doing
    some hOCR-specific preprocessing and then renders them through one of
    several jinja2 templates.

    Note: Empty records are ignored for serialization purposes.

    Args:
        records (iterable): List of kraken.rpred.ocr_record
        image_name (str): Name of the source image
        image_size (tuple): Dimensions of the source image
        writing_mode (str): Sets the principal layout of lines and the
                            direction in which blocks progress. Valid values
                            are horizontal-tb, vertical-rl, and
                            vertical-lr.
        scripts (list): List of scripts contained in the OCR records
        regions (list): Dictionary mapping region types to a list of region
                        polygons.
        template (str): Selector for the serialization format. May be
                        'hocr' or 'alto'.

    Returns:
            (str) rendered template.
    """
    logger.info(f'Serialize {len(records)} records from {image_name} with template {template}.')
    page = {'entities': [],
            'size': image_size,
            'name': image_name,
            'writing_mode': writing_mode,
            'scripts': scripts,
            'date': datetime.datetime.now().isoformat(),
            'base_dir': [rec.base_dir for rec in records][0] if len(records) else None}  # type: dict
    seg_idx = 0
    char_idx = 0
    region_map = {}
    idx = 0
    if regions is not None:
        for id, regs in regions.items():
            for reg in regs:
                region_map[idx] = (id, geom.Polygon(reg), reg)
                idx += 1

    # build region and line type dict
    page['types'] = list(set(line.script for line in records if line.script is not None))
    if regions is not None:
        page['types'].extend(list(regions.keys()))

    is_in_reg = -1
    for idx, record in enumerate(records):
        if record.type == 'baselines':
            l_obj = geom.LineString(record.baseline)
        else:
            l_obj = geom.LineString(record.line)
        reg = list(filter(lambda x: is_in_region(l_obj, x[1][1]), region_map.items()))
        if len(reg) == 0:
            cur_ent = page['entities']
        elif reg[0][0] != is_in_reg:
            reg = reg[0]
            is_in_reg = reg[0]
            region = {'index': reg[0],
                      'bbox': [int(x) for x in reg[1][1].bounds],
                      'boundary': [list(x) for x in reg[1][2]],
                      'region_type': reg[1][0],
                      'lines': [],
                      'type': 'region'
                      }
            page['entities'].append(region)
            cur_ent = region['lines']

        # set field to indicate the availability of baseline segmentation in
        # addition to bounding boxes
        if record.type == 'baselines':
            page['seg_type'] = 'baselines'
        line = {'index': idx,
                'bbox': max_bbox([record.line]),
                'cuts': record.cuts,
                'confidences': record.confidences,
                'recognition': [],
                'boundary': [list(x) for x in record.line],
                'type': 'line'
                }
        if record.script is not None:
            line['script'] = record.script
        if record.type == 'baselines':
            line['baseline'] = [list(x) for x in record.baseline]
        splits = regex.split(r'(\s+)', record.prediction)
        line_offset = 0
        logger.debug(f'Record contains {len(splits)} segments')
        for segment in splits:
            if len(segment) == 0:
                continue
            seg_bbox = max_bbox(record.cuts[line_offset:line_offset + len(segment)])
            seg_struct = {'bbox': seg_bbox,
                          'confidences': record.confidences[line_offset:line_offset + len(segment)],
                          'cuts': record.cuts[line_offset:line_offset + len(segment)],
                          'text': segment,
                          'recognition': [{'bbox': max_bbox([cut]),
                                           'boundary': cut,
                                           'confidence': conf,
                                           'text': char,
                                           'index': cid}
                                          for conf, cut, char, cid in
                                          zip(record.confidences[line_offset:line_offset + len(segment)],
                                              record.cuts[line_offset:line_offset + len(segment)],
                                              segment,
                                              range(char_idx, char_idx + len(segment)))],
                          'index': seg_idx}
            # compute convex hull of all characters in segment
            if record.type == 'baselines':
                pols = []
                for x in record.cuts[line_offset:line_offset + len(segment)]:
                    try:
                        pol = geom.Polygon(x)
                    except ValueError:
                        pol = geom.LineString(x).buffer(0.5, cap_style=2)
                    if pol.area == 0.0:
                        pol = pol.buffer(0.5)
                    # if area is still 0 it's probably a point
                    if pol.area == 0.0:
                        pol = geom.Point(x[0]).buffer(0.5)
                    pols.append(pol)
                pols = unary_union(pols)
                coords = np.array(pols.convex_hull.exterior.coords, dtype=np.uint).tolist()
                seg_struct['boundary'] = coords
            line['recognition'].append(seg_struct)
            char_idx += len(segment)
            seg_idx += 1
            line_offset += len(segment)
        cur_ent.append(line)
    logger.debug('Initializing jinja environment.')
    env = Environment(loader=PackageLoader('kraken', 'templates'),
                      trim_blocks=True,
                      lstrip_blocks=False,
                      autoescape=True)
    env.tests['whitespace'] = str.isspace
    env.filters['rescale'] = _rescale
    logger.debug('Retrieving template.')
    tmpl = env.get_template(template)
    logger.debug('Rendering data.')
    return tmpl.render(page=page)

def render_report(model: str,
                  chars: int,
                  errors: int,
                  char_confusions: Counter,
                  scripts: Counter,
                  insertions: Counter,
                  deletions: int,
                  substitutions: Counter) -> str:
    """
    Renders an accuracy report.

    Args:
        model (str): Model name.
        errors (int): Number of errors on test set.
        char_confusions (dict): Dictionary mapping a tuple (gt, pred) to a
                                number of occurrences.
        scripts (dict): Dictionary counting character per script.
        insertions (dict): Dictionary counting insertion operations per Unicode
                           script
        deletions (int): Number of deletions
        substitutions (dict): Dictionary counting substitution operations per
                              Unicode script.

    Returns:
        A string containing the rendered report.
    """
    logger.info(f'Serializing report for {model}.')

    report = {'model': model,
              'chars': chars,
              'errors': errors,
              'accuracy': (chars-errors)/chars * 100,
              'insertions': sum(insertions.values()),
              'deletions': deletions,
              'substitutions': sum(substitutions.values()),
              'scripts': sorted([{'script': k,
                                  'count': v,
                                  'errors': insertions[k] + substitutions[k],
                                  'accuracy': 100 * (v-(insertions[k] + substitutions[k]))/v} for k, v in scripts.items()],
                                key=lambda x: x['accuracy'],
                                reverse=True),
              'counts': sorted([{'correct': make_printable(k[0]),
                                 'generated': make_printable(k[1]),
                                 'errors': v} for k, v in char_confusions.items() if k[0] != k[1]],
                               key=lambda x: x['errors'],
                               reverse=True)}
    logger.debug('Initializing jinja environment.')
    env = Environment(loader=PackageLoader('kraken', 'templates'),
                      trim_blocks=True,
                      lstrip_blocks=True,
                      autoescape=True)
    logger.debug('Retrieving template.')
    tmpl = env.get_template('report')
    logger.debug('Rendering data.')
    return tmpl.render(report=report)
<|endoftext|>


<commit_before>import sys
import socket
import fcntl
import struct
import random
import os
import shutil
import subprocess
import time
import ipaddress

# Run `python3 -m unittest discover` in this dir to execute tests

default_mount_options_nfs = "nfs hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0"
default_mount_options_cifs = "dir_mode=0777,file_mode=0777,serverino,nofail,vers=3.0"

def get_avere_ips(vserver_string):
    vserver_range = vserver_string.split("-")
    if len(vserver_range) < 2:
        print(
            "Expect vserver ip range to be in the format 'startip-endip' got:{}".format(
                vserver_string
            )
        )
        exit(3)

    all_ips = []
    start_ip = vserver_range[0]
    end_ip = vserver_range[1]

    start_ip_parts = start_ip.split(".")
    start_ip_last_digit = int(start_ip_parts[3])
    ip_prefix = "{}.{}.{}.".format(
        start_ip_parts[0], start_ip_parts[1], start_ip_parts[2]
    )
    end_ip_last_digit = int(end_ip.split(".")[3])

    for ip in range(start_ip_last_digit, end_ip_last_digit + 1):
        all_ips.append(ip_prefix + str(ip))

    return all_ips

def get_ip_address():
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        try:
            # doesn't even have to be reachable
            s.connect(("10.255.255.255", 1))
            return s.getsockname()[0]
        except:
            return "127.0.0.1"

def ip_as_int(ip):
    return int(ipaddress.ip_address(ip))

def remove_lines_containing(file, contains):
    with open(file, "r+") as file:
        d = file.readlines()
        file.seek(0)
        for i in d:
            if contains not in i and i != "\n":
                file.write(i)
        file.truncate()

def print_help():
    print("For example 'sudo python mount.py avere 10.0.1.10-10.0.1.14'")
    print(
        "or 'sudo python mount.py azurefiles <storage-account-name>,<share-name>,<storage-account-key>'"
    )

def install_apt_package(package):
    try:
        print("Attempt to install {}".format(package))
        subprocess.check_call(["apt", "install", package, "-y"])
        print("Install completed successfully")
    except subprocess.CalledProcessError as e:
        print("Failed install {} error: {}".format(package, e))
        exit(4)

# main allows the the mount script to be executable
def main():
    if len(sys.argv) < 3:
        print("Expected arg1: 'mount_type' and arg2 'mount_data'")
        print_help()
        exit(1)

    mount_type = str(sys.argv[1])
    mount_data = str(sys.argv[2])

    mount_share(mount_type, mount_data)

# mount_share allows it to be invoked from other python scripts
def mount_share(mount_type, mount_data):

    if mount_type.lower() != "avere" and mount_type.lower() != "azurefiles":
        print("Expected first arg to be either 'avere' or 'azurefiles'")
        print_help()
        exit(1)

    if mount_data == "":
        print(
            "Expected second arg to be the mounting data. For avere, this is the vserver iprange. For azure files this should be the azure files connection details."
        )
        print_help()
        exit(2)

    print("Mounting type: {}".format(sys.argv[1]))
    print("Mounting data: {}".format(sys.argv[2]))

    mount_point_permissions = 0o0777  # Todo: What permissions does this really need?
    primary_mount_folder = "/media/primarynfs"
    seconday_mount_folder_prefix = "/media/secondarynfs"

    try:
        # Create folder to mount to
        if not os.path.exists(primary_mount_folder):
            os.makedirs(primary_mount_folder)
            os.chmod(primary_mount_folder, mount_point_permissions)

        # Make a backup of the fstab config incase we go wrong
        shutil.copy("/etc/fstab", "/etc/fstab-averescriptbackup")

        # Clear existing NFS mount data to make script idempotent
        remove_lines_containing("/etc/fstab", primary_mount_folder)
        remove_lines_containing("/etc/fstab", seconday_mount_folder_prefix)

        if mount_type.lower() == "azurefiles":
            mount_azurefiles(mount_data, primary_mount_folder)

        if mount_type.lower() == "avere":
            mount_avere(mount_data, primary_mount_folder, mount_point_permissions)

    except IOError as e:
        print("I/O error({0})".format(e))
        exit(1)
    except:
        print("Unexpected error:{0}".format, sys.exc_info())
        raise

    print("Done editing fstab ... attempting mount")

    def mount_all():
        subprocess.check_call(["mount", "-a"])

    retryFunc("mount shares", mount_all, 100)

def retryFunc(desc, funcToRetry, maxRetries):
    # Retry mounting for a while to handle race where VM exists before storage
    # or temporary issue with storage
    print("Attempting, with retries, to: {}".format(desc))
    retryExponentialFactor = 3
    for i in range(1, maxRetries):
        if i == maxRetries:
            print("Failed after max retries")
            exit(3)
        try:
            print("Attempt #{}".format(str(i)))
            funcToRetry()
        except subprocess.CalledProcessError as e:
            print("Failed:{0}".format(e))
            retry_in = i * retryExponentialFactor
            print("retrying in {0}secs".format(retry_in))
            time.sleep(retry_in)
            continue
        else:
            print("Succeeded to: {0} after {1} retries".format(desc, i))
            break

def mount_avere(mount_data, primary_mount_folder, mount_point_permissions):
    # Other apt instances on the machine may be doing an install
    # this means ours will fail so we retry to ensure success
    def install_nfs():
        install_apt_package("nfs-common")

    retryFunc("install nfs-common", install_nfs, 20)

    ips = get_avere_ips(mount_data)
    print("Found ips:{}".format(",".join(ips)))

    # Deterministically select a primary node from the available
    # servers for this vm to use. By using the ip as a seed this ensures
    # re-running will get the same node as primary
    current_ip = get_ip_address()
    current_ip_int = ip_as_int(current_ip)
    print("Using ip as int: {0} for random seed".format((current_ip_int)))
    random.seed(current_ip_int)
    random_node = random.randint(0, len(ips) - 1)

    primary = ips[random_node]
    ips.remove(primary)
    secondarys = ips

    print("Primary node selected: {}".format(primary))
    print("Secondary nodes selected: {}".format(",".join(secondarys)))

    with open("/etc/fstab", "a") as file:

        print("Mounting primary")
        file.write(
            "\n{}:/msazure {} {} \n".format(
                primary, primary_mount_folder, default_mount_options_nfs
            )
        )

        print("Mounting secondarys")
        number = 0
        for ip in secondarys:
            number = number + 1
            folder = "/media/secondarynfs" + str(number)
            if not os.path.exists(folder):
                os.makedirs(folder)
                os.chmod(folder, mount_point_permissions)

            file.write(
                "\n{}:/msazure {} {} \n".format(ip, folder, default_mount_options_nfs)
            )

def mount_azurefiles(mount_data, primary_mount_folder):
    # Other apt instances on the machine may be doing an install
    # this means ours will fail so we retry to ensure success
    def install_cifs():
         install_apt_package("cifs-utils")

    retryFunc("install cifs-utils", install_cifs, 20)

    params = mount_data.split(",")
    if len(params) != 3:
        print("Wrong params for azure files mount, expected 3 as CSV")
        print_help()
        exit(1)

    account_name = params[0]
    share_name = params[1]
    account_key = params[2]

    with open("/etc/fstab", "a") as file:
        print("Mounting primary")
        file.write(
            "\n//{0}.file.core.windows.net/{1} {2} cifs username={0},password={3},{4} \n".format(
                account_name,
                share_name,
                primary_mount_folder,
                account_key,
                default_mount_options_cifs,
            )
        )

if __name__ == "__main__":
    main()

<commit_msg>Change package install func to play nice with retry<commit_after>import sys
import socket
import fcntl
import struct
import random
import os
import shutil
import subprocess
import time
import ipaddress

# Run `python3 -m unittest discover` in this dir to execute tests

default_mount_options_nfs = "nfs hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0"
default_mount_options_cifs = "dir_mode=0777,file_mode=0777,serverino,nofail,vers=3.0"

def get_avere_ips(vserver_string):
    vserver_range = vserver_string.split("-")
    if len(vserver_range) < 2:
        print(
            "Expect vserver ip range to be in the format 'startip-endip' got:{}".format(
                vserver_string
            )
        )
        exit(3)

    all_ips = []
    start_ip = vserver_range[0]
    end_ip = vserver_range[1]

    start_ip_parts = start_ip.split(".")
    start_ip_last_digit = int(start_ip_parts[3])
    ip_prefix = "{}.{}.{}.".format(
        start_ip_parts[0], start_ip_parts[1], start_ip_parts[2]
    )
    end_ip_last_digit = int(end_ip.split(".")[3])

    for ip in range(start_ip_last_digit, end_ip_last_digit + 1):
        all_ips.append(ip_prefix + str(ip))

    return all_ips

def get_ip_address():
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        try:
            # doesn't even have to be reachable
            s.connect(("10.255.255.255", 1))
            return s.getsockname()[0]
        except:
            return "127.0.0.1"

def ip_as_int(ip):
    return int(ipaddress.ip_address(ip))

def remove_lines_containing(file, contains):
    with open(file, "r+") as file:
        d = file.readlines()
        file.seek(0)
        for i in d:
            if contains not in i and i != "\n":
                file.write(i)
        file.truncate()

def print_help():
    print("For example 'sudo python mount.py avere 10.0.1.10-10.0.1.14'")
    print(
        "or 'sudo python mount.py azurefiles <storage-account-name>,<share-name>,<storage-account-key>'"
    )

def install_apt_package(package):
    try:
        print("Attempt to install {}".format(package))
        subprocess.check_call(["apt", "install", package, "-y"])
        print("Install completed successfully")
    except subprocess.CalledProcessError as e:
        print("Failed install {} error: {}".format(package, e))
        raise

# main allows the the mount script to be executable
def main():
    if len(sys.argv) < 3:
        print("Expected arg1: 'mount_type' and arg2 'mount_data'")
        print_help()
        exit(1)

    mount_type = str(sys.argv[1])
    mount_data = str(sys.argv[2])

    mount_share(mount_type, mount_data)

# mount_share allows it to be invoked from other python scripts
def mount_share(mount_type, mount_data):

    if mount_type.lower() != "avere" and mount_type.lower() != "azurefiles":
        print("Expected first arg to be either 'avere' or 'azurefiles'")
        print_help()
        exit(1)

    if mount_data == "":
        print(
            "Expected second arg to be the mounting data. For avere, this is the vserver iprange. For azure files this should be the azure files connection details."
        )
        print_help()
        exit(2)

    print("Mounting type: {}".format(sys.argv[1]))
    print("Mounting data: {}".format(sys.argv[2]))

    mount_point_permissions = 0o0777  # Todo: What permissions does this really need?
    primary_mount_folder = "/media/primarynfs"
    seconday_mount_folder_prefix = "/media/secondarynfs"

    try:
        # Create folder to mount to
        if not os.path.exists(primary_mount_folder):
            os.makedirs(primary_mount_folder)
            os.chmod(primary_mount_folder, mount_point_permissions)

        # Make a backup of the fstab config incase we go wrong
        shutil.copy("/etc/fstab", "/etc/fstab-averescriptbackup")

        # Clear existing NFS mount data to make script idempotent
        remove_lines_containing("/etc/fstab", primary_mount_folder)
        remove_lines_containing("/etc/fstab", seconday_mount_folder_prefix)

        if mount_type.lower() == "azurefiles":
            mount_azurefiles(mount_data, primary_mount_folder)

        if mount_type.lower() == "avere":
            mount_avere(mount_data, primary_mount_folder, mount_point_permissions)

    except IOError as e:
        print("I/O error({0})".format(e))
        exit(1)
    except:
        print("Unexpected error:{0}".format, sys.exc_info())
        raise

    print("Done editing fstab ... attempting mount")

    def mount_all():
        subprocess.check_call(["mount", "-a"])

    retryFunc("mount shares", mount_all, 100)

def retryFunc(desc, funcToRetry, maxRetries):
    # Retry mounting for a while to handle race where VM exists before storage
    # or temporary issue with storage
    print("Attempting, with retries, to: {}".format(desc))
    retryExponentialFactor = 3
    for i in range(1, maxRetries):
        if i == maxRetries:
            print("Failed after max retries")
            exit(3)
        try:
            print("Attempt #{}".format(str(i)))
            funcToRetry()
        except subprocess.CalledProcessError as e:
            print("Failed:{0}".format(e))
            retry_in = i * retryExponentialFactor
            print("retrying in {0}secs".format(retry_in))
            time.sleep(retry_in)
            continue
        else:
            print("Succeeded to: {0} after {1} retries".format(desc, i))
            break

def mount_avere(mount_data, primary_mount_folder, mount_point_permissions):
    # Other apt instances on the machine may be doing an install
    # this means ours will fail so we retry to ensure success
    def install_nfs():
        install_apt_package("nfs-common")

    retryFunc("install nfs-common", install_nfs, 20)

    ips = get_avere_ips(mount_data)
    print("Found ips:{}".format(",".join(ips)))

    # Deterministically select a primary node from the available
    # servers for this vm to use. By using the ip as a seed this ensures
    # re-running will get the same node as primary
    current_ip = get_ip_address()
    current_ip_int = ip_as_int(current_ip)
    print("Using ip as int: {0} for random seed".format((current_ip_int)))
    random.seed(current_ip_int)
    random_node = random.randint(0, len(ips) - 1)

    primary = ips[random_node]
    ips.remove(primary)
    secondarys = ips

    print("Primary node selected: {}".format(primary))
    print("Secondary nodes selected: {}".format(",".join(secondarys)))

    with open("/etc/fstab", "a") as file:

        print("Mounting primary")
        file.write(
            "\n{}:/msazure {} {} \n".format(
                primary, primary_mount_folder, default_mount_options_nfs
            )
        )

        print("Mounting secondarys")
        number = 0
        for ip in secondarys:
            number = number + 1
            folder = "/media/secondarynfs" + str(number)
            if not os.path.exists(folder):
                os.makedirs(folder)
                os.chmod(folder, mount_point_permissions)

            file.write(
                "\n{}:/msazure {} {} \n".format(ip, folder, default_mount_options_nfs)
            )

def mount_azurefiles(mount_data, primary_mount_folder):
    # Other apt instances on the machine may be doing an install
    # this means ours will fail so we retry to ensure success
    def install_cifs():
         install_apt_package("cifs-utils")

    retryFunc("install cifs-utils", install_cifs, 20)

    params = mount_data.split(",")
    if len(params) != 3:
        print("Wrong params for azure files mount, expected 3 as CSV")
        print_help()
        exit(1)

    account_name = params[0]
    share_name = params[1]
    account_key = params[2]

    with open("/etc/fstab", "a") as file:
        print("Mounting primary")
        file.write(
            "\n//{0}.file.core.windows.net/{1} {2} cifs username={0},password={3},{4} \n".format(
                account_name,
                share_name,
                primary_mount_folder,
                account_key,
                default_mount_options_cifs,
            )
        )

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>import pytest
from ..loadxl import ORDER
from encoded.types.experiment import *
pytestmark = pytest.mark.working

@pytest.mark.parametrize('item_type', ORDER)
def test_create_mapping(registry, item_type):
    from snovault.elasticsearch.create_mapping import type_mapping
    from snovault.fourfront_utils import add_default_embeds
    from snovault import TYPES
    mapping = type_mapping(registry[TYPES], item_type)
    assert mapping
    type_info = registry[TYPES].by_item_type[item_type]
    schema = type_info.schema
    embeds = add_default_embeds(type_info.embedded, schema)
    # assert that all embeds exist in mapping for the given type
    for embed in embeds:
        mapping_pointer = mapping
        split_embed = embed.split('.')
        for idx in range(len(split_embed)):
            split_ = split_embed[idx]
            # see if this is last level of embedding- may be a field or object
            if idx == len(split_embed) - 1:
                if 'properties' in mapping_pointer and split_ in mapping_pointer['properties']:
                    final_mapping = mapping_pointer['properties']
                else:
                    final_mapping = mapping_pointer
                assert split_ in final_mapping
                assert 'type' in final_mapping[split_]
            else:
                assert split_ in mapping_pointer['properties']
                mapping_pointer = mapping_pointer['properties'][split_]
<commit_msg>keeping codacy happy<commit_after>import pytest
from ..loadxl import ORDER
from encoded.types.experiment import *
pytestmark = pytest.mark.working

@pytest.mark.parametrize('item_type', ORDER)
def test_create_mapping(registry, item_type):
    from snovault.elasticsearch.create_mapping import type_mapping
    from snovault.fourfront_utils import add_default_embeds
    from snovault import TYPES
    mapping = type_mapping(registry[TYPES], item_type)
    assert mapping
    type_info = registry[TYPES].by_item_type[item_type]
    schema = type_info.schema
    embeds = add_default_embeds(type_info.embedded, schema)
    # assert that all embeds exist in mapping for the given type
    for embed in embeds:
        mapping_pointer = mapping
        split_embed = embed.split('.')
        for idx, split_ in enumerate(split_embed):
            # see if this is last level of embedding- may be a field or object
            if idx == len(split_embed) - 1:
                if 'properties' in mapping_pointer and split_ in mapping_pointer['properties']:
                    final_mapping = mapping_pointer['properties']
                else:
                    final_mapping = mapping_pointer
                assert split_ in final_mapping
                assert 'type' in final_mapping[split_]
            else:
                assert split_ in mapping_pointer['properties']
                mapping_pointer = mapping_pointer['properties'][split_]
<|endoftext|>


<commit_before>"""
Utilities for waffle.

Includes namespacing, caching, and course overrides for waffle flags.

Usage:

For Waffle Flags, first set up the namespace, and then create flags using the
namespace.  For example::

   WAFFLE_FLAG_NAMESPACE = WaffleFlagNamespace(name='my_namespace')

   # Use CourseWaffleFlag when you are in the context of a course.
   SOME_COURSE_FLAG = CourseWaffleFlag(WAFFLE_FLAG_NAMESPACE, 'some_course_feature', __name__)
   # Use WaffleFlag when outside the context of a course.
   SOME_FLAG = WaffleFlag(WAFFLE_FLAG_NAMESPACE, 'some_feature', __name__)

You can check these flags in code using the following::

    SOME_FLAG.is_enabled()
    SOME_COURSE_FLAG.is_enabled(course_key)

To test these WaffleFlags, see testutils.py.

In the above examples, you will use Django Admin "waffle" section to configure
for a flag named: my_namespace.some_course_feature

You could also use the Django Admin "waffle_utils" section to configure a course
override for this same flag (e.g. my_namespace.some_course_feature).

For Waffle Switches, first set up the namespace, and then create the flag name.
For example::

    WAFFLE_SWITCHES = WaffleSwitchNamespace(name=WAFFLE_NAMESPACE)

    ESTIMATE_FIRST_ATTEMPTED = 'estimate_first_attempted'

You can then use the switch as follows::

    WAFFLE_SWITCHES.is_enabled(waffle.ESTIMATE_FIRST_ATTEMPTED)

To test WaffleSwitchNamespace, use the provided context managers.  For example:

    with WAFFLE_SWITCHES.override(waffle.ESTIMATE_FIRST_ATTEMPTED, active=True):
        ...

For long-lived flags, you may want to change the default for devstack, sandboxes,
or new Open edX releases. For help with this, see:
openedx/core/djangoapps/waffle_utils/docs/decisions/0001-refactor-waffle-flag-default.rst

Also see ``WAFFLE_FLAG_CUSTOM_ATTRIBUTES`` and docstring for _set_waffle_flag_attribute
for temporarily instrumenting/monitoring waffle flag usage.

"""
from functools import lru_cache
from contextlib import contextmanager
from weakref import WeakSet

from django.conf import settings
from edx_django_utils.monitoring import set_custom_attribute
from opaque_keys.edx.keys import CourseKey

from edx_toggles.toggles import WaffleFlag as BaseWaffleFlag
from edx_toggles.toggles import WaffleFlagNamespace as BaseWaffleFlagNamespace
from edx_toggles.toggles import WaffleSwitch as BaseWaffleSwitch
from edx_toggles.toggles import WaffleSwitchNamespace as BaseWaffleSwitchNamespace

from openedx.core.lib.cache_utils import get_cache as get_request_cache

class WaffleSwitchNamespace(BaseWaffleSwitchNamespace):
    """
    A waffle switch namespace class that implements request-based caching.
    """

    @property
    def _cached_switches(self):
        """
        Returns a dictionary of all namespaced switches in the request cache.
        """
        return _get_waffle_namespace_request_cache().setdefault("switches", {})

def _get_waffle_namespace_request_cache():
    """
    Returns a request cache shared by all Waffle namespace objects.
    """
    return get_request_cache("WaffleNamespace")

class WaffleSwitch(BaseWaffleSwitch):
    """
    Waffle switch class that benefits from the additional features of the WaffleSwitchNamespace.
    """

    NAMESPACE_CLASS = WaffleSwitchNamespace

class WaffleFlagNamespace(BaseWaffleFlagNamespace):
    """
    Namespace class that implements request-based caching. Also, setup some custom value-checking and processing.
    """

    @property
    def _cached_flags(self):
        """
        Returns a dictionary of all namespaced flags in the request cache.
        """
        return _get_waffle_namespace_request_cache().setdefault("flags", {})

    def _get_flag_active(self, namespaced_flag_name):
        value = super()._get_flag_active(namespaced_flag_name)
        _set_waffle_flag_attribute(namespaced_flag_name, value)
        return value

    def _get_flag_active_default(self, namespaced_flag_name):
        value = super()._get_flag_active_default(namespaced_flag_name)
        set_custom_attribute("warn_flag_no_request_return_value", value)
        return value

# .. toggle_name: WAFFLE_FLAG_CUSTOM_ATTRIBUTES
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
# .. toggle_description: A list of waffle flags to track with custom attributes having
#   values of (True, False, or Both).
# .. toggle_use_cases: opt_in
# .. toggle_creation_date: 2020-06-17
# .. toggle_warnings: Intent is for temporary research (1 day - several weeks) of a flag's usage.
@lru_cache(maxsize=None)
def _get_waffle_flag_custom_attributes_set():
    attributes = getattr(settings, "WAFFLE_FLAG_CUSTOM_ATTRIBUTES", None) or []
    return set(attributes)

def _set_waffle_flag_attribute(name, value):
    """
    For any flag name in settings.WAFFLE_FLAG_CUSTOM_ATTRIBUTES, add name/value
    to cached values and set custom attribute if the value changed.

    The name of the custom attribute will have the prefix ``flag_`` and the
    suffix will match the name of the flag.
    The value of the custom attribute could be False, True, or Both.

    The value Both would mean that the flag had both a True and False
    value at different times during the transaction. This is most
    likely due to having a course override, as is the case with
    CourseWaffleFlag.

    An example NewRelic query to see the values of a flag in different
    environments, if your waffle flag was named ``my.waffle.flag`` might
    look like::

      SELECT count(*) FROM Transaction
      WHERE flag_my.waffle.flag IS NOT NULL
      FACET appName, flag_my.waffle.flag

    Important: Remember to configure ``WAFFLE_FLAG_CUSTOM_ATTRIBUTES`` for
    LMS, Studio and Workers in order to see waffle flag usage in all
    edx-platform environments.
    """
    if name not in _get_waffle_flag_custom_attributes_set():
        return

    flag_attribute_data = _get_waffle_namespace_request_cache().setdefault(
        "flag_attribute", {}
    )
    is_value_changed = True
    if name not in flag_attribute_data:
        # New flag
        flag_attribute_data[name] = str(value)
    else:
        # Existing flag
        if flag_attribute_data[name] == str(value):
            # Same value
            is_value_changed = False
        else:
            # New value
            flag_attribute_data[name] = "Both"

    if is_value_changed:
        attribute_name = "flag_{}".format(name)
        set_custom_attribute(attribute_name, flag_attribute_data[name])

class WaffleFlag(BaseWaffleFlag):
    """
    Waffle flag class that implements custom override method.

    This class should be removed in favour of edx_toggles.toggles.WaffleFlag once we get rid of the WaffleFlagNamespace
    class and the `override` method.
    """
    NAMESPACE_CLASS = WaffleFlagNamespace

    @contextmanager
    def override(self, active=True):
        """
        Shortcut method for `override_waffle_flag`.
        """
        # TODO We can move this import to the top of the file once this code is
        # not all contained within the __init__ module.
        from openedx.core.djangoapps.waffle_utils.testutils import override_waffle_flag

        with override_waffle_flag(self, active):
            yield

class CourseWaffleFlag(WaffleFlag):
    """
    Represents a single waffle flag that can be forced on/off for a course.

    Uses a cached waffle namespace.

    Usage:

       WAFFLE_FLAG_NAMESPACE = WaffleFlagNamespace(name='my_namespace')
       SOME_COURSE_FLAG = CourseWaffleFlag(WAFFLE_FLAG_NAMESPACE, 'some_course_feature', __name__)

    And then we can check this flag in code with::

        SOME_COURSE_FLAG.is_enabled(course_key)

    The Django Admin "waffle_utils" section can be used to configure a course override for this same flag (e.g.
    my_namespace.some_course_feature).
    """

    def _get_course_override_value(self, course_key):
        """
        Returns True/False if the flag was forced on or off for the provided course. Returns None if the flag was not
        overridden.

        Note: Has side effect of caching the override value.

        Arguments:
            course_key (CourseKey): The course to check for override before checking waffle.
        """
        # Import is placed here to avoid model import at project startup.
        from .models import WaffleFlagCourseOverrideModel

        cache_key = "{}.{}".format(self.namespaced_flag_name, str(course_key))
        # pylint: disable=protected-access
        course_override = self.waffle_namespace._cached_flags.get(cache_key)

        if course_override is None:
            course_override = WaffleFlagCourseOverrideModel.override_value(
                self.namespaced_flag_name, course_key
            )
            # pylint: disable=protected-access
            self.waffle_namespace._cached_flags[cache_key] = course_override

        if course_override == WaffleFlagCourseOverrideModel.ALL_CHOICES.on:
            return True
        if course_override == WaffleFlagCourseOverrideModel.ALL_CHOICES.off:
            return False
        return None

    def is_enabled(self, course_key=None):  # pylint: disable=arguments-differ
        """
        Returns whether or not the flag is enabled within the context of a given course.

        Arguments:
            course_key (Optional[CourseKey]): The course to check for override before
                checking waffle. If omitted, check whether the flag is enabled
                outside the context of any course.
        """
        if course_key:
            assert isinstance(
                course_key, CourseKey
            ), "Provided course_key '{}' is not instance of CourseKey.".format(
                course_key
            )
        is_enabled_for_course = self._get_course_override_value(course_key)
        if is_enabled_for_course is not None:
            _set_waffle_flag_attribute(self.namespaced_flag_name, is_enabled_for_course)
            return is_enabled_for_course
        return super().is_enabled()
<commit_msg>Backport override features from edx-toggles to waffle_utils<commit_after>"""
Extra utilities for waffle: most classes are defined in edx_toggles.toggles, but we keep here some extra classes for
usage within edx-platform. These classes cover course override use cases.

Usage:

   WAFFLE_FLAG_NAMESPACE = WaffleFlagNamespace(name='my_namespace')
   # Use CourseWaffleFlag when you are in the context of a course.
   SOME_COURSE_FLAG = CourseWaffleFlag(WAFFLE_FLAG_NAMESPACE, 'some_course_feature', __name__)

You can check this flag in code using the following::

    SOME_COURSE_FLAG.is_enabled(course_key)

To test WaffleSwitchNamespace, use the provided context managers.  For example:

    with WAFFLE_SWITCHES.override(waffle.ESTIMATE_FIRST_ATTEMPTED, active=True):
        ...

Also see ``WAFFLE_FLAG_CUSTOM_ATTRIBUTES`` and docstring for _set_waffle_flag_attribute
for temporarily instrumenting/monitoring waffle flag usage.

"""
import logging
from contextlib import contextmanager

from opaque_keys.edx.keys import CourseKey

from edx_toggles.toggles import WaffleFlag as BaseWaffleFlag
from edx_toggles.toggles import WaffleFlagNamespace
from edx_toggles.toggles import WaffleSwitch as BaseWaffleSwitch
from edx_toggles.toggles import WaffleSwitchNamespace as BaseWaffleSwitchNamespace

log = logging.getLogger(__name__)

class WaffleSwitchNamespace(BaseWaffleSwitchNamespace):
    """
    Waffle switch namespace that implements custom overriding methods. We should eventually get rid of this class.
    """

    @contextmanager
    def override(self, switch_name, active=True):
        """
        Overrides the active value for the given switch for the duration of this
        contextmanager.
        Note: The value is overridden in the request cache AND in the model.
        """
        previous_active = self.is_enabled(switch_name)
        try:
            self.override_for_request(switch_name, active)
            with self.override_in_model(switch_name, active):
                yield
        finally:
            self.override_for_request(switch_name, previous_active)

    def override_for_request(self, switch_name, active=True):
        """
        Overrides the active value for the given switch for the remainder of
        this request (as this is not a context manager).
        Note: The value is overridden in the request cache, not in the model.
        """
        namespaced_switch_name = self._namespaced_name(switch_name)
        self._cached_switches[namespaced_switch_name] = active
        log.info(
            "%sSwitch '%s' set to %s for request.",
            self.log_prefix,
            namespaced_switch_name,
            active,
        )

    @contextmanager
    def override_in_model(self, switch_name, active=True):
        """
        Overrides the active value for the given switch for the duration of this
        contextmanager.
        Note: The value is overridden in the model, not the request cache.
        Note: This should probably be moved to a test class.
        """
        # Import is placed here to avoid model import at project startup.
        # pylint: disable=import-outside-toplevel
        from waffle.testutils import override_switch as waffle_override_switch

        namespaced_switch_name = self._namespaced_name(switch_name)
        with waffle_override_switch(namespaced_switch_name, active):
            log.info(
                "%sSwitch '%s' set to %s in model.",
                self.log_prefix,
                namespaced_switch_name,
                active,
            )
            yield

class WaffleSwitch(BaseWaffleSwitch):
    """
    This class should be removed in favour of edx_toggles.toggles.WaffleSwitch once we get rid of the
    WaffleSwitchNamespace class.
    """

    NAMESPACE_CLASS = WaffleSwitchNamespace

    @contextmanager
    def override(self, active=True):
        with self.waffle_namespace.override(self.switch_name, active):
            yield

class WaffleFlag(BaseWaffleFlag):
    """
    Waffle flag class that implements custom override method.

    This class should be removed in favour of edx_toggles.toggles.WaffleFlag once we get rid of the WaffleFlagNamespace
    class and the `override` method.
    """

    @contextmanager
    def override(self, active=True):
        """
        Shortcut method for `override_waffle_flag`.
        """
        # TODO We can move this import to the top of the file once this code is
        # not all contained within the __init__ module.
        from openedx.core.djangoapps.waffle_utils.testutils import override_waffle_flag

        with override_waffle_flag(self, active):
            yield

class CourseWaffleFlag(WaffleFlag):
    """
    Represents a single waffle flag that can be forced on/off for a course.

    Uses a cached waffle namespace.

    Usage:

       WAFFLE_FLAG_NAMESPACE = WaffleFlagNamespace(name='my_namespace')
       SOME_COURSE_FLAG = CourseWaffleFlag(WAFFLE_FLAG_NAMESPACE, 'some_course_feature', __name__)

    And then we can check this flag in code with::

        SOME_COURSE_FLAG.is_enabled(course_key)

    The Django Admin "waffle_utils" section can be used to configure a course override for this same flag (e.g.
    my_namespace.some_course_feature).
    """

    def _get_course_override_value(self, course_key):
        """
        Returns True/False if the flag was forced on or off for the provided course. Returns None if the flag was not
        overridden.

        Note: Has side effect of caching the override value.

        Arguments:
            course_key (CourseKey): The course to check for override before checking waffle.
        """
        # Import is placed here to avoid model import at project startup.
        from .models import WaffleFlagCourseOverrideModel

        cache_key = "{}.{}".format(self.namespaced_flag_name, str(course_key))
        # pylint: disable=protected-access
        course_override = self.waffle_namespace._cached_flags.get(cache_key)

        if course_override is None:
            course_override = WaffleFlagCourseOverrideModel.override_value(
                self.namespaced_flag_name, course_key
            )
            # pylint: disable=protected-access
            self.waffle_namespace._cached_flags[cache_key] = course_override

        if course_override == WaffleFlagCourseOverrideModel.ALL_CHOICES.on:
            return True
        if course_override == WaffleFlagCourseOverrideModel.ALL_CHOICES.off:
            return False
        return None

    def is_enabled(self, course_key=None):  # pylint: disable=arguments-differ
        """
        Returns whether or not the flag is enabled within the context of a given course.

        Arguments:
            course_key (Optional[CourseKey]): The course to check for override before
                checking waffle. If omitted, check whether the flag is enabled
                outside the context of any course.
        """
        if course_key:
            assert isinstance(
                course_key, CourseKey
            ), "Provided course_key '{}' is not instance of CourseKey.".format(
                course_key
            )
        is_enabled_for_course = self._get_course_override_value(course_key)
        if is_enabled_for_course is not None:
            # pylint: disable=protected-access
            self.namespace._monitor_value(
                self.namespaced_flag_name, is_enabled_for_course
            )
            return is_enabled_for_course
        return super().is_enabled()
<|endoftext|>


<commit_before>from django.template.defaulttags import register

@register.filter
def get_item(dictionary, key):
    return dictionary.get(key)

@register.filter
def word_slice(text, slice):

    def adjust_cut(text, valor, limit=1):
        val = valor
        while text[val-1] != ' ' and val > 1:
            val -= 1
        if val <= limit:
            val = valor
        while text[val] == ' ' and val < len(text):
            val += 1
        return val

    if text == '':
        return text

    slices = slice.split(':')
    inicio = int(slices[0]) if slices[0] != '' else 0
    inicio = adjust_cut(text,  inicio)

    fim = int(slices[1]) if slices[1] != '' else len(text)
    if fim < len(text):
        fim = adjust_cut(text,  fim)

    return text[inicio:fim]

def totalize_data(data, config):
    totrow = data[0].copy()
    for key in totrow:
        totrow[key] = ''

    sum = {key: 0 for key in config['sum']}
    for row in data:
        for key in sum:
            sum[key] += row[key]

    for key in config['descr']:
        totrow[key] = config['descr'][key]

    for key in sum:
        totrow[key] = sum[key]

    for key in config['count']:
        totrow[key] = len(data)

    data.append(totrow)
<commit_msg>test slice values<commit_after>from django.template.defaulttags import register

@register.filter
def get_item(dictionary, key):
    return dictionary.get(key)

@register.filter
def word_slice(text, slice):

    def adjust_cut(text, valor, limit=1):
        val = valor
        while text[val-1] != ' ' and val > 1:
            val -= 1
        if val <= limit:
            val = valor
        while text[val] == ' ' and val < len(text):
            val += 1
        return val

    if text == '':
        return text

    slices = slice.split(':')
    inicio = int(slices[0]) if slices[0] != '' else 0
    if inicio > len(text):
        return ''
    inicio = adjust_cut(text,  inicio)

    fim = int(slices[1]) if slices[1] != '' else len(text)
    if fim < inicio or fim < 0:
        return ''
    if fim < len(text):
        fim = adjust_cut(text,  fim)

    return text[inicio:fim]

def totalize_data(data, config):
    totrow = data[0].copy()
    for key in totrow:
        totrow[key] = ''

    sum = {key: 0 for key in config['sum']}
    for row in data:
        for key in sum:
            sum[key] += row[key]

    for key in config['descr']:
        totrow[key] = config['descr'][key]

    for key in sum:
        totrow[key] = sum[key]

    for key in config['count']:
        totrow[key] = len(data)

    data.append(totrow)
<|endoftext|>


<commit_before>""" Defines Settings and Debug objects """
import datetime
import os
import __main__

class DebugMaster(object):
    """ Used to create the global Debug object """
    _log_file = None
    _debug_level = 0
    _logger = None
    _logger_no_prefix = False
    _logger_methods = {
        "user": None,
        "info": None,
        "action": None,
        "error": None,
        "debug": None
    }
    def user(self, message):
        """ Creates a user log (if user logging is turned on)

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT

        Note: Does *not* use Java string formatting like Sikuli.
        Format your message with Python ``basestring.format()`` instead.
        """
        if Settings.UserLogs:
            self._write_log(Settings.UserLogPrefix, Settings.UserLogTime, message)
    def history(self, message):
        """ Records an Action-level log message

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if Settings.ActionLogs:
            self._write_log("action", Settings.LogTime, message)
    def error(self, message):
        """ Records an Error-level log message

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if Settings.ErrorLogs:
            self._write_log("error", Settings.LogTime, message)
    def info(self, message):
        """ Records an Info-level log message

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if Settings.InfoLogs:
            self._write_log("info", Settings.LogTime, message)
    def on(self, level):
        """ Turns on all debugging messages up to the specified level

        0 = None; 1 = User;
        """
        if isinstance(level, int) and level >= 0 and level <= 3:
            self._debug_level = level
    def off(self):
        """ Turns off all debugging messages """
        self._debug_level = 0
    def log(self, level, message):
        """ Records a Debug-level log message
        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if level <= self._debug_level:
            self._write_log("debug", Settings.LogTime, message)

    def setLogger(self, logger_obj):
        """ Sets log handler to ``logger_obj`` """
        self._logger = logger_obj
    def setLoggerNoPrefix(self, logger_obj):
        """ Sets log handler to ``logger_obj`` """
        self._logger = logger_obj
        self._logger_no_prefix = True
    def setLoggerAll(self, mthd):
        """ Sends all messages to ``logger.[mthd]()`` for handling """
        for key in self._logger_methods:
            self._logger_methods[key] = mthd
    def setLoggerUser(self, mthd):
        """ Sends user messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["user"] = mthd
    def setLoggerInfo(self, mthd):
        """ Sends info messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["info"] = mthd
    def setLoggerAction(self, mthd):
        """ Sends action messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["action"] = mthd
    def setLoggerError(self, mthd):
        """ Sends error messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["error"] = mthd
    def setLoggerDebug(self, mthd):
        """ Sends debug messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["debug"] = mthd
    def setLogFile(self, filepath):
        """ Defines the file to which output log messages should be sent """
        parsed_path = os.path.abspath(filepath)
        # Checks if the provided log filename is in a real directory, and that
        # the filename itself is not a directory.
        if os.path.isdir(os.path.dirname(parsed_path)) and not os.path.isdir(parsed_path):
            self._log_file = parsed_path
        else:
            raise IOError("File not found: " + filepath)
    def _write_log(self, log_type, log_time, message):
        """ Private method to abstract log writing for different types of logs """
        timestamp = datetime.datetime.now().strftime(" %Y-%m-%d %H:%M:%S")
        log_entry = "[{}{}] {}".format(log_type, timestamp if log_time else "", message)
        if self._logger and callable(getattr(self._logger, self._logger_methods[log_type], None)):
            # Check for log handler (sends message only if _logger_no_prefix is True)
            getattr(
                self._logger,
                self._logger_methods[log_type],
                None
                )(message if self._logger_no_prefix else log_entry)
        elif self._log_file:
            # Otherwise write to file, if a file has been specified
            with open(self._log_file) as logfile:
                logfile.write(log_entry)
        else:
            # Otherwise, print to STDOUT
            print(log_entry)

class SettingsMaster(object):
    """ Global settings that Lackey refers to by default """
    ## Logging Settings
    ActionLogs = True # Message prefix: [log]
    InfoLogs = True # Message prefix: [info]
    DebugLogs = False # Message prefix: [debug]
    ErrorLogs = False # Message prefix: [error]
    LogTime = False
    ### User Logging
    UserLogs = True
    UserLogPrefix = "user"
    UserLogTime = True

    ## Region Settings
    MinSimilarity = 0.7
    SlowMotionDelay = 3 # Extra duration of slowed-down visual effects
    WaitScanRate = 3	# Searches per second
    ObserveScanRate = 3 # Searches per second (observers)
    OberveMinChangedPixels = 50 # Threshold to trigger onChange() (not implemented yet)

    ## Keyboard/Mouse Settings
    MoveMouseDelay = 0.3 # Time to take moving mouse to target location
    DelayBeforeMouseDown = 0.3
    DelayBeforeDrag = 0.3
    DelayBeforeDrop = 0.3
    ClickDelay = 0.0 # Resets to 0 after next click
    TypeDelay = 0.0 # Resets to 0 after next keypress

    ## Action Settings
    ShowActions = False

    ## File Settings
    # Path to Sikuli project - might not be current directory
    BundlePath = os.path.dirname(os.path.abspath(os.path.join(os.getcwd(), __main__.__file__)))
    ImagePaths = []
    OcrDataPath = None

    ## Popup settings
    PopupLocation = None

Debug = DebugMaster()
Settings = SettingsMaster()
<commit_msg>Fixed BundlePath when run from interpreter<commit_after>""" Defines Settings and Debug objects """
import datetime
import os
import __main__

class DebugMaster(object):
    """ Used to create the global Debug object """
    _log_file = None
    _debug_level = 0
    _logger = None
    _logger_no_prefix = False
    _logger_methods = {
        "user": None,
        "info": None,
        "action": None,
        "error": None,
        "debug": None
    }
    def user(self, message):
        """ Creates a user log (if user logging is turned on)

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT

        Note: Does *not* use Java string formatting like Sikuli.
        Format your message with Python ``basestring.format()`` instead.
        """
        if Settings.UserLogs:
            self._write_log(Settings.UserLogPrefix, Settings.UserLogTime, message)
    def history(self, message):
        """ Records an Action-level log message

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if Settings.ActionLogs:
            self._write_log("action", Settings.LogTime, message)
    def error(self, message):
        """ Records an Error-level log message

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if Settings.ErrorLogs:
            self._write_log("error", Settings.LogTime, message)
    def info(self, message):
        """ Records an Info-level log message

        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if Settings.InfoLogs:
            self._write_log("info", Settings.LogTime, message)
    def on(self, level):
        """ Turns on all debugging messages up to the specified level

        0 = None; 1 = User;
        """
        if isinstance(level, int) and level >= 0 and level <= 3:
            self._debug_level = level
    def off(self):
        """ Turns off all debugging messages """
        self._debug_level = 0
    def log(self, level, message):
        """ Records a Debug-level log message
        Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is
        defined, sends to STDOUT
        """
        if level <= self._debug_level:
            self._write_log("debug", Settings.LogTime, message)

    def setLogger(self, logger_obj):
        """ Sets log handler to ``logger_obj`` """
        self._logger = logger_obj
    def setLoggerNoPrefix(self, logger_obj):
        """ Sets log handler to ``logger_obj`` """
        self._logger = logger_obj
        self._logger_no_prefix = True
    def setLoggerAll(self, mthd):
        """ Sends all messages to ``logger.[mthd]()`` for handling """
        for key in self._logger_methods:
            self._logger_methods[key] = mthd
    def setLoggerUser(self, mthd):
        """ Sends user messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["user"] = mthd
    def setLoggerInfo(self, mthd):
        """ Sends info messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["info"] = mthd
    def setLoggerAction(self, mthd):
        """ Sends action messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["action"] = mthd
    def setLoggerError(self, mthd):
        """ Sends error messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["error"] = mthd
    def setLoggerDebug(self, mthd):
        """ Sends debug messages to ``logger.[mthd]()`` for handling """
        self._logger_methods["debug"] = mthd
    def setLogFile(self, filepath):
        """ Defines the file to which output log messages should be sent """
        parsed_path = os.path.abspath(filepath)
        # Checks if the provided log filename is in a real directory, and that
        # the filename itself is not a directory.
        if os.path.isdir(os.path.dirname(parsed_path)) and not os.path.isdir(parsed_path):
            self._log_file = parsed_path
        else:
            raise IOError("File not found: " + filepath)
    def _write_log(self, log_type, log_time, message):
        """ Private method to abstract log writing for different types of logs """
        timestamp = datetime.datetime.now().strftime(" %Y-%m-%d %H:%M:%S")
        log_entry = "[{}{}] {}".format(log_type, timestamp if log_time else "", message)
        if self._logger and callable(getattr(self._logger, self._logger_methods[log_type], None)):
            # Check for log handler (sends message only if _logger_no_prefix is True)
            getattr(
                self._logger,
                self._logger_methods[log_type],
                None
                )(message if self._logger_no_prefix else log_entry)
        elif self._log_file:
            # Otherwise write to file, if a file has been specified
            with open(self._log_file) as logfile:
                logfile.write(log_entry)
        else:
            # Otherwise, print to STDOUT
            print(log_entry)

class SettingsMaster(object):
    """ Global settings that Lackey refers to by default """
    ## Logging Settings
    ActionLogs = True # Message prefix: [log]
    InfoLogs = True # Message prefix: [info]
    DebugLogs = False # Message prefix: [debug]
    ErrorLogs = False # Message prefix: [error]
    LogTime = False
    ### User Logging
    UserLogs = True
    UserLogPrefix = "user"
    UserLogTime = True

    ## Region Settings
    MinSimilarity = 0.7
    SlowMotionDelay = 3 # Extra duration of slowed-down visual effects
    WaitScanRate = 3	# Searches per second
    ObserveScanRate = 3 # Searches per second (observers)
    OberveMinChangedPixels = 50 # Threshold to trigger onChange() (not implemented yet)

    ## Keyboard/Mouse Settings
    MoveMouseDelay = 0.3 # Time to take moving mouse to target location
    DelayBeforeMouseDown = 0.3
    DelayBeforeDrag = 0.3
    DelayBeforeDrop = 0.3
    ClickDelay = 0.0 # Resets to 0 after next click
    TypeDelay = 0.0 # Resets to 0 after next keypress

    ## Action Settings
    ShowActions = False

    ## File Settings
    # Path to Sikuli project - might not be current directory
    try:
        BundlePath = os.path.dirname(os.path.abspath(os.path.join(os.getcwd(), __main__.__file__)))
    except AttributeError:
        BundlePath = os.path.dirname(os.path.abspath(os.getcwd()))
    ImagePaths = []
    OcrDataPath = None

    ## Popup settings
    PopupLocation = None

Debug = DebugMaster()
Settings = SettingsMaster()
<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
import lamana.input_
import lamana.distributions
import lamana.constructs
import lamana.theories
import lamana.output_
#from lamana.models import *
#import lamana.ratios
#import lamana.predictions
#import lamana.gamuts

__title__ = 'lamana'
__version__ = '0.4.11.dev0'                                # PEP 440 style
##__version__ = '0.4.11'
__author__ = 'P. Robinson II'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015, P. Robinson II'
<commit_msg>Redo bump version<commit_after># -----------------------------------------------------------------------------
import lamana.input_
import lamana.distributions
import lamana.constructs
import lamana.theories
import lamana.output_
#from lamana.models import *
#import lamana.ratios
#import lamana.predictions
#import lamana.gamuts

__title__ = 'lamana'
__version__ = '0.4.11'                                # PEP 440 style
##__version__ = '0.4.11'
__author__ = 'P. Robinson II'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015, P. Robinson II'
<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
'''A interface between constructs and models modules.'''
# BaseModel(): A super class for user-customized models.
# flake8 constructs.py --ignore=E265,E501,N802,H806

import abc
import importlib
import logging

import six

# TODO: Replace with interactive way to import models
from lamana.models import *
##import lamana.models
from lamana.utils import tools as ut

@six.add_metaclass(abc.ABCMeta)
class BaseModel(object):
    '''Provide attributes for sub-classing custom models.

    Sub-class from here to make a model that interfaces with LamAna.

    Notes
    -----
    This class helps centralize common attributes associated with a custom
    model.  A model is selected in by the `Case.apply()` method and is applied in
    Phase 3 of `constructs.Laminate()`.  It is idiomatic to subclass `BaseModel`
    when making custom models.

    Uses an abstractmethod to enforce implementation of a hook method.  It also
    suggests that BaseModel can only be sub-classed, not instantiated.

    See Also
    --------
    distributions.Case.apply(): model selection
    constructs.Laminate(): creates LMFrame by merging LT data with LFrame
    theories.handshake(): get updated DataFrame and FeatureInput

    '''
    def __init__(self):
        super(BaseModel, self).__init__()
        self.model_name = None
        self.LaminateModel = None
        self.FeatureInput = None

    def __repr__(self):
        return '<{} Model object>'.format(self.__class__.__name__)

    # TODO: Find `_use_model_` more dynamically than forcing to search `Model._use_model`.
    @abc.abstractmethod
    def _use_model_():
        '''Hook method.

        Same name as config.HOOKNAME. This method must be implemented and
        return the following.

        Returns
        -------
        tuple
            Updated (DataFrame, FeatureInput).

        '''
        pass

def handshake(Laminate, adjusted_z=False):
    '''Return updated LaminateModel and FeatureInput objects.

    This key method interfaces between Laminate class and a custom model module.
    Model names are related to the laminate theory, e.g. Classical_LT, Wilson_LT.
    This name is applied by the user upon calling `Case.apply(model='model_name')`
    and is found in the FeatureInput dict.

    Parameters
    ----------
    Laminate : Laminate object
        The entire pre-updated, `Laminate` object is passed in, giving access
        to its methods.  `Laminate.LFrame` only has ID and Dimensional columns,
        so no laminate theory calculations or columns are populated yet.
    adjusted_z : bool, optional; default False
        This option forces the use of the z(m)* column values.  A different
        algorithm was used it calculate the internal values for this variable.

    Notes
    -----
    This method searches for a special hook method named `_use_model_` in the
    specified model module.  This hook may be written as method within a class
    or written as an independent function offering a choice for users to write
    models in either class-style (recommended) or function-style.  This hook
    method simply returns an updated LaminateModel and FeatureInput.

    `handshake()` determines either class- or function-style models by duck typing
    a module name according to the model name provided in the `Case.apply()`` method.
    These modules are located in the `models` directory - a repository for all
    package models.  Assuming a function first, the hook method is sought by
    calling `_use_model_()`; if none is found, the exception is caught and a hook
    method named `<class>._use_model_` is sought next (as of 0.4.5b1).  As of
    0.4.11, hardcoded class names here removed; any pythonic name can be used
    for the <class> parameter.

    Here is the workflow of a model selected by the user and its call in `Laminate`.
    This assumes the model is developed and located in the standard .\models directory.

    1. User selects a model in `lamana.distributions.Case.apply(model='model_name')`;
       Model name is stored in the FeatureInput object then passed into `Laminate`.
    2. Inside `lamana.constructs.Laminate`,  _update_columns._update_calculations(FI)`
       is called, which initiates LT calculations for the given model.
    3. `theories.handshake(L)` is called and searches for the model name in models dir.
    4. Inside the selected model, the hook method `models._use_model_` is called
       (if function-style) or `models.<model_name>._use_model_` (if class-style)

    '''
    HOOKNAME = '_use_model_'                               # looks for this in custom models

    # Find and Import the Model
    model_name = Laminate.FeatureInput['Model']
    modified_name = ''.join(['.', model_name])             # e.g '.Wilson_LT'
    module = importlib.import_module(modified_name, package='lamana.models')

    # Verify Laminate has an empty LMFrame attribute, otherwise Warn
    if any(getattr(Laminate, 'LMFrame')):
        logging.warn('A LaminateModel has been passed to `handshake()`.'
                     ' Expected a pre-updated Laminate object.  Results are unpredictable.')
    try:
        # Look for a hook function
        hook = ut.get_hook_function(module, hookname=HOOKNAME)
        logging.debug('Found a hook function: {}'.format(hook))
    except(AttributeError):
        # This code block obviates hardcoding a specific model class name.
        # Look for a class containing the hook method
        class_obj = ut.get_hook_class(module, hookname=HOOKNAME)
        class_name = getattr(module, class_obj.__name__)
        my_instance = class_name()                         # instantiate the class; important
        hook = getattr(my_instance, HOOKNAME)              # hook method
        logging.debug('Found a hook method {} in {}'.format(hook, class_name))

    # TODO: Add following with args
    #LaminateModel, FeatureInput = hook(Laminate, *args, adjusted_z=False, **kwargs)
    # eqiv. LM, FI = module._use_model_()
    LaminateModel, FeatureInput = hook(Laminate, adjusted_z=False)

    # Make sure the passed FeatureInput has Equal attributes
    ##assert FeatureInput['Parameters']['p'] == Laminate.p

    return(LaminateModel, FeatureInput)

# DEPRECATE : 0.four.11
# def handshake(Laminate, adjusted_z=False):
#     '''Return updated LaminateModel and FeatureInput objects.
#
#     This key method interfaces between Laminate class and a custom model module.
#     Model names are related to the laminate theory, e.g. Classical_LT, Wilson_LT.
#     This name is applied by the user upon calling `Case.apply(model='model_name')`
#     and is found in the FeatureInput dict.
#
#     Parameters
#     ----------
#     Laminate : Laminate object
#         The entire `Laminate` object is passed in, giving access to its methods.
#         `Laminate.LFrame` only has ID and Dimensional columns, so no laminate
#         theory calculations or columns are populated yet.
#     adjusted_z : bool, optional; default False
#         This option forces the use of the z(m)* column values.  A different
#         algorithm was used it calculate the internal values for this variable.
#
#     Notes
#     -----
#     This method searches for a special hook method named `_use_model_` in the
#     specified model module.  This hook may be written as method within a class
#     or written as an independent function offering a choice for users to write
#     models in either class-style (recommended) or function-style.  This hook
#     method simply returns an updated LaminateModel and FeatureInput.
#
#     `handshake()` determines either class- or function-style models by duck typing
#     a module name according to the model name provided in the `Case.apply()`` method.
#     These modules are located in the `models` directory - a repository for all
#     package models.  Assuming a function first, the hook method is sought by
#     calling `_use_model_()`; if none is found, the exception is caught and a hook
#     method named `Model._use_model_` is sought next (as of 0.4.5b1).
#
#     Here is the workflow of a model selected by the user and its call in `Laminate`.
#     This assumes the model is developed and located in the standard .\models directory.
#
#     1. User selects a model in `lamana.distributions.Case.apply(model='model_name')`;
#        Model name is stored in the FeatureInput object then passed into `Laminate`.
#     2. Inside `lamana.constructs.Laminate`,  _update_columns._update_calculations(FI)`
#        is called, which initiates LT calculations for the given model.
#     3. `theories.handshake(L)` is called and searches for the model name in models dir.
#     4. Inside the selected model, the hook method `models._use_model_` is called
#        (if function-style) or `models.<model_name>._use_model_` (if class-style)
#
#     '''
#     # Find and Import the Model
#     model_name = Laminate.FeatureInput['Model']
#     modified_name = ''.join(['.', model_name])                      # '.Wilson_LT'
#     module = importlib.import_module(modified_name, package='lamana.models')
#
#     try:
#         # Look for the function
#         '''Set up to accept *args'''
#         LaminateModel, FeatureInput = module._use_model_(Laminate, adjusted_z=False)
#         #logging.info('Found a function')
#     except(AttributeError):
#         # Catch exceptions, if no function found, find class
#         '''Make smarter to find whatever class has the _use_model hook.'''
#         class_name = getattr(module, 'Model')
#         my_instance = class_name()
#         LaminateModel, FeatureInput = my_instance._use_model_(Laminate, adjusted_z=False)
#         #logging.info('Found a class')
#
#     # Make sure the passed FeatureInput has Equal attributes
#     assert FeatureInput['Parameters']['p'] == Laminate.p
#
#     return(LaminateModel, FeatureInput)
<commit_msg>Remove old code<commit_after># -----------------------------------------------------------------------------
'''A interface between constructs and models modules.'''
# BaseModel(): A super class for user-customized models.
# flake8 constructs.py --ignore=E265,E501,N802,H806

import abc
import importlib
import logging

import six

# TODO: Replace with interactive way to import models
from lamana.models import *
##import lamana.models
from lamana.utils import tools as ut

@six.add_metaclass(abc.ABCMeta)
class BaseModel(object):
    '''Provide attributes for sub-classing custom models.

    Sub-class from here to make a model that interfaces with LamAna.

    Notes
    -----
    This class helps centralize common attributes associated with a custom
    model.  A model is selected in by the `Case.apply()` method and is applied in
    Phase 3 of `constructs.Laminate()`.  It is idiomatic to subclass `BaseModel`
    when making custom models.

    Uses an abstractmethod to enforce implementation of a hook method.  It also
    suggests that BaseModel can only be sub-classed, not instantiated.

    See Also
    --------
    distributions.Case.apply(): model selection
    constructs.Laminate(): creates LMFrame by merging LT data with LFrame
    theories.handshake(): get updated DataFrame and FeatureInput

    '''
    def __init__(self):
        super(BaseModel, self).__init__()
        self.model_name = None
        self.LaminateModel = None
        self.FeatureInput = None

    def __repr__(self):
        return '<{} Model object>'.format(self.__class__.__name__)

    # TODO: Find `_use_model_` more dynamically than forcing to search `Model._use_model`.
    @abc.abstractmethod
    def _use_model_():
        '''Hook method.

        Same name as config.HOOKNAME. This method must be implemented and
        return the following.

        Returns
        -------
        tuple
            Updated (DataFrame, FeatureInput).

        '''
        pass

def handshake(Laminate, adjusted_z=False):
    '''Return updated LaminateModel and FeatureInput objects.

    This key method interfaces between Laminate class and a custom model module.
    Model names are related to the laminate theory, e.g. Classical_LT, Wilson_LT.
    This name is applied by the user upon calling `Case.apply(model='model_name')`
    and is found in the FeatureInput dict.

    Parameters
    ----------
    Laminate : Laminate object
        The entire pre-updated, `Laminate` object is passed in, giving access
        to its methods.  `Laminate.LFrame` only has ID and Dimensional columns,
        so no laminate theory calculations or columns are populated yet.
    adjusted_z : bool, optional; default False
        This option forces the use of the z(m)* column values.  A different
        algorithm was used it calculate the internal values for this variable.

    Notes
    -----
    This method searches for a special hook method named `_use_model_` in the
    specified model module.  This hook may be written as method within a class
    or written as an independent function offering a choice for users to write
    models in either class-style (recommended) or function-style.  This hook
    method simply returns an updated LaminateModel and FeatureInput.

    `handshake()` determines either class- or function-style models by duck typing
    a module name according to the model name provided in the `Case.apply()`` method.
    These modules are located in the `models` directory - a repository for all
    package models.  Assuming a function first, the hook method is sought by
    calling `_use_model_()`; if none is found, the exception is caught and a hook
    method named `<class>._use_model_` is sought next (as of 0.4.5b1).  As of
    0.4.11, hardcoded class names here removed; any pythonic name can be used
    for the <class> parameter.

    Here is the workflow of a model selected by the user and its call in `Laminate`.
    This assumes the model is developed and located in the standard .\models directory.

    1. User selects a model in `lamana.distributions.Case.apply(model='model_name')`;
       Model name is stored in the FeatureInput object then passed into `Laminate`.
    2. Inside `lamana.constructs.Laminate`,  _update_columns._update_calculations(FI)`
       is called, which initiates LT calculations for the given model.
    3. `theories.handshake(L)` is called and searches for the model name in models dir.
    4. Inside the selected model, the hook method `models._use_model_` is called
       (if function-style) or `models.<model_name>._use_model_` (if class-style)

    '''
    HOOKNAME = '_use_model_'                               # looks for this in custom models

    # Find and Import the Model
    model_name = Laminate.FeatureInput['Model']
    modified_name = ''.join(['.', model_name])             # e.g '.Wilson_LT'
    module = importlib.import_module(modified_name, package='lamana.models')

    # Verify Laminate has an empty LMFrame attribute, otherwise Warn
    if any(getattr(Laminate, 'LMFrame')):
        logging.warn('A LaminateModel has been passed to `handshake()`.'
                     ' Expected a pre-updated Laminate object.  Results are unpredictable.')
    try:
        # Look for a hook function
        hook = ut.get_hook_function(module, hookname=HOOKNAME)
        logging.debug('Found a hook function: {}'.format(hook))
    except(AttributeError):
        # This code block obviates hardcoding a specific model class name.
        # Look for a class containing the hook method
        class_obj = ut.get_hook_class(module, hookname=HOOKNAME)
        class_name = getattr(module, class_obj.__name__)
        my_instance = class_name()                         # instantiate the class; important
        hook = getattr(my_instance, HOOKNAME)              # hook method
        logging.debug('Found a hook method {} in {}'.format(hook, class_name))

    # TODO: Add following with args
    #LaminateModel, FeatureInput = hook(Laminate, *args, adjusted_z=False, **kwargs)
    # eqiv. LM, FI = module._use_model_()
    LaminateModel, FeatureInput = hook(Laminate, adjusted_z=False)

    # Make sure the passed FeatureInput has Equal attributes
    ##assert FeatureInput['Parameters']['p'] == Laminate.p

    return(LaminateModel, FeatureInput)
<|endoftext|>


<commit_before># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
"""Evaluate pre-trained model trained for ppl metric.
This seq2seq model was trained on convai2:self.
"""

from projects.convai2.baselines.download_models import download
from parlai.core.params import ParlaiParser
from examples.eval_model import setup_args, eval_model
from parlai.agents.seq2seq.seq2seq import Seq2seqAgent

if __name__ == '__main__':
    parser = setup_args()
    parser.set_defaults(
        task='convai2:self',
        model='seq2seq',
        model_file='models:convai2/seq2seq/convai2_self_seq2seq_model',
        dict_file='models:convai2/seq2seq/dict_convai2_self/dict_convai2_self',
        datatype='valid',
        batchsize=128,
    )
    opt = parser.parse_args()
    download(opt, 'convai2/seq2seq', 'convai2_self_seq2seq_model.tgz')
    download(opt, 'convai2/seq2seq', 'dict_convai2_self')
    eval_model(parser, printargs=False)
<commit_msg>bug fix (#668)<commit_after># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
"""Evaluate pre-trained model trained for ppl metric.
This seq2seq model was trained on convai2:self.
"""

from projects.convai2.baselines.download_models import download
from parlai.core.params import ParlaiParser
from examples.eval_model import setup_args, eval_model
from parlai.agents.seq2seq.seq2seq import Seq2seqAgent

if __name__ == '__main__':
    parser = setup_args()
    parser.set_defaults(
        task='convai2:self',
        model='seq2seq',
        model_file='models:convai2/seq2seq/convai2_self_seq2seq_model/convai2_self_seq2seq_model',
        dict_file='models:convai2/seq2seq/dict_convai2_self/dict_convai2_self',
        datatype='valid',
        batchsize=128,
    )
    opt = parser.parse_args()
    download(opt, 'convai2/seq2seq', 'convai2_self_seq2seq_model.tgz')
    download(opt, 'convai2/seq2seq', 'dict_convai2_self')
    eval_model(parser, printargs=False)
<|endoftext|>