<commit_before>from __future__ import print_function

import argparse
import os
import struct
import sys
import yaml

def read_from_config_file(configfile):
    """
    Read the format patterns from the given YAML configuration file.

    Parameters
    ----------
    configfile : str
        Path to YAML config file to read from.

    Returns
    -------
    formats : list of str
        A list of strings of the form
        ["<pattern1>:<count1>", "<pattern2>:<count2>", ...]. Each instance of
        ":<countN>" can be omitted for format patterns that only occur once.
    """
    with open(configfile, 'r') as f:
        formats = yaml.load(f)['formats']

    return formats

def write_to_config_file(configfile, formats):
    """
    Read the format patterns from the given YAML configuration file.

    Parameters
    ----------
    configfile : str
        Path to YAML config file to read from.
    formats : list of str
        A list of strings of the form
        ["<pattern1>:<count1>", "<pattern2>:<count2>", ...]. Each instance of
        ":<countN>" can be omitted for format patterns that only occur once.
    """
    with open(configfile, 'w') as f:
        yaml.dump({'formats': formats}, f, default_flow_style=False)

def gen_format_string(formats, size=None):
    """
    Generate a format string specifying the a byte alignment given a list of the
    form ["<pattern1>:<count1>", "<pattern2>:<count2>", ...]. For example,

    >>> gen_format_string(["id4s:2", "f2d"])

    would return "id4sid4sf2d".

    For convenience, a special wildcard (*) character can be used once to
    automatically determine the count of one pattern. As an example,

    >>> gen_format_string(["i", "8s:*"])

    would return "i8s8s" if size is 20 bytes.

    Parameters
    ----------
    formats : list of str
        A list of strings of the form
        ["<pattern1>:<count1>", "<pattern2>:<count2>", ...], where each count is
        the number of repeated occurences of each format pattern. Each instance
        of ":<countN>" can be omitted for format patterns that only occur once.
        In the special case that a <countN> "*", the remainder
        of the available bytes in the file is used to automatically calculate
        the count. This can only be done once per list of formats.
    size : int, optional
        size of the source file in bytes. Only needed if wilcard '*' character
        is used in counts.

    Returns
    -------
    fmt : str
        The full format string.
    formats : list of str
        Formats list with wildcards expanded to actual counts.
    """
    # Fallback format
    if formats == ['*']:
        return '*', formats

    fmt = ''
    # Use this to keep track of size expended by format so far.
    cumsize = 0
    special_pattern = None
    special_index = None
    for i, pattern_info in enumerate(formats):
        pattern_info = pattern_info.split(':')
        pattern = pattern_info[0]

        # count is 1 if not given.
        if len(pattern_info) == 1:
            cumsize += struct.calcsize(pattern)
            fmt += pattern
        elif pattern_info[1][-1] == '*':
            if special_pattern:
                raise struct.error('Wildcard (*) character may only be used once')

            # We will need the current pattern and count values for later
            # when the remaining size is fully calculated.
            special_pattern = pattern
            special_index = i

            # Until remaining size is fully calculated, set a placeholder.
            fmt += '{0}'
        else:
            count = int(pattern_info[1])
            result = count*pattern
            cumsize += struct.calcsize(result)
            fmt += result

    # We are now ready to allocate the remaining bytes
    # for the special '*' pattern.
    if special_pattern:
        # Make sure source is specified.
        if size is None:
            raise struct.error('source must be specified if wildcard (*) '
                               'character is in format counts.')

        # Calculate the count such that pattern evenly fits in remaining size
        remaining = size - cumsize
        chunksize = struct.calcsize(special_pattern)
        count = remaining / chunksize
        if remaining % chunksize != 0:
            raise struct.error('Given chunksize of {0} bytes does not divide '
                               'evenly into remaining number of bytes.'
                               .format(chunksize))

        result = count*special_pattern
        fmt = fmt.format(result)

        # Update formats list to expand * character. This is needed because otherwise
        # the executable script will not run if the format saved in a config file
        # contains the * character when source is not given in the script
        # (eg, bconv is ran only to manipulate the defaultformat string)
        formats[special_index] = formats[special_index].replace('*', str(count))

    # Final sanity check: Ensure format string size and source file size match.
    fmt_size = struct.calcsize('=' + fmt)
    if size and size != fmt_size:
        raise struct.error('Format string size and chunk size do not match.\n'
                           'Expected: {0}, Got: {1}'.format(size, fmt_size))

    return fmt, formats

def convert(source, destination=None, byte_order=None, fmt='*'):
    """
    Converts the given file (specified by source) from one byte order
    to another. For example, to convert a file in the current working directory
    from big to little endian, call

    >>> convert('a.bin', 'little')

    By default, the source file is overwritten. To prevent this, a destination
    path may optionally be specified. Additionally, not specifying order
    will assume that you are trying to convert to the native byte order of your
    platform. Eg,

    >>> convert('a.bin', 'b.bin')

    will convert a.bin from big endian to little endian and store the result
    in b.bin on x86 platforms.

    Parameters
    ----------
    source : str
        The path to the binary file to be converted.
    destination: str, optional
        The path to the converted output file. If not specified, source is
        overwritten.
    byte_order: {None, 'little', 'big'}
        The byte order to convert to (endianness). If not specified, the
        native byte ordering of your platform is used (eg, 'little' on x86).
        "Do nothing" conversion operations are not allowed, so the input file
        given by source will be assumed to be formatted in the opposite
        byte order.
    fmt: str, optional
        Format string. See documentation for the python struct module for
        valid examples. The string should span the entire size of the file you
        are converting. The default format is "Nc", where N is the size of the
        file in bytes. Thus the bytes of the entire file are reversed in one go.
    """
    # This format string gives the number of bytes and type
    # for each piece of data in the record.
    if fmt == '*':
        num = os.path.getsize(source)
        fmt = num*'c'

    # Set default destination path to source (overwrite)
    if destination is None:
        destination = source

    # Set default byte order to native
    if byte_order is None:
        byte_order = sys.byteorder

    # This tells the struct library the byte order when packing/unpacking
    if byte_order == 'little':
        fmt_in = '>{0}'.format(fmt)
        fmt_out = '<{0}'.format(fmt)
    else:
        fmt_in = '<{0}'.format(fmt)
        fmt_out = '>{0}'.format(fmt)

    # Read and convert the data from source file
    with open(source, 'rb') as f:
        stream = f.read()
        data = struct.unpack(fmt_in, stream)

    # Write converted data to destination file
    with open(destination, 'wb') as f:
        f.write(struct.pack(fmt_out, *data))

def main():
    """
    CLI script for convert function.
    """
    description = 'Convert files from one byte order to another.'
    parser = argparse.ArgumentParser(prog='bconv', description=description)
    parser.add_argument('source', nargs='?', help='Path to input file')
    parser.add_argument('destination', nargs='?', help='Path to output file')
    parser.add_argument('--byte-order', dest='order', help='Output file byte order')
    parser.add_argument('-l', '--little-endian', dest='little', action='store_true',
                        help='Use little endian byte ordering for destination.')
    parser.add_argument('-b', '--big-endian', dest='big', action='store_true',
                        help='Use big endian byte ordering for destination.')
    parser.add_argument('-f', '--format', nargs='*', help='Format string.'
                        ' See documentation for the python standard library'
                        ' struct module for valid examples. To shorten the format'
                        ' string with common patterns, you may use the following'
                        ' notation:\n'
                        ' <pattern1>:<count1> <pattern2>:<count2> ...\n Omitting'
                        ' the : assumes a count of 1. As an example, "-f d i2s:2" is'
                        ' equivalent to "-f di2si2s". In addition, one of the'
                        ' counts may have a special wildcard character (*).'
                        ' When source is given, the count is automatically'
                        ' converted such that the number of bytes represented by'
                        ' pattern*count fits into the remainder of available bytes.'
                        ' For example, "bconv a.bin -p -f i 4s:*" will print'
                        ' "i4s4s" when a.bin is 12 bytes.')
    parser.add_argument('-d', '--set-default-format', dest='store', action='store_true',
                        help='If true, the format specified with the -f or -c'
                        ' switch is stored in the ~/.bconvrc file.'
                        ' Any formats containing a "*" are expanded.')
    parser.add_argument('-c', '--configfile', help='If specified, override the '
                        'default ~/.bconvrc for loading the the format string. '
                        'This is useful in place of the -f switch for really '
                        'long format strings.')
    parser.add_argument('-p', '--print-format', dest='printf', action='store_true',
                        help='Print the currently used format string. Omitting'
                        ' all other arguments prints the format string stored in'
                        ' the ~/.bconv file that comes with this program.')

    # Now process the arguments
    args = parser.parse_args()

    # Print help message if arguments are default.
    if not (args.source or args.destination or args.order or args.big
            or args.little or args.format or args.store or args.configfile
            or args.printf):
        parser.print_usage()
        sys.exit()

    # Input file
    source = args.source
    size = None

    # Ensure source is valid if given
    if source:
        try:
            size = os.path.getsize(source)
        except OSError as e:
            parser.error(e)

    # Output file
    destination = args.destination

    # Byte order
    if ((args.order is not None and (args.big or args.little))
        or (args.big and args.little)):
        parser.error('Cannot specify multiple byte-orders.')

    if args.big:
        byte_order = 'big'

    elif args.little:
        byte_order = 'little'

    else:
        byte_order = args.order

    # Format string
    configfile = os.join(os.path.expanduser('~'), '.bconvrc')
    if args.format and args.configfile:
        parser.error('Multiple format strings specified, please use only one of'
                     ' -f and -c when overriding default format.')
    elif args.format:
        formats = args.format

    elif args.configfile:
        try:
            configfile = args.configfile
            formats = read_from_config_file(args.configfile)

        except IOError as e:
            parser.error(e)

    else:
        try:
            formats = read_from_config_file(configfile)
        except IOError as e:
            # Do this in case default config file is corrupted or doesn't exist.
            formats = ['*']

    # Finally we can generate the full format string
    try:
        fmt, expanded_formats = gen_format_string(formats, size)
    except struct.error as e:
        paraser.error(e)

    # Store default format in config file
    if args.store or (args.format is None and expanded_formats != formats):
        write_to_config_file(configfile, expanded_formats)

    # Print the format string
    if args.printf:
        print('Current format: {0}'.format(fmt))

    # If source isn't given, we're done.
    if not source:
        return

    # Do the actual byte order conversion.
    try:
        convert(source, destination, byte_order, fmt)
    except IOError as e:
        # source input file can't be read
        parser.error(e)
    except struct.error as e:
        # badly formed format string
        parser.error(e)

if __name__ == '__main__':
    main()
<commit_msg>Minor edits<commit_after>from __future__ import print_function

import argparse
import os
import struct
import sys
import yaml

def read_from_config_file(configfile):
    """
    Read the format patterns from the given YAML configuration file.

    Parameters
    ----------
    configfile : str
        Path to YAML config file to read from.

    Returns
    -------
    formats : list of str
        A list of strings of the form
        ["<pattern1>:<count1>", "<pattern2>:<count2>", ...]. Each instance of
        ":<countN>" can be omitted for format patterns that only occur once.
    """
    with open(configfile, 'r') as f:
        formats = yaml.load(f)['formats']

    return formats

def write_to_config_file(configfile, formats):
    """
    Read the format patterns from the given YAML configuration file.

    Parameters
    ----------
    configfile : str
        Path to YAML config file to read from.
    formats : list of str
        A list of strings of the form
        ["<pattern1>:<count1>", "<pattern2>:<count2>", ...]. Each instance of
        ":<countN>" can be omitted for format patterns that only occur once.
    """
    with open(configfile, 'w') as f:
        yaml.dump({'formats': formats}, f, default_flow_style=False)

def gen_format_string(formats, size=None):
    """
    Generate a format string specifying the byte alignment given a list of the
    form ["<pattern1>:<count1>", "<pattern2>:<count2>", ...]. For example,

    >>> gen_format_string(["id4s:2", "f2d"])

    would return "id4sid4sf2d".

    For convenience, a special wildcard (*) character can be used once to
    automatically determine the count of one pattern. As an example,

    >>> gen_format_string(["i", "8s:*"])

    would return "i8s8s" if size is 20 bytes.

    Parameters
    ----------
    formats : list of str
        A list of strings of the form
        ["<pattern1>:<count1>", "<pattern2>:<count2>", ...], where each count is
        the number of repeated occurences of each format pattern. Each instance
        of ":<countN>" can be omitted for format patterns that only occur once.
        In the special case that a <countN> "*", the remainder
        of the available bytes in the file is used to automatically calculate
        the count. This can only be done once per list of formats.
    size : int, optional
        size of the source file in bytes. Only needed if wilcard '*' character
        is used in counts.

    Returns
    -------
    fmt : str
        The full format string.
    formats : list of str
        Formats list with wildcards expanded to actual counts.
    """
    # Fallback format
    if formats == ['*']:
        return '*', formats

    fmt = ''
    # Use this to keep track of size expended by format so far.
    cumsize = 0
    special_pattern = None
    special_index = None
    for i, pattern_info in enumerate(formats):
        pattern_info = pattern_info.split(':')
        pattern = pattern_info[0]

        # count is 1 if not given.
        if len(pattern_info) == 1:
            cumsize += struct.calcsize(pattern)
            fmt += pattern
        elif pattern_info[1][-1] == '*':
            if special_pattern:
                raise struct.error('Wildcard (*) character may only be used once')

            # We will need the current pattern and count values for later
            # when the remaining size is fully calculated.
            special_pattern = pattern
            special_index = i

            # Until remaining size is fully calculated, set a placeholder.
            fmt += '{0}'
        else:
            count = int(pattern_info[1])
            result = count*pattern
            cumsize += struct.calcsize(result)
            fmt += result

    # We are now ready to allocate the remaining bytes
    # for the special '*' pattern.
    if special_pattern:
        # Make sure source is specified.
        if size is None:
            raise struct.error('source must be specified if wildcard (*) '
                               'character is in format counts.')

        # Calculate the count such that pattern evenly fits in remaining size
        remaining = size - cumsize
        chunksize = struct.calcsize(special_pattern)
        count = remaining / chunksize
        if remaining % chunksize != 0:
            raise struct.error('Given chunksize of {0} bytes does not divide '
                               'evenly into remaining number of bytes.'
                               .format(chunksize))

        result = count*special_pattern
        fmt = fmt.format(result)

        # Update formats list to expand * character. This is needed because otherwise
        # the executable script will not run if the format saved in a config file
        # contains the * character when source is not given in the script
        # (eg, bconv is ran only to manipulate the defaultformat string)
        formats[special_index] = formats[special_index].replace('*', str(count))

    # Final sanity check: Ensure format string size and source file size match.
    fmt_size = struct.calcsize('=' + fmt)
    if size and size != fmt_size:
        raise struct.error('Format string size and chunk size do not match.\n'
                           'Expected: {0}, Got: {1}'.format(size, fmt_size))

    return fmt, formats

def convert(source, destination=None, byte_order=None, fmt='*'):
    """
    Converts the given file (specified by source) from one byte order
    to another. For example, to convert a file in the current working directory
    from big to little endian, call

    >>> convert('a.bin', 'little')

    By default, the source file is overwritten. To prevent this, a destination
    path may optionally be specified. Additionally, not specifying order
    will assume that you are trying to convert to the native byte order of your
    platform. Eg,

    >>> convert('a.bin', 'b.bin')

    will convert a.bin from big endian to little endian and store the result
    in b.bin on x86 platforms.

    Parameters
    ----------
    source : str
        The path to the binary file to be converted.
    destination: str, optional
        The path to the converted output file. If not specified, source is
        overwritten.
    byte_order: {None, 'little', 'big'}
        The byte order to convert to (endianness). If not specified, the
        native byte ordering of your platform is used (eg, 'little' on x86).
        "Do nothing" conversion operations are not allowed, so the input file
        given by source will be assumed to be formatted in the opposite
        byte order.
    fmt: str, optional
        Format string. See documentation for the python struct module for
        valid examples. The string should span the entire size of the file you
        are converting. The default format is "Nc", where N is the size of the
        file in bytes. Thus the bytes of the entire file are reversed in one go.
    """
    # This format string gives the number of bytes and type
    # for each piece of data in the record.
    if fmt == '*':
        num = os.path.getsize(source)
        fmt = num*'c'

    # Set default destination path to source (overwrite)
    if destination is None:
        destination = source

    # Set default byte order to native
    if byte_order is None:
        byte_order = sys.byteorder

    # This tells the struct library the byte order when packing/unpacking
    if byte_order == 'little':
        fmt_in = '>{0}'.format(fmt)
        fmt_out = '<{0}'.format(fmt)
    else:
        fmt_in = '<{0}'.format(fmt)
        fmt_out = '>{0}'.format(fmt)

    # Read and convert the data from source file
    with open(source, 'rb') as f:
        stream = f.read()
        data = struct.unpack(fmt_in, stream)

    # Write converted data to destination file
    with open(destination, 'wb') as f:
        f.write(struct.pack(fmt_out, *data))

def main():
    """
    CLI script for convert function.
    """
    description = 'Convert files from one byte order to another.'
    parser = argparse.ArgumentParser(prog='bconv', description=description)
    parser.add_argument('source', nargs='?', help='Path to input file')
    parser.add_argument('destination', nargs='?', help='Path to output file')
    parser.add_argument('--byte-order', dest='order', help='Output file byte order')
    parser.add_argument('-l', '--little-endian', dest='little', action='store_true',
                        help='Use little endian byte ordering for destination.')
    parser.add_argument('-b', '--big-endian', dest='big', action='store_true',
                        help='Use big endian byte ordering for destination.')
    parser.add_argument('-f', '--format', nargs='*', help='Format string.'
                        ' See documentation for the python standard library'
                        ' struct module for valid examples. To shorten the format'
                        ' string with common patterns, you may use the following'
                        ' notation:\n'
                        ' <pattern1>:<count1> <pattern2>:<count2> ...\n Omitting'
                        ' the : assumes a count of 1. As an example, "-f d i2s:2" is'
                        ' equivalent to "-f di2si2s". In addition, one of the'
                        ' counts may have a special wildcard character (*).'
                        ' When source is given, the count is automatically'
                        ' converted such that the number of bytes represented by'
                        ' pattern*count fits into the remainder of available bytes.'
                        ' For example, "bconv a.bin -p -f i 4s:*" will print'
                        ' "i4s4s" when a.bin is 12 bytes.')
    parser.add_argument('-d', '--set-default-format', dest='store', action='store_true',
                        help='If true, the format specified with the -f or -c'
                        ' switch is stored in the ~/.bconvrc file.'
                        ' Any formats containing a "*" are expanded.')
    parser.add_argument('-c', '--configfile', help='If specified, override the '
                        'default ~/.bconvrc for loading the the format string. '
                        'This is useful in place of the -f switch for really '
                        'long format strings.')
    parser.add_argument('-p', '--print-format', dest='printf', action='store_true',
                        help='Print the currently used format string. Omitting'
                        ' all other arguments prints the format string stored in'
                        ' the ~/.bconv file that comes with this program.')

    # Now process the arguments
    args = parser.parse_args()

    # Print help message if arguments are default.
    if not (args.source or args.destination or args.order or args.big
            or args.little or args.format or args.store or args.configfile
            or args.printf):
        parser.print_usage()
        sys.exit()

    # Input file
    source = args.source
    size = None

    # Ensure source is valid if given
    if source:
        try:
            size = os.path.getsize(source)
        except OSError as e:
            parser.error(e)

    # Output file
    destination = args.destination

    # Byte order
    if ((args.order is not None and (args.big or args.little))
        or (args.big and args.little)):
        parser.error('Cannot specify multiple byte-orders.')

    if args.big:
        byte_order = 'big'

    elif args.little:
        byte_order = 'little'

    else:
        byte_order = args.order

    # Format string
    configfile = os.join(os.path.expanduser('~'), '.bconvrc')
    if args.format and args.configfile:
        parser.error('Multiple format strings specified, please use only one of'
                     ' -f and -c when overriding default format.')
    elif args.format:
        formats = args.format

    elif args.configfile:
        try:
            configfile = args.configfile
            formats = read_from_config_file(args.configfile)

        except IOError as e:
            parser.error(e)

    else:
        try:
            formats = read_from_config_file(configfile)
        except IOError as e:
            # Do this in case default config file is corrupted or doesn't exist.
            formats = ['*']

    # Finally we can generate the full format string
    try:
        fmt, expanded_formats = gen_format_string(formats, size)
    except struct.error as e:
        paraser.error(e)

    # Store default format in config file
    if args.store or (args.format is None and expanded_formats != formats):
        write_to_config_file(configfile, expanded_formats)

    # Print the format string
    if args.printf:
        print('Current format: {0}'.format(fmt))

    # If source isn't given, we're done.
    if not source:
        return

    # Do the actual byte order conversion.
    try:
        convert(source, destination, byte_order, fmt)
    except IOError as e:
        # source input file can't be read
        parser.error(e)
    except struct.error as e:
        # badly formed format string
        parser.error(e)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import xmltodict, os, json
import sys

if len(sys.argv) > 1:
  code_dir = sys.argv[1]
else:
  code_dir = "code"

for folder, subs, files in os.walk(code_dir):
    for f in files:
        path = os.path.join(folder, f)
        (root, ext) = os.path.splitext(path)
        if ext == '.xml':
            jsonpath = path.replace(ext, '.json')
            d = xmltodict.parse(open(path))
            json.dump(d, open(jsonpath, 'w'))
<commit_msg>and remove the old convert.py script<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from logging import getLogger, StreamHandler, DEBUG
import configparser
import os
import shutil
import glob
import functools
from enum import Enum, unique
from pathlib import Path
# library
import cv2
import numpy as np
# Myapp library
from hsvcolor import HSVcolor
from iimagefillter import GrayScaleFilter, AdaptiveThresholdFilter, IImageFilter, ImageStream
from classifier import Classifier

logger = getLogger('myapp.tweetbot')
if __name__ == "__main__":
    handler = StreamHandler()
    handler.setLevel(DEBUG)
    logger.setLevel(DEBUG)
    logger.addHandler(handler)
@unique
class ImageType(Enum):
    RAW = 1
    NUMBER = 2
    HINTS = 4
    PLAN = 8
    HINT_RAW = RAW | HINTS
    HINT_NUMBER = NUMBER | HINTS
class BaseFilter(IImageFilter):
    def __init__(self):
        super().__init__('BaseFilter')
    def filtered(self, stream):
        result = stream
        return result
class AppImageFilter(IImageFilter):
    def __init__(self, image_type, hsv):
        super().__init__('AppImageFilter')
        self._contryMask = {'netzawar':(HSVcolor(175, 55, 0), HSVcolor(255, 255, 255)),
                            'casedria':(HSVcolor(53, 0, 0), HSVcolor(79, 255, 255)),
                            'geburand':(HSVcolor(120, 0, 100), HSVcolor(150, 255, 255)),
                            'hordine':(HSVcolor(24, 0, 249), HSVcolor(30, 255, 255)),
                            'ielsord':(HSVcolor(79, 0, 0), HSVcolor(112, 255, 255))}
        self.image_type = image_type
        self.hsv = hsv
    def filtered(self, stream):
        binary = stream
        # white color
        white = self.__getWhiteMasking(self.hsv)
        binary = cv2.bitwise_and(binary, binary, mask=white)
        if self.image_type == ImageType.PLAN:
            height, width = stream.shape[:2]
            cv2.rectangle(binary, (0, 0), (min(1000, width), height), (0,0,0), -1)
            return binary

        #fez country color mask pattern
        binary = self.bitwise_not(binary, self._contryMask['netzawar'])
        binary = self.bitwise_not(binary, self._contryMask['geburand'])
        binary = self.bitwise_not(binary, self._contryMask['ielsord'])
        binary = self.bitwise_not(binary, self._contryMask['casedria'])
        binary = self.bitwise_not(binary, self._contryMask['hordine'])
        # １位より下を黒色で塗りつぶしてマスク。
        height, width = stream.shape[:2]
        cv2.rectangle(binary, (0, min(500, height)), (width,height), (0,0,0), -1)
        return binary
    def bitwise_not(self, binary, mask_range):
        lower, upper = mask_range
        return cv2.bitwise_not(binary, binary, mask=self.__inRange(self.hsv, lower, upper))
    def __getWhiteMasking(self, hsv):
        sensitivity = 15
        lower = HSVcolor(0, 0, 255 - sensitivity)
        upper = HSVcolor(255, sensitivity, 255)
        return self.__inRange(hsv, lower, upper)
    def __inRange(self, hsv, lower, upper):
        return cv2.inRange(hsv, lower.to_np(), upper.to_np())
class DataProcessor(object):
    def __init__(self, name, image_type):
        self.__name = name
        self.image_type = image_type
        self.color = None
        self.__hsv = None
    @property
    def name(self):
        return self.__name
    @property
    def hsv(self):
        if self.__hsv is None:
            self.__hsv = cv2.cvtColor(self.color, cv2.COLOR_BGR2HSV)
        return self.__hsv
    def prepare(self):
        """
            @return color image
        """
        filename = self.name
        zoom = 2
        c = cv2.imread(filename)
        if c is None:
            logger.info("notfound:{0}".format(self.name))
            return c
        if self.image_type == ImageType.NUMBER or self.image_type == ImageType.HINT_NUMBER:
            """
                small size image.
                    AKAZE#detectAndCompute at keypoints of 0.
                Ensure image scall zooming
            """
            zoom = 10
        self.color = cv2.resize(c, (c.shape[1]*zoom, c.shape[0]*zoom), interpolation=cv2.INTER_CUBIC)
        return self.color
    def batch(self):
        """
            masking)
                step1:white color
                step2:color -> grayscale -> adaptiveThreshold -> binary
                step3:country color
                step4:fill min(500, height)
            @return binary image
        """
        if self.image_type == ImageType.NUMBER or self.image_type == ImageType.HINT_NUMBER:
            return self.color
        stream = ImageStream()
        stream.data = self.color
        stream.addFilter(GrayScaleFilter())
        stream.addFilter(AdaptiveThresholdFilter())
        stream.addFilter(AppImageFilter(self.image_type, self.hsv))
        return stream.tofiltered()
class country(object):
    def __init__(self, config):
        self.hints = config['WORK_FOLDER']['HINTS']
        d = dict()
        names = config['COUNTRY']['NAMES'].split('|')
        for ele in names:
            c = ele.split(":")
            d[c[0]] = c[1]
        self.names = d
        self.detector = cv2.AKAZE_create()
        self.classifier = Classifier()
        self.numberclassifier = Classifier()
        self.__load(self.classifier, ImageType.HINT_RAW)
        for i in range(10):
            self.__load(self.numberclassifier, ImageType.HINT_NUMBER, i)

    def __load(self, classifier, image_type, n=None):
        """
            loading descriptors image and label
        """
        loadPath = Path(self.hints)
        if n is not None:
            loadPath  = Path(self.hints, 'number', str(n))
        features = []
        labels = []
        for media in glob.iglob(os.path.join(str(loadPath), "*.png")):
            (keypoints, descriptors) = self.__cachedetect(media, image_type)
            if image_type == ImageType.HINT_NUMBER:
                #logger.info(keypoints)
                pass
            features.append(descriptors)
            if image_type == ImageType.HINT_NUMBER:
                labels.append(n)
            else:
                labels.append(os.path.basename(media))
        classifier.fit(features, labels)
        logger.info('load classifier')
        logger.info(classifier.labels)
    @functools.lru_cache(maxsize=8)
    def __cachedetect(self, media, image_type=None):
        pro = DataProcessor(media, image_type)
        if pro.prepare() is None:
            logger.error('image error:{0}'.format(media))
            return None, None
        batch = pro.batch()
        if image_type == ImageType.RAW:
            cv2.imwrite('./base_color.png', pro.color)
            cv2.imwrite('./base_binary.png', batch)
        elif image_type == ImageType.NUMBER \
                or image_type == ImageType.HINT_NUMBER  \
                or image_type == ImageType.HINT_RAW:
            pass
        else:
            cv2.imwrite('./binary_{0}'.format(os.path.basename(media)), batch)
        return self.detector.detectAndCompute(batch, None)
    def getCountry(self, src):
        """
            @return country list
        """
        (keypoints, descriptors) = self.__cachedetect(src, ImageType.RAW)
        if keypoints is None:
            return None
        d = self.classifier.predict(descriptors)
        logger.info(d)
        #d2 = self.classifier.predict(descriptors, top_n=1)
        #logger.info(d2)
        return d
    def getNumber(self, src):
        (keypoints, descriptors) = self.__cachedetect(src, ImageType.NUMBER)
        if keypoints is None:
            return None
        d3 = self.numberclassifier.predict(descriptors, top_n=100)
        #logger.info(d3)
        return d3
    def classify(features):
        label = ''
        return label
    def classifyNumberImage(self):
        basepath = './test/'
        count = 0
        for media in glob.iglob(os.path.join(basepath, "*.png")):
            if os.path.getsize(media) > 2048:
                continue
            count += 1
            d = self.getNumber(media)
            logger.info(d)
            s = sum([v for m,v in d.items()])
            if s == 0:
                logger.warning('keypoint NotFound:{0}'.format(media))
                continue
            for k in d.keys():
                prefix = str(k)
                filename = os.path.join(basepath + prefix + '/', os.path.basename(media))
                directory = os.path.dirname(filename)
                if not os.path.exists(directory):
                    os.mkdir(directory)
                shutil.copy2(media, filename)
                break
        return None
    def getName(self, n):
        return self.names[n]

class Clipping(object):
    def __init__(self, media, target=None):
        self.media = media
        self.target = target
        pro = DataProcessor(self.media, ImageType.PLAN)
        pro.prepare()
        self.binary = pro.batch()
        self.color = pro.color
        self.drawClipSource = True
    def loadImage(self):
        pass
    def imagefilter(self):
        pass
    def number(self):
        """
            fillter
                1,頂点数が3未満
                2,面積が50未満
        """
        cv2.imwrite('./test/binary_color{0}'.format(os.path.basename(self.media)), self.color)
        cv2.imwrite('./test/binary_{0}'.format(os.path.basename(self.media)), self.binary)
        image, contours, hierarchy = cv2.findContours(self.binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        rect = []
        for c in contours:
            approx = cv2.approxPolyDP(c,1,True)
            if len(approx) < 3:
                continue
            area = cv2.contourArea(approx)
            if area < 50:
                continue
            if self.drawClipSource:
                #cv2.drawContours(self.color, [c], -1, (0,255,0), 3)
                pass
            rect.append(cv2.boundingRect(approx))

        rect = sorted(rect, key=lambda x:(x[1], x[0]))
        self.splitImage(rect)
        return ''
    def splitImage(self, rect):
        srcPath = Path(self.media)
        srcfilename = os.path.basename(srcPath.stem)
        for i, value  in enumerate(rect):
            x, y = value[0], value[1]
            w, h = value[2], value[3]
            p1 = (x, y)
            p2 = (x+w, y+h)
            cv2.rectangle(self.color, p1, p2, (0,255,0), 2)
            roi = self.color[y:y+h,x:x+w]
            #roi = roi.shape[:2]
            #image = cv2.resize(self.color[y:y+h,x:x+w],(32,32))
            image = roi
            cv2.imwrite('./test/{0}_{1}_{2}{3}'.format(srcfilename, i, value, srcPath.suffix), image)
def main():
    config = configparser.ConfigParser()
    with open('./setting.ini', 'r', encoding='utf-8-sig') as f:
        config.read_file(f)
    c = country(config)
    # benchMark
    ele = ['./backup/hints/201702190825_0565e4fcbc166f00577cbd1f9a76f8c7.png',
        #    './backup/test/201702191909_ac08ccbbb04f2a1feeb4f8aaa08ae008.png',
        #     './backup/test/201702192006_2e268d7508c20aa00b22dfd41639d65e.png',
        ] * 2
    for l in ele:
        logger.info(l)
        for k in c.getCountry(l).keys():
            country_name = c.getName(k)
            logger.info(country_name)
            break
    #return
    ele = ['./dat/number/sample/1.png',
           './dat/number/sample/1_1.png',
          ] * 1
    for l in ele:
        logger.info(l)
        d = c.getNumber(l)
    ele = ['./backup/hints/201702190825_0565e4fcbc166f00577cbd1f9a76f8c7.png',
           './backup/201702161156_919c7be6531ffeb5cfeb1e6e701556d3.png',
           './images/201702212104_4f1abbaf37872a27bec7f908a527f512.png',
           './backup/201702161344_b17e5061a7083b933531d75808f3de48.png',
           './backup/201702161659_b17e5061a7083b933531d75808f3de48.png',
          ] * 5
    for l in ele:
        clip = Clipping(l)
        clip.number()

    c.classifyNumberImage()
if __name__ == "__main__":
    main()
<commit_msg>:art:Clean county.py<commit_after># -*- coding: utf-8 -*-
from logging import getLogger, StreamHandler, DEBUG
import configparser
import os
import shutil
import tempfile
import glob
import functools
from enum import Enum, unique
from pathlib import Path
# library
import cv2
import numpy as np
from PIL import Image
# Myapp library
from hsvcolor import HSVcolor
from iimagefillter import GrayScaleFilter, AdaptiveThresholdFilter, IImageFilter, ImageStream
from classifier import Classifier
from ocrengine import OCREngine

logger = getLogger('myapp.tweetbot')
if __name__ == "__main__":
    handler = StreamHandler()
    handler.setLevel(DEBUG)
    logger.setLevel(DEBUG)
    logger.addHandler(handler)
@unique
class ImageType(Enum):
    RAW = 1
    NUMBER = 2
    HINTS = 4
    PLAN = 8
    HINT_RAW = RAW | HINTS
    HINT_NUMBER = NUMBER | HINTS
class ROIFilter(IImageFilter):
    def __init__(self):
        super().__init__('BaseFilter')
    def filtered(self, stream):
        start_x = 240
        start_y = 350
        end_x = 920
        end_y = 1500
        return stream[start_y:end_x, start_x:end_y]
class AppImageFilter(IImageFilter):
    def __init__(self, image_type, hsv):
        super().__init__('AppImageFilter')
        self._contryMask = {'netzawar':(HSVcolor(175, 55, 0), HSVcolor(255, 255, 255)),
                            'casedria':(HSVcolor(53, 0, 0), HSVcolor(79, 255, 255)),
                            'geburand':(HSVcolor(120, 0, 100), HSVcolor(150, 255, 255)),
                            'hordine':(HSVcolor(24, 0, 249), HSVcolor(30, 255, 255)),
                            'ielsord':(HSVcolor(79, 0, 0), HSVcolor(112, 255, 255))}
        self.image_type = image_type
        self.hsv = hsv
    def filtered(self, stream):
        binary = stream

        # white color
        white = self.__getWhiteMasking(self.hsv)
        binary = cv2.bitwise_and(binary, binary, mask=white)
        if self.image_type == ImageType.PLAN:
            height, width = stream.shape[:2]
            cv2.rectangle(binary, (0, 0), (min(1000, width), height), (0,0,0), -1)
            return binary

        lower, upper = HSVcolor(0, 0, 0), HSVcolor(30, 66, 255)
        binary = cv2.bitwise_and(binary, self.__inRange(self.hsv, lower, upper))

        #fez country color mask pattern
        #binary = self.bitwise_not(binary, self._contryMask['netzawar'])
        #binary = self.bitwise_not(binary, self._contryMask['geburand'])
        #binary = self.bitwise_not(binary, self._contryMask['ielsord'])
        #binary = self.bitwise_not(binary, self._contryMask['casedria'])
        #binary = self.bitwise_not(binary, self._contryMask['hordine'])
        # image out range fill
        height, width = stream.shape[:2]
        start_x = 240
        start_y = 350
        end_x = 920
        end_y = 1500
        binary = binary[start_y:end_x, start_x:end_y]
        return binary
    def bitwise_not(self, binary, mask_range):
        lower, upper = mask_range
        return cv2.bitwise_not(binary, binary, mask=self.__inRange(self.hsv, lower, upper))
    def __getWhiteMasking(self, hsv):
        sensitivity = 15
        lower = HSVcolor(0, 0, 255 - sensitivity)
        upper = HSVcolor(255, sensitivity, 255)
        return self.__inRange(hsv, lower, upper)
    def __inRange(self, hsv, lower, upper):
        return cv2.inRange(hsv, lower.to_np(), upper.to_np())
class DataProcessor(object):
    def __init__(self, name, image_type):
        self.__name = name
        self.image_type = image_type
        self.color = None
        self.__hsv = None
    @property
    def name(self):
        return self.__name
    @property
    def hsv(self):
        if self.__hsv is None:
            self.__hsv = cv2.cvtColor(self.color, cv2.COLOR_BGR2HSV)
        return self.__hsv
    def prepare(self):
        """
            @return color image
        """
        filename = self.name
        zoom = 2
        c = cv2.imread(filename)
        if c is None:
            raise FileNotFoundError(self.name)
        if self.image_type == ImageType.NUMBER or self.image_type == ImageType.HINT_NUMBER:
            """
                small size image.
                    AKAZE#detectAndCompute at keypoints of 0.
                Ensure image scall zooming
            """
            zoom = 10
        self.color = cv2.resize(c, (c.shape[1]*zoom, c.shape[0]*zoom), interpolation=cv2.INTER_CUBIC)
        return self.color
    def batch(self):
        """
            masking)
                step1:white color
                step2:color -> grayscale -> adaptiveThreshold -> binary
                step3:country color
                step4:fill min(500, height)
            @return binary image
        """
        if self.image_type == ImageType.NUMBER or self.image_type == ImageType.HINT_NUMBER:
            return self.color
        stream = ImageStream()
        stream.data = self.color
        #stream.addFilter(ROIFilter())
        stream.addFilter(GrayScaleFilter())
        stream.addFilter(AdaptiveThresholdFilter())
        stream.addFilter(AppImageFilter(self.image_type, self.hsv))
        return stream.tofiltered()
class country(object):
    def __init__(self, config):
        self.hints = config['WORK_FOLDER']['HINTS']
        d = dict()
        names = config['COUNTRY']['NAMES'].split('|')
        for ele in names:
            c = ele.split(":")
            d[c[0]] = c[1]
        self.names = d
        self.detector = cv2.AKAZE_create()
        self.classifier = Classifier()
        self.numberclassifier = Classifier()
        self.__load(self.classifier, ImageType.HINT_RAW)
        for i in range(10):
            self.__load(self.numberclassifier, ImageType.HINT_NUMBER, i)

    def __load(self, classifier, image_type, n=None):
        """
            loading descriptors image and label
        """
        loadPath = Path(self.hints)
        if n is not None:
            loadPath  = Path(self.hints, 'number', str(n))
        features = []
        labels = []
        for media in glob.iglob(os.path.join(str(loadPath), "*.png")):
            (keypoints, descriptors) = self.__cachedetect(media, image_type)
            if image_type == ImageType.HINT_NUMBER:
                #logger.info(keypoints)
                pass
            features.append(descriptors)
            if image_type == ImageType.HINT_NUMBER:
                labels.append(n)
            else:
                labels.append(os.path.basename(media))
        classifier.fit(features, labels)
        #logger.info('load classifier')
        #logger.info(classifier.labels)
    @functools.lru_cache(maxsize=8)
    def __cachedetect(self, media, image_type=None):
        pro = DataProcessor(media, image_type)
        if pro.prepare() is None:
            logger.error('image error:{0}'.format(media))
            return None, None
        batch = pro.batch()
        if image_type == ImageType.RAW:
            cv2.imwrite('./base_color.png', pro.color)
            cv2.imwrite('./base_binary.png', batch)
        elif image_type == ImageType.NUMBER \
                or image_type == ImageType.HINT_NUMBER  \
                or image_type == ImageType.HINT_RAW:
            pass
        else:
            cv2.imwrite('./binary_{0}'.format(os.path.basename(media)), batch)
        return self.detector.detectAndCompute(batch, None)
    def getCountry(self, src):
        """
            @return country list
        """
        (keypoints, descriptors) = self.__cachedetect(src, ImageType.RAW)
        if keypoints is None:
            return None
        d = self.classifier.predict(descriptors)

        pro = DataProcessor(src, ImageType.RAW)
        if pro.prepare() is None:
            logger.error('image error:{0}'.format(src))
            return None, None
        batch = pro.batch()
        baseName = Path(src)
        ocr = OCREngine()
        with tempfile.NamedTemporaryFile(delete=False, suffix=baseName.suffix) as temp:
             temp_file_name = temp.name
             logger.info(temp_file_name)
             cv2.imwrite(temp_file_name, batch)

        document = ocr.recognize(temp_file_name)
        for text in [document.rank1, document.rank2, document.rank3
                     , document.rank4, document.rank5]:
            logger.info('{0}/{1}'.format(text['name'], text['score']))

        #TextFormat()

        logger.info(d)
        #d2 = self.classifier.predict(descriptors, top_n=1)
        #logger.info(d2)
        return d
    def getNumber(self, src):
        (keypoints, descriptors) = self.__cachedetect(src, ImageType.NUMBER)
        if keypoints is None:
            return None
        d3 = self.numberclassifier.predict(descriptors, top_n=100)
        #logger.info(d3)
        return d3
    def getName(self, n):
        return self.names[n]

class Clipping(object):
    def __init__(self, media, target=None):
        self.media = media
        self.target = target
        pro = DataProcessor(self.media, ImageType.PLAN)
        pro.prepare()
        self.binary = pro.batch()
        self.color = pro.color
        self.drawClipSource = True
    def loadImage(self):
        pass
    def imagefilter(self):
        pass
    def number(self):
        """
            fillter
                1,頂点数が3未満
                2,面積が50未満
        """
        cv2.imwrite('./test/binary_color{0}'.format(os.path.basename(self.media)), self.color)
        cv2.imwrite('./test/binary_{0}'.format(os.path.basename(self.media)), self.binary)
        image, contours, hierarchy = cv2.findContours(self.binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        rect = []
        for c in contours:
            approx = cv2.approxPolyDP(c,1,True)
            if len(approx) < 3:
                continue
            area = cv2.contourArea(approx)
            if area < 50:
                continue
            if self.drawClipSource:
                cv2.drawContours(self.color, [c], -1, (0,255,0), 3)
                pass
            rect.append(cv2.boundingRect(approx))

        rect = sorted(rect, key=lambda x:(x[1], x[0]))
        self.splitImage(rect)
        return ''
    def splitImage(self, rect):
        srcPath = Path(self.media)
        srcfilename = os.path.basename(srcPath.stem)
        for i, value  in enumerate(rect):
            x, y = value[0], value[1]
            w, h = value[2], value[3]
            p1 = (x, y)
            p2 = (x+w, y+h)
            cv2.rectangle(self.color, p1, p2, (0,255,0), 2)
            roi = self.color[y:y+h,x:x+w]
            #roi = roi.shape[:2]
            #image = cv2.resize(self.color[y:y+h,x:x+w],(32,32))
            image = roi
            cv2.imwrite('./test/{0}_{1}_{2}{3}'.format(srcfilename, i, value, srcPath.suffix), image)
def main():
    config = configparser.ConfigParser()
    with open('./setting.ini', 'r', encoding='utf-8-sig') as f:
        config.read_file(f)
    c = country(config)
    # benchMark
    ele = ['./backup/hints/201702190825_0565e4fcbc166f00577cbd1f9a76f8c7.png',
        #    './backup/test/201702191909_ac08ccbbb04f2a1feeb4f8aaa08ae008.png',
        #     './backup/test/201702192006_2e268d7508c20aa00b22dfd41639d65e.png',
        ] * 2
    for l in ele:
        logger.info(l)
        for k in c.getCountry(l).keys():
            country_name = c.getName(k)
            logger.info(country_name)
            break
if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# CTK: Cherokee Toolkit
#
# Authors:
#      Alvaro Lopez Ortega <alvaro@alobbs.com>
#
# Copyright (C) 2009 Alvaro Lopez Ortega
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of version 2 of the GNU General Public
# License as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
#

import re
import string

from consts import *
from util import json_dump
from PageCleaner import Postprocess

widget_uniq_id = 1;

SYNC_JS_LOAD_JS = """
if (typeof (__loaded_%(name)s) == 'undefined') {
    $.ajax ({url: '%(url)s', type:'get', dataType:'script', async:false, global:false, cache:true, ifModified:true});
    __loaded_%(name)s = true;
}
"""

class RenderResponse:
    def __init__ (self, html='', js='', headers=[], helps=[]):
        self.html    = html
        self.js      = js
        self.headers = headers[:]
        self.helps   = helps[:]

    def clean_up_headers (self):
        noDupes = []
        [noDupes.append(i) for i in self.headers if not noDupes.count(i)]
        self.headers = noDupes

    def __add__ (self, other):
        assert isinstance(other, RenderResponse)

        # New response obj
        i = RenderResponse()

        # Append the new response
        i.html    = self.html    + other.html
        i.js      = self.js      + other.js
        i.headers = self.headers + other.headers
        i.helps   = self.helps   + other.helps

        # Sort the headers
        i.clean_up_headers()
        return i

    def toStr (self):
        txt = self.html
        alpha = string.letters + string.digits

        # Synchronous JS loading..
        # Jquery MUST be loaded for this code to work
        sync_js_load = ""

        for js_header in filter(lambda x: x.startswith('<script'), self.headers):
            url_js  = re.findall (r'src="(.+)"', js_header)[0]
            name_js = ''.join (map (lambda x: ('_',x)[x in alpha], re.findall (r'src=".+/(.+)\.js"', js_header)[0]))
            sync_js_load += SYNC_JS_LOAD_JS %({'name':name_js, 'url':url_js})

        # Render the Javascript block
        if self.js or sync_js_load:
            txt += HTML_JS_ON_READY_BLOCK %(sync_js_load + self.js)

        return Postprocess(txt)

    def toJSON (self):
        tmp = filter (lambda x: x, [x.toJSON() for x in self.helps])
        if tmp:
            help = []
            for e in reduce(lambda x,y: x+y, tmp):
                if not e[0] in [x[0] for x in help]:
                    help.append(e)
        else:
            help = []

        return json_dump({'html':    self.html,
                          'js':      Postprocess(self.js),
                          'headers': self.headers,
                          'helps':   help})

class Widget:
    def __init__ (self):
        global widget_uniq_id;

        widget_uniq_id += 1;
        self.uniq_id = widget_uniq_id;
        self.id      = "widget_%d" %(widget_uniq_id)
        self.binds   = []

    def Render (self):
        render = RenderResponse()

        for event, js in self.binds:
            render.js += "$('#%s').bind('%s', function(event){ %s });\n" %(self.id, event, js)

        return render

    # Javacript events
    #
    def bind (self, event, js):
        self.binds.append ((event, js))

    def JS_to_trigger (self, event, param=None, selector=None):
        if not selector:
            selector = "$('#%s')" %(self.id)

        if param:
            return "%s.trigger('%s', %s);" %(selector, event, param)
        else:
            return "%s.trigger('%s');" %(selector, event)

<commit_msg>Adds to helper methods: Widget::JS_to_show() and Widget::JS_to_hide()<commit_after># -*- coding: utf-8 -*-

# CTK: Cherokee Toolkit
#
# Authors:
#      Alvaro Lopez Ortega <alvaro@alobbs.com>
#
# Copyright (C) 2009 Alvaro Lopez Ortega
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of version 2 of the GNU General Public
# License as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
#

import re
import string

from consts import *
from util import json_dump
from PageCleaner import Postprocess

widget_uniq_id = 1;

SYNC_JS_LOAD_JS = """
if (typeof (__loaded_%(name)s) == 'undefined') {
    $.ajax ({url: '%(url)s', type:'get', dataType:'script', async:false, global:false, cache:true, ifModified:true});
    __loaded_%(name)s = true;
}
"""

class RenderResponse:
    def __init__ (self, html='', js='', headers=[], helps=[]):
        self.html    = html
        self.js      = js
        self.headers = headers[:]
        self.helps   = helps[:]

    def clean_up_headers (self):
        noDupes = []
        [noDupes.append(i) for i in self.headers if not noDupes.count(i)]
        self.headers = noDupes

    def __add__ (self, other):
        assert isinstance(other, RenderResponse)

        # New response obj
        i = RenderResponse()

        # Append the new response
        i.html    = self.html    + other.html
        i.js      = self.js      + other.js
        i.headers = self.headers + other.headers
        i.helps   = self.helps   + other.helps

        # Sort the headers
        i.clean_up_headers()
        return i

    def toStr (self):
        txt = self.html
        alpha = string.letters + string.digits

        # Synchronous JS loading..
        # Jquery MUST be loaded for this code to work
        sync_js_load = ""

        for js_header in filter(lambda x: x.startswith('<script'), self.headers):
            url_js  = re.findall (r'src="(.+)"', js_header)[0]
            name_js = ''.join (map (lambda x: ('_',x)[x in alpha], re.findall (r'src=".+/(.+)\.js"', js_header)[0]))
            sync_js_load += SYNC_JS_LOAD_JS %({'name':name_js, 'url':url_js})

        # Render the Javascript block
        if self.js or sync_js_load:
            txt += HTML_JS_ON_READY_BLOCK %(sync_js_load + self.js)

        return Postprocess(txt)

    def toJSON (self):
        tmp = filter (lambda x: x, [x.toJSON() for x in self.helps])
        if tmp:
            help = []
            for e in reduce(lambda x,y: x+y, tmp):
                if not e[0] in [x[0] for x in help]:
                    help.append(e)
        else:
            help = []

        return json_dump({'html':    self.html,
                          'js':      Postprocess(self.js),
                          'headers': self.headers,
                          'helps':   help})

class Widget:
    def __init__ (self):
        global widget_uniq_id;

        widget_uniq_id += 1;
        self.uniq_id = widget_uniq_id;
        self.id      = "widget_%d" %(widget_uniq_id)
        self.binds   = []

    def Render (self):
        render = RenderResponse()

        for event, js in self.binds:
            render.js += "$('#%s').bind('%s', function(event){ %s });\n" %(self.id, event, js)

        return render

    # Javacript events
    #
    def bind (self, event, js):
        self.binds.append ((event, js))

    def JS_to_trigger (self, event, param=None, selector=None):
        if not selector:
            selector = "$('#%s')" %(self.id)

        if param:
            return "%s.trigger('%s', %s);" %(selector, event, param)
        else:
            return "%s.trigger('%s');" %(selector, event)

    def JS_to_show (self, how='', selector=None):
        if not selector:
            selector = "$('#%s')" %(self.id)
        return '%s.show(%s);' %(selector, how)

    def JS_to_hide (self, how='', selector=None):
        if not selector:
            selector = "$('#%s')" %(self.id)
        return '%s.hide(%s);' %(selector, how)
<|endoftext|>


<commit_before>"""
Copyright (C) 2015  Genome Research Ltd.

Author: Irina Colgiu <ic4@sanger.ac.uk>

This program is part of meta-check

meta-check is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

This file has been created on Nov 16, 2015.
"""

from metadata.seqscape_metadata.seqscape_metadata import SeqscapeRawMetadata, SeqscapeEntitiesFetched

from sequencescape import connect_to_sequencescape, Sample, Study, Library

class SeqscapeRawMetadataProvider:

    @classmethod
    def _get_connection(cls, host, port, db_name, user):
        return connect_to_sequencescape("mysql://" + user + ":@" + host + ":" + str(port) + "/" + db_name)

    @classmethod
    def _retrieve_samples(cls, ss_connection, sample_names, sample_ids, sample_accession_nrs):
        print("SAMPLES NAMESSS: ", sample_names)
        samples_by_name = ss_connection.sample.get_by_name(sample_names)
        samples_fetched_by_name = SeqscapeEntitiesFetched(samples_by_name,
                                                                    query_ids=sample_names,
                                                                    query_id_type='name',
                                                                    query_entity_type='sample',
                                                                    fetched_entity_type='sample')

        samples_by_id = ss_connection.sample.get_by_id(sample_ids)
        samples_fetched_by_id = SeqscapeEntitiesFetched(samples_by_id,
                                                                  query_ids=sample_ids,
                                                                  query_id_type='internal_id',
                                                                  query_entity_type='sample',
                                                                  fetched_entity_type='sample')

        samples_by_accession_nr = ss_connection.sample.get_by_accession_number(sample_accession_nrs)
        samples_fetched_by_accession_nr = SeqscapeEntitiesFetched(samples_by_accession_nr,
                                                                            query_ids=sample_accession_nrs,
                                                                            query_id_type='accession_number',
                                                                            query_entity_type='sample',
                                                                            fetched_entity_type='sample')

        return samples_fetched_by_name, samples_fetched_by_id, samples_fetched_by_accession_nr

    @classmethod
    def _retrieve_studies(cls, ss_connection, study_names, study_ids, study_accession_nrs):
        studies_by_name = ss_connection.study.get_by_name(study_names)
        studies_fetched_by_name = SeqscapeEntitiesFetched(studies_by_name,
                                                                    query_ids=study_names,
                                                                    query_id_type='name',
                                                                    query_entity_type='study',
                                                                    fetched_entity_type='study'
        )

        studies_by_accession_nr = ss_connection.study.get_by_accession_number(study_accession_nrs)
        studies_fetched_by_accession_nr = SeqscapeEntitiesFetched(studies_by_accession_nr,
                                                                            query_ids=study_accession_nrs,
                                                                            query_id_type='accession_number',
                                                                            query_entity_type='study',
                                                                            fetched_entity_type='study'
        )

        studies_by_id = ss_connection.study.get_by_id(study_ids)
        studies_fetched_by_id = SeqscapeEntitiesFetched(studies_by_id,
                                                                  query_ids=study_ids,
                                                                  query_id_type='internal_id',
                                                                  query_entity_type='study',
                                                                  fetched_entity_type='study'
        )

        return studies_fetched_by_name, studies_fetched_by_id, studies_fetched_by_accession_nr

    @classmethod
    def _retrieve_libraries(cls, ss_connection, library_names, library_ids):
        libraries_by_id = ss_connection.library.get_by_id(library_ids)
        if not libraries_by_id:
            libraries_by_id = ss_connection.well.get_by_id(library_ids)
        if not libraries_by_id:
            libraries_by_id = ss_connection.multiplexed_library(library_ids)

        libraries_fetched_by_id = SeqscapeEntitiesFetched(libraries_by_id,
                                                                    query_ids=library_ids,
                                                                    query_id_type='internal_id',
                                                                    query_entity_type='library',
                                                                    fetched_entity_type='library')

        # if i_meta.libraries['name']:
        libraries_by_name = ss_connection.library.get_by_name(library_names)
        libraries_fetched_by_name = SeqscapeEntitiesFetched(libraries_by_name,
                                                                      query_ids=library_names,
                                                                      query_id_type='name',
                                                                      query_entity_type='library',
                                                                      fetched_entity_type='library')
        return libraries_fetched_by_name, libraries_fetched_by_id

    @classmethod
    def _retrieve_samples_for_studies(cls, ss_connection, studies):
        return ss_connection.sample.get_associated_with_study(studies)

    @classmethod
    def _retrieve_studies_for_samples(cls, ss_connection, samples):
        return ss_connection.study.get_associated_with_sample(samples)

    @classmethod
    def retrieve_raw_metadata(cls, samples, libraries, studies):
        import config

        ss_connection = cls._get_connection(config.SEQSC_HOST, config.SEQSC_PORT, config.SEQSC_DB_NAME,
                                            config.SEQSC_USER)

        ss_connection.sample.get_by_name("1866STDY5139782")

        samples_fetched_by_names, samples_fetched_by_ids, samples_fetched_by_accession_nrs = \
            cls._retrieve_samples(ss_connection, samples['name'], samples['internal_id'], samples['accession_number'])

        studies_fetched_by_names, studies_fetched_by_ids, studies_fetched_by_accession_nrs = \
            cls._retrieve_studies(ss_connection, studies['name'], studies['internal_id'], studies['accession_number'])

        libraries_fetched_by_names, libraries_fetched_by_ids = \
            cls._retrieve_libraries(ss_connection, libraries['name'], libraries['internal_id'])

        raw_meta = SeqscapeRawMetadata()
        raw_meta.add_fetched_entities(samples_fetched_by_names)
        raw_meta.add_fetched_entities(samples_fetched_by_accession_nrs)
        raw_meta.add_fetched_entities(samples_fetched_by_ids)

        raw_meta.add_fetched_entities(studies_fetched_by_accession_nrs)
        raw_meta.add_fetched_entities(studies_fetched_by_ids)
        raw_meta.add_fetched_entities(studies_fetched_by_names)

        raw_meta.add_fetched_entities(libraries_fetched_by_names)
        raw_meta.add_fetched_entities(libraries_fetched_by_ids)

        # Getting the sample-study associations:
        studies_set = raw_meta.get_entities_without_duplicates_by_entity_type('study')
#        print("UNIQ STUDIES: :::::::::::::::::::::", studies_set)
        samples_for_study = cls._retrieve_samples_for_studies(ss_connection, studies_set)

        raw_meta.add_fetched_entities_by_association(samples_for_study)
#        print("SAMPLES FOR STUDYYYYYYYYYYYYYYYYYYYY : ", samples_for_study)

        samples_set = raw_meta.get_entities_without_duplicates_by_entity_type('sample')
#        print("Samples UNIQ:::::::::::::::::::::", samples_set)
        studies_for_samples = cls._retrieve_studies_for_samples(ss_connection, samples_set)

        raw_meta.add_fetched_entities_by_association(studies_for_samples)
#        print("Studies for SAMPLESSSS: ", studies_for_samples)

        return raw_meta<commit_msg>Renamed methods and cleaned code from comments<commit_after>"""
Copyright (C) 2015  Genome Research Ltd.

Author: Irina Colgiu <ic4@sanger.ac.uk>

This program is part of meta-check

meta-check is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

This file has been created on Nov 16, 2015.
"""

from metadata.seqscape_metadata.seqscape_metadata import SeqscapeRawMetadata, SeqscapeEntitiesFetched
import config
from sequencescape import connect_to_sequencescape, Sample, Study, Library

class SeqscapeRawMetadataProvider:

    @classmethod
    def _get_connection(cls, host, port, db_name, user):
        return connect_to_sequencescape("mysql://" + user + ":@" + host + ":" + str(port) + "/" + db_name)

    @classmethod
    def _fetch_samples(cls, ss_connection, sample_names, sample_ids, sample_accession_nrs):
        print("SAMPLES NAMESSS: ", sample_names)
        samples_by_name = ss_connection.sample.get_by_name(sample_names)
        samples_fetched_by_name = SeqscapeEntitiesFetched(samples_by_name,
                                                                    query_ids=sample_names,
                                                                    query_id_type='name',
                                                                    query_entity_type='sample',
                                                                    fetched_entity_type='sample')

        samples_by_id = ss_connection.sample.get_by_id(sample_ids)
        samples_fetched_by_id = SeqscapeEntitiesFetched(samples_by_id,
                                                                  query_ids=sample_ids,
                                                                  query_id_type='internal_id',
                                                                  query_entity_type='sample',
                                                                  fetched_entity_type='sample')

        samples_by_accession_nr = ss_connection.sample.get_by_accession_number(sample_accession_nrs)
        samples_fetched_by_accession_nr = SeqscapeEntitiesFetched(samples_by_accession_nr,
                                                                            query_ids=sample_accession_nrs,
                                                                            query_id_type='accession_number',
                                                                            query_entity_type='sample',
                                                                            fetched_entity_type='sample')

        return samples_fetched_by_name, samples_fetched_by_id, samples_fetched_by_accession_nr

    @classmethod
    def _fetch_studies(cls, ss_connection, study_names, study_ids, study_accession_nrs):
        studies_by_name = ss_connection.study.get_by_name(study_names)
        studies_fetched_by_name = SeqscapeEntitiesFetched(studies_by_name,
                                                                    query_ids=study_names,
                                                                    query_id_type='name',
                                                                    query_entity_type='study',
                                                                    fetched_entity_type='study'
        )

        studies_by_accession_nr = ss_connection.study.get_by_accession_number(study_accession_nrs)
        studies_fetched_by_accession_nr = SeqscapeEntitiesFetched(studies_by_accession_nr,
                                                                            query_ids=study_accession_nrs,
                                                                            query_id_type='accession_number',
                                                                            query_entity_type='study',
                                                                            fetched_entity_type='study'
        )

        studies_by_id = ss_connection.study.get_by_id(study_ids)
        studies_fetched_by_id = SeqscapeEntitiesFetched(studies_by_id,
                                                                  query_ids=study_ids,
                                                                  query_id_type='internal_id',
                                                                  query_entity_type='study',
                                                                  fetched_entity_type='study'
        )

        return studies_fetched_by_name, studies_fetched_by_id, studies_fetched_by_accession_nr

    @classmethod
    def _fetch_libraries(cls, ss_connection, library_names, library_ids):
        libraries_by_id = ss_connection.library.get_by_id(library_ids)
        if not libraries_by_id:
            libraries_by_id = ss_connection.well.get_by_id(library_ids)
        if not libraries_by_id:
            libraries_by_id = ss_connection.multiplexed_library(library_ids)

        libraries_fetched_by_id = SeqscapeEntitiesFetched(libraries_by_id,
                                                                    query_ids=library_ids,
                                                                    query_id_type='internal_id',
                                                                    query_entity_type='library',
                                                                    fetched_entity_type='library')

        libraries_by_name = ss_connection.library.get_by_name(library_names)
        libraries_fetched_by_name = SeqscapeEntitiesFetched(libraries_by_name,
                                                                      query_ids=library_names,
                                                                      query_id_type='name',
                                                                      query_entity_type='library',
                                                                      fetched_entity_type='library')
        return libraries_fetched_by_name, libraries_fetched_by_id

    @classmethod
    def _fetch_samples_for_studies(cls, ss_connection, studies):
        return ss_connection.sample.get_associated_with_study(studies)

    @classmethod
    def _fetch_studies_for_samples(cls, ss_connection, samples):
        return ss_connection.study.get_associated_with_sample(samples)

    @classmethod
    def fetch(cls, samples, libraries, studies) ->SeqscapeRawMetadata:
        ss_connection = cls._get_connection(config.SEQSC_HOST, config.SEQSC_PORT, config.SEQSC_DB_NAME,
                                            config.SEQSC_USER)

        samples_fetched_by_names, samples_fetched_by_ids, samples_fetched_by_accession_nrs = \
            cls._fetch_samples(ss_connection, samples['name'], samples['internal_id'], samples['accession_number'])

        studies_fetched_by_names, studies_fetched_by_ids, studies_fetched_by_accession_nrs = \
            cls._fetch_studies(ss_connection, studies['name'], studies['internal_id'], studies['accession_number'])

        libraries_fetched_by_names, libraries_fetched_by_ids = \
            cls._fetch_libraries(ss_connection, libraries['name'], libraries['internal_id'])

        raw_meta = SeqscapeRawMetadata()
        raw_meta.add_fetched_entities(samples_fetched_by_names)
        raw_meta.add_fetched_entities(samples_fetched_by_accession_nrs)
        raw_meta.add_fetched_entities(samples_fetched_by_ids)

        raw_meta.add_fetched_entities(studies_fetched_by_accession_nrs)
        raw_meta.add_fetched_entities(studies_fetched_by_ids)
        raw_meta.add_fetched_entities(studies_fetched_by_names)

        raw_meta.add_fetched_entities(libraries_fetched_by_names)
        raw_meta.add_fetched_entities(libraries_fetched_by_ids)

        # Getting the sample-study associations:
        studies_set = raw_meta.get_entities_without_duplicates_by_entity_type('study')
        samples_for_study = cls._fetch_samples_for_studies(ss_connection, studies_set)
        raw_meta.add_fetched_entities_by_association(samples_for_study)

        samples_set = raw_meta.get_entities_without_duplicates_by_entity_type('sample')
        studies_for_samples = cls._fetch_studies_for_samples(ss_connection, samples_set)
        raw_meta.add_fetched_entities_by_association(studies_for_samples)

        return raw_meta<|endoftext|>


<commit_before>"""
   calculate the autocorrelation of a *sorted* bed file with a set
   of *distance* lags.
"""
import argparse
from chart import chart
import sys
import os
import numpy as np
from itertools import groupby, combinations
from _common import bediter, pairwise

def acf(fnames, lags, col_num0):
    acfs = {}
    for lag_min, lag_max in pairwise(lags):
        xs, ys = [], [] # these hold the lagged values.
        for fname in fnames:
            # groupby chromosome.
            for key, chromlist in groupby(bediter(fname, col_num0), lambda a: a["chrom"]):
                chromlist = list(chromlist)
                for ix, xbed in enumerate(chromlist):
                    # find all lines within lag of xbed.
                    for iy in xrange(ix + 1, len(chromlist)):
                        ybed = chromlist[iy]
                        # y is always > x so dist calc is simplified.
                        # too close:
                        if ybed['start'] - xbed['end'] < lag_min: continue
                        # too far.
                        if ybed['start'] - xbed['end'] > lag_max: break

                        xs.append(xbed['p'])
                        ys.append(ybed['p'])

        acfs[(lag_min, lag_max)] = (np.corrcoef(xs, ys)[0, 1], len(xs))
    return sorted(acfs.items())

def run(args):
    d = map(int, args.d.split(":"))
    d[1] += 1 # adjust for non-inclusive end-points...
    assert len(d) == 3
    lags = range(*d)
    acf_vals = acf(args.files, lags, args.c - 1)
    values = [float(v[0]) for k, v in acf_vals]
    xlabels = "|".join("%s-%s" % k for k, v in acf_vals)
    print "#", chart(values, xlabels)
    print "lag_min\tlag_max\tcorrelation\tN"
    for k,v in sorted(acf_vals):
        print "%i\t%i\t%.4g\t%i" % (k[0], k[1], v[0], v[1])

def main():
    p = argparse.ArgumentParser(description=__doc__,
                   formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("-d", dest="d", help="start:stop:stepsize of distance. e.g."
            " %(default)s means check acf at distances of:"
            "[15, 65, 115, 165, 215, 265, 315, 365, 415, 465]",
            type=str, default="15:500:50")
    p.add_argument("-c", dest="c", help="column number that has the value to take the"
            " acf", type=int, default=4)
    p.add_argument('files', nargs='+', help='files to process')
    args = p.parse_args()
    if (len(args.files) == 0):
        sys.exit(not p.print_help())
    return run(args)

if __name__ == "__main__":
    import doctest
    if doctest.testmod(optionflags=doctest.ELLIPSIS |\
                                   doctest.NORMALIZE_WHITESPACE).failed == 0:
        main()
<commit_msg>stub code for durbin watson.<commit_after>"""
   calculate the autocorrelation of a *sorted* bed file with a set
   of *distance* lags.
"""
import argparse
from chart import chart
import sys
import os
import numpy as np
from itertools import groupby, combinations
from _common import bediter, pairwise

try:
    import scikits.statsmodels.api as sm
    from scikits.statsmodels.stats.stattools import durbin_watson

    def dw(xs, ys):
        model = sm.OLS(xs, ys)
        r = model.fit()
        return durbin_watson(r.resid)
    HAS_DW = False # leave for now, not working.
except ImportError:
    def dw(xs, ys): return None
    HAS_DW = False

def acf(fnames, lags, col_num0):
    acfs = {}
    for lag_min, lag_max in pairwise(lags):
        xs, ys = [], [] # these hold the lagged values.
        for fname in fnames:
            # groupby chromosome.
            for key, chromlist in groupby(bediter(fname, col_num0), lambda a: a["chrom"]):
                chromlist = list(chromlist)
                for ix, xbed in enumerate(chromlist):
                    # find all lines within lag of xbed.
                    for iy in xrange(ix + 1, len(chromlist)):
                        ybed = chromlist[iy]
                        # y is always > x so dist calc is simplified.
                        # too close:
                        if ybed['start'] - xbed['end'] < lag_min: continue
                        # too far.
                        if ybed['start'] - xbed['end'] > lag_max: break

                        xs.append(xbed['p'])
                        ys.append(ybed['p'])
        xs, ys = np.array(xs), np.array(ys)
        acfs[(lag_min, lag_max)] = (np.corrcoef(xs, ys)[0, 1], len(xs),
                                    dw(xs, ys))
    return sorted(acfs.items())

def run(args):
    d = map(int, args.d.split(":"))
    d[1] += 1 # adjust for non-inclusive end-points...
    assert len(d) == 3
    lags = range(*d)
    acf_vals = acf(args.files, lags, args.c - 1)
    values = [float(v[0]) for k, v in acf_vals]
    xlabels = "|".join("%s-%s" % k for k, v in acf_vals)
    print "#", chart(values, xlabels)
    print "lag_min\tlag_max\tcorrelation\tN" + \
                      ("\tdurbin-watson" if HAS_DW else "")
    for k,v in sorted(acf_vals):
        line = "%i\t%i\t%.4g\t%i" % (k[0], k[1], v[0], v[1])
        if HAS_DW: line += "\t%.2f" % v[2]
        print line

def main():
    p = argparse.ArgumentParser(description=__doc__,
                   formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("-d", dest="d", help="start:stop:stepsize of distance. e.g."
            " %(default)s means check acf at distances of:"
            "[15, 65, 115, 165, 215, 265, 315, 365, 415, 465]",
            type=str, default="15:500:50")
    p.add_argument("-c", dest="c", help="column number that has the value to take the"
            " acf", type=int, default=4)
    p.add_argument('files', nargs='+', help='files to process')
    args = p.parse_args()
    if (len(args.files) == 0):
        sys.exit(not p.print_help())
    return run(args)

if __name__ == "__main__":
    import doctest
    if doctest.testmod(optionflags=doctest.ELLIPSIS |\
                                   doctest.NORMALIZE_WHITESPACE).failed == 0:
        main()
<|endoftext|>


<commit_before>from functools import wraps
import uuid
import logging

from django.conf import settings
from django import forms

from .utils import to_import_string, import_attribute

logger = logging.getLogger('cq')

def to_func_name(func):
    # Try to convert to a name before returing. Will default
    # to the import string.
    return TaskFunc.get_name(to_import_string(func))

def to_signature(func, args, kwargs):
    return {
        'func_name': to_func_name(func),
        'args': args,
        'kwargs': kwargs
    }

def from_signature(sig):
    func = TaskFunc.get_task_func(sig['func_name'])
    return (func, tuple(sig.get('args', ())), sig.get('kwargs', {}))

class TaskFunc(object):
    task_table = {}
    name_table = {}

    def __init__(self, name=None, atomic=True):
        self.name = name
        self.atomic = atomic

    def __call__(self, func):
        @wraps(func)
        def _delay(*args, **kwargs):
            if getattr(settings, 'CQ_SERIAL', False):
                # Create a SerialTask here to make sure we end up
                # returning the task instead of the result.
                kwargs['task'] = SerialTask()
                return self.wrapper(func, args, kwargs)
            else:
                from .models import delay
                return delay(func, args, kwargs)

        @wraps(func)
        def wrapper(*args, **kwargs):
            return self.wrapper(func, args, kwargs)

        # Add the task to the tables.
        self.func = wrapper
        func_name = to_import_string(func)
        self.task_table[func_name] = self
        logger.debug('Adding task definition: {}'.format(func_name))
        if self.name and self.name != func_name:
            self.task_table[self.name] = self
            logger.debug('Adding task definition: {}'.format(self.name))
        if self.name:
            self.name_table[self.name] = func_name

        wrapper.delay = _delay
        return wrapper

    def wrapper(self, func, args, kwargs):
        task = kwargs.pop('task', None)
        direct = task is None
        serial = isinstance(task, SerialTask)
        if direct or serial:
            task = SerialTask()
        try:
            result = func(task, *args, **kwargs)
        except Exception as err:
            if serial:
                for func, args, kwargs in task._errbacks:
                    func(*((task, err,) + tuple(args)), **kwargs)
            raise
        if direct or serial:
            while isinstance(result, SerialTask):
                result = result.result
            task.result = result
            if direct:
                return task.result
            else:
                return task
        else:
            return result

    @classmethod
    def get_task(cls, name):
        return cls.task_table[name]

    @classmethod
    def get_task_func(cls, name):
        try:
            return cls.get_task(name).func
        except KeyError:
            pass
        return import_attribute(name)

    @classmethod
    def get_name(cls, func_name):
        try:
            task = cls.task_table[func_name]
        except KeyError:
            return func_name
        return task.name or func_name

class SerialTask(object):
    """
    """
    def __init__(self, result=None):
        self.id = uuid.uuid4()
        self.result = result
        self._errbacks = []

    def subtask(self, func, args=(), kwargs={}):
        # Note: A serial task will automatically be created.
        return func(*args, task=self, **kwargs)

    def chain(self, func, args=(), kwargs={}):
        all_args = args
        if self.result is not None:
            all_args = (self.result,) + args
        # Note: A serial task will automatically be created.
        return func(*all_args, task=self, **kwargs)

    def errorback(self, func, args=(), kwargs={}):
        self._errbacks.append((func, args, kwargs))

    def log(self, msg, level=logging.INFO, origin=None):
        logger.log(level, msg)

class FuncNameWidget(forms.TextInput):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._name = 'func_name'
        self._list = sorted(list(TaskFunc.task_table.keys()))
        self.attrs.update({'list': 'list__%s' % self._name})

    def render(self, name, value, attrs=None):
        text_html = super().render(name, value, attrs=attrs)
        data_list = '<datalist id="list__%s">' % self._name
        for item in self._list:
            data_list += '<option value="%s">' % item
        data_list += '</datalist>'
        return (text_html + data_list)
<commit_msg>Updating serial tasks to handle results.<commit_after>from functools import wraps
import uuid
import logging

from django.conf import settings
from django import forms

from .utils import to_import_string, import_attribute

logger = logging.getLogger('cq')

def to_func_name(func):
    # Try to convert to a name before returing. Will default
    # to the import string.
    return TaskFunc.get_name(to_import_string(func))

def to_signature(func, args, kwargs):
    return {
        'func_name': to_func_name(func),
        'args': args,
        'kwargs': kwargs
    }

def from_signature(sig):
    func = TaskFunc.get_task_func(sig['func_name'])
    return (func, tuple(sig.get('args', ())), sig.get('kwargs', {}))

class TaskFunc(object):
    task_table = {}
    name_table = {}

    def __init__(self, name=None, atomic=True):
        self.name = name
        self.atomic = atomic

    def __call__(self, func):
        @wraps(func)
        def _delay(*args, **kwargs):
            if getattr(settings, 'CQ_SERIAL', False):
                # Create a SerialTask here to make sure we end up
                # returning the task instead of the result.
                kwargs['task'] = SerialTask()
                return self.wrapper(func, args, kwargs)
            else:
                from .models import delay
                return delay(func, args, kwargs)

        @wraps(func)
        def wrapper(*args, **kwargs):
            return self.wrapper(func, args, kwargs)

        # Add the task to the tables.
        self.func = wrapper
        func_name = to_import_string(func)
        self.task_table[func_name] = self
        logger.debug('Adding task definition: {}'.format(func_name))
        if self.name and self.name != func_name:
            self.task_table[self.name] = self
            logger.debug('Adding task definition: {}'.format(self.name))
        if self.name:
            self.name_table[self.name] = func_name

        wrapper.delay = _delay
        return wrapper

    def wrapper(self, func, args, kwargs):
        task = kwargs.pop('task', None)
        direct = task is None
        serial = isinstance(task, SerialTask)
        if direct or serial:
            task = SerialTask(parent=task, previous=task)
        try:
            result = func(task, *args, **kwargs)
        except Exception as err:
            if serial:
                for func, args, kwargs in task._errbacks:
                    func(*((task, err,) + tuple(args)), **kwargs)
            raise
        if direct or serial:
            while isinstance(result, SerialTask):
                result = result.result
            task.result = result
            if direct:
                return task.result
            else:
                return task
        else:
            return result

    @classmethod
    def get_task(cls, name):
        return cls.task_table[name]

    @classmethod
    def get_task_func(cls, name):
        try:
            return cls.get_task(name).func
        except KeyError:
            pass
        return import_attribute(name)

    @classmethod
    def get_name(cls, func_name):
        try:
            task = cls.task_table[func_name]
        except KeyError:
            return func_name
        return task.name or func_name

class SerialTask(object):
    """
    """
    def __init__(self, result=None, parent=None, previous=None):
        self.id = uuid.uuid4()
        self.result = result
        self.parent = parent
        self.previous = previous
        self._errbacks = []

    def subtask(self, func, args=(), kwargs={}):
        # Note: A serial task will automatically be created.
        return func(*args, task=self, **kwargs)

    def chain(self, func, args=(), kwargs={}):
        # Note: A serial task will automatically be created.
        return func(*args, task=self, **kwargs)

    def errorback(self, func, args=(), kwargs={}):
        self._errbacks.append((func, args, kwargs))

    def log(self, msg, level=logging.INFO, origin=None):
        logger.log(level, msg)

class FuncNameWidget(forms.TextInput):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._name = 'func_name'
        self._list = sorted(list(TaskFunc.task_table.keys()))
        self.attrs.update({'list': 'list__%s' % self._name})

    def render(self, name, value, attrs=None):
        text_html = super().render(name, value, attrs=attrs)
        data_list = '<datalist id="list__%s">' % self._name
        for item in self._list:
            data_list += '<option value="%s">' % item
        data_list += '</datalist>'
        return (text_html + data_list)
<|endoftext|>