<commit_before>from itertools import product, combinations_with_replacement
from copy import deepcopy
from numpy import array, ndarray

# Psudonym for None that states an argument is required to be supplied by the
# user
RequiredArg = None

def is_iterable(obj):
    '''Returns True if object is iterable and not a str or dict.'''
    return hasattr(obj, '__iter__') and not bad_iterable_type(obj)

def bad_iterable_type(obj):
    '''Returns True if str or dict.'''
    return isinstance(obj, str) or isinstance(obj, dict)

def has_str_elems(obj):
    '''Returns True if all elements of iterable are str.'''
    return all([isinstance(elem, str) for elem in obj])

def is_good_iterable(obj):
    '''Returns True if object is iterable with respect to types.'''
    return is_iterable(obj) and has_str_elems(obj)

def to_camel_case(string):
    return string.replace('_', ' ').title().replace(' ', '')

def proper_type_return(val):
    '''Expects and requires a dictionary with type keys.'''
    if len(val) == 0:
        return None
    elif len(val) == 1:
        return list(val.values())[0]
    else:
        return val

class _ValidatedDefaultDict:

    def __init__(self, *args, **kwargs):
        if len(kwargs) != 0 and len(args) != 0:
            raise ValueError("An unnamed argument and keyword arguments "
                             "cannot both be specified.")
        if len(kwargs) == 0 and len(args) == 0:
            raise ValueError("Either an unnamed argument or keyword "
                             "arguments must be specified.")
        if len(args) > 1:
            raise ValueError("Only one unnamed argument allowed.")
        if len(kwargs) > 0:
            self._default = kwargs
            self._dft_constructor = dict
        else:
            dft = args[0]
            self._default = dft
            if isinstance(dft, ndarray):
                self._dft_constructor = array
            else:
                self._dft_constructor = type(dft)

    def _cast_to_dft(self, val):
        '''Attempts to convert val to expected value type.'''
        try:
            val = self._dft_constructor(val)
        except TypeError:
            raise TypeError("Provided type {} cannot be cast to initialized "
                            "required type {}".format(type(val),
                                                      type(self._default))
                            )
        else:
            return val

    def _validate_values(self, val):
        val = self._cast_to_dft(val)
        if type(self._default) == dict:
            dft_copy = self.default
            dft_keys = set(dft_copy)
            if len(dft_keys.intersection(val.keys())) != len(val.keys()):
                raise ValueError("Keys must be a subset of available keys ")
            dft_copy.update(val)
            val = dft_copy
        return val

    # Add function to validate dictionary keys' value types as well
    # Could follow current model on the args based type checking

    def _validate_and_split_key(self, key):
        '''Validate key given regardless of key length.'''
        if self._len_keys == 1:
            return self._validate_and_split_len_one(key)
        else:
            return self._validate_and_split_len(key)

    def _validate_and_split_len_one(self, key):
        '''Validate single type keys.

        Accepted input is a type string, and arbitrarily nested interators that
        culminate in str types.
        '''
        if isinstance(key, str):
            return [key]
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len_one(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _validate_and_split_len(self, key):
        '''Validate all key lengths greater than one, N.

        Valid input is an arbitrarily deep series of iterables that culminate
        in N length tuples, this includes an iterable depth of zero. The N
        length tuples can contain for each member either a type string or an
        iterable of type strings.
        '''
        if isinstance(key, tuple) and len(key) == self._len_keys:
            fst, snd = key
            if any([not is_good_iterable(v) and not isinstance(v, str)
                    for v in key]):
                raise KeyError("The key {} is not valid.".format(key))
            key = list(key)
            for ind in range(len(key)):
                if isinstance(key[ind], str):
                    key[ind] = [key[ind]]
            return list(product(*key))
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _yield_keys(self, key):
        '''Returns the generated keys in proper sorted order.

        The order is necessary so ('A', 'B') is equivalent to ('B', A').
        '''
        if self._len_keys > 1:
            keys = self._validate_and_split_key(key)
            for key in keys:
                yield tuple(sorted(list(key)))
        else:
            yield from self._validate_and_split_key(key)

    @property
    def default(self):
        return deepcopy(self._default)

    @default.setter
    def default(self, new_default):
        new_default = self._cast_to_dft(new_default)
        if isinstance(new_default, dict):
            keys = set(self._default.keys())
            provided_keys = set(new_default.keys())
            if keys.intersection(provided_keys) != provided_keys:
                raise ValueError("New default must a subset of current keys.")
            self._default.update(new_default)
        else:
            self._default = new_default

class TypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, *args, len_keys, **kwargs):

        # Validate proper key constraint
        if len_keys < 1 or len_keys != int(len_keys):
            raise ValueError("len_keys must be a positive integer.")
        self._len_keys = len_keys
        super().__init__(*args, **kwargs)
        self._dict = dict()

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = self._dict[key]
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            self._dict[key] = val

    def keys(self):
        yield from self._dict.keys()

class AttachedTypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, cpp_obj, param_name,
                 type_kind, type_param_dict, sim):
        # store info to communicate with c++
        self._cpp_obj = cpp_obj
        self._param_name = param_name
        self._sim = sim
        self._type_kind = type_kind
        self._len_keys = type_param_dict._len_keys
        # Get default data
        self._default = type_param_dict.default
        self._dft_constructor = type_param_dict._dft_constructor
        # add all types to c++
        for key in self.keys():
            try:
                self[key] = type_param_dict[key]
            except ValueError as verr:
                raise ValueError("Type {} ".format(key) + verr.args[0])

    def to_dettached(self):
        if isinstance(self.default, dict):
            type_param_dict = TypeParameterDict(**self.default,
                                                len_keys=self._len_keys)
        else:
            type_param_dict = TypeParameterDict(self.default,
                                                len_keys=self._len_keys)
        for key in self.keys():
            type_param_dict[key] = self[key]
        return type_param_dict

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = getattr(self._cpp_obj, self._getter)(key)
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            getattr(self._cpp_obj, self._setter)(key, val)

    def _yield_keys(self, key):
        '''Includes key check for existing simulation keys.'''
        curr_keys = self.keys()
        for key in super()._yield_keys(key):
            if key not in curr_keys:
                raise KeyError("Type {} does not exist in the "
                               "system.".format(key))
            else:
                yield key

    @property
    def _setter(self):
        return 'set' + to_camel_case(self._param_name)

    @property
    def _getter(self):
        return 'get' + to_camel_case(self._param_name)

    def keys(self):
        single_keys = getattr(self._sim.state, self._type_kind)
        if self._len_keys == 2:
            return combinations_with_replacement(single_keys, 2)
        else:
            return single_keys
<commit_msg>Fix bug in TypeParameterDict where default was not used in getattr<commit_after>from itertools import product, combinations_with_replacement
from copy import deepcopy
from numpy import array, ndarray

# Psudonym for None that states an argument is required to be supplied by the
# user
RequiredArg = None

def is_iterable(obj):
    '''Returns True if object is iterable and not a str or dict.'''
    return hasattr(obj, '__iter__') and not bad_iterable_type(obj)

def bad_iterable_type(obj):
    '''Returns True if str or dict.'''
    return isinstance(obj, str) or isinstance(obj, dict)

def has_str_elems(obj):
    '''Returns True if all elements of iterable are str.'''
    return all([isinstance(elem, str) for elem in obj])

def is_good_iterable(obj):
    '''Returns True if object is iterable with respect to types.'''
    return is_iterable(obj) and has_str_elems(obj)

def to_camel_case(string):
    return string.replace('_', ' ').title().replace(' ', '')

def proper_type_return(val):
    '''Expects and requires a dictionary with type keys.'''
    if len(val) == 0:
        return None
    elif len(val) == 1:
        return list(val.values())[0]
    else:
        return val

class _ValidatedDefaultDict:

    def __init__(self, *args, **kwargs):
        if len(kwargs) != 0 and len(args) != 0:
            raise ValueError("An unnamed argument and keyword arguments "
                             "cannot both be specified.")
        if len(kwargs) == 0 and len(args) == 0:
            raise ValueError("Either an unnamed argument or keyword "
                             "arguments must be specified.")
        if len(args) > 1:
            raise ValueError("Only one unnamed argument allowed.")
        if len(kwargs) > 0:
            self._default = kwargs
            self._dft_constructor = dict
        else:
            dft = args[0]
            self._default = dft
            if isinstance(dft, ndarray):
                self._dft_constructor = array
            else:
                self._dft_constructor = type(dft)

    def _cast_to_dft(self, val):
        '''Attempts to convert val to expected value type.'''
        try:
            val = self._dft_constructor(val)
        except TypeError:
            raise TypeError("Provided type {} cannot be cast to initialized "
                            "required type {}".format(type(val),
                                                      type(self._default))
                            )
        else:
            return val

    def _validate_values(self, val):
        val = self._cast_to_dft(val)
        if type(self._default) == dict:
            dft_copy = self.default
            dft_keys = set(dft_copy)
            if len(dft_keys.intersection(val.keys())) != len(val.keys()):
                raise ValueError("Keys must be a subset of available keys ")
            dft_copy.update(val)
            val = dft_copy
        return val

    # Add function to validate dictionary keys' value types as well
    # Could follow current model on the args based type checking

    def _validate_and_split_key(self, key):
        '''Validate key given regardless of key length.'''
        if self._len_keys == 1:
            return self._validate_and_split_len_one(key)
        else:
            return self._validate_and_split_len(key)

    def _validate_and_split_len_one(self, key):
        '''Validate single type keys.

        Accepted input is a type string, and arbitrarily nested interators that
        culminate in str types.
        '''
        if isinstance(key, str):
            return [key]
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len_one(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _validate_and_split_len(self, key):
        '''Validate all key lengths greater than one, N.

        Valid input is an arbitrarily deep series of iterables that culminate
        in N length tuples, this includes an iterable depth of zero. The N
        length tuples can contain for each member either a type string or an
        iterable of type strings.
        '''
        if isinstance(key, tuple) and len(key) == self._len_keys:
            fst, snd = key
            if any([not is_good_iterable(v) and not isinstance(v, str)
                    for v in key]):
                raise KeyError("The key {} is not valid.".format(key))
            key = list(key)
            for ind in range(len(key)):
                if isinstance(key[ind], str):
                    key[ind] = [key[ind]]
            return list(product(*key))
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _yield_keys(self, key):
        '''Returns the generated keys in proper sorted order.

        The order is necessary so ('A', 'B') is equivalent to ('B', A').
        '''
        if self._len_keys > 1:
            keys = self._validate_and_split_key(key)
            for key in keys:
                yield tuple(sorted(list(key)))
        else:
            yield from self._validate_and_split_key(key)

    @property
    def default(self):
        return deepcopy(self._default)

    @default.setter
    def default(self, new_default):
        new_default = self._cast_to_dft(new_default)
        if isinstance(new_default, dict):
            keys = set(self._default.keys())
            provided_keys = set(new_default.keys())
            if keys.intersection(provided_keys) != provided_keys:
                raise ValueError("New default must a subset of current keys.")
            self._default.update(new_default)
        else:
            self._default = new_default

class TypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, *args, len_keys, **kwargs):

        # Validate proper key constraint
        if len_keys < 1 or len_keys != int(len_keys):
            raise ValueError("len_keys must be a positive integer.")
        self._len_keys = len_keys
        super().__init__(*args, **kwargs)
        self._dict = dict()

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            try:
                vals[key] = self._dict[key]
            except KeyError:
                vals[key] = self.default
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            self._dict[key] = val

    def keys(self):
        yield from self._dict.keys()

class AttachedTypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, cpp_obj, param_name,
                 type_kind, type_param_dict, sim):
        # store info to communicate with c++
        self._cpp_obj = cpp_obj
        self._param_name = param_name
        self._sim = sim
        self._type_kind = type_kind
        self._len_keys = type_param_dict._len_keys
        # Get default data
        self._default = type_param_dict.default
        self._dft_constructor = type_param_dict._dft_constructor
        # add all types to c++
        for key in self.keys():
            try:
                self[key] = type_param_dict[key]
            except ValueError as verr:
                raise ValueError("Type {} ".format(key) + verr.args[0])

    def to_dettached(self):
        if isinstance(self.default, dict):
            type_param_dict = TypeParameterDict(**self.default,
                                                len_keys=self._len_keys)
        else:
            type_param_dict = TypeParameterDict(self.default,
                                                len_keys=self._len_keys)
        for key in self.keys():
            type_param_dict[key] = self[key]
        return type_param_dict

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = getattr(self._cpp_obj, self._getter)(key)
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            getattr(self._cpp_obj, self._setter)(key, val)

    def _yield_keys(self, key):
        '''Includes key check for existing simulation keys.'''
        curr_keys = self.keys()
        for key in super()._yield_keys(key):
            if key not in curr_keys:
                raise KeyError("Type {} does not exist in the "
                               "system.".format(key))
            else:
                yield key

    @property
    def _setter(self):
        return 'set' + to_camel_case(self._param_name)

    @property
    def _getter(self):
        return 'get' + to_camel_case(self._param_name)

    def keys(self):
        single_keys = getattr(self._sim.state, self._type_kind)
        if self._len_keys == 2:
            return combinations_with_replacement(single_keys, 2)
        else:
            return single_keys
<|endoftext|>


<commit_before>import os

import h5py
from annotypes import add_call_types, Anno, TYPE_CHECKING
from vdsgen import InterleaveVDSGenerator, \
    ReshapeVDSGenerator

from malcolm.core import APartName, Future, Info, PartRegistrar
from malcolm.modules import builtin, scanning

if TYPE_CHECKING:
    from typing import List, Dict

    PartInfo = Dict[str, List[Info]]

# If the HDF writer doesn't get new frames in this time (seconds), consider it
# stalled and raise
FRAME_TIMEOUT = 60

VDS_DATASET_NAME = "data"
VDS_UID_NAME = "uid"
VDS_SUM_NAME = "sum"

with Anno("Directory to write data to"):
    AFileDir = str
with Anno("Argument for fileTemplate, normally filename without extension"):
    AFileName = str
with Anno("File Name"):
    AFileTemplate = str
with Anno("Argument for fileTemplate, normally filename without extension"):
    AFormatName = str

def greater_than_zero(v):
    # type: (int) -> bool
    return v > 0

def files_shape(frames, block_size, file_count):
    # all files get at least per_file blocks
    per_file = int(frames) / int(file_count * block_size)
    # this is the remainder once per_file blocks have been distributed
    remainder = int(frames) % int(file_count * block_size)

    # distribute the remainder
    remainders = [block_size if remains > block_size else remains
                  for remains in range(remainder, 0, -block_size)]
    # pad the remainders list with zeros
    remainders += [0] * (file_count - len(remainders))

    shape = tuple(per_file * block_size + remainders[i]
                  for i in range(file_count))
    return shape

def one_vds(vds_folder, vds_name, files, width, height,
            shape, generator, alternates, block_size, node, d_type):
    # this vds reshapes from 1 file per data writer to a single 1D data set
    gen = InterleaveVDSGenerator(
        vds_folder,
        files=files,
        source={'height': width,
                'width': height,
                'dtype': d_type,
                'shape': shape
                },
        output=vds_name,
        source_node=node,
        target_node="process/" + node + "_interleave",
        block_size=block_size,
        log_level=1)
    gen.generate_vds()

    # this VDS shapes the data to match the dimensions of the scan
    gen = ReshapeVDSGenerator(path=vds_folder,
                              files=[vds_name],
                              source_node="process/" + node + "_interleave",
                              target_node=node,
                              output=vds_name,
                              shape=generator.shape,
                              alternate=alternates,
                              log_level=1)

    gen.generate_vds()

def create_vds(generator, raw_name, vds_path, child):
    vds_folder, vds_name = os.path.split(vds_path)

    image_width = child.imageWidth.value
    image_height = child.imageHeight.value
    block_size = child.blockSize.value
    hdf_count = child.numProcesses.value
    data_type = child.dataType.value

    # hdf_shape tuple represents the number of images in each file
    hdf_shape = files_shape(generator.size, block_size, hdf_count)

    alternates = (gen.alternate for gen in generator.generators)
    if any(alternates):
        raise ValueError("Snake scans are not yet supported")
    else:
        alternates = None

    files = [os.path.join(
        vds_folder, '{}_{:06d}.h5'.format(raw_name, i + 1))
        for i in range(hdf_count)]
    shape = (hdf_shape, image_height, image_width)

    # prepare a vds for the image data
    one_vds(vds_folder, vds_name, files, image_width, image_height,
            shape, generator, alternates, block_size,
            VDS_DATASET_NAME, data_type)

    shape = (hdf_shape, 1, 1)

    # prepare a vds for the unique IDs
    one_vds(vds_folder, vds_name, files, 1, 1,
            shape, generator, alternates, block_size,
            VDS_UID_NAME, 'uint64')
    # prepare a vds for the sums
    one_vds(vds_folder, vds_name, files, 1, 1,
            shape, generator, alternates, block_size,
            VDS_SUM_NAME, 'uint64')

set_bases = ["/entry/detector/", "/entry/sum/", "/entry/uid/"]
set_data = ["/data", "/sum", "/uid"]

def add_nexus_nodes(generator, vds_file_path):
    """ Add in the additional information to make this into a standard nexus
    format file:-
    (a) create the standard structure under the 'entry' group with a
    subgroup for each dataset. 'set_bases' lists the data sets we make here.
    (b) save a dataset for each axis in each of the dimensions of the scan
    representing the demand position at every point in the scan.
    """

    # create the axes dimensions attribute, a comma separated list giving size
    # of the axis dimensions padded with . for the detector dimensions and
    # multidimensional dimensions
    pad_dims = []
    for d in generator.dimensions:
        if len(d.axes) == 1:
            pad_dims.append("%s_set" % d.axes[0])
        else:
            pad_dims.append(".")

    pad_dims += ["."] * 2  # assume a 2 dimensional detector

    with h5py.File(vds_file_path, 'r+', libver="latest") as vds:
        for data, node in zip(set_data, set_bases):
            # create a group for this entry
            vds.require_group(node)
            # points to the axis demand data sets
            vds[node].attrs["axes"] = ','.join(pad_dims)

            data_node_name = node.split('/')[-2]
            # points to the detector dataset for this entry
            vds[node].attrs["signal"] = data_node_name
            # a hard link from this entry 'signal' to the actual data
            vds[node + data_node_name] = vds[data]

            axis_sets = {}
            # iterate the axes in each dimension of the generator to create the
            # axis information nodes
            for i, d in enumerate(generator.dimensions):
                for axis in d.axes:
                    # add signal data dimension for axis
                    axis_indices = '{}_set_indices'.format(axis)
                    vds[node].attrs[axis_indices] = str(i)

                    # demand positions for axis
                    axis_set = '{}_set'.format(axis)
                    if axis_sets.get(axis_set):
                        # link to the first entry's demand list
                        vds[node + axis_set] = axis_sets[axis_set]
                    else:
                        # create the demand list for the first entry only
                        axis_demands = d.get_positions(axis)
                        vds.create_dataset(
                            node + axis_set, data=axis_demands)
                        vds[node + axis_set].attrs["units"] = \
                            generator.units[axis]
                    axis_sets[axis_set] = vds[node + axis_set]

# We will set these attributes on the child block, so don't save them
@builtin.util.no_save("fileName", "filePath", "numCapture")
class OdinWriterPart(builtin.parts.ChildPart):
    """Part for controlling an `hdf_writer_block` in a Device"""

    def __init__(self, name, mri):
        # type: (APartName, scanning.parts.AMri) -> None
        super(OdinWriterPart, self).__init__(name, mri)
        # Future for the start action
        self.start_future = None  # type: Future
        self.array_future = None  # type: Future
        self.done_when_reaches = 0
        # CompletedSteps = arrayCounter + self.uniqueid_offset
        self.unique_id_offset = 0
        # The HDF5 layout file we write to say where the datasets go
        self.layout_filename = None  # type: str
        # Hooks
        self.register_hooked(scanning.hooks.ConfigureHook, self.configure)
        self.register_hooked((scanning.hooks.PostRunArmedHook,
                              scanning.hooks.SeekHook), self.seek)
        self.register_hooked((scanning.hooks.RunHook,
                              scanning.hooks.ResumeHook), self.run)
        self.register_hooked(scanning.hooks.PostRunReadyHook,
                             self.post_run_ready)
        self.register_hooked(scanning.hooks.AbortHook, self.abort)
        self.register_hooked(scanning.hooks.PauseHook, self.pause)

    @add_call_types
    def reset(self, context):
        # type: (scanning.hooks.AContext) -> None
        super(OdinWriterPart, self).reset(context)
        self.abort(context)

    def setup(self, registrar):
        # type: (PartRegistrar) -> None
        super(OdinWriterPart, self).setup(registrar)
        # Tell the controller to expose some extra configure parameters
        registrar.report(scanning.hooks.ConfigureHook.create_info(
            self.configure))

    @add_call_types
    def pause(self, context):
        # type: (scanning.hooks.AContext) -> None
        raise NotImplementedError("Seek not implemented")

    # Allow CamelCase as these parameters will be serialized
    # noinspection PyPep8Naming
    @add_call_types
    def configure(self,
                  context,  # type: scanning.hooks.AContext
                  completed_steps,  # type: scanning.hooks.ACompletedSteps
                  steps_to_do,  # type: scanning.hooks.AStepsToDo
                  generator,  # type: scanning.hooks.AGenerator
                  fileDir,  # type: AFileDir
                  formatName="odin",  # type: AFormatName
                  fileTemplate="%s.hdf",  # type: AFileTemplate
                  ):
        # type: (...) -> scanning.hooks.UInfos

        # On initial configure, expect to get the demanded number of frames
        self.done_when_reaches = completed_steps + steps_to_do
        self.unique_id_offset = 0
        child = context.block_view(self.mri)
        file_dir = fileDir.rstrip(os.sep)

        # derive file path from template as AreaDetector would normally do
        fileName = fileTemplate.replace('%s', formatName)

        # this is path to the requested file which will be a VDS
        vds_full_filename = os.path.join(fileDir, fileName)

        # this is the path to underlying file the odin writer will write to
        raw_file_name = fileTemplate.replace('%s', formatName + '_raw_data')
        raw_file_basename, _ = os.path.splitext(raw_file_name)

        assert "." in vds_full_filename, \
            "File extension for %r should be supplied" % vds_full_filename
        futures = child.put_attribute_values_async(dict(
            numCapture=steps_to_do,
            filePath=file_dir + os.sep,
            fileName=raw_file_basename))
        context.wait_all_futures(futures)

        # Start the plugin
        self.start_future = child.start_async()
        # Start a future waiting for the first array
        self.array_future = child.when_value_matches_async(
            "numCaptured", greater_than_zero)

        create_vds(generator, raw_file_basename,
                   vds_full_filename, child)
        add_nexus_nodes(generator, vds_full_filename)

        return None

    @add_call_types
    def seek(self,
             context,  # type: scanning.hooks.AContext
             completed_steps,  # type: scanning.hooks.ACompletedSteps
             steps_to_do,  # type: scanning.hooks.AStepsToDo
             ):
        # type: (...) -> None
        # This is rewinding or setting up for another batch, so the detector
        # will skip to a uniqueID that has not been produced yet
        self.unique_id_offset = completed_steps - self.done_when_reaches
        self.done_when_reaches += steps_to_do
        child = context.block_view(self.mri)
        # Just reset the array counter_block
        child.arrayCounter.put_value(0)
        # Start a future waiting for the first array
        self.array_future = child.when_value_matches_async(
            "numCaptured", greater_than_zero)

    @add_call_types
    def run(self, context):
        # type: (scanning.hooks.AContext) -> None
        context.wait_all_futures(self.array_future)
        context.unsubscribe_all()
        child = context.block_view(self.mri)
        child.numCaptured.subscribe_value(self.update_completed_steps)
        child.when_value_matches(
            "numCaptured", self.done_when_reaches, event_timeout=FRAME_TIMEOUT)

    @add_call_types
    def post_run_ready(self, context):
        # type: (scanning.hooks.AContext) -> None
        # If this is the last one, wait until the file is closed
        context.wait_all_futures(self.start_future)

    @add_call_types
    def abort(self, context):
        # type: (scanning.hooks.AContext) -> None
        child = context.block_view(self.mri)
        child.stop()

    def update_completed_steps(self, value):
        # type: (int) -> None
        completed_steps = value + self.unique_id_offset
        self.registrar.report(scanning.infos.RunProgressInfo(completed_steps))
<commit_msg>fixes to odinwriter VDS generation after testing with new Odin<commit_after>import os

import h5py
from annotypes import add_call_types, Anno, TYPE_CHECKING
from vdsgen import InterleaveVDSGenerator, \
    ReshapeVDSGenerator

from malcolm.core import APartName, Future, Info, PartRegistrar
from malcolm.modules import builtin, scanning

if TYPE_CHECKING:
    from typing import List, Dict

    PartInfo = Dict[str, List[Info]]

# If the HDF writer doesn't get new frames in this time (seconds), consider it
# stalled and raise
FRAME_TIMEOUT = 60

with Anno("Directory to write data to"):
    AFileDir = str
with Anno("Argument for fileTemplate, normally filename without extension"):
    AFileName = str
with Anno("File Name"):
    AFileTemplate = str
with Anno("Argument for fileTemplate, normally filename without extension"):
    AFormatName = str

def greater_than_zero(v):
    # type: (int) -> bool
    return v > 0

def files_shape(frames, block_size, file_count):
    # all files get at least per_file blocks
    per_file = int(frames) / int(file_count * block_size)
    # this is the remainder once per_file blocks have been distributed
    remainder = int(frames) % int(file_count * block_size)

    # distribute the remainder
    remainders = [block_size if remains > block_size else remains
                  for remains in range(remainder, 0, -block_size)]
    # pad the remainders list with zeros
    remainders += [0] * (file_count - len(remainders))

    shape = tuple(per_file * block_size + remainders[i]
                  for i in range(file_count))
    return shape

def one_vds(vds_folder, vds_name, files, width, height,
            shape, generator, alternates, block_size, source_node,
            target_node, d_type):
    # this vds reshapes from 1 file per data writer to a single 1D data set
    gen = InterleaveVDSGenerator(
        vds_folder,
        files=files,
        source={'height': width,
                'width': height,
                'dtype': d_type,
                'shape': shape
                },
        output=vds_name,
        source_node=source_node,
        target_node="process/" + target_node + "_interleave",
        block_size=block_size,
        log_level=1)
    gen.generate_vds()

    # this VDS shapes the data to match the dimensions of the scan
    gen = ReshapeVDSGenerator(path=vds_folder,
                              files=[vds_name],
                              source_node="process/" + target_node + \
                               "_interleave",
                              target_node=target_node,
                              output=vds_name,
                              shape=generator.shape,
                              alternate=alternates,
                              log_level=1)

    gen.generate_vds()

def create_vds(generator, raw_name, vds_path, child):
    vds_folder, vds_name = os.path.split(vds_path)

    image_width = child.imageWidth.value
    image_height = child.imageHeight.value
    #image_width = 2069
    #image_height = 1793
    block_size = child.blockSize.value
    hdf_count = child.numProcesses.value
    data_type = child.dataType.value

    # hdf_shape tuple represents the number of images in each file
    hdf_shape = files_shape(generator.size, block_size, hdf_count)

    alternates = (gen.alternate for gen in generator.generators)
    if any(alternates):
        raise ValueError("Snake scans are not yet supported")
    else:
        alternates = None

    files = [os.path.join(
        vds_folder, '{}_{:06d}.h5'.format(raw_name, i + 1))
        for i in range(hdf_count)]
    shape = (hdf_shape, image_height, image_width)

    # prepare a vds for the image data
    one_vds(vds_folder, vds_name, files, image_width, image_height,
            shape, generator, alternates, block_size,
            'data', 'data', data_type)

    shape = (hdf_shape, 1, 1)

    # prepare a vds for the unique IDs
    one_vds(vds_folder, vds_name, files, 1, 1,
            shape, generator, alternates, block_size,
            'UID', 'uid', 'uint64')
    # prepare a vds for the sums
    one_vds(vds_folder, vds_name, files, 1, 1,
            shape, generator, alternates, block_size,
            'SUM', 'sum', 'uint64')

set_bases = ["/entry/detector/", "/entry/sum/", "/entry/uid/"]
set_data = ["/data", "/sum", "/uid"]

def add_nexus_nodes(generator, vds_file_path):
    """ Add in the additional information to make this into a standard nexus
    format file:-
    (a) create the standard structure under the 'entry' group with a
    subgroup for each dataset. 'set_bases' lists the data sets we make here.
    (b) save a dataset for each axis in each of the dimensions of the scan
    representing the demand position at every point in the scan.
    """

    # create the axes dimensions attribute, a comma separated list giving size
    # of the axis dimensions padded with . for the detector dimensions and
    # multidimensional dimensions
    pad_dims = []
    for d in generator.dimensions:
        if len(d.axes) == 1:
            pad_dims.append("%s_set" % d.axes[0])
        else:
            pad_dims.append(".")

    pad_dims += ["."] * 2  # assume a 2 dimensional detector

    with h5py.File(vds_file_path, 'r+', libver="latest") as vds:
        for data, node in zip(set_data, set_bases):
            # create a group for this entry
            vds.require_group(node)
            # points to the axis demand data sets
            vds[node].attrs["axes"] = ','.join(pad_dims)

            data_node_name = node.split('/')[-2]
            # points to the detector dataset for this entry
            vds[node].attrs["signal"] = data_node_name
            # a hard link from this entry 'signal' to the actual data
            vds[node + data_node_name] = vds[data]

            axis_sets = {}
            # iterate the axes in each dimension of the generator to create the
            # axis information nodes
            for i, d in enumerate(generator.dimensions):
                for axis in d.axes:
                    # add signal data dimension for axis
                    axis_indices = '{}_set_indices'.format(axis)
                    vds[node].attrs[axis_indices] = str(i)

                    # demand positions for axis
                    axis_set = '{}_set'.format(axis)
                    if axis_sets.get(axis_set):
                        # link to the first entry's demand list
                        vds[node + axis_set] = axis_sets[axis_set]
                    else:
                        # create the demand list for the first entry only
                        axis_demands = d.get_positions(axis)
                        vds.create_dataset(
                            node + axis_set, data=axis_demands)
                        vds[node + axis_set].attrs["units"] = \
                            generator.units[axis]
                    axis_sets[axis_set] = vds[node + axis_set]

# We will set these attributes on the child block, so don't save them
@builtin.util.no_save("fileName", "filePath", "numCapture")
class OdinWriterPart(builtin.parts.ChildPart):
    """Part for controlling an `hdf_writer_block` in a Device"""

    def __init__(self, name, mri):
        # type: (APartName, scanning.parts.AMri) -> None
        super(OdinWriterPart, self).__init__(name, mri)
        # Future for the start action
        self.start_future = None  # type: Future
        self.array_future = None  # type: Future
        self.done_when_reaches = 0
        # CompletedSteps = arrayCounter + self.uniqueid_offset
        self.unique_id_offset = 0
        # The HDF5 layout file we write to say where the datasets go
        self.layout_filename = None  # type: str
        # Hooks
        self.register_hooked(scanning.hooks.ConfigureHook, self.configure)
        self.register_hooked((scanning.hooks.PostRunArmedHook,
                              scanning.hooks.SeekHook), self.seek)
        self.register_hooked((scanning.hooks.RunHook,
                              scanning.hooks.ResumeHook), self.run)
        self.register_hooked(scanning.hooks.PostRunReadyHook,
                             self.post_run_ready)
        self.register_hooked(scanning.hooks.AbortHook, self.abort)
        self.register_hooked(scanning.hooks.PauseHook, self.pause)

    @add_call_types
    def reset(self, context):
        # type: (scanning.hooks.AContext) -> None
        super(OdinWriterPart, self).reset(context)
        self.abort(context)

    def setup(self, registrar):
        # type: (PartRegistrar) -> None
        super(OdinWriterPart, self).setup(registrar)
        # Tell the controller to expose some extra configure parameters
        registrar.report(scanning.hooks.ConfigureHook.create_info(
            self.configure))

    @add_call_types
    def pause(self, context):
        # type: (scanning.hooks.AContext) -> None
        raise NotImplementedError("Seek not implemented")

    # Allow CamelCase as these parameters will be serialized
    # noinspection PyPep8Naming
    @add_call_types
    def configure(self,
                  context,  # type: scanning.hooks.AContext
                  completed_steps,  # type: scanning.hooks.ACompletedSteps
                  steps_to_do,  # type: scanning.hooks.AStepsToDo
                  generator,  # type: scanning.hooks.AGenerator
                  fileDir,  # type: AFileDir
                  formatName="odin",  # type: AFormatName
                  fileTemplate="%s.hdf",  # type: AFileTemplate
                  ):
        # type: (...) -> scanning.hooks.UInfos

        # On initial configure, expect to get the demanded number of frames
        self.done_when_reaches = completed_steps + steps_to_do
        self.unique_id_offset = 0
        child = context.block_view(self.mri)
        file_dir = fileDir.rstrip(os.sep)

        # derive file path from template as AreaDetector would normally do
        fileName = fileTemplate.replace('%s', formatName)

        # this is path to the requested file which will be a VDS
        vds_full_filename = os.path.join(fileDir, fileName)

        # this is the path to underlying file the odin writer will write to
        raw_file_name = fileTemplate.replace('%s', formatName + '_raw_data')
        raw_file_basename, _ = os.path.splitext(raw_file_name)

        assert "." in vds_full_filename, \
            "File extension for %r should be supplied" % vds_full_filename
        futures = child.put_attribute_values_async(dict(
            numCapture=steps_to_do,
            filePath=file_dir + os.sep,
            fileName=raw_file_basename))
        context.wait_all_futures(futures)

        # Start the plugin
        self.start_future = child.start_async()
        # Start a future waiting for the first array
        self.array_future = child.when_value_matches_async(
            "numCaptured", greater_than_zero)

        create_vds(generator, raw_file_basename,
                   vds_full_filename, child)
        add_nexus_nodes(generator, vds_full_filename)

        return None

    @add_call_types
    def seek(self,
             context,  # type: scanning.hooks.AContext
             completed_steps,  # type: scanning.hooks.ACompletedSteps
             steps_to_do,  # type: scanning.hooks.AStepsToDo
             ):
        # type: (...) -> None
        # This is rewinding or setting up for another batch, so the detector
        # will skip to a uniqueID that has not been produced yet
        self.unique_id_offset = completed_steps - self.done_when_reaches
        self.done_when_reaches += steps_to_do
        child = context.block_view(self.mri)
        # Just reset the array counter_block
        child.arrayCounter.put_value(0)
        # Start a future waiting for the first array
        self.array_future = child.when_value_matches_async(
            "numCaptured", greater_than_zero)

    @add_call_types
    def run(self, context):
        # type: (scanning.hooks.AContext) -> None
        context.wait_all_futures(self.array_future)
        context.unsubscribe_all()
        child = context.block_view(self.mri)
        child.numCaptured.subscribe_value(self.update_completed_steps)
        child.when_value_matches(
            "numCaptured", self.done_when_reaches, event_timeout=FRAME_TIMEOUT)

    @add_call_types
    def post_run_ready(self, context):
        # type: (scanning.hooks.AContext) -> None
        # If this is the last one, wait until the file is closed
        context.wait_all_futures(self.start_future)

    @add_call_types
    def abort(self, context):
        # type: (scanning.hooks.AContext) -> None
        child = context.block_view(self.mri)
        child.stop()

    def update_completed_steps(self, value):
        # type: (int) -> None
        completed_steps = value + self.unique_id_offset
        self.registrar.report(scanning.infos.RunProgressInfo(completed_steps))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import alabaster
from sprockets.mixins import metrics

project = 'sprockets.mixins.metrics'
copyright = 'AWeber Communications, Inc.'
version = metrics.__version__
release = '.'.join(str(v) for v in metrics.version_info[0:2])

needs_sphinx = '1.0'
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.viewcode',
]

templates_path = []
source_suffix = '.rst'
source_encoding = 'utf-8-sig'
master_doc = 'index'
exclude_patterns = []
pygments_style = 'sphinx'
html_style = 'custom.css'
html_static_path = ['.']
html_theme = 'alabaster'
html_theme_path = [alabaster.get_path()]
html_sidebars = {
    '**': ['about.html', 'navigation.html'],
}
html_theme_options = {
    'github_user': 'sprockets',
    'github_repo': 'sprockets.mixins.metrics',
    'description': 'Application metrics tracker',
    'github_banner': True,
    'travis_button': True,
}

intersphinx_mapping = {
    'python': ('http://docs.python.org/3/', None),
    'tornado': ('http://tornadoweb.org/en/latest/', None),
}
<commit_msg>docs: Add codecov badge.<commit_after># -*- coding: utf-8 -*-
import alabaster
from sprockets.mixins import metrics

project = 'sprockets.mixins.metrics'
copyright = 'AWeber Communications, Inc.'
version = metrics.__version__
release = '.'.join(str(v) for v in metrics.version_info[0:2])

needs_sphinx = '1.0'
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.viewcode',
]

templates_path = []
source_suffix = '.rst'
source_encoding = 'utf-8-sig'
master_doc = 'index'
exclude_patterns = []
pygments_style = 'sphinx'
html_style = 'custom.css'
html_static_path = ['.']
html_theme = 'alabaster'
html_theme_path = [alabaster.get_path()]
html_sidebars = {
    '**': ['about.html', 'navigation.html'],
}
html_theme_options = {
    'github_user': 'sprockets',
    'github_repo': 'sprockets.mixins.metrics',
    'description': 'Application metrics tracker',
    'github_banner': True,
    'travis_button': True,
    'codecov_button': True,
}

intersphinx_mapping = {
    'python': ('http://docs.python.org/3/', None),
    'tornado': ('http://tornadoweb.org/en/latest/', None),
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Remember and restore audio stream / subtitle settings between individual
episodes of a tv show or movie
"""
from __future__ import unicode_literals

import xbmc

import json

from resources.lib.globals import g
import resources.lib.common as common

from .action_manager import PlaybackActionManager

STREAMS = {
    'audio': {
        'current': 'currentaudiostream',
        'setter': xbmc.Player.setAudioStream,
    },
    'subtitle': {
        'current': 'currentsubtitle',
        'setter': xbmc.Player.setSubtitleStream,
    },
    'subtitleenabled': {
        'current': 'subtitleenabled',
        'setter': xbmc.Player.showSubtitles
    }
}

class StreamContinuityManager(PlaybackActionManager):
    """
    Detects changes in audio / subtitle streams during playback, saves them
    for the currently playing show and restores them on subsequent episodes.
    """

    def __init__(self):
        super(StreamContinuityManager, self).__init__()
        self.storage = common.PersistentStorage(__name__, no_save_on_destroy=True)
        self.current_videoid = None
        self.current_streams = {}
        self.player = xbmc.Player()
        self.did_restore = False
        self.resume = {}

    @property
    def sc_settings(self):
        """Stored stream settings for the current videoid"""

        return g.SHARED_DB.get_stream_continuity(g.LOCAL_DB.get_active_profile_guid(),
                                                 self.current_videoid.value, {})

    def _initialize(self, data):
        videoid = common.VideoId.from_dict(data['videoid'])
        if videoid.mediatype in [common.VideoId.MOVIE, common.VideoId.EPISODE]:
            self.did_restore = False
            self.current_videoid = videoid
        else:
            self.enabled = False

    def _on_playback_started(self, player_state):
        xbmc.sleep(500)  # Wait for slower systems
        for stype in STREAMS:
            self._set_current_stream(stype, player_state)
            self._restore_stream(stype)
        if (self.sc_settings.get('subtitleenabled', None) is None
                and g.ADDON.getSettingBool('forced_subtitle_workaround')):
            # Use the workaround only when the user did not change the show subtitle setting
            _show_only_forced_subtitle()
        self.did_restore = True

    def _on_tick(self, player_state):
        if not self.did_restore:
            common.debug('Did not restore streams yet, ignoring tick')
            return
        for stype in STREAMS:
            current_stream = self.current_streams[stype]
            player_stream = player_state.get(STREAMS[stype]['current'])
            if player_stream != current_stream:
                common.debug('{} has changed from {} to {}'
                             .format(stype, current_stream, player_stream))
                self._set_current_stream(stype, player_state)
                self._save_changed_stream(stype, player_stream)

    def _set_current_stream(self, stype, player_state):
        self.current_streams.update({
            stype: player_state.get(STREAMS[stype]['current'])
        })

    def _restore_stream(self, stype):
        common.debug('Trying to restore {}...'.format(stype))
        set_stream = STREAMS[stype]['setter']
        stored_stream = self.sc_settings.get(stype)
        if (stored_stream is not None and
                self.current_streams[stype] != stored_stream):
            # subtitleenabled is boolean and not a dict
            set_stream(self.player, (stored_stream['index']
                                     if isinstance(stored_stream, dict)
                                     else stored_stream))
            self.current_streams[stype] = stored_stream
            common.debug('Restored {} to {}'.format(stype, stored_stream))

    def _save_changed_stream(self, stype, stream):
        common.debug('Save changed stream {} for {}'.format(stream, stype))
        new_sc_settings = self.sc_settings.copy()
        new_sc_settings[stype] = stream
        g.SHARED_DB.set_stream_continuity(g.LOCAL_DB.get_active_profile_guid(),
                                          self.current_videoid.value,
                                          new_sc_settings)

    def __repr__(self):
        return ('enabled={}, current_videoid={}'
                .format(self.enabled, self.current_videoid))

def _show_only_forced_subtitle():
    # When we have "forced only" subtitle setting in Kodi Player, Kodi use this behavior:
    # 1) try to select forced subtitle that matches audio language
    # 2) when missing, try to select the first "regular" subtitle that matches audio language
    # This Kodi behavior is totally non sense.
    # If forced is selected you must not view the regular subtitles
    # There is no other solution than to disable the subtitles manually.
    manifest_data = json.loads(common.load_file('manifest.json'))
    common.fix_locale_languages(manifest_data['timedtexttracks'])
    audio_language = common.get_kodi_audio_language()
    if not any(text_track.get('isForcedNarrative', False) is True and
               text_track['language'] == audio_language
               for text_track in manifest_data['timedtexttracks']):
        xbmc.Player().showSubtitles(False)
<commit_msg>Fix stream_continuity for tvshows<commit_after># -*- coding: utf-8 -*-

"""
Remember and restore audio stream / subtitle settings between individual
episodes of a tv show or movie
"""
from __future__ import unicode_literals

import xbmc

import json

from resources.lib.globals import g
import resources.lib.common as common

from .action_manager import PlaybackActionManager

STREAMS = {
    'audio': {
        'current': 'currentaudiostream',
        'setter': xbmc.Player.setAudioStream,
    },
    'subtitle': {
        'current': 'currentsubtitle',
        'setter': xbmc.Player.setSubtitleStream,
    },
    'subtitleenabled': {
        'current': 'subtitleenabled',
        'setter': xbmc.Player.showSubtitles
    }
}

class StreamContinuityManager(PlaybackActionManager):
    """
    Detects changes in audio / subtitle streams during playback, saves them
    for the currently playing show and restores them on subsequent episodes.
    """

    def __init__(self):
        super(StreamContinuityManager, self).__init__()
        self.storage = common.PersistentStorage(__name__, no_save_on_destroy=True)
        self.current_videoid = None
        self.current_streams = {}
        self.player = xbmc.Player()
        self.did_restore = False
        self.resume = {}

    @property
    def sc_settings(self):
        """Stored stream settings for the current videoid"""

        return g.SHARED_DB.get_stream_continuity(g.LOCAL_DB.get_active_profile_guid(),
                                                 self.current_videoid.value, {})

    def _initialize(self, data):
        videoid = common.VideoId.from_dict(data['videoid'])
        if videoid.mediatype in [common.VideoId.MOVIE, common.VideoId.EPISODE]:
            self.did_restore = False
            self.current_videoid = videoid \
                if videoid.mediatype == common.VideoId.MOVIE \
                else videoid.derive_parent(0)
        else:
            self.enabled = False

    def _on_playback_started(self, player_state):
        xbmc.sleep(500)  # Wait for slower systems
        for stype in STREAMS:
            self._set_current_stream(stype, player_state)
            self._restore_stream(stype)
        if (self.sc_settings.get('subtitleenabled', None) is None
                and g.ADDON.getSettingBool('forced_subtitle_workaround')):
            # Use the workaround only when the user did not change the show subtitle setting
            _show_only_forced_subtitle()
        self.did_restore = True

    def _on_tick(self, player_state):
        if not self.did_restore:
            common.debug('Did not restore streams yet, ignoring tick')
            return
        for stype in STREAMS:
            current_stream = self.current_streams[stype]
            player_stream = player_state.get(STREAMS[stype]['current'])
            if player_stream != current_stream:
                common.debug('{} has changed from {} to {}'
                             .format(stype, current_stream, player_stream))
                self._set_current_stream(stype, player_state)
                self._save_changed_stream(stype, player_stream)

    def _set_current_stream(self, stype, player_state):
        self.current_streams.update({
            stype: player_state.get(STREAMS[stype]['current'])
        })

    def _restore_stream(self, stype):
        common.debug('Trying to restore {}...'.format(stype))
        set_stream = STREAMS[stype]['setter']
        stored_stream = self.sc_settings.get(stype)
        if (stored_stream is not None and
                self.current_streams[stype] != stored_stream):
            # subtitleenabled is boolean and not a dict
            set_stream(self.player, (stored_stream['index']
                                     if isinstance(stored_stream, dict)
                                     else stored_stream))
            self.current_streams[stype] = stored_stream
            common.debug('Restored {} to {}'.format(stype, stored_stream))

    def _save_changed_stream(self, stype, stream):
        common.debug('Save changed stream {} for {}'.format(stream, stype))
        new_sc_settings = self.sc_settings.copy()
        new_sc_settings[stype] = stream
        g.SHARED_DB.set_stream_continuity(g.LOCAL_DB.get_active_profile_guid(),
                                          self.current_videoid.value,
                                          new_sc_settings)

    def __repr__(self):
        return ('enabled={}, current_videoid={}'
                .format(self.enabled, self.current_videoid))

def _show_only_forced_subtitle():
    # When we have "forced only" subtitle setting in Kodi Player, Kodi use this behavior:
    # 1) try to select forced subtitle that matches audio language
    # 2) when missing, try to select the first "regular" subtitle that matches audio language
    # This Kodi behavior is totally non sense.
    # If forced is selected you must not view the regular subtitles
    # There is no other solution than to disable the subtitles manually.
    manifest_data = json.loads(common.load_file('manifest.json'))
    common.fix_locale_languages(manifest_data['timedtexttracks'])
    audio_language = common.get_kodi_audio_language()
    if not any(text_track.get('isForcedNarrative', False) is True and
               text_track['language'] == audio_language
               for text_track in manifest_data['timedtexttracks']):
        xbmc.Player().showSubtitles(False)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
@file
@brief Helpers for the hackathon 2017 (Label Emmaüs).
"""
import os
from io import StringIO, BytesIO
import ijson
from pyquickhelper.loghelper import noLOG

def enumerate_json_items(filename, encoding=None, fLOG=noLOG):
    """
    Enumerates items from a JSON file or string.

    @param      filename        filename or string or stream to parse
    @param      encoding        encoding
    @param      fLOG            logging function
    @return                     iterator on records at first level.

    It assumes the syntax follows the format: ``[ {"id":1, ...}, {"id": 2, ...}, ...]``.

    .. exref::
        :title: Processes a json file by streaming.

        The module :epkg:`ijson` can read a JSON file by streaming.
        This module is needed because a record can be written on multiple lines.
        This function leverages it produces the following results.

        .. runpython::
            :showcode:

            from ensae_projects.hackathon import enumerate_json_items

            text_json = '''
                [
                {
                    "glossary": {
                        "title": "example glossary",
                        "GlossDiv": {
                            "title": "S",
                            "GlossList": [{
                                "GlossEntry": {
                                    "ID": "SGML",
                                    "SortAs": "SGML",
                                    "GlossTerm": "Standard Generalized Markup Language",
                                    "Acronym": "SGML",
                                    "Abbrev": "ISO 8879:1986",
                                    "GlossDef": {
                                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
                                        "GlossSeeAlso": ["GML", "XML"]
                                    },
                                    "GlossSee": "markup"
                                }
                            }]
                        }
                    }
                },
                {
                    "glossary": {
                        "title": "example glossary",
                        "GlossDiv": {
                            "title": "S",
                            "GlossList": {
                                "GlossEntry": [{
                                    "ID": "SGML",
                                    "SortAs": "SGML",
                                    "GlossTerm": "Standard Generalized Markup Language",
                                    "Acronym": "SGML",
                                    "Abbrev": "ISO 8879:1986",
                                    "GlossDef": {
                                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
                                        "GlossSeeAlso": ["GML", "XML"]
                                    },
                                    "GlossSee": "markup"
                                }]
                            }
                        }
                    }
                }
                ]
            '''

            for item in enumerate_json_items(text_json):
                print('------------')
                print(item)
    """
    if isinstance(filename, str):
        if "{" not in filename and os.path.exists(filename):
            with open(filename, "rb", encoding=encoding) as f:
                for el in enumerate_json_items(f, encoding=encoding, fLOG=fLOG):
                    yield el
        elif isinstance(filename, str):
            st = StringIO(filename)
            for el in enumerate_json_items(st, encoding=encoding, fLOG=fLOG):
                yield el
        elif isinstance(filename, bytes):
            st = BytesIO(filename)
            for el in enumerate_json_items(st, encoding=encoding, fLOG=fLOG):
                yield el
        else:
            raise TypeError("Unable to process type '{}'.".format(type(filename)))
    else:
        parser = ijson.parse(filename)
        current = None
        curkey = None
        stack = []
        nbyield = 0
        for i, (_, event, value) in enumerate(parser):
            if i % 1000000 == 0:
                fLOG("[enumerate_json_items] i={0} yielded={1}".format(
                    i, nbyield))
            if event == "start_array":
                if curkey is None:
                    current = []
                else:
                    if not isinstance(current, dict):
                        raise RuntimeError(
                            "Type issue {0}".format(type(current)))
                    c = []
                    current[curkey] = c
                    current = c
                curkey = None
                stack.append(current)
            elif event == "end_array":
                stack.pop()
                if len(stack) == 0:
                    # We should be done.
                    current = None
                else:
                    current = stack[-1]
            elif event == "start_map":
                c = {}
                if curkey is None:
                    current.append(c)
                else:
                    current[curkey] = c  # pylint: disable=E1137
                stack.append(c)
                current = c
                curkey = None
            elif event == "end_map":
                stack.pop()
                current = stack[-1]
                if len(stack) == 1:
                    nbyield += 1
                    yield current[-1]
                    # We clear the memory.
                    current.clear()
            elif event == "map_key":
                curkey = value
            elif event in {"string", "number", "boolean"}:
                if curkey is None:
                    current.append(value)
                else:
                    current[curkey] = value  # pylint: disable=E1137
                    curkey = None
            elif event == "null":
                if curkey is None:
                    current.append(None)
                else:
                    current[curkey] = None  # pylint: disable=E1137
                    curkey = None
            else:
                raise ValueError("Unknown event '{0}'".format(event))

def extract_images_from_json_2017(filename, encoding=None, fLOG=noLOG):
    """
    Extracts fields from a JSON files such as images.

    @param      filename    filename
    @param      encoding    encoding
    @param      fLOG        logging function
    @return                 iterator on images

    ..warning:: Copy between two iterations?

        If you plan to store the enumerated dictionaries, you should
        copy them because dictionary are reused.

    One example on dummy data implementing a subset of the fields
    the JSON contains. This can be easily converted into a dataframe.

    .. runpython::
        :showcode:

        from ensae_projects.hackathon import extract_images_from_json_2017

        text_json = '''
            [
               {"assigned_images": [],
                "best_offer": {"created_on": "2016-11-04T23:20:53+01:00", "images": [], "offer_longitude": null, "availability": "in_stock",
                               "start_selling_date": null, "delay_before_shipping": 0.00, "free_return": null, "free_shipping": null,
                               "assigned_images": [{"image_path": "https://coucou.JPEG"}],
                               "id": 1306501, "eco_tax": 0.000000, "keywords": ["boutique", "test"],
                "sku": "AAAA27160018",
                "product": {"pk": 2550, "external_id": null, "id": 2580},
                "description": "livre l", "last_modified": "2016-11-04T23:27:01+01:00",
                "name": "les names", "language": "fr"}, "id": 25540,
                "description": "livre 2", "slug": "les-l",
                "application_categories": [280, 283], "product_type": "physical",
                "name": "les l n", "language": "fr", "popularity": 99, "gender": null
                }
            ]
            '''

        items = []
        for item in extract_images_from_json_2017(text_json):
            print(item)
            items.append(item)

        from pandas import DataFrame
        df = DataFrame(items)
        print(df)
    """
    for record in enumerate_json_items(filename, encoding=encoding, fLOG=fLOG):
        images = []
        if "best_offer" in record and record["best_offer"]:
            best = record["best_offer"]
            if "assigned_images" in best and best["assigned_images"]:
                images.extend(best["assigned_images"])
        else:
            continue
        product = best.get("product")
        if product is None:
            continue
        if "assigned_images" in record and record["assigned_images"]:
            images.extend(record["assigned_images"])
        res = {}
        res["product_pk"] = product.get("pk")
        res["product_id"] = product.get("id")
        res["id2"] = record.get("id2")
        res["sku"] = best.get("sku")
        res["created_on"] = record.get("created_on")
        res["keywords"] = record.get("keywords")
        if isinstance(res["keywords"], list):
            res['keywords'] = ";".join(res['keywords'])
        res["availability"] = best.get("availability")
        res["eco_tax"] = best.get("eco_tax")
        res["restock_date"] = best.get("restock_date")
        res["status"] = best.get("status")
        res["number_of_items"] = best.get("number_of_items")
        res["price_with_vat"] = best.get("price_with_vat")
        res["price_without_vat"] = best.get("price_without_vat")
        res["previous_price_without_vat"] = best.get(
            "previous_price_without_vat")
        res["max_order_quantity"] = best.get("max_order_quantity")
        res["stock"] = best.get("stock")
        res["start_selling_date"] = best.get("start_selling_date")
        res["description"] = record.get("description")
        if isinstance(res["description"], str):
            res["description"] = res["description"].replace(
                "\n", "\\n").replace("\t", "\\t").replace("\r", "")
        res["last_modified"] = best.get("last_modified")
        res["name"] = record.get("name")
        res["product_type"] = record.get("product_type")
        res["gender"] = record.get("gender")
        res["popularity"] = record.get("popularity")
        res["application_categories"] = record.get("application_categories")
        if isinstance(res["application_categories"], list):
            res["application_categories"] = ",".join(
                map(str, res["application_categories"]))
        res["language"] = record.get("language")
        paths = list(im.get("image_path") for im in images)
        done = set()
        for p in paths:
            if p and p not in done:
                res["image_path"] = p
                yield res
                done.add(p)
<commit_msg>Update json_helper.py<commit_after># -*- coding: utf-8 -*-
"""
@file
@brief Helpers for the hackathon 2017 (Label Emmaüs).
"""
import os
from io import BytesIO
import ijson
from pyquickhelper.loghelper import noLOG

def enumerate_json_items(filename, encoding=None, fLOG=noLOG):
    """
    Enumerates items from a JSON file or string.

    @param      filename        filename or string or stream to parse
    @param      encoding        encoding
    @param      fLOG            logging function
    @return                     iterator on records at first level.

    It assumes the syntax follows the format: ``[ {"id":1, ...}, {"id": 2, ...}, ...]``.

    .. exref::
        :title: Processes a json file by streaming.

        The module :epkg:`ijson` can read a JSON file by streaming.
        This module is needed because a record can be written on multiple lines.
        This function leverages it produces the following results.

        .. runpython::
            :showcode:

            from ensae_projects.hackathon import enumerate_json_items

            text_json = '''
                [
                {
                    "glossary": {
                        "title": "example glossary",
                        "GlossDiv": {
                            "title": "S",
                            "GlossList": [{
                                "GlossEntry": {
                                    "ID": "SGML",
                                    "SortAs": "SGML",
                                    "GlossTerm": "Standard Generalized Markup Language",
                                    "Acronym": "SGML",
                                    "Abbrev": "ISO 8879:1986",
                                    "GlossDef": {
                                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
                                        "GlossSeeAlso": ["GML", "XML"]
                                    },
                                    "GlossSee": "markup"
                                }
                            }]
                        }
                    }
                },
                {
                    "glossary": {
                        "title": "example glossary",
                        "GlossDiv": {
                            "title": "S",
                            "GlossList": {
                                "GlossEntry": [{
                                    "ID": "SGML",
                                    "SortAs": "SGML",
                                    "GlossTerm": "Standard Generalized Markup Language",
                                    "Acronym": "SGML",
                                    "Abbrev": "ISO 8879:1986",
                                    "GlossDef": {
                                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
                                        "GlossSeeAlso": ["GML", "XML"]
                                    },
                                    "GlossSee": "markup"
                                }]
                            }
                        }
                    }
                }
                ]
            '''

            for item in enumerate_json_items(text_json):
                print('------------')
                print(item)
    """
    if isinstance(filename, str):
        if "{" not in filename and os.path.exists(filename):
            with open(filename, "rb", encoding=encoding) as f:
                for el in enumerate_json_items(f, encoding=encoding, fLOG=fLOG):
                    yield el
        elif isinstance(filename, str):
            st = BytesIO(filename.encode('utf-8'))
            for el in enumerate_json_items(st, encoding=encoding, fLOG=fLOG):
                yield el
        else:
            raise TypeError("Unable to process type '{}'.".format(type(filename)))
    else:
        parser = ijson.parse(filename)
        current = None
        curkey = None
        stack = []
        nbyield = 0
        for i, (_, event, value) in enumerate(parser):
            if i % 1000000 == 0:
                fLOG("[enumerate_json_items] i={0} yielded={1}".format(
                    i, nbyield))
            if event == "start_array":
                if curkey is None:
                    current = []
                else:
                    if not isinstance(current, dict):
                        raise RuntimeError(
                            "Type issue {0}".format(type(current)))
                    c = []
                    current[curkey] = c
                    current = c
                curkey = None
                stack.append(current)
            elif event == "end_array":
                stack.pop()
                if len(stack) == 0:
                    # We should be done.
                    current = None
                else:
                    current = stack[-1]
            elif event == "start_map":
                c = {}
                if curkey is None:
                    current.append(c)
                else:
                    current[curkey] = c  # pylint: disable=E1137
                stack.append(c)
                current = c
                curkey = None
            elif event == "end_map":
                stack.pop()
                current = stack[-1]
                if len(stack) == 1:
                    nbyield += 1
                    yield current[-1]
                    # We clear the memory.
                    current.clear()
            elif event == "map_key":
                curkey = value
            elif event in {"string", "number", "boolean"}:
                if curkey is None:
                    current.append(value)
                else:
                    current[curkey] = value  # pylint: disable=E1137
                    curkey = None
            elif event == "null":
                if curkey is None:
                    current.append(None)
                else:
                    current[curkey] = None  # pylint: disable=E1137
                    curkey = None
            else:
                raise ValueError("Unknown event '{0}'".format(event))

def extract_images_from_json_2017(filename, encoding=None, fLOG=noLOG):
    """
    Extracts fields from a JSON files such as images.

    @param      filename    filename
    @param      encoding    encoding
    @param      fLOG        logging function
    @return                 iterator on images

    ..warning:: Copy between two iterations?

        If you plan to store the enumerated dictionaries, you should
        copy them because dictionary are reused.

    One example on dummy data implementing a subset of the fields
    the JSON contains. This can be easily converted into a dataframe.

    .. runpython::
        :showcode:

        from ensae_projects.hackathon import extract_images_from_json_2017

        text_json = '''
            [
               {"assigned_images": [],
                "best_offer": {"created_on": "2016-11-04T23:20:53+01:00", "images": [], "offer_longitude": null, "availability": "in_stock",
                               "start_selling_date": null, "delay_before_shipping": 0.00, "free_return": null, "free_shipping": null,
                               "assigned_images": [{"image_path": "https://coucou.JPEG"}],
                               "id": 1306501, "eco_tax": 0.000000, "keywords": ["boutique", "test"],
                "sku": "AAAA27160018",
                "product": {"pk": 2550, "external_id": null, "id": 2580},
                "description": "livre l", "last_modified": "2016-11-04T23:27:01+01:00",
                "name": "les names", "language": "fr"}, "id": 25540,
                "description": "livre 2", "slug": "les-l",
                "application_categories": [280, 283], "product_type": "physical",
                "name": "les l n", "language": "fr", "popularity": 99, "gender": null
                }
            ]
            '''

        items = []
        for item in extract_images_from_json_2017(text_json):
            print(item)
            items.append(item)

        from pandas import DataFrame
        df = DataFrame(items)
        print(df)
    """
    for record in enumerate_json_items(filename, encoding=encoding, fLOG=fLOG):
        images = []
        if "best_offer" in record and record["best_offer"]:
            best = record["best_offer"]
            if "assigned_images" in best and best["assigned_images"]:
                images.extend(best["assigned_images"])
        else:
            continue
        product = best.get("product")
        if product is None:
            continue
        if "assigned_images" in record and record["assigned_images"]:
            images.extend(record["assigned_images"])
        res = {}
        res["product_pk"] = product.get("pk")
        res["product_id"] = product.get("id")
        res["id2"] = record.get("id2")
        res["sku"] = best.get("sku")
        res["created_on"] = record.get("created_on")
        res["keywords"] = record.get("keywords")
        if isinstance(res["keywords"], list):
            res['keywords'] = ";".join(res['keywords'])
        res["availability"] = best.get("availability")
        res["eco_tax"] = best.get("eco_tax")
        res["restock_date"] = best.get("restock_date")
        res["status"] = best.get("status")
        res["number_of_items"] = best.get("number_of_items")
        res["price_with_vat"] = best.get("price_with_vat")
        res["price_without_vat"] = best.get("price_without_vat")
        res["previous_price_without_vat"] = best.get(
            "previous_price_without_vat")
        res["max_order_quantity"] = best.get("max_order_quantity")
        res["stock"] = best.get("stock")
        res["start_selling_date"] = best.get("start_selling_date")
        res["description"] = record.get("description")
        if isinstance(res["description"], str):
            res["description"] = res["description"].replace(
                "\n", "\\n").replace("\t", "\\t").replace("\r", "")
        res["last_modified"] = best.get("last_modified")
        res["name"] = record.get("name")
        res["product_type"] = record.get("product_type")
        res["gender"] = record.get("gender")
        res["popularity"] = record.get("popularity")
        res["application_categories"] = record.get("application_categories")
        if isinstance(res["application_categories"], list):
            res["application_categories"] = ",".join(
                map(str, res["application_categories"]))
        res["language"] = record.get("language")
        paths = list(im.get("image_path") for im in images)
        done = set()
        for p in paths:
            if p and p not in done:
                res["image_path"] = p
                yield res
                done.add(p)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# blessings documentation build configuration file, created by
# sphinx-quickstart on Thu Mar 31 13:40:27 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

import blessings

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Blessings'
copyright = u'2011, Erik Rose'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.3'
# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'blessingsdoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'blessings.tex', u'Blessings Documentation',
   u'Erik Rose', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'Blessings', u'Blessings Documentation',
     [u'Erik Rose'], 1)
]
<commit_msg>Bump version to 1.4 in docs config.<commit_after># -*- coding: utf-8 -*-
#
# blessings documentation build configuration file, created by
# sphinx-quickstart on Thu Mar 31 13:40:27 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

import blessings

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Blessings'
copyright = u'2011, Erik Rose'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.4'
# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'blessingsdoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'blessings.tex', u'Blessings Documentation',
   u'Erik Rose', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'Blessings', u'Blessings Documentation',
     [u'Erik Rose'], 1)
]
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# :LICENSE: MIT

__author__ = "Saurav Ghosh"
__email__ = "sauravcsvt@vt.edu"

import os
os.system("unzip Dis2Vec.zip -d ./")
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
sys.path.append("./Dis2Vec/")
import cPickle as pickle
from gensim.models import Word2Vec
import time
import logging
import argparse
logging.basicConfig()

def perform_word2vec(sent_list, vec_dim, win_size, count_min, SG_negative, iter_corpus, spm, opm, smoothing, vocab_file):

    start_time = time.time()
    model_HM = Word2Vec(sent_list, size=vec_dim, window=win_size,
                        min_count=count_min, sample=1e-05, workers=100,
                        sg=1, hs=0, negative=SG_negative, iter=iter_corpus, sampling_param=spm, objective_param=opm, smoothing=smoothing, vocab_file=vocab_file)
    end_time = time.time()
    print "Total time taken is: " + str((end_time - start_time) / 3600.) + " hours"
    out_folder = './output/'
    if not os.path.isdir(out_folder):
        os.makedirs(out_folder)
    if model_HM.sample == 0:
        model_HM.save(out_folder + 'model_all_vocab_' + str(vec_dim) + "_" + str(win_size) + "_" + str(count_min) + "_" + str(SG_negative) + "_" + str(spm) + "_" + str(opm) + "_" + str(smoothing) + "_" + str(iter_corpus) + '.word2vec')
    else:
        model_HM.save(out_folder + 'model_all_vocab_' + str(vec_dim) + "_" + str(win_size) + "_" + str(count_min) + "_" + str(SG_negative) + "_" + str(spm) + "_" + str(opm) + "_" + str(smoothing) + "_" + str(iter_corpus) + 'w_sample.word2vec')

def parse_args():

    ap = argparse.ArgumentParser()
    ap.add_argument("-ic", "--inputcorpus", type = str, required = True, help = "Input corpus which should be a list of sentences as input where each sentence is a list of tokens. file should be in .pkl format")
    ap.add_argument("-v", "--domainvocab", type = str, required = True, help = "Domain-specific vocabulary. file should be in .pkl format")
    return ap.parse_args()

def main():

    _arg = parse_args()
    sentences_corpus = pickle.load(open(_arg.inputcorpus, "r")) # Input corpus (list of sentences as input where each sentence is a list of tokens. file should be in .pkl format).
    domain_vocab_file = _arg.domainvocab # domain-specific vocabulary. file should be in .pkl format
    param_list = {"dim": 300, "win": [15], "min_cnt": 5, "neg": [1, 5, 15], "iter": 1, "spm": [0.3, 0.5, 0.7], "opm": [0.3, 0.5, 0.7], "smoothing": [0.75, 1.0]}
    for param_win in param_list['win']:
        for param_neg in param_list['neg']:
            for param_spm in param_list['spm']:
                for param_opm in param_list['opm']:
                    for param_smooth in param_list["smoothing"]:
                        perform_word2vec(sentences_corpus, param_list["dim"], param_win, param_list["min_cnt"],
                                         param_neg, param_list["iter"], param_spm, param_opm, param_smooth, domain_vocab_file)

if __name__ == "__main__":
    main()
<commit_msg>another commit<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# :LICENSE: MIT

__author__ = "Saurav Ghosh"
__email__ = "sauravcsvt@vt.edu"

import os
os.system("unzip Dis2Vec.zip -d ./")
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
sys.path.append("./Dis2Vec/")
import cPickle as pickle
from gensim.models import Word2Vec
import time
import logging
import argparse
logging.basicConfig()

class Dis2Vec(object):

    def __init__(self, **kwargs):
        self.params = kwargs

    def run_Dis2Vec(self):

        start_time = time.time()
        model_HM = Word2Vec(self.params["sent"], size=self.params["dim"], window=self.params["win"],
                            min_count=self.params["min_cnt"], sample=self.params["sample"], workers=100,
                            sg=1, hs=0, negative=self.params["neg"], iter=self.params["iter"],
                            sampling_param=self.params["spm"], objective_param=self.params["opm"],
                            smoothing=self.params["smoothing"], vocab_file=self.params["vocab"])
        end_time = time.time()
        print "Total time taken is: " + str((end_time - start_time) / 3600.) + " hours"
        out_folder = './output/'
        if not os.path.isdir(out_folder):
            os.makedirs(out_folder)
        if model_HM.sample == 0:
            model_HM.save(out_folder + 'model_Dis2Vec.word2vec')
        else:
            model_HM.save(out_folder + 'model_Dis2Vec_w_sample.word2vec')

def parse_args():

    ap = argparse.ArgumentParser()
    ap.add_argument("-ic", "--inputcorpus", type = str, required = True, help = "Input corpus which should be a list of sentences as input where each sentence is a list of tokens. file should be in .pkl format")
    ap.add_argument("-v", "--domainvocab", type = str, required = True, help = "Domain-specific vocabulary. file should be in .pkl format")
    return ap.parse_args()

def main():

    _arg = parse_args()
    sentences_corpus = pickle.load(open(_arg.inputcorpus, "r")) # Input corpus (list of sentences as input where each sentence is a list of tokens. file should be in .pkl format).
    domain_vocab_file = _arg.domainvocab # domain-specific vocabulary. file should be in .pkl format
    kwargs = {"sent": sentences_corpus, "vocab": domain_vocab_file,
              "dim": 300, "win": 15, "min_cnt": 5, "neg": 1, "iter": 1,
              "spm": 0.3, "opm": 0.7, "smoothing": 1.0, "sample": 1e-05}
    Dis2Vec(**kwargs).run_Dis2Vec()

if __name__ == "__main__":
    main()
<|endoftext|>