<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

""":mod:`openquake.hazardlib.calc.hazard_curve` implements
:func:`calc_hazard_curves`. Here is an example of a classical PSHA
parallel calculator computing the hazard curves per each realization in less
than 20 lines of code:

.. code-block:: python

   import sys
   import logging
   from openquake.baselib import parallel
   from openquake.hazardlib.calc.filters import SourceFilter
   from openquake.hazardlib.calc.hazard_curve import calc_hazard_curves
   from openquake.commonlib import readinput

   def main(job_ini):
       logging.basicConfig(level=logging.INFO)
       oq = readinput.get_oqparam(job_ini)
       sitecol = readinput.get_site_collection(oq)
       src_filter = SourceFilter(sitecol, oq.maximum_distance)
       csm = readinput.get_composite_source_model(oq).filter(src_filter)
       rlzs_assoc = csm.info.get_rlzs_assoc()
       for i, sm in enumerate(csm.source_models):
           for rlz in rlzs_assoc.rlzs_by_smodel[i]:
               gsim_by_trt = rlzs_assoc.gsim_by_trt[rlz.ordinal]
               hcurves = calc_hazard_curves(
                   sm.src_groups, src_filter, oq.imtls,
                   gsim_by_trt, oq.truncation_level,
                   parallel.Starmap.apply)
           print('rlz=%s, hcurves=%s' % (rlz, hcurves))

   if __name__ == '__main__':
       main(sys.argv[1])  # path to a job.ini file

NB: the implementation in the engine is smarter and more
efficient. Here we start a parallel computation per each realization,
the engine manages all the realizations at once.
"""
from __future__ import division
import sys
import time
import operator
import numpy

from openquake.baselib.python3compat import raise_, zip
from openquake.baselib.performance import Monitor
from openquake.baselib.general import DictArray, groupby
from openquake.baselib.parallel import Sequential
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib.gsim.base import GroundShakingIntensityModel
from openquake.hazardlib.calc.filters import SourceFilter, FarAwayRupture
from openquake.hazardlib.imt import from_string
from openquake.hazardlib.sourceconverter import SourceGroup

def zero_curves(num_sites, imtls):
    """
    :param num_sites: the number of sites
    :param imtls: the intensity measure levels dictionary
    :returns: an array of zero curves with length num_sites
    """
    # numpy dtype for the hazard curves
    imt_dt = numpy.dtype([(imt, float, 1 if imls is None else len(imls))
                          for imt, imls in imtls.items()])
    return numpy.zeros(num_sites, imt_dt)

def rupture_weight_pairs(src):
    """
    Generator yielding (rupture, weight) for each rupture in the source
    """
    if hasattr(src, 'weights'):
        for pair in zip(src.iter_ruptures(), src.weights):
            yield pair
    weight = 1. / (src.num_ruptures or src.count_ruptures())
    for rup in src.iter_ruptures():
        yield rup, weight

# NB: it is important for this to be fast since it is inside an inner loop
def get_probability_no_exceedance(
        rupture, sctx, rctx, dctx, imtls, gsims, trunclevel, pne_mons):
    """
    :param rupture: a Rupture instance
    :param sctx: the corresponding SiteContext instance
    :param rctx: the corresponding RuptureContext instance
    :param dctx: the corresponding DistanceContext instance
    :param imtls: a dictionary-like object providing the intensity levels
    :param gsims: the list of GSIMs to use
    :param trunclevel: the truncation level
    :param pne_mons: monitors for the probability of no exceedance
    :returns: an array of shape (num_sites, num_levels, num_gsims)
    """
    pne_array = numpy.zeros((len(sctx.sites), len(imtls.array), len(gsims)))
    for i, gsim in enumerate(gsims):
        with pne_mons[i]:
            pnos = []  # list of arrays nsites x nlevels
            for imt in imtls:
                poes = gsim.get_poes(
                    sctx, rctx, dctx, from_string(imt), imtls[imt], trunclevel)
                pnos.append(rupture.get_probability_no_exceedance(poes))
            pne_array[:, :, i] = numpy.concatenate(pnos, axis=1)
    return pne_array

def poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon, pne_mons,
            bbs=(), rup_indep=True, disagg_mon=None):
    """
    Compute the ProbabilityMap generated by the given source. Also,
    store some information in the monitors and optionally in the
    bounding boxes.
    """
    pmap = ProbabilityMap.build(
        len(imtls.array), len(cmaker.gsims), s_sites.sids, initvalue=rup_indep)
    try:
        for rup, weight in rupture_weight_pairs(src):
            with ctx_mon:  # compute distances
                try:
                    sctx, rctx, dctx = cmaker.make_contexts(s_sites, rup)
                except FarAwayRupture:
                    continue
            # compute probabilities and updates the pmap
            pnes = get_probability_no_exceedance(
                rup, sctx, rctx, dctx, imtls, cmaker.gsims, trunclevel,
                pne_mons)
            for sid, pne in zip(sctx.sites.sids, pnes):
                if rup_indep:
                    pmap[sid].array *= pne
                else:
                    pmap[sid].array += pne * weight
            # add optional disaggregation information (bounding boxes)
            if bbs:
                with disagg_mon:
                    sids = set(sctx.sites.sids)
                    jb_dists = dctx.rjb
                    closest_points = rup.surface.get_closest_points(
                        sctx.sites.mesh)
                    bs = [bb for bb in bbs if bb.site_id in sids]
                    # NB: the assert below is always true; we are
                    # protecting against possible refactoring errors
                    assert len(bs) == len(jb_dists) == len(closest_points)
                    for bb, dist, p in zip(bs, jb_dists, closest_points):
                        bb.update([dist], [p.longitude], [p.latitude])
    except Exception as err:
        etype, err, tb = sys.exc_info()
        msg = '%s (source id=%s)' % (str(err), src.source_id)
        raise_(etype, msg, tb)
    return ~pmap

# this is used by the engine
def pmap_from_grp(
        sources, source_site_filter, imtls, gsims, truncation_level=None,
        bbs=(), monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.sources
        trt = sources[0].tectonic_region_type
        mutex_weight = {src.source_id: weight for src, weight in
                        zip(group.sources, group.srcs_weights)}
        grp_probability = group.grp_probability
    else:  # list of sources
        trt = sources[0].tectonic_region_type
        group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
        grp_probability = 1
    maxdist = source_site_filter.integration_distance
    if hasattr(gsims, 'keys'):  # dictionary trt -> gsim
        gsims = [gsims[trt]]
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mons = [monitor('%s.get_poes' % gsim, measuremem=False)
                    for gsim in gsims]
        disagg_mon = monitor('get closest points', measuremem=False)
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = sources[0].src_group_id
        for src, s_sites in source_site_filter(sources):
            t0 = time.time()
            poemap = poe_map(
                src, s_sites, imtls, cmaker, truncation_level, ctx_mon,
                pne_mons, bbs, group.rup_interdep == 'indep', disagg_mon)
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap.setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mons[0].counts}
        return pmap * grp_probability

def calc_hazard_curves(
        groups, ss_filter, imtls, gsim_by_trt, truncation_level=None,
        apply=Sequential.apply):
    """
    Compute hazard curves on a list of sites, given a set of seismic source
    groups and a dictionary of ground shaking intensity models (one per
    tectonic region type).

    Probability of ground motion exceedance is computed in different ways
    depending if the sources are independent or mutually exclusive.

    :param groups:
        A sequence of groups of seismic sources objects (instances of
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param ss_filter:
        A source filter over the site collection or the site collection itself
    :param imtls:
        Dictionary mapping intensity measure type strings
        to lists of intensity measure levels.
    :param gsim_by_trt:
        Dictionary mapping tectonic region types (members
        of :class:`openquake.hazardlib.const.TRT`) to
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE` objects.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param maximum_distance:
        The integration distance, if any
    :returns:
        An array of size N, where N is the number of sites, which elements
        are records with fields given by the intensity measure types; the
        size of each field is given by the number of levels in ``imtls``.
    """
    # This is ensuring backward compatibility i.e. processing a list of
    # sources
    if not isinstance(groups[0], SourceGroup):  # sent a list of sources
        dic = groupby(groups, operator.attrgetter('tectonic_region_type'))
        groups = [SourceGroup(trt, dic[trt], 'src_group', 'indep', 'indep')
                  for trt in dic]
    if hasattr(ss_filter, 'sitecol'):  # a filter, as it should be
        sitecol = ss_filter.sitecol
    else:  # backward compatibility, a site collection was passed
        sitecol = ss_filter
        ss_filter = SourceFilter(sitecol, {})

    imtls = DictArray(imtls)
    pmap = ProbabilityMap(len(imtls.array), 1)
    # Processing groups with homogeneous tectonic region
    for group in groups:
        pmap |= apply(
            pmap_from_grp,
            (group, ss_filter, imtls, gsim_by_trt, truncation_level),
            weight=operator.attrgetter('weight')).reduce(operator.or_)
    return pmap.convert(imtls, len(sitecol.complete))
<commit_msg>Managed the case of missing grp_probability<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

""":mod:`openquake.hazardlib.calc.hazard_curve` implements
:func:`calc_hazard_curves`. Here is an example of a classical PSHA
parallel calculator computing the hazard curves per each realization in less
than 20 lines of code:

.. code-block:: python

   import sys
   import logging
   from openquake.baselib import parallel
   from openquake.hazardlib.calc.filters import SourceFilter
   from openquake.hazardlib.calc.hazard_curve import calc_hazard_curves
   from openquake.commonlib import readinput

   def main(job_ini):
       logging.basicConfig(level=logging.INFO)
       oq = readinput.get_oqparam(job_ini)
       sitecol = readinput.get_site_collection(oq)
       src_filter = SourceFilter(sitecol, oq.maximum_distance)
       csm = readinput.get_composite_source_model(oq).filter(src_filter)
       rlzs_assoc = csm.info.get_rlzs_assoc()
       for i, sm in enumerate(csm.source_models):
           for rlz in rlzs_assoc.rlzs_by_smodel[i]:
               gsim_by_trt = rlzs_assoc.gsim_by_trt[rlz.ordinal]
               hcurves = calc_hazard_curves(
                   sm.src_groups, src_filter, oq.imtls,
                   gsim_by_trt, oq.truncation_level,
                   parallel.Starmap.apply)
           print('rlz=%s, hcurves=%s' % (rlz, hcurves))

   if __name__ == '__main__':
       main(sys.argv[1])  # path to a job.ini file

NB: the implementation in the engine is smarter and more
efficient. Here we start a parallel computation per each realization,
the engine manages all the realizations at once.
"""
from __future__ import division
import sys
import time
import operator
import numpy

from openquake.baselib.python3compat import raise_, zip
from openquake.baselib.performance import Monitor
from openquake.baselib.general import DictArray, groupby
from openquake.baselib.parallel import Sequential
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib.gsim.base import GroundShakingIntensityModel
from openquake.hazardlib.calc.filters import SourceFilter, FarAwayRupture
from openquake.hazardlib.imt import from_string
from openquake.hazardlib.sourceconverter import SourceGroup

def zero_curves(num_sites, imtls):
    """
    :param num_sites: the number of sites
    :param imtls: the intensity measure levels dictionary
    :returns: an array of zero curves with length num_sites
    """
    # numpy dtype for the hazard curves
    imt_dt = numpy.dtype([(imt, float, 1 if imls is None else len(imls))
                          for imt, imls in imtls.items()])
    return numpy.zeros(num_sites, imt_dt)

def rupture_weight_pairs(src):
    """
    Generator yielding (rupture, weight) for each rupture in the source
    """
    if hasattr(src, 'weights'):
        for pair in zip(src.iter_ruptures(), src.weights):
            yield pair
    weight = 1. / (src.num_ruptures or src.count_ruptures())
    for rup in src.iter_ruptures():
        yield rup, weight

# NB: it is important for this to be fast since it is inside an inner loop
def get_probability_no_exceedance(
        rupture, sctx, rctx, dctx, imtls, gsims, trunclevel, pne_mons):
    """
    :param rupture: a Rupture instance
    :param sctx: the corresponding SiteContext instance
    :param rctx: the corresponding RuptureContext instance
    :param dctx: the corresponding DistanceContext instance
    :param imtls: a dictionary-like object providing the intensity levels
    :param gsims: the list of GSIMs to use
    :param trunclevel: the truncation level
    :param pne_mons: monitors for the probability of no exceedance
    :returns: an array of shape (num_sites, num_levels, num_gsims)
    """
    pne_array = numpy.zeros((len(sctx.sites), len(imtls.array), len(gsims)))
    for i, gsim in enumerate(gsims):
        with pne_mons[i]:
            pnos = []  # list of arrays nsites x nlevels
            for imt in imtls:
                poes = gsim.get_poes(
                    sctx, rctx, dctx, from_string(imt), imtls[imt], trunclevel)
                pnos.append(rupture.get_probability_no_exceedance(poes))
            pne_array[:, :, i] = numpy.concatenate(pnos, axis=1)
    return pne_array

def poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon, pne_mons,
            bbs=(), rup_indep=True, disagg_mon=None):
    """
    Compute the ProbabilityMap generated by the given source. Also,
    store some information in the monitors and optionally in the
    bounding boxes.
    """
    pmap = ProbabilityMap.build(
        len(imtls.array), len(cmaker.gsims), s_sites.sids, initvalue=rup_indep)
    try:
        for rup, weight in rupture_weight_pairs(src):
            with ctx_mon:  # compute distances
                try:
                    sctx, rctx, dctx = cmaker.make_contexts(s_sites, rup)
                except FarAwayRupture:
                    continue
            # compute probabilities and updates the pmap
            pnes = get_probability_no_exceedance(
                rup, sctx, rctx, dctx, imtls, cmaker.gsims, trunclevel,
                pne_mons)
            for sid, pne in zip(sctx.sites.sids, pnes):
                if rup_indep:
                    pmap[sid].array *= pne
                else:
                    pmap[sid].array += pne * weight
            # add optional disaggregation information (bounding boxes)
            if bbs:
                with disagg_mon:
                    sids = set(sctx.sites.sids)
                    jb_dists = dctx.rjb
                    closest_points = rup.surface.get_closest_points(
                        sctx.sites.mesh)
                    bs = [bb for bb in bbs if bb.site_id in sids]
                    # NB: the assert below is always true; we are
                    # protecting against possible refactoring errors
                    assert len(bs) == len(jb_dists) == len(closest_points)
                    for bb, dist, p in zip(bs, jb_dists, closest_points):
                        bb.update([dist], [p.longitude], [p.latitude])
    except Exception as err:
        etype, err, tb = sys.exc_info()
        msg = '%s (source id=%s)' % (str(err), src.source_id)
        raise_(etype, msg, tb)
    return ~pmap

# this is used by the engine
def pmap_from_grp(
        sources, source_site_filter, imtls, gsims, truncation_level=None,
        bbs=(), monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.sources
        trt = sources[0].tectonic_region_type
        mutex_weight = {src.source_id: weight for src, weight in
                        zip(group.sources, group.srcs_weights)}
        grp_probability = (group.grp_probability
                           if group.grp_probability is not None else 1)
    else:  # list of sources
        trt = sources[0].tectonic_region_type
        group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
        grp_probability = 1
    maxdist = source_site_filter.integration_distance
    if hasattr(gsims, 'keys'):  # dictionary trt -> gsim
        gsims = [gsims[trt]]
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mons = [monitor('%s.get_poes' % gsim, measuremem=False)
                    for gsim in gsims]
        disagg_mon = monitor('get closest points', measuremem=False)
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = sources[0].src_group_id
        for src, s_sites in source_site_filter(sources):
            t0 = time.time()
            poemap = poe_map(
                src, s_sites, imtls, cmaker, truncation_level, ctx_mon,
                pne_mons, bbs, group.rup_interdep == 'indep', disagg_mon)
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap.setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mons[0].counts}
        return pmap * grp_probability

def calc_hazard_curves(
        groups, ss_filter, imtls, gsim_by_trt, truncation_level=None,
        apply=Sequential.apply):
    """
    Compute hazard curves on a list of sites, given a set of seismic source
    groups and a dictionary of ground shaking intensity models (one per
    tectonic region type).

    Probability of ground motion exceedance is computed in different ways
    depending if the sources are independent or mutually exclusive.

    :param groups:
        A sequence of groups of seismic sources objects (instances of
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param ss_filter:
        A source filter over the site collection or the site collection itself
    :param imtls:
        Dictionary mapping intensity measure type strings
        to lists of intensity measure levels.
    :param gsim_by_trt:
        Dictionary mapping tectonic region types (members
        of :class:`openquake.hazardlib.const.TRT`) to
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE` objects.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param maximum_distance:
        The integration distance, if any
    :returns:
        An array of size N, where N is the number of sites, which elements
        are records with fields given by the intensity measure types; the
        size of each field is given by the number of levels in ``imtls``.
    """
    # This is ensuring backward compatibility i.e. processing a list of
    # sources
    if not isinstance(groups[0], SourceGroup):  # sent a list of sources
        dic = groupby(groups, operator.attrgetter('tectonic_region_type'))
        groups = [SourceGroup(trt, dic[trt], 'src_group', 'indep', 'indep')
                  for trt in dic]
    if hasattr(ss_filter, 'sitecol'):  # a filter, as it should be
        sitecol = ss_filter.sitecol
    else:  # backward compatibility, a site collection was passed
        sitecol = ss_filter
        ss_filter = SourceFilter(sitecol, {})

    imtls = DictArray(imtls)
    pmap = ProbabilityMap(len(imtls.array), 1)
    # Processing groups with homogeneous tectonic region
    for group in groups:
        pmap |= apply(
            pmap_from_grp,
            (group, ss_filter, imtls, gsim_by_trt, truncation_level),
            weight=operator.attrgetter('weight')).reduce(operator.or_)
    return pmap.convert(imtls, len(sitecol.complete))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
try:
    os.chdir(os.path.dirname(__file__))
except OSError:
    pass
except:
    exit()
import numpy as np
import sys
import audioLoopback, channelModel, maskNoise, wifi80211
import audio, util
import pylab as pl
import iir

wifi = wifi80211.WiFi_802_11()

fn = '35631__reinsamba__crystal-glass.wav'
Fs = 48000.
Fc = 19000. #Fs/4
upsample_factor = 16
mask_noise = maskNoise.prepareMaskNoise(fn, Fs, Fc, upsample_factor)
mask_noise = mask_noise[:int(Fs)]
mask_noise[int(Fs*.5):] *= 1-np.arange(int(Fs*.5))/float(Fs*.5)
lsnr = None
mask_noise = None

rate = 0
length = 16

def test(visualize=False):
    input_octets = np.random.random_integers(0,255,length)
    output = wifi.encode(input_octets, rate)
    if lsnr is not None:
        input = channelModel.channelModel(output, lsnr)
        bitrate = None
    elif Fs is not None:
        input = audioLoopback.audioLoopback(output, Fs, Fc, upsample_factor, mask_noise)
        bitrate = input_octets.size*8 * Fs / float(output.size) / upsample_factor
    try:
        results, _ = wifi.decode(input, visualize)
        return len(results), bitrate
    except Exception, e:
        print e
        return False, bitrate

def transmit(message):
    length = len(message)
    input_octets = np.array(map(ord, message))
    output = wifi.encode(input_octets, rate)
    audioLoopback.audioOut(output, Fs, Fc, upsample_factor, mask_noise)

def presentResults(results, drawFunc):
    _results = results
    _drawFunc = drawFunc
    def f():
        if len(_results):
            for result in _results:
                payload, _, _, lsnr_estimate = result
                print repr(''.join(map(chr, payload))) + (' @ %.3f dB' % lsnr_estimate)
        #else:
        #    decoderDiagnostics()
        #    pl.draw()
        if drawFunc is not None:
            pl.figure(1)
            drawFunc()
            pl.draw()
    return f

badPacketWaveforms = []

class ContinuousReceiver(audioLoopback.AudioBuffer):
    def init(self):
        packetLength = wifi.encode(np.zeros(length, int), rate).size
        self.kwargs['maximum'] = int(packetLength*4)
        self.kwargs['trigger'] = int(packetLength*2)
        self.dtype = np.complex64
        super(ContinuousReceiver, self).init()
        self.inputProcessor = audioLoopback.InputProcessor(Fs, Fc, upsample_factor)
    def trigger_received(self):
        input = self.peek(self.maximum)
        #print '%.0f%%' % ((float(self.length)/self.maximum) * 100)
        #print '\r\x1b[K' + ('.' * int(30 + 10*np.log10(np.var(input)))),
        endIndex = 0
        try:
            results, drawFunc = wifi.decode(input, visualize, visualize)
            for result in results:
                _, startIndex, endIndex, _ = result
            if not len(results):
                badPacketWaveforms.append(input)
                del badPacketWaveforms[:-10]
            self.onMainThread(presentResults(results, drawFunc))
        except Exception, e:
            badPacketWaveforms.append(input)
            del badPacketWaveforms[:-10]
            print repr(e)
        if endIndex:
            return endIndex
        else:
            return self.trigger/2

class ContinuousTransmitter(audio.stream.ThreadedStream):
    def init(self):
        self.channels = 2
        super(ContinuousTransmitter, self).init()
        self.i = 0
        cutoff = Fc - Fs/upsample_factor
        self.hp = [iir.highpass(cutoff/Fs, continuous=True, dtype=np.float64) for i in xrange(2)]
    def thread_produce(self):
        input_octets = ord('A') + np.random.random_integers(0,25,length)
        input_octets[:6] = map(ord, '%06d' % self.i)
        self.i += 1
        output = wifi.encode(input_octets, rate)
        output = audioLoopback.processOutput(output, Fs, Fc, upsample_factor, None)
        return np.hstack((self.hp[0](output[:,0])[:,np.newaxis],
                          self.hp[1](output[:,1])[:,np.newaxis]))

def startListening():
    audio.record(ContinuousReceiver(), Fs)

def startTransmitting():
    audio.play(ContinuousTransmitter(), Fs)

def decoderDiagnostics(waveform=None):
    if waveform is None:
        waveform = badPacketWaveforms[-1]
    Fs_eff = Fs/upsample_factor
    ac = wifi.autocorrelate(waveform)
    ac_t = np.arange(ac.size)*16/Fs_eff
    synch = wifi.synchronize(waveform)/float(Fs_eff)
    pl.figure(2)
    pl.clf()
    pl.subplot(211)
    pl.specgram(waveform, NFFT=64, noverlap=64-1, Fc=Fc, Fs=Fs_eff, interpolation='nearest', window=lambda x:x)
    pl.xlim(0, waveform.size/Fs_eff)
    yl = pl.ylim(); pl.vlines(synch, *yl); pl.ylim(*yl)
    pl.subplot(212)
    pl.plot(ac_t, ac)
    yl = pl.ylim(); pl.vlines(synch, *yl); pl.ylim(*yl)
    pl.xlim(0, waveform.size/Fs_eff)

visualize = False

if __name__ == '__main__':
    if len(sys.argv) > 1:
        args = sys.argv[1:]
        while True:
            if args[0] == '--Fc':
                Fc = float(args[1])
                args = args[2:]
            elif args[0] == '--visualize':
                visualize = True
                args = args[1:]
            elif args[0] == '--length':
                length = int(args[1])
                args = args[2:]
            else:
                break
        if args[0] == '--rx':
            try:
                startListening()
            except KeyboardInterrupt:
                pass
            decoderDiagnostics()
        elif args[0] == '--tx':
            startTransmitting()
        elif args[0] == '--wav-in' and len(args) > 1:
            input, Fs = util.readwave(args[1])
            input = audioLoopback.processInput(input, Fs, Fc, upsample_factor)
            for payload,_,_,lsnr_estimate in wifi.decode(input)[0]:
                print repr(''.join(map(chr, payload))) + (' @ %.1f dB' % lsnr_estimate)
        elif args[0] == '--wav-out' and len(args) > 1:
            fn = args[1]
            args = args[2:]
            packets = 1
            while len(args):
                if args[0] == '--packets':
                    packets = int(args[1])
                    args = args[2:]
            outputChunks = []
            for i in xrange(packets):
                input_octets = ord('A') + np.random.random_integers(0,25,length)
                input_octets[:6] = map(ord, '%06d' % i)
                output = audioLoopback.processOutput(wifi.encode(input_octets, rate), Fs, Fc, upsample_factor, mask_noise)
                output *= 1 / np.abs(output).max()
                outputChunks.append(output)
                outputChunks.append(np.zeros((Fs/10, outputChunks[0].shape[1])))
            outputChunks.append(np.zeros((Fs/2, outputChunks[0].shape[1])))
            output = np.vstack(outputChunks)
            bytesPerSample = 3
            util.writewave(fn, output, Fs, bytesPerSample)
<commit_msg>Added switch for high-pass output filtering<commit_after>#!/usr/bin/env python
import os
try:
    os.chdir(os.path.dirname(__file__))
except OSError:
    pass
except:
    exit()
import numpy as np
import sys
import audioLoopback, channelModel, maskNoise, wifi80211
import audio, util
import pylab as pl
import iir

wifi = wifi80211.WiFi_802_11()

fn = '35631__reinsamba__crystal-glass.wav'
Fs = 48000.
Fc = 19000. #Fs/4
upsample_factor = 16
mask_noise = maskNoise.prepareMaskNoise(fn, Fs, Fc, upsample_factor)
mask_noise = mask_noise[:int(Fs)]
mask_noise[int(Fs*.5):] *= 1-np.arange(int(Fs*.5))/float(Fs*.5)
lsnr = None
mask_noise = None

rate = 0
length = 16

def test(visualize=False):
    input_octets = np.random.random_integers(0,255,length)
    output = wifi.encode(input_octets, rate)
    if lsnr is not None:
        input = channelModel.channelModel(output, lsnr)
        bitrate = None
    elif Fs is not None:
        input = audioLoopback.audioLoopback(output, Fs, Fc, upsample_factor, mask_noise)
        bitrate = input_octets.size*8 * Fs / float(output.size) / upsample_factor
    try:
        results, _ = wifi.decode(input, visualize)
        return len(results), bitrate
    except Exception, e:
        print e
        return False, bitrate

def transmit(message):
    length = len(message)
    input_octets = np.array(map(ord, message))
    output = wifi.encode(input_octets, rate)
    audioLoopback.audioOut(output, Fs, Fc, upsample_factor, mask_noise)

def presentResults(results, drawFunc):
    _results = results
    _drawFunc = drawFunc
    def f():
        if len(_results):
            for result in _results:
                payload, _, _, lsnr_estimate = result
                print repr(''.join(map(chr, payload))) + (' @ %.3f dB' % lsnr_estimate)
        #else:
        #    decoderDiagnostics()
        #    pl.draw()
        if drawFunc is not None:
            pl.figure(1)
            drawFunc()
            pl.draw()
    return f

badPacketWaveforms = []

class ContinuousReceiver(audioLoopback.AudioBuffer):
    def init(self):
        packetLength = wifi.encode(np.zeros(length, int), rate).size
        self.kwargs['maximum'] = int(packetLength*4)
        self.kwargs['trigger'] = int(packetLength*2)
        self.dtype = np.complex64
        super(ContinuousReceiver, self).init()
        self.inputProcessor = audioLoopback.InputProcessor(Fs, Fc, upsample_factor)
    def trigger_received(self):
        input = self.peek(self.maximum)
        #print '%.0f%%' % ((float(self.length)/self.maximum) * 100)
        #print '\r\x1b[K' + ('.' * int(30 + 10*np.log10(np.var(input)))),
        endIndex = 0
        try:
            results, drawFunc = wifi.decode(input, visualize, visualize)
            for result in results:
                _, startIndex, endIndex, _ = result
            if not len(results):
                badPacketWaveforms.append(input)
                del badPacketWaveforms[:-10]
            self.onMainThread(presentResults(results, drawFunc))
        except Exception, e:
            badPacketWaveforms.append(input)
            del badPacketWaveforms[:-10]
            print repr(e)
        if endIndex:
            return endIndex
        else:
            return self.trigger/2

class ContinuousTransmitter(audio.stream.ThreadedStream):
    def init(self):
        self.channels = 2
        super(ContinuousTransmitter, self).init()
        self.i = 0
        cutoff = Fc - Fs/upsample_factor
        self.hp = [iir.highpass(cutoff/Fs, continuous=True, dtype=np.float64) for i in xrange(2)]
    def thread_produce(self):
        input_octets = ord('A') + np.random.random_integers(0,25,length)
        input_octets[:6] = map(ord, '%06d' % self.i)
        self.i += 1
        output = wifi.encode(input_octets, rate)
        output = audioLoopback.processOutput(output, Fs, Fc, upsample_factor, None)
        if False:
            return output
        else:
            return np.hstack((self.hp[0](output[:,0])[:,np.newaxis],
                              self.hp[1](output[:,1])[:,np.newaxis]))

def startListening():
    audio.record(ContinuousReceiver(), Fs)

def startTransmitting():
    audio.play(ContinuousTransmitter(), Fs)

def decoderDiagnostics(waveform=None):
    if waveform is None:
        waveform = badPacketWaveforms[-1]
    Fs_eff = Fs/upsample_factor
    ac = wifi.autocorrelate(waveform)
    ac_t = np.arange(ac.size)*16/Fs_eff
    synch = wifi.synchronize(waveform)/float(Fs_eff)
    pl.figure(2)
    pl.clf()
    pl.subplot(211)
    pl.specgram(waveform, NFFT=64, noverlap=64-1, Fc=Fc, Fs=Fs_eff, interpolation='nearest', window=lambda x:x)
    pl.xlim(0, waveform.size/Fs_eff)
    yl = pl.ylim(); pl.vlines(synch, *yl); pl.ylim(*yl)
    pl.subplot(212)
    pl.plot(ac_t, ac)
    yl = pl.ylim(); pl.vlines(synch, *yl); pl.ylim(*yl)
    pl.xlim(0, waveform.size/Fs_eff)

visualize = False

if __name__ == '__main__':
    if len(sys.argv) > 1:
        args = sys.argv[1:]
        while True:
            if args[0] == '--Fc':
                Fc = float(args[1])
                args = args[2:]
            elif args[0] == '--visualize':
                visualize = True
                args = args[1:]
            elif args[0] == '--length':
                length = int(args[1])
                args = args[2:]
            else:
                break
        if args[0] == '--rx':
            try:
                startListening()
            except KeyboardInterrupt:
                pass
            decoderDiagnostics()
        elif args[0] == '--tx':
            startTransmitting()
        elif args[0] == '--wav-in' and len(args) > 1:
            input, Fs = util.readwave(args[1])
            input = audioLoopback.processInput(input, Fs, Fc, upsample_factor)
            for payload,_,_,lsnr_estimate in wifi.decode(input)[0]:
                print repr(''.join(map(chr, payload))) + (' @ %.1f dB' % lsnr_estimate)
        elif args[0] == '--wav-out' and len(args) > 1:
            fn = args[1]
            args = args[2:]
            packets = 1
            while len(args):
                if args[0] == '--packets':
                    packets = int(args[1])
                    args = args[2:]
            outputChunks = []
            for i in xrange(packets):
                input_octets = ord('A') + np.random.random_integers(0,25,length)
                input_octets[:6] = map(ord, '%06d' % i)
                output = audioLoopback.processOutput(wifi.encode(input_octets, rate), Fs, Fc, upsample_factor, mask_noise)
                output *= 1 / np.abs(output).max()
                outputChunks.append(output)
                outputChunks.append(np.zeros((Fs/10, outputChunks[0].shape[1])))
            outputChunks.append(np.zeros((Fs/2, outputChunks[0].shape[1])))
            output = np.vstack(outputChunks)
            bytesPerSample = 3
            util.writewave(fn, output, Fs, bytesPerSample)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os,types
from xml.dom import minidom

import netsvc
import tools
import print_fnc

from osv.orm import browse_null, browse_record
import pooler

class InheritDict(dict):
    # Might be usefull when we're doing name lookup for call or eval.

    def __init__(self, parent=None):
        self.parent = parent

    def __getitem__(self, name):
        if name in self:
            return super(InheritDict, self).__getitem__(name)
        else:
            if not self.parent:
                raise KeyError
            else:
                return self.parent[name]

def tounicode(val):
    if isinstance(val, str):
        unicode_val = unicode(val, 'utf-8')
    elif isinstance(val, unicode):
        unicode_val = val
    else:
        unicode_val = unicode(val)
    return unicode_val

class document(object):
    def __init__(self, cr, uid, datas, func=False):
        # create a new document
        self.cr = cr
        self.pool = pooler.get_pool(cr.dbname)
        self.doc = minidom.Document()
        self.func = func or {}
        self.datas = datas
        self.uid = uid
        self.bin_datas = {}

    def node_attrs_get(self, node):
        attrs = {}
        nattr = node.attributes
        for i in range(nattr.length):
            attr = nattr.item(i)
            attrs[attr.localName] = attr.nodeValue
#           attrs[attr.name] = attr.nodeValue
        return attrs

    def get_value(self, browser, field_path):
        fields = field_path.split('.')

        if not len(fields):
            return ''

        value = browser
        for f in fields:
            if isinstance(value, list):
                if len(value)==0:
                    return ''
                value = value[0]
            if isinstance(value, browse_null):
                return ''
            else:
                value = value[f]

        if isinstance(value, browse_null) or (type(value)==bool and not value):
            return ''
        else:
            return value

    def get_value2(self, browser, field_path):
        value = self.get_value(browser, field_path)
        if isinstance(value, browse_record):
            return value.id
        elif isinstance(value, browse_null):
            return False
        else:
            return value

    def eval(self, record, expr):
#TODO: support remote variables (eg address.title) in expr
# how to do that: parse the string, find dots, replace those dotted variables by temporary
# "simple ones", fetch the value of those variables and add them (temporarily) to the _data
# dictionary passed to eval

#FIXME: it wont work if the data hasn't been fetched yet... this could
# happen if the eval node is the first one using this browse_record
# the next line is a workaround for the problem: it causes the resource to be loaded
#Pinky: Why not this ? eval(expr, browser) ?
#       name = browser.name
#       data_dict = browser._data[self.get_value(browser, 'id')]
        return eval(expr, {}, {'obj': record})

    def parse_node(self, node, parent, browser, datas=None):
        # node is the node of the xml template to be parsed
        # parent = the parent node in the xml data tree we are creating

        if node.nodeType == node.ELEMENT_NODE:

            # convert the attributes of the node to a dictionary

            attrs = self.node_attrs_get(node)
            if 'type' in attrs:
                if attrs['type']=='field':
                    value = self.get_value(browser, attrs['name'])
#TODO: test this
                    if value == '' and 'default' in attrs:
                        value = attrs['default']
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    el_txt = self.doc.createTextNode(tounicode(value))
                    el.appendChild(el_txt)

#TODO: test this
                    for key, value in attrs.iteritems():
                        if key not in ('type', 'name', 'default'):
                            el.setAttribute(key, value)

                elif attrs['type']=='attachment':
                    if isinstance(browser, list):
                        model = browser[0]._table_name
                    else:
                        model = browser._table_name

                    value = self.get_value(browser, attrs['name'])

                    service = netsvc.LocalService("object_proxy")
                    ids = service.execute(self.cr.dbname, self.uid, 'ir.attachment', 'search', [('res_model','=',model),('res_id','=',int(value))])
                    datas = service.execute(self.cr.dbname, self.uid, 'ir.attachment', 'read', ids)

                    if len(datas):
                        # if there are several, pick first
                        datas = datas[0]
                        fname = str(datas['datas_fname'])
                        ext = fname.split('.')[-1].lower()
                        if ext in ('jpg','jpeg', 'png'):
                            import base64
                            from StringIO import StringIO
                            dt = base64.decodestring(datas['datas'])
                            fp = StringIO()
                            fp.write(dt)
                            i = str(len(self.bin_datas))
                            self.bin_datas[i] = fp

                            el = self.doc.createElement(node.localName)
                            parent.appendChild(el)
                            # node content is the length of the image
                            el_txt = self.doc.createTextNode(i)
                            el.appendChild(el_txt)

                elif attrs['type']=='data':
#TODO: test this
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    txt = self.datas.get('form', {}).get(attrs['name'], '')
                    el_txt = self.doc.createTextNode(tounicode(txt))
                    el.appendChild(el_txt)

                elif attrs['type']=='function':
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    if attrs['name'] in self.func:
                        txt = self.func[attrs['name']](node)
                    else:
                        txt = print_fnc.print_fnc(attrs['name'], node)
                    el_txt = self.doc.createTextNode(txt)
                    el.appendChild(el_txt)

                elif attrs['type']=='eval':
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    value = self.eval(browser, attrs['expr'])
                    el_txt = self.doc.createTextNode(str(value))
                    el.appendChild(el_txt)

                elif attrs['type']=='fields':
                    fields = attrs['name'].split(',')
                    vals = {}
                    for b in browser:
                        value = tuple([self.get_value2(b, f) for f in fields])
                        if not value in vals:
                            vals[value]=[]
                        vals[value].append(b)
                    keys = vals.keys()
                    keys.sort()

                    if 'order' in attrs and attrs['order']=='desc':
                        keys.reverse()

                    v_list = [vals[k] for k in keys]
                    for v in v_list:
                        el = self.doc.createElement(node.localName)
                        parent.appendChild(el)
                        el_cld = node.firstChild
                        while el_cld:
                            self.parse_node(el_cld, el, v)
                            el_cld = el_cld.nextSibling

                elif attrs['type']=='call':
                    if len(attrs['args']):
#TODO: test this
                        # fetches the values of the variables which names where passed in the args attribute
                        args = [self.eval(browser, arg) for arg in attrs['args'].split(',')]
                    else:
                        args = []

                    # get the object
                    if attrs.has_key('model'):
                        obj = self.pool.get(attrs['model'])
                    else:
                        if isinstance(browser, list):
                            obj = browser[0]._table
                        else:
                            obj = browser._table

                    # get the ids
                    if attrs.has_key('ids'):
                        ids = self.eval(browser, attrs['ids'])
                    else:
                        if isinstance(browser, list):
                            ids = [b.id for b in browser]
                        else:
                            ids = [browser.id]

                    # call the method itself
                    newdatas = getattr(obj, attrs['name'])(self.cr, self.uid, ids, *args)

                    def parse_result_tree(node, parent, datas):
                        if node.nodeType == node.ELEMENT_NODE:
                            el = self.doc.createElement(node.localName)
                            parent.appendChild(el)
                            atr = self.node_attrs_get(node)
                            if 'value' in atr:
                                if not isinstance(datas[atr['value']], (str, unicode)):
                                    txt = self.doc.createTextNode(str(datas[atr['value']]))
                                else:
                                    txt = self.doc.createTextNode(datas[atr['value']].decode('utf-8'))
                                el.appendChild(txt)
                            else:
                                el_cld = node.firstChild
                                while el_cld:
                                    parse_result_tree(el_cld, el, datas)
                                    el_cld = el_cld.nextSibling
                        elif node.nodeType==node.TEXT_NODE:
                            el = self.doc.createTextNode(node.nodeValue)
                            parent.appendChild(el)
                        else:
                            pass

                    if not isinstance(newdatas, list):
                        newdatas = [newdatas]
                    for newdata in newdatas:
                        parse_result_tree(node, parent, newdata)

                elif attrs['type']=='zoom':
                    value = self.get_value(browser, attrs['name'])

                    if value:
                        if not isinstance(value, list):
                            v_list = [value]
                        else:
                            v_list = value
                        for v in v_list:
                            el = self.doc.createElement(node.localName)
                            parent.appendChild(el)
                            el_cld = node.firstChild
                            while el_cld:
                                self.parse_node(el_cld, el, v)
                                el_cld = el_cld.nextSibling
            else:
                # if there is no "type" attribute in the node, copy it to the xml data and parse its childs
                el = self.doc.createElement(node.localName)
                parent.appendChild(el)
                el_cld = node.firstChild
                while el_cld:
                    self.parse_node(el_cld, el, browser)
                    el_cld = el_cld.nextSibling

        elif node.nodeType==node.TEXT_NODE:
            # if it's a text node, copy it to the xml data
            el = self.doc.createTextNode(node.nodeValue)
            parent.appendChild(el)
        else:
            pass

    def xml_get(self):
        return self.doc.toxml('utf-8')

    def parse_tree(self, ids, model, context=None):
        if not context:
            context={}
        browser = self.pool.get(model).browse(self.cr, self.uid, ids, context)
        self.parse_node(self.dom.documentElement, self.doc, browser)

    def parse_string(self, xml, ids, model, context=None):
        if not context:
            context={}
        # parses the xml template to memory
        self.dom = minidom.parseString(xml)

        # create the xml data from the xml template
        self.parse_tree(ids, model, context)

    def parse(self, filename, ids, model, context=None):
        if not context:
            context={}
        # parses the xml template to memory
        self.dom = minidom.parseString(tools.file_open(filename).read())

        # create the xml data from the xml template
        self.parse_tree(ids, model, context)

    def close(self):
        self.doc = None
        self.dom = None

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[FIX] Bug 364835 : corrected non-enlgish character problem for parsing report(ref:Ferdinand)<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os,types
from xml.dom import minidom

import netsvc
import tools
import print_fnc

from osv.orm import browse_null, browse_record
import pooler

class InheritDict(dict):
    # Might be usefull when we're doing name lookup for call or eval.

    def __init__(self, parent=None):
        self.parent = parent

    def __getitem__(self, name):
        if name in self:
            return super(InheritDict, self).__getitem__(name)
        else:
            if not self.parent:
                raise KeyError
            else:
                return self.parent[name]

def tounicode(val):
    if isinstance(val, str):
        unicode_val = unicode(val, 'utf-8')
    elif isinstance(val, unicode):
        unicode_val = val
    else:
        unicode_val = unicode(val)
    return unicode_val

class document(object):
    def __init__(self, cr, uid, datas, func=False):
        # create a new document
        self.cr = cr
        self.pool = pooler.get_pool(cr.dbname)
        self.doc = minidom.Document()
        self.func = func or {}
        self.datas = datas
        self.uid = uid
        self.bin_datas = {}

    def node_attrs_get(self, node):
        attrs = {}
        nattr = node.attributes
        for i in range(nattr.length):
            attr = nattr.item(i)
            attrs[attr.localName] = attr.nodeValue
#           attrs[attr.name] = attr.nodeValue
        return attrs

    def get_value(self, browser, field_path):
        fields = field_path.split('.')

        if not len(fields):
            return ''

        value = browser
        for f in fields:
            if isinstance(value, list):
                if len(value)==0:
                    return ''
                value = value[0]
            if isinstance(value, browse_null):
                return ''
            else:
                value = value[f]

        if isinstance(value, browse_null) or (type(value)==bool and not value):
            return ''
        else:
            return value

    def get_value2(self, browser, field_path):
        value = self.get_value(browser, field_path)
        if isinstance(value, browse_record):
            return value.id
        elif isinstance(value, browse_null):
            return False
        else:
            return value

    def eval(self, record, expr):
#TODO: support remote variables (eg address.title) in expr
# how to do that: parse the string, find dots, replace those dotted variables by temporary
# "simple ones", fetch the value of those variables and add them (temporarily) to the _data
# dictionary passed to eval

#FIXME: it wont work if the data hasn't been fetched yet... this could
# happen if the eval node is the first one using this browse_record
# the next line is a workaround for the problem: it causes the resource to be loaded
#Pinky: Why not this ? eval(expr, browser) ?
#       name = browser.name
#       data_dict = browser._data[self.get_value(browser, 'id')]
        return eval(expr, {}, {'obj': record})

    def parse_node(self, node, parent, browser, datas=None):
        # node is the node of the xml template to be parsed
        # parent = the parent node in the xml data tree we are creating

        if node.nodeType == node.ELEMENT_NODE:

            # convert the attributes of the node to a dictionary

            attrs = self.node_attrs_get(node)
            if 'type' in attrs:
                if attrs['type']=='field':
                    value = self.get_value(browser, attrs['name'])
#TODO: test this
                    if value == '' and 'default' in attrs:
                        value = attrs['default']
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    el_txt = self.doc.createTextNode(tounicode(value))
                    el.appendChild(el_txt)

#TODO: test this
                    for key, value in attrs.iteritems():
                        if key not in ('type', 'name', 'default'):
                            el.setAttribute(key, value)

                elif attrs['type']=='attachment':
                    if isinstance(browser, list):
                        model = browser[0]._table_name
                    else:
                        model = browser._table_name

                    value = self.get_value(browser, attrs['name'])

                    service = netsvc.LocalService("object_proxy")
                    ids = service.execute(self.cr.dbname, self.uid, 'ir.attachment', 'search', [('res_model','=',model),('res_id','=',int(value))])
                    datas = service.execute(self.cr.dbname, self.uid, 'ir.attachment', 'read', ids)

                    if len(datas):
                        # if there are several, pick first
                        datas = datas[0]
                        fname = str(datas['datas_fname'])
                        ext = fname.split('.')[-1].lower()
                        if ext in ('jpg','jpeg', 'png'):
                            import base64
                            from StringIO import StringIO
                            dt = base64.decodestring(datas['datas'])
                            fp = StringIO()
                            fp.write(dt)
                            i = str(len(self.bin_datas))
                            self.bin_datas[i] = fp

                            el = self.doc.createElement(node.localName)
                            parent.appendChild(el)
                            # node content is the length of the image
                            el_txt = self.doc.createTextNode(i)
                            el.appendChild(el_txt)

                elif attrs['type']=='data':
#TODO: test this
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    txt = self.datas.get('form', {}).get(attrs['name'], '')
                    el_txt = self.doc.createTextNode(tounicode(txt))
                    el.appendChild(el_txt)

                elif attrs['type']=='function':
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    if attrs['name'] in self.func:
                        txt = self.func[attrs['name']](node)
                    else:
                        txt = print_fnc.print_fnc(attrs['name'], node)
                    el_txt = self.doc.createTextNode(txt)
                    el.appendChild(el_txt)

                elif attrs['type']=='eval':
                    el = self.doc.createElement(node.localName)
                    parent.appendChild(el)
                    value = self.eval(browser, attrs['expr'])
                    el_txt = self.doc.createTextNode(str(value))
                    el.appendChild(el_txt)

                elif attrs['type']=='fields':
                    fields = attrs['name'].split(',')
                    vals = {}
                    for b in browser:
                        value = tuple([self.get_value2(b, f) for f in fields])
                        if not value in vals:
                            vals[value]=[]
                        vals[value].append(b)
                    keys = vals.keys()
                    keys.sort()

                    if 'order' in attrs and attrs['order']=='desc':
                        keys.reverse()

                    v_list = [vals[k] for k in keys]
                    for v in v_list:
                        el = self.doc.createElement(node.localName)
                        parent.appendChild(el)
                        el_cld = node.firstChild
                        while el_cld:
                            self.parse_node(el_cld, el, v)
                            el_cld = el_cld.nextSibling

                elif attrs['type']=='call':
                    if len(attrs['args']):
#TODO: test this
                        # fetches the values of the variables which names where passed in the args attribute
                        args = [self.eval(browser, arg) for arg in attrs['args'].split(',')]
                    else:
                        args = []

                    # get the object
                    if attrs.has_key('model'):
                        obj = self.pool.get(attrs['model'])
                    else:
                        if isinstance(browser, list):
                            obj = browser[0]._table
                        else:
                            obj = browser._table

                    # get the ids
                    if attrs.has_key('ids'):
                        ids = self.eval(browser, attrs['ids'])
                    else:
                        if isinstance(browser, list):
                            ids = [b.id for b in browser]
                        else:
                            ids = [browser.id]

                    # call the method itself
                    newdatas = getattr(obj, attrs['name'])(self.cr, self.uid, ids, *args)

                    def parse_result_tree(node, parent, datas):
                        if node.nodeType == node.ELEMENT_NODE:
                            el = self.doc.createElement(node.localName)
                            parent.appendChild(el)
                            atr = self.node_attrs_get(node)
                            if 'value' in atr:
                                if not isinstance(datas[atr['value']], (str, unicode)):
                                    txt = self.doc.createTextNode(str(datas[atr['value']]))
                                else:
#                                    txt = self.doc.createTextNode(datas[atr['value']].decode('utf-8'))
                                    txt = self.doc.createTextNode(datas[atr['value']])
                                el.appendChild(txt)
                            else:
                                el_cld = node.firstChild
                                while el_cld:
                                    parse_result_tree(el_cld, el, datas)
                                    el_cld = el_cld.nextSibling
                        elif node.nodeType==node.TEXT_NODE:
                            el = self.doc.createTextNode(node.nodeValue)
                            parent.appendChild(el)
                        else:
                            pass

                    if not isinstance(newdatas, list):
                        newdatas = [newdatas]
                    for newdata in newdatas:
                        parse_result_tree(node, parent, newdata)

                elif attrs['type']=='zoom':
                    value = self.get_value(browser, attrs['name'])

                    if value:
                        if not isinstance(value, list):
                            v_list = [value]
                        else:
                            v_list = value
                        for v in v_list:
                            el = self.doc.createElement(node.localName)
                            parent.appendChild(el)
                            el_cld = node.firstChild
                            while el_cld:
                                self.parse_node(el_cld, el, v)
                                el_cld = el_cld.nextSibling
            else:
                # if there is no "type" attribute in the node, copy it to the xml data and parse its childs
                el = self.doc.createElement(node.localName)
                parent.appendChild(el)
                el_cld = node.firstChild
                while el_cld:
                    self.parse_node(el_cld, el, browser)
                    el_cld = el_cld.nextSibling

        elif node.nodeType==node.TEXT_NODE:
            # if it's a text node, copy it to the xml data
            el = self.doc.createTextNode(node.nodeValue)
            parent.appendChild(el)
        else:
            pass

    def xml_get(self):
        return self.doc.toxml('utf-8')

    def parse_tree(self, ids, model, context=None):
        if not context:
            context={}
        browser = self.pool.get(model).browse(self.cr, self.uid, ids, context)
        self.parse_node(self.dom.documentElement, self.doc, browser)

    def parse_string(self, xml, ids, model, context=None):
        if not context:
            context={}
        # parses the xml template to memory
        self.dom = minidom.parseString(xml)

        # create the xml data from the xml template
        self.parse_tree(ids, model, context)

    def parse(self, filename, ids, model, context=None):
        if not context:
            context={}
        # parses the xml template to memory
        self.dom = minidom.parseString(tools.file_open(filename).read())

        # create the xml data from the xml template
        self.parse_tree(ids, model, context)

    def close(self):
        self.doc = None
        self.dom = None

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>"""Builds the convolutional neural network model.

Matthew Alger
The Australian National University
2016
"""

import argparse

def main(n_filters, conv_size, pool_size, dropout,
         patch_size, out_path=None):
    # Imports must be in the function, or whenever we import this module, Keras
    # will dump to stdout.
    import keras.layers.core as core
    from keras.layers import Input, Dense
    import keras.layers.convolutional as conv
    from keras.models import Model

    im_in = Input(shape=(1, patch_size, patch_size))
    # 1 x 32 x 32
    conv1 = conv.Convolution2D(filters=n_filters,
                               kernel_size=(conv_size, conv_size),
                               border_mode='valid',
                               activation='relu')(im_in)
    # 32 x 28 x 28
    pool1 = conv.MaxPooling2D(pool_size=(pool_size, pool_size))(conv1)
    # 32 x 14 x 14
    conv2 = conv.Convolution2D(filters=n_filters,
                               kernel_size=(conv_size, conv_size),
                               border_mode='valid',
                               activation='relu')(pool1)
    # 32 x 10 x 10
    pool2 = conv.MaxPooling2D(pool_size=(pool_size, pool_size))(conv2)
    # 32 x 5 x 5
    conv3 = conv.Convolution2D(filters=n_filters,
                               kernel_size=(conv_size, conv_size),
                               border_mode='valid', activation='relu')(pool2)
    # 32 x 1 x 1
    dropout = core.Dropout(dropout)(conv3)
    flatten = core.Flatten()(dropout)
    lr = Dense(1, activation='sigmoid')(flatten)

    model = Model(inputs=[im_in], outputs=[lr])
    model.compile(loss='binary_crossentropy', optimizer='adadelta')

    model_json = model.to_json()

    if out_path is not None:
        with open(out_path, 'w') as f:
            f.write(model_json)

    return model_json

def _populate_parser(parser):
    parser.description = 'Compiles a convolutional neural network to JSON.'
    parser.add_argument('--out_path', help='path to output model JSON',
                        default='data/model.json')
    parser.add_argument('--n_filters', help='number of convolutional filters',
                        default=32, type=int)
    parser.add_argument('--conv_size', help='size of convolutional filters',
                        default=4, type=int)
    parser.add_argument('--pool_size', help='size of max pool',
                        default=2, type=int)
    parser.add_argument('--dropout', help='dropout percentage',
                        default=0.25, type=float)
    parser.add_argument('--patch_size', help='size of image patches',
                        default=32, type=int)

def _main(args):
    main(args.n_filters, args.conv_size, args.pool_size, args.dropout,
         args.patch_size, out_path=args.out_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    _populate_parser(parser)
    args = parser.parse_args()
    _main(args)
<commit_msg>Added explicit channel order<commit_after>"""Builds the convolutional neural network model.

Matthew Alger
The Australian National University
2016
"""

import argparse

def main(n_filters, conv_size, pool_size, dropout,
         patch_size, out_path=None):
    # Imports must be in the function, or whenever we import this module, Keras
    # will dump to stdout.
    import keras.layers.core as core
    from keras.layers import Input, Dense
    import keras.layers.convolutional as conv
    from keras.models import Model

    im_in = Input(shape=(1, patch_size, patch_size))
    # 1 x 32 x 32
    conv1 = conv.Convolution2D(filters=n_filters,
                               kernel_size=(conv_size, conv_size),
                               border_mode='valid',
                               activation='relu',
                               data_format='channels_first')(im_in)
    # 32 x 28 x 28
    pool1 = conv.MaxPooling2D(pool_size=(pool_size, pool_size),
                              data_format='channels_first')(conv1)
    # 32 x 14 x 14
    conv2 = conv.Convolution2D(filters=n_filters,
                               kernel_size=(conv_size, conv_size),
                               border_mode='valid',
                               activation='relu',
                               data_format='channels_first')(pool1)
    # 32 x 10 x 10
    pool2 = conv.MaxPooling2D(pool_size=(pool_size, pool_size),
                              data_format='channels_first')(conv2)
    # 32 x 5 x 5
    conv3 = conv.Convolution2D(filters=n_filters,
                               kernel_size=(conv_size, conv_size),
                               border_mode='valid', activation='relu',
                               data_format='channels_first')(pool2)
    # 32 x 1 x 1
    dropout = core.Dropout(dropout)(conv3)
    flatten = core.Flatten()(dropout)
    lr = Dense(1, activation='sigmoid')(flatten)

    model = Model(inputs=[im_in], outputs=[lr])
    model.compile(loss='binary_crossentropy', optimizer='adadelta')

    model_json = model.to_json()

    if out_path is not None:
        with open(out_path, 'w') as f:
            f.write(model_json)

    return model_json

def _populate_parser(parser):
    parser.description = 'Compiles a convolutional neural network to JSON.'
    parser.add_argument('--out_path', help='path to output model JSON',
                        default='data/model.json')
    parser.add_argument('--n_filters', help='number of convolutional filters',
                        default=32, type=int)
    parser.add_argument('--conv_size', help='size of convolutional filters',
                        default=4, type=int)
    parser.add_argument('--pool_size', help='size of max pool',
                        default=2, type=int)
    parser.add_argument('--dropout', help='dropout percentage',
                        default=0.25, type=float)
    parser.add_argument('--patch_size', help='size of image patches',
                        default=32, type=int)

def _main(args):
    main(args.n_filters, args.conv_size, args.pool_size, args.dropout,
         args.patch_size, out_path=args.out_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    _populate_parser(parser)
    args = parser.parse_args()
    _main(args)
<|endoftext|>