<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import math

class OGRError(Exception):
    pass

def bbox_to_pixel_offsets(gt, bbox, rshape):
    originX = gt[0]
    originY = gt[3]
    pixel_width = gt[1]
    pixel_height = gt[5]

    col1 = int(math.floor((bbox[0] - originX) / pixel_width))
    col2 = int(math.ceil((bbox[2] - originX) / pixel_width))

    row1 = int(math.floor((bbox[3] - originY) / pixel_height))
    row2 = int(math.ceil((bbox[1] - originY) / pixel_height))

    # "Clip" the geometry bounds to the overall raster bounding box
    # This should avoid any rasterIO errors for partially overlapping polys
    if col1 < 0:
        col1 = 0
    if col2 > rshape[0]:
        col2 = rshape[0]
    if row1 < 0:
        row1 = 0
    if row2 > rshape[1]:
        row2 = rshape[1]

    cols = col2 - col1
    rows = row2 - row1

    return (col1, row1, cols, rows)

def pixel_offsets_to_window(offsets):
    """
    Convert (col1, row1, cols, rows)
    to a rasterio-compatible window
    https://github.com/mapbox/rasterio/blob/master/docs/windowed-rw.rst#windows
    """
    if len(offsets) != 4:
        raise ValueError("offset should be a 4-element tuple")
    x1, y1, xsize, ysize = offsets
    col1, row1, cols, rows = offsets
    return ((row1, row1 + rows), (col1, col1 + cols))

def raster_extent_as_bounds(gt, shape):
    x1 = gt[0]
    x2 = gt[0] + (gt[1] * shape[0])
    y1 = gt[3] + (gt[5] * shape[1])
    y2 = gt[3]
    return (x1, y1, x2, y2)

def get_percentile(stat):
    if not stat.startswith('percentile_'):
        raise ValueError("must start with 'percentile_'")
    qstr = stat.replace("percentile_", '')
    q = float(qstr)
    if q > 100.0:
        raise ValueError('percentiles must be <= 100')
    if q < 0.0:
        raise ValueError('percentiles must be >= 0')
    return q

def feature_to_geojson(feature):
    """ This duplicates the feature.ExportToJson ogr method
    but is safe across gdal versions since it was fixed only in 1.8+
    see http://trac.osgeo.org/gdal/ticket/3870"""

    geom = feature.GetGeometryRef()
    if geom is not None:
        geom_json_string = geom.ExportToJson()
        geom_json_object = json.loads(geom_json_string)
    else:
        geom_json_object = None

    output = {
        'type': 'Feature',
        'geometry': geom_json_object,
        'properties': {}
    }

    fid = feature.GetFID()
    if fid:
        output['fid'] = fid

    for key in list(feature.keys()):
        output['properties'][key] = feature.GetField(key)

    return output

def shapely_to_ogr_type(shapely_type):
    from osgeo import ogr
    if shapely_type == "Polygon":
        return ogr.wkbPolygon
    elif shapely_type == "LineString":
        return ogr.wkbLineString
    elif shapely_type == "MultiPolygon":
        return ogr.wkbMultiPolygon
    elif shapely_type == "MultiLineString":
        return ogr.wkbLineString
    raise TypeError("shapely type %s not supported" % shapely_type)

def parse_geo(thing):
    """ Given a python object, try to get a geo-json like mapping from it
    """
    from shapely.geos import ReadingError
    from shapely import wkt, wkb

    # object implementing geo_interface
    try:
        geo = thing.__geo_interface__
        return geo
    except AttributeError:
        pass

    # wkb
    try:
        shape = wkb.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError):
        pass

    # wkt
    try:
        shape = wkt.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError, AttributeError):
        pass

    # geojson-like python mapping
    valid_types = ["Feature", "Point", "LineString", "Polygon",
                   "MultiPoint", "MultiLineString", "MultiPolygon"]
    try:
        assert thing['type'] in valid_types
        return thing
    except (AssertionError, TypeError):
        pass

    # geojson string
    try:
        maybe_geo = json.loads(thing)
        assert maybe_geo['type'] in valid_types + ["FeatureCollection"]
        return maybe_geo
    except (ValueError, AssertionError, TypeError):
        pass

    raise ValueError("Can't parse %s as a geo-like object" % thing)

def get_ogr_ds(vds):
    from osgeo import ogr
    if not isinstance(vds, str):
        raise OGRError("OGR cannot open %r: not a string" % vds)

    ds = ogr.Open(vds)
    if not ds:
        raise OGRError("OGR cannot open %r" % vds)

    return ds

def ogr_srs(vector, layer_num):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    return layer.GetSpatialRef()

def ogr_records(vector, layer_num=0):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    if layer.GetFeatureCount() == 0:
        raise OGRError("No Features")
    feature = layer.GetNextFeature()
    while feature is not None:
        yield feature_to_geojson(feature)
        feature = layer.GetNextFeature()

def geo_records(vectors):
    for vector in vectors:
        yield parse_geo(vector)

def get_features(vectors, layer_num=0):
    from osgeo import osr
    spatial_ref = osr.SpatialReference()
    if isinstance(vectors, str):
        try:
            # either an OGR layer ...
            get_ogr_ds(vectors)
            features_iter = ogr_records(vectors, layer_num)
            spatial_ref = ogr_srs(vectors, layer_num)
            strategy = "ogr"
        except (OGRError, AttributeError):
            # ... or a single string to be parsed as wkt/wkb/json
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, bytes):
        # wkb
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    elif hasattr(vectors, '__geo_interface__'):
        geotype = vectors.__geo_interface__['type']
        if geotype.lower() == 'featurecollection':
            # ... a featurecollection
            features_iter = geo_records(vectors.__geo_interface__['features'])
            strategy = "geo_featurecollection"
        else:
            # ... or an single object
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, dict):
        # ... or an python mapping
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    else:
        # ... or an iterable of objects
        features_iter = geo_records(vectors)
        strategy = "iter_geo"

    return features_iter, strategy, spatial_ref

def rasterize_geom(geom, src_offset, new_gt, all_touched):
    from rasterio import features
    geoms = [(geom, 1)]
    rv_array = features.rasterize(
        geoms,
        out_shape=(src_offset[3], src_offset[2]),
        transform=new_gt,
        fill=0,
        all_touched=all_touched)
    return rv_array
<commit_msg>use Affine for rasterization; finishes #64<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import math
from rasterio import features
from affine import Affine

class OGRError(Exception):
    pass

def bbox_to_pixel_offsets(gt, bbox, rshape):
    originX = gt[0]
    originY = gt[3]
    pixel_width = gt[1]
    pixel_height = gt[5]

    col1 = int(math.floor((bbox[0] - originX) / pixel_width))
    col2 = int(math.ceil((bbox[2] - originX) / pixel_width))

    row1 = int(math.floor((bbox[3] - originY) / pixel_height))
    row2 = int(math.ceil((bbox[1] - originY) / pixel_height))

    # "Clip" the geometry bounds to the overall raster bounding box
    # This should avoid any rasterIO errors for partially overlapping polys
    if col1 < 0:
        col1 = 0
    if col2 > rshape[0]:
        col2 = rshape[0]
    if row1 < 0:
        row1 = 0
    if row2 > rshape[1]:
        row2 = rshape[1]

    cols = col2 - col1
    rows = row2 - row1

    return (col1, row1, cols, rows)

def pixel_offsets_to_window(offsets):
    """
    Convert (col1, row1, cols, rows)
    to a rasterio-compatible window
    https://github.com/mapbox/rasterio/blob/master/docs/windowed-rw.rst#windows
    """
    if len(offsets) != 4:
        raise ValueError("offset should be a 4-element tuple")
    x1, y1, xsize, ysize = offsets
    col1, row1, cols, rows = offsets
    return ((row1, row1 + rows), (col1, col1 + cols))

def raster_extent_as_bounds(gt, shape):
    x1 = gt[0]
    x2 = gt[0] + (gt[1] * shape[0])
    y1 = gt[3] + (gt[5] * shape[1])
    y2 = gt[3]
    return (x1, y1, x2, y2)

def get_percentile(stat):
    if not stat.startswith('percentile_'):
        raise ValueError("must start with 'percentile_'")
    qstr = stat.replace("percentile_", '')
    q = float(qstr)
    if q > 100.0:
        raise ValueError('percentiles must be <= 100')
    if q < 0.0:
        raise ValueError('percentiles must be >= 0')
    return q

def feature_to_geojson(feature):
    """ This duplicates the feature.ExportToJson ogr method
    but is safe across gdal versions since it was fixed only in 1.8+
    see http://trac.osgeo.org/gdal/ticket/3870"""

    geom = feature.GetGeometryRef()
    if geom is not None:
        geom_json_string = geom.ExportToJson()
        geom_json_object = json.loads(geom_json_string)
    else:
        geom_json_object = None

    output = {
        'type': 'Feature',
        'geometry': geom_json_object,
        'properties': {}
    }

    fid = feature.GetFID()
    if fid:
        output['fid'] = fid

    for key in list(feature.keys()):
        output['properties'][key] = feature.GetField(key)

    return output

def shapely_to_ogr_type(shapely_type):
    from osgeo import ogr
    if shapely_type == "Polygon":
        return ogr.wkbPolygon
    elif shapely_type == "LineString":
        return ogr.wkbLineString
    elif shapely_type == "MultiPolygon":
        return ogr.wkbMultiPolygon
    elif shapely_type == "MultiLineString":
        return ogr.wkbLineString
    raise TypeError("shapely type %s not supported" % shapely_type)

def parse_geo(thing):
    """ Given a python object, try to get a geo-json like mapping from it
    """
    from shapely.geos import ReadingError
    from shapely import wkt, wkb

    # object implementing geo_interface
    try:
        geo = thing.__geo_interface__
        return geo
    except AttributeError:
        pass

    # wkb
    try:
        shape = wkb.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError):
        pass

    # wkt
    try:
        shape = wkt.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError, AttributeError):
        pass

    # geojson-like python mapping
    valid_types = ["Feature", "Point", "LineString", "Polygon",
                   "MultiPoint", "MultiLineString", "MultiPolygon"]
    try:
        assert thing['type'] in valid_types
        return thing
    except (AssertionError, TypeError):
        pass

    # geojson string
    try:
        maybe_geo = json.loads(thing)
        assert maybe_geo['type'] in valid_types + ["FeatureCollection"]
        return maybe_geo
    except (ValueError, AssertionError, TypeError):
        pass

    raise ValueError("Can't parse %s as a geo-like object" % thing)

def get_ogr_ds(vds):
    from osgeo import ogr
    if not isinstance(vds, str):
        raise OGRError("OGR cannot open %r: not a string" % vds)

    ds = ogr.Open(vds)
    if not ds:
        raise OGRError("OGR cannot open %r" % vds)

    return ds

def ogr_srs(vector, layer_num):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    return layer.GetSpatialRef()

def ogr_records(vector, layer_num=0):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    if layer.GetFeatureCount() == 0:
        raise OGRError("No Features")
    feature = layer.GetNextFeature()
    while feature is not None:
        yield feature_to_geojson(feature)
        feature = layer.GetNextFeature()

def geo_records(vectors):
    for vector in vectors:
        yield parse_geo(vector)

def get_features(vectors, layer_num=0):
    from osgeo import osr
    spatial_ref = osr.SpatialReference()
    if isinstance(vectors, str):
        try:
            # either an OGR layer ...
            get_ogr_ds(vectors)
            features_iter = ogr_records(vectors, layer_num)
            spatial_ref = ogr_srs(vectors, layer_num)
            strategy = "ogr"
        except (OGRError, AttributeError):
            # ... or a single string to be parsed as wkt/wkb/json
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, bytes):
        # wkb
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    elif hasattr(vectors, '__geo_interface__'):
        geotype = vectors.__geo_interface__['type']
        if geotype.lower() == 'featurecollection':
            # ... a featurecollection
            features_iter = geo_records(vectors.__geo_interface__['features'])
            strategy = "geo_featurecollection"
        else:
            # ... or an single object
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, dict):
        # ... or an python mapping
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    else:
        # ... or an iterable of objects
        features_iter = geo_records(vectors)
        strategy = "iter_geo"

    return features_iter, strategy, spatial_ref

def rasterize_geom(geom, src_offset, new_gt, all_touched):
    geoms = [(geom, 1)]
    affinetrans = Affine.from_gdal(*new_gt)
    rv_array = features.rasterize(
        geoms,
        out_shape=(src_offset[3], src_offset[2]),
        transform=affinetrans,
        fill=0,
        all_touched=all_touched)
    return rv_array
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    drift - Tenancy logic
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    Figure out which tenant is in play
"""
from __future__ import absolute_import

import os
import logging
import socket
import warnings

from flask import request, has_request_context
from werkzeug.local import LocalProxy

log = logging.getLogger(__name__)

def get_tenant_name(request_headers=None):
    """Figure out tenant name in the following order:
    1. Tenant name explicitly specified in request header under the key 'Drift-Header'.
    2. Tenant name is the left most part of the host name specified in the request header.
    3. Default tenant name is specified in environment variable 'DRIFT_DEFAULT_TENANT'.
    """
    if request_headers:
        if 'Drift-Tenant' in request_headers:
            return request_headers['Drift-tenant']
        if 'Host' in request_headers:
            host_parts = split_host(request_headers['Host'])
            if host_parts['tenant']:
                return host_parts['tenant']

    return os.environ.get('DRIFT_DEFAULT_TENANT')

# Flask specific block begins
def _flask_get_tenant_name():
    """Calls get_tenant_name with request header and host if applicable.
    """
    request_headers = request.headers if has_request_context() else None
    return get_tenant_name(request_headers)

current_tenant_name = LocalProxy(_flask_get_tenant_name)

# Deprecated name as it's a bit misleading.
def _get_tenant_from_hostname(*args, **kw):
    warnings.warn(
        "'tenant_from_hostname' has been renamed 'current_tenant_name'.",
        DeprecationWarning
    )
    return _flask_get_tenant_name()

tenant_from_hostname = LocalProxy(_get_tenant_from_hostname)
# Flask specific block ends

def split_host(host):
    """Split 'host' into tenant, domain and port if possible."""
    ret = {
        'host': host,
        'tenant': None,
        'domain': None,
        'port': None,
    }

    # IPv6 barf because of https://tools.ietf.org/html/rfc3986#section-3.2.2
    if '[' in host and ']' in host:
        return ret  # Quickly and dirtily assume IPv6

    if ':' in host:
        ret['host'], ret['port'] = host.split(':', 1)
    if '.' in host and not _is_valid_ipv4_address(host):
        ret['tenant'], ret['domain'] = host.split('.', 1)

    return ret

def _is_valid_ipv4_address(address):
    try:
        socket.inet_pton(socket.AF_INET, address)
    except AttributeError:  # no inet_pton here, sorry
        try:
            socket.inet_aton(address)
        except socket.error:
            return False
        return address.count('.') == 3
    except socket.error:  # not a valid address
        return False

    return True

<commit_msg>Turn off deprecation warning for 'tenant_from_hostname' as it's too chatty<commit_after># -*- coding: utf-8 -*-
"""
    drift - Tenancy logic
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    Figure out which tenant is in play
"""
from __future__ import absolute_import

import os
import logging
import socket
import warnings

from flask import request, has_request_context
from werkzeug.local import LocalProxy

log = logging.getLogger(__name__)

def get_tenant_name(request_headers=None):
    """Figure out tenant name in the following order:
    1. Tenant name explicitly specified in request header under the key 'Drift-Header'.
    2. Tenant name is the left most part of the host name specified in the request header.
    3. Default tenant name is specified in environment variable 'DRIFT_DEFAULT_TENANT'.
    """
    if request_headers:
        if 'Drift-Tenant' in request_headers:
            return request_headers['Drift-tenant']
        if 'Host' in request_headers:
            host_parts = split_host(request_headers['Host'])
            if host_parts['tenant']:
                return host_parts['tenant']

    return os.environ.get('DRIFT_DEFAULT_TENANT')

# Flask specific block begins
def _flask_get_tenant_name():
    """Calls get_tenant_name with request header and host if applicable.
    """
    request_headers = request.headers if has_request_context() else None
    return get_tenant_name(request_headers)

current_tenant_name = LocalProxy(_flask_get_tenant_name)

# Deprecated name as it's a bit misleading.
def _get_tenant_from_hostname(*args, **kw):
    # warnings.warn(
    #     "'tenant_from_hostname' has been renamed 'current_tenant_name'.",
    #     DeprecationWarning,
    #     stacklevel=2
    # )
    return _flask_get_tenant_name()

tenant_from_hostname = LocalProxy(_get_tenant_from_hostname)
# Flask specific block ends

def split_host(host):
    """Split 'host' into tenant, domain and port if possible."""
    ret = {
        'host': host,
        'tenant': None,
        'domain': None,
        'port': None,
    }

    # IPv6 barf because of https://tools.ietf.org/html/rfc3986#section-3.2.2
    if '[' in host and ']' in host:
        return ret  # Quickly and dirtily assume IPv6

    if ':' in host:
        ret['host'], ret['port'] = host.split(':', 1)
    if '.' in host and not _is_valid_ipv4_address(host):
        ret['tenant'], ret['domain'] = host.split('.', 1)

    return ret

def _is_valid_ipv4_address(address):
    try:
        socket.inet_pton(socket.AF_INET, address)
    except AttributeError:  # no inet_pton here, sorry
        try:
            socket.inet_aton(address)
        except socket.error:
            return False
        return address.count('.') == 3
    except socket.error:  # not a valid address
        return False

    return True

<|endoftext|>


<commit_before>import argparse
from dataset.ingest.labeled import ingestImagery
from builder.args import addLoggingParams, addUnsupDataParams, \
                         addDebuggingParams
from builder.profiler import setupLogging

def buildCSV(csvFile, nets, target, data, percReturned=100., debug=False) :
    '''Test the imagery for how close it is to the target data. This also sorts
       the results according to closeness, so we can create a tiled tip-sheet.
    '''
    import numpy as np
    import pandas as pd

    # reload the target directory into the networks
    [net.loadFeatureMatrix(target) for net in nets]

    # split the data
    imagery = data[0]
    labels = data[1]

    # setup the dataframe with appropriate column names
    cols = ['Similarity', 'Batch', 'Index', 'True Label']
    cols.extend(['F' + str(ii) for ii in \
                 xrange(np.prod(nets[0].getNetworkOutputSize()[1:]))])
    sims = pd.DataFrame()

    batchSize = imagery.shape[1]
    for ii, batch in enumerate(imagery) :
        # only store the encoding from the first network
        cos, enc = nets[0].closenessAndEncoding(batch)

        # average the results found by multiple networks
        [net.closeness(batch, cos) for net in nets[1:]]
        cos /= float(len(nets))

        # collect the row for each entry of the batch
        batchList = []
        for jj in xrange(batchSize) :
            tmpList = [cos[jj], ii, jj, labels[ii][jj]]
            tmpList.extend(enc[jj])
            batchList.append(tmpList)

        # write a batch of data to the data frame
        sims = sims.append(batchList)

    # write the output to a csv
    sims.to_csv(csvFile)#, columns=cols)

if __name__ == '__main__' :
    '''This application tests how close the examples are to a provided target
       set. This averages the results against several networks and then
       reorders the chips according to the most likeness to the target set.
    '''

    # add the parent directory to grab the common functions
    import sys, os
    sys.path.append(os.path.dirname(os.path.dirname(
        os.path.abspath(__file__))))
    from classifySAEApp import createNetworks

    parser = argparse.ArgumentParser()
    addLoggingParams(parser)
    addUnsupDataParams(parser, 'saeClass', multiLoad=True)
    addDebuggingParams(parser)
    parser.add_argument('--percent', dest='percentReturned', type=float,
                        default=100.,
                        help='Return some percentage of the highest related ' +
                             'examples. All others will not be returned to ' +
                             'the user.')
    parser.add_argument('--csv', dest='csvFile',type=str, default='output.csv',
                        help='Name of the CSV file output.')
    options = parser.parse_args()

    # setup the logger
    log, prof = setupLogging(options, 'SAE-Classification Benchmark')

    # NOTE: The pickleDataset will silently use previously created pickles if
    #       one exists (for efficiency). So watch out for stale pickles!
    train, test, labels = ingestImagery(filepath=options.data, shared=False,
                                        batchSize=options.batchSize,
                                        holdoutPercentage=options.holdout,
                                        log=log)

    # load all networks initialized to the target imagery
    nets = createNetworks(options.maxTarget, options.batchSize,
                          options.synapse, prof, options.debug)

    # test the training data for similarity to the target
    buildCSV(options.csvFile. nets, options.targetDir, test,
             options.percentReturned, options.debug)
<commit_msg>updates to get the columns and indices outputs to the csv properly.<commit_after>import argparse
from dataset.ingest.labeled import ingestImagery
from builder.args import addLoggingParams, addUnsupDataParams, \
                         addDebuggingParams
from builder.profiler import setupLogging

def buildCSV(csvFile, nets, target, data, percReturned=100., debug=False) :
    '''Test the imagery for how close it is to the target data. This also sorts
       the results according to closeness, so we can create a tiled tip-sheet.
    '''
    import numpy as np
    import pandas as pd

    # reload the target directory into the networks
    [net.loadFeatureMatrix(target) for net in nets]

    # split the data
    imagery = data[0]
    labels = data[1]

    # setup the dataframe with appropriate column names
    cols = ['Similarity', 'Batch', 'Index', 'True Label']
    cols.extend(['F' + str(ii) for ii in \
                 xrange(np.prod(nets[0].getNetworkOutputSize()[1:]))])
    sims = pd.DataFrame(columns=cols)

    batchSize = imagery.shape[1]
    for ii, batch in enumerate(imagery) :
        # only store the encoding from the first network
        cos, enc = nets[0].closenessAndEncoding(batch)

        # average the results found by multiple networks
        [net.closeness(batch, cos) for net in nets[1:]]
        cos /= float(len(nets))

        # collect the row for each entry of the batch
        batchList = []
        for jj in xrange(batchSize) :
            tmpList = [cos[jj], ii, jj, labels[ii][jj]]
            tmpList.extend(enc[jj])
            batchList.append(tmpList)

        # write a batch of data to the data frame
        sims = sims.append(pd.DataFrame(batchList, columns=cols),
                           ignore_index=True)

    # write the output to a csv
    sims.to_csv(csvFile, encoding='utf-8')

if __name__ == '__main__' :
    '''This application tests how close the examples are to a provided target
       set. This averages the results against several networks and then
       reorders the chips according to the most likeness to the target set.
    '''

    # add the parent directory to grab the common functions
    import sys, os
    sys.path.append(os.path.dirname(os.path.dirname(
        os.path.abspath(__file__))))
    from classifySAEApp import createNetworks

    parser = argparse.ArgumentParser()
    addLoggingParams(parser)
    addUnsupDataParams(parser, 'saeClass', multiLoad=True)
    addDebuggingParams(parser)
    parser.add_argument('--percent', dest='percentReturned', type=float,
                        default=100.,
                        help='Return some percentage of the highest related ' +
                             'examples. All others will not be returned to ' +
                             'the user.')
    parser.add_argument('--csv', dest='csvFile',type=str, default='output.csv',
                        help='Name of the CSV file output.')
    options = parser.parse_args()

    # setup the logger
    log, prof = setupLogging(options, 'SAE-Classification Benchmark')

    # NOTE: The pickleDataset will silently use previously created pickles if
    #       one exists (for efficiency). So watch out for stale pickles!
    train, test, labels = ingestImagery(filepath=options.data, shared=False,
                                        batchSize=options.batchSize,
                                        holdoutPercentage=options.holdout,
                                        log=log)

    # load all networks initialized to the target imagery
    nets = createNetworks(options.maxTarget, options.batchSize,
                          options.synapse, prof, options.debug)

    # test the training data for similarity to the target
    buildCSV(options.csvFile, nets, options.targetDir, test,
             options.percentReturned, options.debug)
<|endoftext|>


<commit_before># website patch

import webnotes
from webnotes.model.doc import Document

def execute():
	add_website_manager()
	return
	cleanup_file_data()
	update_patch_log()
	from webnotes.modules import reload_doc
	reload_doc('website', 'Role', 'Website Manager')
	reload_doc('website', 'Module Def', 'Website')
	reload_doc('website', 'doctype', 'website_settings')
	reload_doc('website', 'doctype', 'style_settings')
	reload_doc('website', 'doctype', 'top_bar_item')
	reload_doc('website', 'doctype', 'contact_us_settings')
	reload_doc('website', 'doctype', 'about_us_settings')
	reload_doc('website', 'doctype', 'products_settings')
	reload_doc('website', 'doctype', 'blog')
	reload_doc('website', 'doctype', 'blog_subscriber')
	reload_doc('website', 'doctype', 'product')
	reload_doc('website', 'doctype', 'web_page')

	reload_doc('website', 'page', 'contact')
	reload_doc('website', 'page', 'about')
	reload_doc('website', 'page', 'blog')
	reload_doc('website', 'page', 'unsubscribe')
	reload_doc('website', 'page', 'products')

def add_website_manager():
	"""add website manager to system manager"""
	for i in webnotes.conn.sql("""select distinct parent from tabUserRole
		where role='System Manager'"""):
		if not webnotes.conn.sql("""select parent from tabUserRole
			where role='Website Manager' and parent=%s""", i[0]):
			d = Document('UserRole')
			d.parent = i[0]
			d.role = 'Website Manager'
			d.save(1)

def cleanup_file_data():
	webnotes.conn.commit()
	webnotes.conn.sql("""alter table `tabFile Data` drop column blob_content""")
	webnotes.conn.begin()

def update_patch_log():
	webnotes.conn.commit()
	webnotes.conn.sql("""alter table __PatchLog engine=InnoDB""")
	webnotes.conn.begin()
<commit_msg>patch update<commit_after># website patch

import webnotes
from webnotes.model.doc import Document

def execute():
	add_website_manager()
	cleanup_file_data()
	update_patch_log()
	from webnotes.modules import reload_doc
	reload_doc('website', 'Role', 'Website Manager')
	reload_doc('website', 'Module Def', 'Website')
	reload_doc('website', 'doctype', 'website_settings')
	reload_doc('website', 'doctype', 'style_settings')
	reload_doc('website', 'doctype', 'top_bar_item')
	reload_doc('website', 'doctype', 'contact_us_settings')
	reload_doc('website', 'doctype', 'about_us_settings')
	reload_doc('website', 'doctype', 'products_settings')
	reload_doc('website', 'doctype', 'blog')
	reload_doc('website', 'doctype', 'blog_subscriber')
	reload_doc('website', 'doctype', 'product')
	reload_doc('website', 'doctype', 'web_page')

	reload_doc('website', 'page', 'contact')
	reload_doc('website', 'page', 'about')
	reload_doc('website', 'page', 'blog')
	reload_doc('website', 'page', 'unsubscribe')
	reload_doc('website', 'page', 'products')

def add_website_manager():
	"""add website manager to system manager"""
	for i in webnotes.conn.sql("""select distinct parent from tabUserRole
		where role='System Manager'"""):
		if not webnotes.conn.sql("""select parent from tabUserRole
			where role='Website Manager' and parent=%s""", i[0]):
			d = Document('UserRole')
			d.parent = i[0]
			d.role = 'Website Manager'
			d.save(1)

def cleanup_file_data():
	webnotes.conn.commit()
	webnotes.conn.sql("""alter table `tabFile Data` drop column blob_content""")
	webnotes.conn.begin()

def update_patch_log():
	webnotes.conn.commit()
	webnotes.conn.sql("""alter table __PatchLog engine=InnoDB""")
	webnotes.conn.begin()
<|endoftext|>


<commit_before>

'''

Implementation of the Delta-Variance Method from Stutzki et al. 1998.

'''

import numpy as np
from scipy.stats import nanmean
from scipy.spatial.distance import euclidean
from astropy.convolution import convolve_fft
from astropy import units as u

class DeltaVariance(object):

    """

    docstring for DeltaVariance

    """

    def __init__(self, img, header, weights, diam_ratio=1.5, lags=None):
        super(DeltaVariance, self).__init__()

        self.img = img
        self.header = header
        self.weights = weights
        self.diam_ratio = diam_ratio

        self.nanflag = False
        if np.isnan(self.img).any():
            self.nanflag = True

        if lags is None:
            try:
                self.ang_size = self.header["CDELT2"] * u.deg
                min_size = (0.1 * u.arcmin) / self.ang_size.to(u.arcmin)
                # Can't be smaller than one pixel, set to 3 to
                # avoid noisy pixels. Can't be bigger than half the image
                #(deals with issue from sim cubes).
                if min_size < 3.0 or min_size > min(self.img.shape) / 2.:
                    min_size = 3.0
            except KeyError:
                print "No CDELT2 in header. Using pixel scales."
                self.ang_size = 1.0 * u.astrophys.pixel
                min_size = 3.0
            self.lags = np.logspace(np.log10(min_size),
                                    np.log10(min(self.img.shape) / 2.), 25)
        else:
            self.lags = lags

        self.convolved_arrays = []
        self.convolved_weights = []
        self.delta_var = np.empty((len(self.lags, )))
        self.delta_var_error = np.empty((2, len(self.lags)))

    def do_convolutions(self):
        for i, lag in enumerate(self.lags):
            core = core_kernel(lag, self.img.shape[0], self.img.shape[1])
            annulus = annulus_kernel(
                lag, self.diam_ratio, self.img.shape[0], self.img.shape[1])

            # Extend to avoid boundary effects from non-periodicity
            pad_weights = np.pad(self.weights, int(lag), padwithzeros)
            pad_img = np.pad(self.img, int(lag), padwithzeros) * pad_weights

            interpolate_nan = False
            if self.nanflag:
                interpolate_nan = True

            img_core = convolve_fft(
                pad_img, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            img_annulus = convolve_fft(
                pad_img, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_core = convolve_fft(
                pad_weights, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_annulus = convolve_fft(
                pad_weights, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)

            weights_core[np.where(weights_core == 0)] = np.NaN
            weights_annulus[np.where(weights_annulus == 0)] = np.NaN

            self.convolved_arrays.append(
                (img_core / weights_core) - (img_annulus / weights_annulus))
            self.convolved_weights.append(weights_core * weights_annulus)

        return self

    def compute_deltavar(self, nsamples=5000, alpha=0.05):

        for i, (convolved_array, convolved_weight) in \
         enumerate(zip(self.convolved_arrays, self.convolved_weights)):

            delta_var_val = _delvar(convolved_array, convolved_weight)

            # bootstrap to find an error
            bootstrap_delvar = np.empty((nsamples, ))
            for n in range(nsamples):
                resample = bootstrap_resample(convolved_array)
                bootstrap_delvar[n] = _delvar(resample, convolved_weight)

            stat = np.sort(bootstrap_delvar)
            error = (stat[int((alpha/2.0)*nsamples)],
                     stat[int((1-alpha/2.0)*nsamples)])

            self.delta_var[i] = delta_var_val
            self.delta_var_error[:, i] = error

        return self

    def run(self, verbose=False, ang_units=True):

        self.do_convolutions()
        self.compute_deltavar()

        if ang_units:
            self.lags *= self.ang_size

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar = [np.abs(self.delta_var -
                                self.delta_var_error[0, :]),
                         np.abs(self.delta_var +
                                self.delta_var_error[1, :])]
            p.errorbar(self.lags, self.delta_var, yerr=error_bar, fmt="bD-")
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def core_kernel(lag, x_size, y_size):
    '''
    Core Kernel for convolution.

    INPUTS
    ------

    lag : float

    OUTPUTS
    -------

    kernel : numpy.ndarray
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))
    kernel = ((4 / np.pi * lag)) * \
        np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)

    return kernel / np.sum(kernel)

def annulus_kernel(lag, diam_ratio, x_size, y_size):
    '''

    Annulus Kernel for convolution.

    INPUTS
    ------

    lag - float
          size of kernel, same as width

    diam_ratio - float
                 ratio between kernel diameters

    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))

    inner = np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)
    outer = np.exp(-(x ** 2. + y ** 2.) / (diam_ratio * lag / 2.) ** 2.)

    kernel = 4 / (np.pi * lag * (diam_ratio ** 2. - 1)) * (outer - inner)

    return kernel / np.sum(kernel)

def padwithzeros(vector, pad_width, iaxis, kwargs):
    vector[:pad_width[0]] = 0
    vector[-pad_width[1]:] = 0
    return vector

class DeltaVariance_Distance(object):

    """

    """

    def __init__(self, dataset1, weights1, dataset2, weights2, diam_ratio=1.5,
                 lags=None, fiducial_model=None):
        super(DeltaVariance_Distance, self).__init__()

        if fiducial_model is not None:
            self.delvar1 = fiducial_model
        else:
            self.delvar1 = DeltaVariance(dataset1[0], dataset1[1], weights1,
                                         diam_ratio=diam_ratio, lags=lags)
            self.delvar1.run()

        self.delvar2 = DeltaVariance(dataset2[0], dataset2[1], weights2,
                                     diam_ratio=diam_ratio, lags=lags)
        self.delvar2.run()

        self.distance = None

    def distance_metric(self, verbose=False):

        errors1 = np.abs(self.delvar1.delta_var_error[1, :] -
                         self.delvar1.delta_var_error[0, :])
        errors2 = np.abs(self.delvar2.delta_var_error[1, :] -
                         self.delvar2.delta_var_error[0, :])

        self.distance = euclidean(np.log10(self.delvar1.delta_var / errors1),
                                  np.log10(self.delvar2.delta_var / errors2))

        if verbose:
            import matplotlib.pyplot as p

            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar1 = [np.abs(self.delvar1.delta_var -
                                 self.delvar1.delta_var_error[0, :]),
                          np.abs(self.delvar1.delta_var +
                                 self.delvar1.delta_var_error[1, :])]
            p.errorbar(self.delvar1.lags, self.delvar1.delta_var,
                       yerr=error_bar1, fmt="bD-",
                       label="Delta Var 1")
            error_bar2 = [np.abs(self.delvar2.delta_var -
                                 self.delvar2.delta_var_error[0, :]),
                          np.abs(self.delvar2.delta_var +
                                 self.delvar2.delta_var_error[1, :])]
            p.errorbar(self.delvar2.lags, self.delvar2.delta_var,
                       yerr=error_bar2, fmt="gD-",
                       label="Delta Var 2")
            ax.legend(loc='best')
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def _delvar(array, weight):
    '''
    Computes the delta variance of the given array.
    '''
    avg_value = nanmean(array, axis=None)

    val = np.nansum((array - avg_value) ** 2. * weight) /\
        np.nansum(weight)

    return val

def bootstrap_resample(X, n=None):
    """
    Code based on: http://nbviewer.ipython.org/gist/aflaxman/6871948
    Bootstrap resample an array_like
    Parameters
    ----------
    X : array_like
      data to resample
    n : int, optional
      length of resampled array, equal to len(X) if n==None
    Results
    -------
    returns X_resamples
    """
    if n is None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = X[resample_i]
    return X_resample
<commit_msg>Change error type.<commit_after>

'''

Implementation of the Delta-Variance Method from Stutzki et al. 1998.

'''

import numpy as np
from scipy.stats import nanmean
from scipy.spatial.distance import euclidean
from astropy.convolution import convolve_fft
from astropy import units as u

class DeltaVariance(object):

    """

    docstring for DeltaVariance

    """

    def __init__(self, img, header, weights, diam_ratio=1.5, lags=None):
        super(DeltaVariance, self).__init__()

        self.img = img
        self.header = header
        self.weights = weights
        self.diam_ratio = diam_ratio

        self.nanflag = False
        if np.isnan(self.img).any():
            self.nanflag = True

        if lags is None:
            try:
                self.ang_size = self.header["CDELT2"] * u.deg
                min_size = (0.1 * u.arcmin) / self.ang_size.to(u.arcmin)
                # Can't be smaller than one pixel, set to 3 to
                # avoid noisy pixels. Can't be bigger than half the image
                #(deals with issue from sim cubes).
                if min_size < 3.0 or min_size > min(self.img.shape) / 2.:
                    min_size = 3.0
            except ValueError:
                print "No CDELT2 in header. Using pixel scales."
                self.ang_size = 1.0 * u.astrophys.pixel
                min_size = 3.0
            self.lags = np.logspace(np.log10(min_size),
                                    np.log10(min(self.img.shape) / 2.), 25)
        else:
            self.lags = lags

        self.convolved_arrays = []
        self.convolved_weights = []
        self.delta_var = np.empty((len(self.lags, )))
        self.delta_var_error = np.empty((2, len(self.lags)))

    def do_convolutions(self):
        for i, lag in enumerate(self.lags):
            core = core_kernel(lag, self.img.shape[0], self.img.shape[1])
            annulus = annulus_kernel(
                lag, self.diam_ratio, self.img.shape[0], self.img.shape[1])

            # Extend to avoid boundary effects from non-periodicity
            pad_weights = np.pad(self.weights, int(lag), padwithzeros)
            pad_img = np.pad(self.img, int(lag), padwithzeros) * pad_weights

            interpolate_nan = False
            if self.nanflag:
                interpolate_nan = True

            img_core = convolve_fft(
                pad_img, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            img_annulus = convolve_fft(
                pad_img, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_core = convolve_fft(
                pad_weights, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_annulus = convolve_fft(
                pad_weights, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)

            weights_core[np.where(weights_core == 0)] = np.NaN
            weights_annulus[np.where(weights_annulus == 0)] = np.NaN

            self.convolved_arrays.append(
                (img_core / weights_core) - (img_annulus / weights_annulus))
            self.convolved_weights.append(weights_core * weights_annulus)

        return self

    def compute_deltavar(self, nsamples=5000, alpha=0.05):

        for i, (convolved_array, convolved_weight) in \
         enumerate(zip(self.convolved_arrays, self.convolved_weights)):

            delta_var_val = _delvar(convolved_array, convolved_weight)

            # bootstrap to find an error
            bootstrap_delvar = np.empty((nsamples, ))
            for n in range(nsamples):
                resample = bootstrap_resample(convolved_array)
                bootstrap_delvar[n] = _delvar(resample, convolved_weight)

            stat = np.sort(bootstrap_delvar)
            error = (stat[int((alpha/2.0)*nsamples)],
                     stat[int((1-alpha/2.0)*nsamples)])

            self.delta_var[i] = delta_var_val
            self.delta_var_error[:, i] = error

        return self

    def run(self, verbose=False, ang_units=True):

        self.do_convolutions()
        self.compute_deltavar()

        if ang_units:
            self.lags *= self.ang_size

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar = [np.abs(self.delta_var -
                                self.delta_var_error[0, :]),
                         np.abs(self.delta_var +
                                self.delta_var_error[1, :])]
            p.errorbar(self.lags, self.delta_var, yerr=error_bar, fmt="bD-")
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def core_kernel(lag, x_size, y_size):
    '''
    Core Kernel for convolution.

    INPUTS
    ------

    lag : float

    OUTPUTS
    -------

    kernel : numpy.ndarray
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))
    kernel = ((4 / np.pi * lag)) * \
        np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)

    return kernel / np.sum(kernel)

def annulus_kernel(lag, diam_ratio, x_size, y_size):
    '''

    Annulus Kernel for convolution.

    INPUTS
    ------

    lag - float
          size of kernel, same as width

    diam_ratio - float
                 ratio between kernel diameters

    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))

    inner = np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)
    outer = np.exp(-(x ** 2. + y ** 2.) / (diam_ratio * lag / 2.) ** 2.)

    kernel = 4 / (np.pi * lag * (diam_ratio ** 2. - 1)) * (outer - inner)

    return kernel / np.sum(kernel)

def padwithzeros(vector, pad_width, iaxis, kwargs):
    vector[:pad_width[0]] = 0
    vector[-pad_width[1]:] = 0
    return vector

class DeltaVariance_Distance(object):

    """

    """

    def __init__(self, dataset1, weights1, dataset2, weights2, diam_ratio=1.5,
                 lags=None, fiducial_model=None):
        super(DeltaVariance_Distance, self).__init__()

        if fiducial_model is not None:
            self.delvar1 = fiducial_model
        else:
            self.delvar1 = DeltaVariance(dataset1[0], dataset1[1], weights1,
                                         diam_ratio=diam_ratio, lags=lags)
            self.delvar1.run()

        self.delvar2 = DeltaVariance(dataset2[0], dataset2[1], weights2,
                                     diam_ratio=diam_ratio, lags=lags)
        self.delvar2.run()

        self.distance = None

    def distance_metric(self, verbose=False):

        errors1 = np.abs(self.delvar1.delta_var_error[1, :] -
                         self.delvar1.delta_var_error[0, :])
        errors2 = np.abs(self.delvar2.delta_var_error[1, :] -
                         self.delvar2.delta_var_error[0, :])

        self.distance = euclidean(np.log10(self.delvar1.delta_var / errors1),
                                  np.log10(self.delvar2.delta_var / errors2))

        if verbose:
            import matplotlib.pyplot as p

            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar1 = [np.abs(self.delvar1.delta_var -
                                 self.delvar1.delta_var_error[0, :]),
                          np.abs(self.delvar1.delta_var +
                                 self.delvar1.delta_var_error[1, :])]
            p.errorbar(self.delvar1.lags, self.delvar1.delta_var,
                       yerr=error_bar1, fmt="bD-",
                       label="Delta Var 1")
            error_bar2 = [np.abs(self.delvar2.delta_var -
                                 self.delvar2.delta_var_error[0, :]),
                          np.abs(self.delvar2.delta_var +
                                 self.delvar2.delta_var_error[1, :])]
            p.errorbar(self.delvar2.lags, self.delvar2.delta_var,
                       yerr=error_bar2, fmt="gD-",
                       label="Delta Var 2")
            ax.legend(loc='best')
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def _delvar(array, weight):
    '''
    Computes the delta variance of the given array.
    '''
    avg_value = nanmean(array, axis=None)

    val = np.nansum((array - avg_value) ** 2. * weight) /\
        np.nansum(weight)

    return val

def bootstrap_resample(X, n=None):
    """
    Code based on: http://nbviewer.ipython.org/gist/aflaxman/6871948
    Bootstrap resample an array_like
    Parameters
    ----------
    X : array_like
      data to resample
    n : int, optional
      length of resampled array, equal to len(X) if n==None
    Results
    -------
    returns X_resamples
    """
    if n is None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = X[resample_i]
    return X_resample
<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

'''

Implementation of the Delta-Variance Method from Stutzki et al. 1998.

'''

import numpy as np
from scipy.stats import nanmean
from astropy.convolution import convolve_fft
from astropy import units as u

class DeltaVariance(object):

    """

    The delta-variance technique as described in Ossenkopf et al. (2008).

    Parameters
    ----------

    img : numpy.ndarray
        The image calculate the delta-variance of.
    header : FITS header
        Image header.
    weights : numpy.ndarray
        Weights to be used.
    diam_ratio : float, optional
        The ratio between the kernel sizes.
    lags : numpy.ndarray or list, optional
        The pixel scales to compute the delta-variance at.

    """

    def __init__(self, img, header, weights=None, diam_ratio=1.5, lags=None):
        super(DeltaVariance, self).__init__()

        self.img = img
        self.header = header
        self.diam_ratio = diam_ratio

        if weights is None:
            self.weights = np.ones(img.shape)
        else:
            self.weights = weights

        self.nanflag = False
        if np.isnan(self.img).any() or np.isnan(self.weights).any():
            self.nanflag = True

        if lags is None:
            try:
                self.ang_size = self.header["CDELT2"] * u.deg
                min_size = (0.1 * u.arcmin) / self.ang_size.to(u.arcmin)
                # Can't be smaller than one pixel, set to 3 to
                # avoid noisy pixels. Can't be bigger than half the image
                #(deals with issue from sim cubes).
                if min_size < 3.0 or min_size > min(self.img.shape) / 2.:
                    min_size = 3.0
            except ValueError:
                print "No CDELT2 in header. Using pixel scales."
                self.ang_size = 1.0 * u.astrophys.pixel
                min_size = 3.0
            self.lags = np.logspace(np.log10(min_size),
                                    np.log10(min(self.img.shape) / 2.), 25)
        else:
            self.lags = lags

        self.convolved_arrays = []
        self.convolved_weights = []
        self.delta_var = np.empty((len(self.lags, )))
        self.delta_var_error = np.empty((2, len(self.lags)))

    def do_convolutions(self):
        for i, lag in enumerate(self.lags):
            core = core_kernel(lag, self.img.shape[0], self.img.shape[1])
            annulus = annulus_kernel(
                lag, self.diam_ratio, self.img.shape[0], self.img.shape[1])

            # Extend to avoid boundary effects from non-periodicity
            pad_weights = np.pad(self.weights, int(lag), padwithzeros)
            pad_img = np.pad(self.img, int(lag), padwithzeros) * pad_weights

            interpolate_nan = False
            if self.nanflag:
                interpolate_nan = True

            img_core = convolve_fft(
                pad_img, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            img_annulus = convolve_fft(
                pad_img, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_core = convolve_fft(
                pad_weights, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_annulus = convolve_fft(
                pad_weights, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)

            weights_core[np.where(weights_core == 0)] = np.NaN
            weights_annulus[np.where(weights_annulus == 0)] = np.NaN

            self.convolved_arrays.append(
                (img_core / weights_core) - (img_annulus / weights_annulus))
            self.convolved_weights.append(weights_core * weights_annulus)

        return self

    def compute_deltavar(self, bootstrap=False, nsamples=100, alpha=0.05):

        for i, (convolved_array, convolved_weight) in \
         enumerate(zip(self.convolved_arrays, self.convolved_weights)):

            delta_var_val = _delvar(convolved_array, convolved_weight)

            # bootstrap to find an error
            if bootstrap:
                bootstrap_delvar = np.empty((nsamples, ))
                for n in range(nsamples):
                    resample = bootstrap_resample(convolved_array)
                    bootstrap_delvar[n] = _delvar(resample, convolved_weight)

                stat = np.sort(bootstrap_delvar)
                error = (stat[int((alpha/2.0)*nsamples)],
                         stat[int((1-alpha/2.0)*nsamples)])
            else:
                error = (0.0, 0.0)

            self.delta_var[i] = delta_var_val
            self.delta_var_error[:, i] = error

        return self

    def run(self, bootstrap=False, verbose=False, ang_units=True):

        self.do_convolutions()
        self.compute_deltavar(bootstrap=bootstrap)

        if ang_units:
            self.lags *= self.ang_size

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar = [np.abs(self.delta_var -
                                self.delta_var_error[0, :]),
                         np.abs(self.delta_var +
                                self.delta_var_error[1, :])]
            p.errorbar(self.lags, self.delta_var, yerr=error_bar, fmt="bD-")
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def core_kernel(lag, x_size, y_size):
    '''
    Core Kernel for convolution.

    Parameters
    ----------

    lag : int
        Size of the lag. Set the kernel size.
    x_size : int
        Grid size to use in the x direction
    y_size_size : int
        Grid size to use in the y_size direction

    Returns
    -------

    kernel : numpy.ndarray
        Normalized kernel.
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))
    kernel = ((4 / np.pi * lag)) * \
        np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)

    return kernel / np.sum(kernel)

def annulus_kernel(lag, diam_ratio, x_size, y_size):
    '''

    Annulus Kernel for convolution.

    Parameters
    ----------
    lag : int
        Size of the lag. Set the kernel size.
    diam_ratio : float
                 Ratio between kernel diameters.
    x_size : int
        Grid size to use in the x direction
    y_size_size : int
        Grid size to use in the y_size direction

    Returns
    -------

    kernel : numpy.ndarray
        Normalized kernel.
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))

    inner = np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)
    outer = np.exp(-(x ** 2. + y ** 2.) / (diam_ratio * lag / 2.) ** 2.)

    kernel = 4 / (np.pi * lag * (diam_ratio ** 2. - 1)) * (outer - inner)

    return kernel / np.sum(kernel)

def padwithzeros(vector, pad_width, iaxis, kwargs):
    '''
    Pad array with zeros.
    '''
    vector[:pad_width[0]] = 0
    vector[-pad_width[1]:] = 0
    return vector

class DeltaVariance_Distance(object):

    """
    Compares 2 datasets using delta-variance. The distance between them is
    given by the Euclidean distance between the curves weighted by the
    bootstrapped errors.

    Parameters
    ----------

    dataset1 : FITS hdu
        Contains the data and header for one dataset.
    dataset2 : FITS hdu
        See above.
    weights1 : numpy.ndarray
        Weights for dataset1.
    weights2 : numpy.ndarray
        See above.
    diam_ratio : float, optional
        The ratio between the kernel sizes.
    lags : numpy.ndarray or list, optional
        The pixel scales to compute the delta-variance at.
    fiducial_model : DeltaVariance
        A computed DeltaVariance model. Used to avoid recomputing.
    """

    def __init__(self, dataset1, dataset2, weights1=None, weights2=None,
                 diam_ratio=1.5, lags=None, fiducial_model=None):
        super(DeltaVariance_Distance, self).__init__()

        if fiducial_model is not None:
            self.delvar1 = fiducial_model
        else:
            self.delvar1 = DeltaVariance(dataset1[0], dataset1[1],
                                         weights=weights1,
                                         diam_ratio=diam_ratio, lags=lags)
            self.delvar1.run()

        self.delvar2 = DeltaVariance(dataset2[0], dataset2[1],
                                     weights=weights2,
                                     diam_ratio=diam_ratio, lags=lags)
        self.delvar2.run()

        self.distance = None

    def distance_metric(self, verbose=False):
        '''
        Applies the Euclidean distance to the delta-variance curves.

        Parameters
        ----------
        verbose : bool, optional
            Enables plotting.
        '''

        errors1 = np.abs(self.delvar1.delta_var_error[1, :] -
                         self.delvar1.delta_var_error[0, :])
        errors2 = np.abs(self.delvar2.delta_var_error[1, :] -
                         self.delvar2.delta_var_error[0, :])

        # Check for NaNs and negatives
        nans1 = np.logical_or(np.isnan(self.delvar1.delta_var),
                              self.delvar1.delta_var <= 0.0)
        nans2 = np.logical_or(np.isnan(self.delvar2.delta_var),
                              self.delvar2.delta_var <= 0.0)

        if nans1.any() or nans2.any():
            all_nans = np.logical_or(nans1, nans2)

            deltavar1 = np.log10(self.delvar1.delta_var)[~all_nans]
            deltavar2 = np.log10(self.delvar2.delta_var)[~all_nans]

        else:

            deltavar1 = np.log10(self.delvar1.delta_var)
            deltavar2 = np.log10(self.delvar2.delta_var)

        self.distance = np.linalg.norm(deltavar1 - deltavar2)

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar1 = [np.abs(self.delvar1.delta_var -
                                 self.delvar1.delta_var_error[0, :]),
                          np.abs(self.delvar1.delta_var +
                                 self.delvar1.delta_var_error[1, :])]
            p.errorbar(self.delvar1.lags, self.delvar1.delta_var,
                       yerr=error_bar1, fmt="bD-",
                       label="Delta Var 1")
            error_bar2 = [np.abs(self.delvar2.delta_var -
                                 self.delvar2.delta_var_error[0, :]),
                          np.abs(self.delvar2.delta_var +
                                 self.delvar2.delta_var_error[1, :])]
            p.errorbar(self.delvar2.lags, self.delvar2.delta_var,
                       yerr=error_bar2, fmt="gD-",
                       label="Delta Var 2")
            ax.legend(loc='best')
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def _delvar(array, weight):
    '''
    Computes the delta variance of the given array.
    '''
    avg_value = nanmean(array, axis=None)

    val = np.nansum((array - avg_value) ** 2. * weight) /\
        np.nansum(weight)

    return val

def bootstrap_resample(X, n=None):
    """
    Code based on: http://nbviewer.ipython.org/gist/aflaxman/6871948
    Bootstrap resample an array_like
    Parameters
    ----------
    X : array_like
      data to resample
    n : int, optional
      length of resampled array, equal to len(X) if n==None

    Returns
    -------
    returns X_resamples
    """
    if n is None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = X[resample_i]
    return X_resample
<commit_msg>Fix errorbars in delta-variance plots<commit_after># Licensed under an MIT open source license - see LICENSE

'''

Implementation of the Delta-Variance Method from Stutzki et al. 1998.

'''

import numpy as np
from scipy.stats import nanmean
from astropy.convolution import convolve_fft
from astropy import units as u

class DeltaVariance(object):

    """

    The delta-variance technique as described in Ossenkopf et al. (2008).

    Parameters
    ----------

    img : numpy.ndarray
        The image calculate the delta-variance of.
    header : FITS header
        Image header.
    weights : numpy.ndarray
        Weights to be used.
    diam_ratio : float, optional
        The ratio between the kernel sizes.
    lags : numpy.ndarray or list, optional
        The pixel scales to compute the delta-variance at.

    """

    def __init__(self, img, header, weights=None, diam_ratio=1.5, lags=None):
        super(DeltaVariance, self).__init__()

        self.img = img
        self.header = header
        self.diam_ratio = diam_ratio

        if weights is None:
            self.weights = np.ones(img.shape)
        else:
            self.weights = weights

        self.nanflag = False
        if np.isnan(self.img).any() or np.isnan(self.weights).any():
            self.nanflag = True

        if lags is None:
            try:
                self.ang_size = self.header["CDELT2"] * u.deg
                min_size = (0.1 * u.arcmin) / self.ang_size.to(u.arcmin)
                # Can't be smaller than one pixel, set to 3 to
                # avoid noisy pixels. Can't be bigger than half the image
                #(deals with issue from sim cubes).
                if min_size < 3.0 or min_size > min(self.img.shape) / 2.:
                    min_size = 3.0
            except ValueError:
                print "No CDELT2 in header. Using pixel scales."
                self.ang_size = 1.0 * u.astrophys.pixel
                min_size = 3.0
            self.lags = np.logspace(np.log10(min_size),
                                    np.log10(min(self.img.shape) / 2.), 25)
        else:
            self.lags = lags

        self.convolved_arrays = []
        self.convolved_weights = []
        self.delta_var = np.empty((len(self.lags, )))
        self.delta_var_error = np.empty((2, len(self.lags)))

    def do_convolutions(self):
        for i, lag in enumerate(self.lags):
            core = core_kernel(lag, self.img.shape[0], self.img.shape[1])
            annulus = annulus_kernel(
                lag, self.diam_ratio, self.img.shape[0], self.img.shape[1])

            # Extend to avoid boundary effects from non-periodicity
            pad_weights = np.pad(self.weights, int(lag), padwithzeros)
            pad_img = np.pad(self.img, int(lag), padwithzeros) * pad_weights

            interpolate_nan = False
            if self.nanflag:
                interpolate_nan = True

            img_core = convolve_fft(
                pad_img, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            img_annulus = convolve_fft(
                pad_img, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_core = convolve_fft(
                pad_weights, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_annulus = convolve_fft(
                pad_weights, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)

            weights_core[np.where(weights_core == 0)] = np.NaN
            weights_annulus[np.where(weights_annulus == 0)] = np.NaN

            self.convolved_arrays.append(
                (img_core / weights_core) - (img_annulus / weights_annulus))
            self.convolved_weights.append(weights_core * weights_annulus)

        return self

    def compute_deltavar(self, bootstrap=False, nsamples=100, alpha=0.05):

        for i, (convolved_array, convolved_weight) in \
         enumerate(zip(self.convolved_arrays, self.convolved_weights)):

            delta_var_val = _delvar(convolved_array, convolved_weight)

            # bootstrap to find an error
            if bootstrap:
                bootstrap_delvar = np.empty((nsamples, ))
                for n in range(nsamples):
                    resample = bootstrap_resample(convolved_array)
                    bootstrap_delvar[n] = _delvar(resample, convolved_weight)

                stat = np.sort(bootstrap_delvar)
                error = (stat[int((alpha/2.0)*nsamples)],
                         stat[int((1-alpha/2.0)*nsamples)])
            else:
                error = (0.0, 0.0)

            self.delta_var[i] = delta_var_val
            self.delta_var_error[:, i] = error

        return self

    def run(self, bootstrap=False, verbose=False, ang_units=True):

        self.do_convolutions()
        self.compute_deltavar(bootstrap=bootstrap)

        if ang_units:
            self.lags *= self.ang_size

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar = [self.delta_var_error[0, :],
                         self.delta_var_error[1, :]]
            p.errorbar(self.lags, self.delta_var, yerr=error_bar, fmt="bD-")
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def core_kernel(lag, x_size, y_size):
    '''
    Core Kernel for convolution.

    Parameters
    ----------

    lag : int
        Size of the lag. Set the kernel size.
    x_size : int
        Grid size to use in the x direction
    y_size_size : int
        Grid size to use in the y_size direction

    Returns
    -------

    kernel : numpy.ndarray
        Normalized kernel.
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))
    kernel = ((4 / np.pi * lag)) * \
        np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)

    return kernel / np.sum(kernel)

def annulus_kernel(lag, diam_ratio, x_size, y_size):
    '''

    Annulus Kernel for convolution.

    Parameters
    ----------
    lag : int
        Size of the lag. Set the kernel size.
    diam_ratio : float
                 Ratio between kernel diameters.
    x_size : int
        Grid size to use in the x direction
    y_size_size : int
        Grid size to use in the y_size direction

    Returns
    -------

    kernel : numpy.ndarray
        Normalized kernel.
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))

    inner = np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)
    outer = np.exp(-(x ** 2. + y ** 2.) / (diam_ratio * lag / 2.) ** 2.)

    kernel = 4 / (np.pi * lag * (diam_ratio ** 2. - 1)) * (outer - inner)

    return kernel / np.sum(kernel)

def padwithzeros(vector, pad_width, iaxis, kwargs):
    '''
    Pad array with zeros.
    '''
    vector[:pad_width[0]] = 0
    vector[-pad_width[1]:] = 0
    return vector

class DeltaVariance_Distance(object):

    """
    Compares 2 datasets using delta-variance. The distance between them is
    given by the Euclidean distance between the curves weighted by the
    bootstrapped errors.

    Parameters
    ----------

    dataset1 : FITS hdu
        Contains the data and header for one dataset.
    dataset2 : FITS hdu
        See above.
    weights1 : numpy.ndarray
        Weights for dataset1.
    weights2 : numpy.ndarray
        See above.
    diam_ratio : float, optional
        The ratio between the kernel sizes.
    lags : numpy.ndarray or list, optional
        The pixel scales to compute the delta-variance at.
    fiducial_model : DeltaVariance
        A computed DeltaVariance model. Used to avoid recomputing.
    """

    def __init__(self, dataset1, dataset2, weights1=None, weights2=None,
                 diam_ratio=1.5, lags=None, fiducial_model=None):
        super(DeltaVariance_Distance, self).__init__()

        if fiducial_model is not None:
            self.delvar1 = fiducial_model
        else:
            self.delvar1 = DeltaVariance(dataset1[0], dataset1[1],
                                         weights=weights1,
                                         diam_ratio=diam_ratio, lags=lags)
            self.delvar1.run()

        self.delvar2 = DeltaVariance(dataset2[0], dataset2[1],
                                     weights=weights2,
                                     diam_ratio=diam_ratio, lags=lags)
        self.delvar2.run()

        self.distance = None

    def distance_metric(self, verbose=False):
        '''
        Applies the Euclidean distance to the delta-variance curves.

        Parameters
        ----------
        verbose : bool, optional
            Enables plotting.
        '''

        errors1 = np.abs(self.delvar1.delta_var_error[1, :] -
                         self.delvar1.delta_var_error[0, :])
        errors2 = np.abs(self.delvar2.delta_var_error[1, :] -
                         self.delvar2.delta_var_error[0, :])

        # Check for NaNs and negatives
        nans1 = np.logical_or(np.isnan(self.delvar1.delta_var),
                              self.delvar1.delta_var <= 0.0)
        nans2 = np.logical_or(np.isnan(self.delvar2.delta_var),
                              self.delvar2.delta_var <= 0.0)

        if nans1.any() or nans2.any():
            all_nans = np.logical_or(nans1, nans2)

            deltavar1 = np.log10(self.delvar1.delta_var)[~all_nans]
            deltavar2 = np.log10(self.delvar2.delta_var)[~all_nans]

        else:

            deltavar1 = np.log10(self.delvar1.delta_var)
            deltavar2 = np.log10(self.delvar2.delta_var)

        self.distance = np.linalg.norm(deltavar1 - deltavar2)

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar1 = [self.delvar1.delta_var_error[0, :],
                         self.delvar1.delta_var_error[1, :]]
            p.errorbar(self.delvar1.lags, self.delvar1.delta_var,
                       yerr=error_bar1, fmt="bD-",
                       label="Delta Var 1")
            error_bar = [self.delvar2.delta_var_error[0, :],
                         self.delvar2.delta_var_error[1, :]]
            p.errorbar(self.delvar2.lags, self.delvar2.delta_var,
                       yerr=error_bar2, fmt="gD-",
                       label="Delta Var 2")
            ax.legend(loc='best')
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def _delvar(array, weight):
    '''
    Computes the delta variance of the given array.
    '''
    avg_value = nanmean(array, axis=None)

    val = np.nansum((array - avg_value) ** 2. * weight) /\
        np.nansum(weight)

    return val

def bootstrap_resample(X, n=None):
    """
    Code based on: http://nbviewer.ipython.org/gist/aflaxman/6871948
    Bootstrap resample an array_like
    Parameters
    ----------
    X : array_like
      data to resample
    n : int, optional
      length of resampled array, equal to len(X) if n==None

    Returns
    -------
    returns X_resamples
    """
    if n is None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = X[resample_i]
    return X_resample
<|endoftext|>


<commit_before>import os
import sys
import warnings

from distutils.core import Distribution as _Distribution
from distutils.errors import DistutilsFileError, DistutilsSetupError
from setuptools.dist import _get_unpatched

from .extern import six
from .util import DefaultGetDict, IgnoreDict, cfg_to_args

_Distribution = _get_unpatched(_Distribution)

def d2to1(dist, attr, value):
    """Implements the actual d2to1 setup() keyword.  When used, this should be
    the only keyword in your setup() aside from `setup_requires`.

    If given as a string, the value of d2to1 is assumed to be the relative path
    to the setup.cfg file to use.  Otherwise, if it evaluates to true, it
    simply assumes that d2to1 should be used, and the default 'setup.cfg' is
    used.

    This works by reading the setup.cfg file, parsing out the supported
    metadata and command options, and using them to rebuild the
    `DistributionMetadata` object and set the newly added command options.

    The reason for doing things this way is that a custom `Distribution` class
    will not play nicely with setup_requires; however, this implementation may
    not work well with distributions that do use a `Distribution` subclass.
    """

    if not value:
        return
    if isinstance(value, six.string_types):
        path = os.path.abspath(value)
    else:
        path = os.path.abspath('setup.cfg')
    if not os.path.exists(path):
        raise DistutilsFileError(
            'The setup.cfg file %s does not exist.' % path)

    # Converts the setup.cfg file to setup() arguments
    try:
        attrs = cfg_to_args(path)
    except:
        e = sys.exc_info()[1]
        raise DistutilsSetupError(
            'Error parsing %s: %s: %s' % (path, e.__class__.__name__,
                                          six.u(str(e))))

    # Repeat some of the Distribution initialization code with the newly
    # provided attrs
    if attrs:
        # Skips 'options' and 'licence' support which are rarely used; may add
        # back in later if demanded
        for key, val in six.iteritems(attrs):
            if hasattr(dist.metadata, 'set_' + key):
                getattr(dist.metadata, 'set_' + key)(val)
            elif hasattr(dist.metadata, key):
                setattr(dist.metadata, key, val)
            elif hasattr(dist, key):
                setattr(dist, key, val)
            else:
                msg = 'Unknown distribution option: %s' % repr(key)
                warnings.warn(msg)

    # Re-finalize the underlying Distribution
    _Distribution.finalize_options(dist)

    # This bit comes out of distribute/setuptools
    if isinstance(dist.metadata.version, six.integer_types + (float,)):
        # Some people apparently take "version number" too literally :)
        dist.metadata.version = str(dist.metadata.version)

    # This bit of hackery is necessary so that the Distribution will ignore
    # normally unsupport command options (namely pre-hooks and post-hooks).
    # dist.command_options is normally a dict mapping command names to dicts of
    # their options.  Now it will be a defaultdict that returns IgnoreDicts for
    # the each command's options so we can pass through the unsupported options
    ignore = ['pre_hook.*', 'post_hook.*']
    dist.command_options = DefaultGetDict(lambda: IgnoreDict(ignore))
<commit_msg>Remove extra complexity of six.u wrapper.<commit_after>import os
import sys
import warnings

from distutils.core import Distribution as _Distribution
from distutils.errors import DistutilsFileError, DistutilsSetupError
from setuptools.dist import _get_unpatched

from .extern import six
from .util import DefaultGetDict, IgnoreDict, cfg_to_args

_Distribution = _get_unpatched(_Distribution)

def d2to1(dist, attr, value):
    """Implements the actual d2to1 setup() keyword.  When used, this should be
    the only keyword in your setup() aside from `setup_requires`.

    If given as a string, the value of d2to1 is assumed to be the relative path
    to the setup.cfg file to use.  Otherwise, if it evaluates to true, it
    simply assumes that d2to1 should be used, and the default 'setup.cfg' is
    used.

    This works by reading the setup.cfg file, parsing out the supported
    metadata and command options, and using them to rebuild the
    `DistributionMetadata` object and set the newly added command options.

    The reason for doing things this way is that a custom `Distribution` class
    will not play nicely with setup_requires; however, this implementation may
    not work well with distributions that do use a `Distribution` subclass.
    """

    if not value:
        return
    if isinstance(value, six.string_types):
        path = os.path.abspath(value)
    else:
        path = os.path.abspath('setup.cfg')
    if not os.path.exists(path):
        raise DistutilsFileError(
            'The setup.cfg file %s does not exist.' % path)

    # Converts the setup.cfg file to setup() arguments
    try:
        attrs = cfg_to_args(path)
    except:
        e = sys.exc_info()[1]
        raise DistutilsSetupError(
            'Error parsing %s: %s: %s' % (path, e.__class__.__name__,
                                          e.args[0]))

    # Repeat some of the Distribution initialization code with the newly
    # provided attrs
    if attrs:
        # Skips 'options' and 'licence' support which are rarely used; may add
        # back in later if demanded
        for key, val in six.iteritems(attrs):
            if hasattr(dist.metadata, 'set_' + key):
                getattr(dist.metadata, 'set_' + key)(val)
            elif hasattr(dist.metadata, key):
                setattr(dist.metadata, key, val)
            elif hasattr(dist, key):
                setattr(dist, key, val)
            else:
                msg = 'Unknown distribution option: %s' % repr(key)
                warnings.warn(msg)

    # Re-finalize the underlying Distribution
    _Distribution.finalize_options(dist)

    # This bit comes out of distribute/setuptools
    if isinstance(dist.metadata.version, six.integer_types + (float,)):
        # Some people apparently take "version number" too literally :)
        dist.metadata.version = str(dist.metadata.version)

    # This bit of hackery is necessary so that the Distribution will ignore
    # normally unsupport command options (namely pre-hooks and post-hooks).
    # dist.command_options is normally a dict mapping command names to dicts of
    # their options.  Now it will be a defaultdict that returns IgnoreDicts for
    # the each command's options so we can pass through the unsupported options
    ignore = ['pre_hook.*', 'post_hook.*']
    dist.command_options = DefaultGetDict(lambda: IgnoreDict(ignore))
<|endoftext|>


<commit_before>class Finder(object):
    def __init__(self, target):
        self.target = target
        self.index = -1

    def feed(self, char):
        if not self.target:
            return

        self.index += 1

        if self.index >= len(self.target):
            return

        if self.target[self.index] != char:
            self.index = -1
            if char == self.target[0]:
                self.feed(char)

    def reset(self):
        self.index = -1

    @property
    def finding(self):
        return self.index > -1 and self.index < len(self.target) - 1

    @property
    def found(self):
        return self.index == len(self.target) - 1
<commit_msg>avoid calling length every time in finder<commit_after>class Finder(object):
    def __init__(self, target):
        self.target = target
        self.length = len(self.target)
        self.index = -1

    def feed(self, char):
        self.index += 1

        if self.index >= self.length:
            return

        if self.target[self.index] != char:
            self.index = -1
            if char == self.target[0]:
                self.feed(char)

    def reset(self):
        self.index = -1

    @property
    def finding(self):
        return self.index > -1 and self.index < self.length - 1

    @property
    def found(self):
        return self.index == self.length - 1
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from copy import deepcopy
from collections import defaultdict
from functools import wraps

from .expressions import RExpression

class Empty(object):
    def __repr__(self):
        return "<empty>"

empty = Empty()

class Dependency(object):
    def __init__(self, name, expression=None, message=None):
        self.name = name
        self.expression = expression
        self.message = message

    def get_key(self):
        return self.name

    def get_value(self):
        return self.expression or empty

    def get_error_message(self):
        return self.message

class Requires(object):
    """
    """

    empty = empty

    def __init__(self, from_, dep, message=None):
        # process from_ partial
        self._from_keys = set()
        from_name = self._get_from_name(from_)
        if from_name:
            self._from_keys.add(from_name)

        dep_obj = self._get_dep_object(from_, dep, message)
        self.adj = {from_: (dep_obj, )}

    def _get_from_name(self, from_):
        if isinstance(from_, RExpression):
            # partial on the from_ value
            from_names = from_.get_fields()
            assert len(from_names) == 1, "from_ must only contain one field"
            from_name = list(from_names)[0]
            return from_name
        return None

    def _get_dep_object(self, from_, dep, message):
        if isinstance(dep, RExpression):
            # Complex dependency
            dep_names = dep.get_fields() - {from_}
            assert len(dep_names) <= 1, "Currently do not support complex dependencies in expressions"
            if len(dep_names) == 1:
                dep_name = list(dep_names)[0]
                # expression dependecy
                return Dependency(name=dep_name, expression=dep, message=message)
            # self expression depedency
            return Dependency(name=from_, expression=dep, message=message)
        # flat full dependency
        return Dependency(name=dep, message=message)

    def __add__(self, other):
        assert isinstance(other, Requires)
        combined = list(self.adj.items()) + list(other.adj.items())
        d = defaultdict(list)
        for key, value in combined:
            d[key].extend(list(value))
        self._from_keys = self._from_keys.union(other._from_keys)
        new = deepcopy(self)
        new.adj = d
        return new

    def deps(self, key, value, seen=None):
        # DFS of the dependency graph

        partial_key = (key, value) if key in self._from_keys else None

        # use of seen so we terminate and don't blow the stack
        seen = seen if seen else set()
        seen.add(partial_key or key)

        if key not in self.adj and partial_key is None:
            # key has no dependencies or partial dependencies
            return []

        rels = self.adj.get(key, list())

        if partial_key:
            # We need to resolve the partial dependency
            # to see if it is valid.
            lookup = dict((partial_key,))
            partial_rels = []
            for exp in self.adj.keys():
                if isinstance(exp, RExpression):
                    if exp(lookup):
                        dependencies_from_partial = self.adj.get(exp)
                        for dep in dependencies_from_partial:
                            partial_rels.append(dep)
            rels.extend(partial_rels)

        deps = []
        for dep in rels:
            if dep not in seen:
                deps.append(dep)
                child_deps = self.deps(dep, value, seen)
                deps.extend(child_deps)
        return deps

    def _validate(self, key, data):
        for dep in self.deps(key, data.get(key, self.empty)):

            dep_key = dep.get_key()
            dep_value = dep.get_value()

            if dep_key not in data:
                raise RequirementError("%s requires '%s' to be present" % (key, dep_key))
            if dep_value is not self.empty:
                if not dep_value(data):
                    error_message = dep.get_error_message() or dep_value.error(key, dep_key, data)
                    raise RequirementError(error_message)

    def validate(self, data):
        for key in data.keys():
            self._validate(key, data)

class RequirementError(Exception):
    pass

def validate(requires):
    def validate_decorator(func):
        @wraps(func)
        def func_wrapper(*args, **kwargs):
            requires.validate(kwargs)
            return func(*args, **kwargs)
        return func_wrapper
    return validate_decorator
<commit_msg>Pass key information into RequirementError (#16)<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from copy import deepcopy
from collections import defaultdict
from functools import wraps

from .expressions import RExpression

class Empty(object):
    def __repr__(self):
        return "<empty>"

empty = Empty()

class Dependency(object):
    def __init__(self, name, expression=None, message=None):
        self.name = name
        self.expression = expression
        self.message = message

    def get_key(self):
        return self.name

    def get_value(self):
        return self.expression or empty

    def get_error_message(self):
        return self.message

class Requires(object):
    """
    """

    empty = empty

    def __init__(self, from_, dep, message=None):
        # process from_ partial
        self._from_keys = set()
        from_name = self._get_from_name(from_)
        if from_name:
            self._from_keys.add(from_name)

        dep_obj = self._get_dep_object(from_, dep, message)
        self.adj = {from_: (dep_obj, )}

    def _get_from_name(self, from_):
        if isinstance(from_, RExpression):
            # partial on the from_ value
            from_names = from_.get_fields()
            assert len(from_names) == 1, "from_ must only contain one field"
            from_name = list(from_names)[0]
            return from_name
        return None

    def _get_dep_object(self, from_, dep, message):
        if isinstance(dep, RExpression):
            # Complex dependency
            dep_names = dep.get_fields() - {from_}
            assert len(dep_names) <= 1, "Currently do not support complex dependencies in expressions"
            if len(dep_names) == 1:
                dep_name = list(dep_names)[0]
                # expression dependecy
                return Dependency(name=dep_name, expression=dep, message=message)
            # self expression depedency
            return Dependency(name=from_, expression=dep, message=message)
        # flat full dependency
        return Dependency(name=dep, message=message)

    def __add__(self, other):
        assert isinstance(other, Requires)
        combined = list(self.adj.items()) + list(other.adj.items())
        d = defaultdict(list)
        for key, value in combined:
            d[key].extend(list(value))
        self._from_keys = self._from_keys.union(other._from_keys)
        new = deepcopy(self)
        new.adj = d
        return new

    def deps(self, key, value, seen=None):
        # DFS of the dependency graph

        partial_key = (key, value) if key in self._from_keys else None

        # use of seen so we terminate and don't blow the stack
        seen = seen if seen else set()
        seen.add(partial_key or key)

        if key not in self.adj and partial_key is None:
            # key has no dependencies or partial dependencies
            return []

        rels = self.adj.get(key, list())

        if partial_key:
            # We need to resolve the partial dependency
            # to see if it is valid.
            lookup = dict((partial_key,))
            partial_rels = []
            for exp in self.adj.keys():
                if isinstance(exp, RExpression):
                    if exp(lookup):
                        dependencies_from_partial = self.adj.get(exp)
                        for dep in dependencies_from_partial:
                            partial_rels.append(dep)
            rels.extend(partial_rels)

        deps = []
        for dep in rels:
            if dep not in seen:
                deps.append(dep)
                child_deps = self.deps(dep, value, seen)
                deps.extend(child_deps)
        return deps

    def _validate(self, field, data):
        for dependency in self.deps(field, data.get(field, self.empty)):

            dependency_name = dependency.get_key()
            dependency_value = dependency.get_value()

            if dependency_name not in data:
                raise RequirementError(
                    field,
                    dependency_name,
                    None,
                    "%s requires '%s' to be present" % (field, dependency_name)
                )

            if dependency_value is not self.empty:
                if not dependency_value(data):
                    error_message = dependency.get_error_message() or dependency_value.error(field, dependency_name, data)
                    raise RequirementError(
                        field,
                        dependency_name,
                        dependency_value,
                        error_message
                    )

    def validate(self, data):
        for key in data.keys():
            self._validate(key, data)

class RequirementError(Exception):

    def __init__(self, field=None, dependency_name=None, dependency_value=None, *args, **kwargs):
        super(RequirementError, self).__init__(*args, **kwargs)
        self.field = field
        self.dependency_name = dependency_name
        self.dependency_value = dependency_value

def validate(requires):
    def validate_decorator(func):
        @wraps(func)
        def func_wrapper(*args, **kwargs):
            requires.validate(kwargs)
            return func(*args, **kwargs)
        return func_wrapper
    return validate_decorator
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Agile Business Group sagl (<http://www.agilebg.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, orm
import openerp.addons.decimal_precision as dp

class StockPartialPickingLine(orm.TransientModel):
    _inherit = 'stock.partial.picking.line'

    def on_change_product_uos_qty(
            self, cr, uid, ids,
            product_uos_qty, move_id, context=None):
        result = {}
        move_obj = self.pool[('stock.move')].browse(
            cr, uid, move_id, context=context)
        result['value'] = {'quantity': move_obj.product_qty*(
            product_uos_qty/move_obj.product_uos_qty)}
        return result

    _columns = {
        'product_uos': fields.many2one('product.uom', 'Product UOS'),
        'product_uos_qty': fields.float(
            'Quantity (UOS)',
            digits_compute=dp.get_precision('Product Unit of Measure')),
    }

class StockPartialPicking(orm.TransientModel):
    _inherit = 'stock.partial.picking'

    def _partial_move_for(self, cr, uid, move, context=None):
        partial_move = super(
            StockPartialPicking,
            self)._partial_move_for(cr, uid, move)
        partial_move.update({
            'product_uos': move.product_uos.id,
            'product_uos_qty': move.product_uos_qty
        })
        return partial_move
<commit_msg>[FIX] Delete () in pool<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Agile Business Group sagl (<http://www.agilebg.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, orm
import openerp.addons.decimal_precision as dp

class StockPartialPickingLine(orm.TransientModel):
    _inherit = 'stock.partial.picking.line'

    def on_change_product_uos_qty(
            self, cr, uid, ids,
            product_uos_qty, move_id, context=None):
        result = {}
        move_obj = self.pool['stock.move'].browse(
            cr, uid, move_id, context=context)
        result['value'] = {'quantity': move_obj.product_qty*(
            product_uos_qty/move_obj.product_uos_qty)}
        return result

    _columns = {
        'product_uos': fields.many2one('product.uom', 'Product UOS'),
        'product_uos_qty': fields.float(
            'Quantity (UOS)',
            digits_compute=dp.get_precision('Product Unit of Measure')),
    }

class StockPartialPicking(orm.TransientModel):
    _inherit = 'stock.partial.picking'

    def _partial_move_for(self, cr, uid, move, context=None):
        partial_move = super(
            StockPartialPicking,
            self)._partial_move_for(cr, uid, move)
        partial_move.update({
            'product_uos': move.product_uos.id,
            'product_uos_qty': move.product_uos_qty
        })
        return partial_move
<|endoftext|>