<commit_before>from __future__ import print_function, division, absolute_import

import os
import hashlib
import datetime
import time

import uuid
import hashlib
import base64

import urlparse

import simplejson as json

import redis
Redis = redis.StrictRedis()

from flask import (Flask, g, render_template, url_for, request, make_response,
                   redirect, flash, jsonify)

from wqflask import app

from pprint import pformat as pf

from wqflask.database import db_session

from wqflask import model

from utility import Bunch, Struct, hmac
from utility.formatting import numify

from base import trait
from base.data_set import create_dataset

import logging
from utility.logger import getLogger
logger = getLogger(__name__)

def process_traits(unprocessed_traits):
    if isinstance(unprocessed_traits, basestring):
        unprocessed_traits = unprocessed_traits.split(",")
    traits = set()
    for trait in unprocessed_traits:
        data, _separator, hmac = trait.rpartition(':')
        data = data.strip()
        if g.user_session.logged_in:
          assert hmac == hmac.data_hmac(data), "Data tampering?"
        traits.add(str(data))

    return traits

def report_change(len_before, len_now):
    new_length = len_now - len_before
    if new_length:
        flash("We've added {} to your collection.".format(
            numify(new_length, 'new trait', 'new traits')))
    else:
        logger.debug("No new traits were added.")

@app.route("/collections/store_trait_list", methods=('POST',))
def store_traits_list():
   params = request.form

   traits = params['traits']
   hash = params['hash']

   Redis.set(hash, traits)

   return hash

@app.route("/collections/add")
def collections_add():

    collections = g.user_session.user_collections
    if len(collections) < 1:
        collection_name = "Default Collection"
        uc_id = g.user_session.add_collection(collection_name, set())
        collections = g.user_session.user_collections

    #ZS: One of these might be unnecessary
    if 'traits' in request.args:
        traits=request.args['traits']
        return render_template("collections/add.html",
                                traits = traits,
                                collections = collections,
                              )
    else:
        hash = request.args['hash']
        return render_template("collections/add.html",
                                hash = hash,
                                collections = collections,
                              )

@app.route("/collections/new")
def collections_new():
    params = request.args

    if "sign_in" in params:
        return redirect(url_for('login'))
    if "create_new" in params:
        collection_name = params['new_collection']
        if collection_name.strip() == "":
            collection_name = datetime.datetime.utcnow().strftime('Collection_%b_%d_%H:%M')
        return create_new(collection_name)
    elif "add_to_existing" in params:
        if 'existing_collection' not in params:
            collections = g.user_session.user_collections
            for collection in collections:
                if collection["name"] == "Default Collection":
                    collection_id = collection["id"]
                    collection_name = collection["name"]
                    default_collection_exists = True
            if not default_collection_exists:
                return create_new("Default Collection")
        else:
            collection_id = params['existing_collection'].split(":")[0]
            collection_name = params['existing_collection'].split(":")[1]

        if "hash" in params:
            unprocessed_traits = Redis.get(params['hash'])
        else:
            unprocessed_traits = params['traits']
        traits = list(process_traits(unprocessed_traits))
        g.user_session.add_traits_to_collection(collection_id, traits)
        return redirect(url_for('view_collection', uc_id=collection_id))
    else:
        CauseAnError

def create_new(collection_name):
    params = request.args

    if "hash" in params:
        unprocessed_traits = Redis.get(params['hash'])
        Redis.delete(params['hash'])
    else:
        unprocessed_traits = params['traits']

    traits = process_traits(unprocessed_traits)

    uc_id = g.user_session.add_collection(collection_name, traits)

    return redirect(url_for('view_collection', uc_id=uc_id))

@app.route("/collections/list")
def list_collections():
    params = request.args

    user_collections = list(g.user_session.user_collections)
    return render_template("collections/list.html",
                            params = params,
                            collections = user_collections,
                            )

@app.route("/collections/remove", methods=('POST',))
def remove_traits():
    params = request.form

    uc_id = params['uc_id']
    traits_to_remove = params.getlist('traits[]')
    traits_to_remove = process_traits(traits_to_remove)
    logger.debug("\n\n  after processing, traits_to_remove:", traits_to_remove)

    members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove)

    # We need to return something so we'll return this...maybe in the future
    # we can use it to check the results
    return str(len(members_now))

@app.route("/collections/delete", methods=('POST',))
def delete_collection():
    params = request.form
    uc_id = ""

    uc_id = params['uc_id']
    if len(uc_id.split(":")) > 1:
        for this_uc_id in uc_id.split(":"):
            collection_name = g.user_session.delete_collection(this_uc_id)
    else:
        collection_name = g.user_session.delete_collection(uc_id)

    if uc_id != "":
        if len(uc_id.split(":")) > 1:
            flash("We've deleted the selected collections.", "alert-info")
        else:
            flash("We've deleted the collection: {}.".format(uc_id), "alert-info")
    else:
        flash("We've deleted the collection: {}.".format(collection_name), "alert-info")

    return redirect(url_for('list_collections'))

@app.route("/collections/view")
def view_collection():
    params = request.args

    uc_id = params['uc_id']
    uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next()
    traits = uc["members"]

    trait_obs = []
    json_version = []

    for atrait in traits:
        name, dataset_name = atrait.split(':')
        if dataset_name == "Temp":
            group = name.split("_")[2]
            dataset = create_dataset(dataset_name, dataset_type = "Temp", group_name = group)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
        else:
            dataset = create_dataset(dataset_name)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
            trait_ob = trait.retrieve_trait_info(trait_ob, dataset, get_qtl_info=True)
        trait_obs.append(trait_ob)

        json_version.append(trait.jsonable(trait_ob))

    collection_info = dict(trait_obs=trait_obs,
                           uc = uc)

    if "json" in params:
        return json.dumps(json_version)
    else:
        return render_template("collections/view.html",
                           **collection_info
                           )

@app.route("/collections/change_name", methods=('POST',))
def change_collection_name():
    params = request.form

    collection_id = params['collection_id']
    new_name = params['new_name']

    g.user_session.change_collection_name(collection_id, new_name)

    return new_name

<commit_msg>Added fix for issue when creating collections (related to hmac import change)<commit_after>from __future__ import print_function, division, absolute_import

import os
import hashlib
import datetime
import time

import uuid
import hashlib
import base64

import urlparse

import simplejson as json

import redis
Redis = redis.StrictRedis()

from flask import (Flask, g, render_template, url_for, request, make_response,
                   redirect, flash, jsonify)

from wqflask import app

from pprint import pformat as pf

from wqflask.database import db_session

from wqflask import model

from utility import Bunch, Struct, hmac
from utility.formatting import numify

from base import trait
from base.data_set import create_dataset

import logging
from utility.logger import getLogger
logger = getLogger(__name__)

def process_traits(unprocessed_traits):
    if isinstance(unprocessed_traits, basestring):
        unprocessed_traits = unprocessed_traits.split(",")
    traits = set()
    for trait in unprocessed_traits:
        data, _separator, the_hmac = trait.rpartition(':')
        data = data.strip()
        if g.user_session.logged_in:
          assert the_hmac == hmac.hmac_creation(data), "Data tampering?"
        traits.add(str(data))

    return traits

def report_change(len_before, len_now):
    new_length = len_now - len_before
    if new_length:
        flash("We've added {} to your collection.".format(
            numify(new_length, 'new trait', 'new traits')))
    else:
        logger.debug("No new traits were added.")

@app.route("/collections/store_trait_list", methods=('POST',))
def store_traits_list():
   params = request.form

   traits = params['traits']
   hash = params['hash']

   Redis.set(hash, traits)

   return hash

@app.route("/collections/add")
def collections_add():

    collections = g.user_session.user_collections
    if len(collections) < 1:
        collection_name = "Default Collection"
        uc_id = g.user_session.add_collection(collection_name, set())
        collections = g.user_session.user_collections

    #ZS: One of these might be unnecessary
    if 'traits' in request.args:
        traits=request.args['traits']
        return render_template("collections/add.html",
                                traits = traits,
                                collections = collections,
                              )
    else:
        hash = request.args['hash']
        return render_template("collections/add.html",
                                hash = hash,
                                collections = collections,
                              )

@app.route("/collections/new")
def collections_new():
    params = request.args

    if "sign_in" in params:
        return redirect(url_for('login'))
    if "create_new" in params:
        collection_name = params['new_collection']
        if collection_name.strip() == "":
            collection_name = datetime.datetime.utcnow().strftime('Collection_%b_%d_%H:%M')
        return create_new(collection_name)
    elif "add_to_existing" in params:
        if 'existing_collection' not in params:
            collections = g.user_session.user_collections
            for collection in collections:
                if collection["name"] == "Default Collection":
                    collection_id = collection["id"]
                    collection_name = collection["name"]
                    default_collection_exists = True
            if not default_collection_exists:
                return create_new("Default Collection")
        else:
            collection_id = params['existing_collection'].split(":")[0]
            collection_name = params['existing_collection'].split(":")[1]

        if "hash" in params:
            unprocessed_traits = Redis.get(params['hash'])
        else:
            unprocessed_traits = params['traits']
        traits = list(process_traits(unprocessed_traits))
        g.user_session.add_traits_to_collection(collection_id, traits)
        return redirect(url_for('view_collection', uc_id=collection_id))
    else:
        CauseAnError

def create_new(collection_name):
    params = request.args

    if "hash" in params:
        unprocessed_traits = Redis.get(params['hash'])
        Redis.delete(params['hash'])
    else:
        unprocessed_traits = params['traits']

    traits = process_traits(unprocessed_traits)

    uc_id = g.user_session.add_collection(collection_name, traits)

    return redirect(url_for('view_collection', uc_id=uc_id))

@app.route("/collections/list")
def list_collections():
    params = request.args

    user_collections = list(g.user_session.user_collections)
    return render_template("collections/list.html",
                            params = params,
                            collections = user_collections,
                            )

@app.route("/collections/remove", methods=('POST',))
def remove_traits():
    params = request.form

    uc_id = params['uc_id']
    traits_to_remove = params.getlist('traits[]')
    traits_to_remove = process_traits(traits_to_remove)
    logger.debug("\n\n  after processing, traits_to_remove:", traits_to_remove)

    members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove)

    # We need to return something so we'll return this...maybe in the future
    # we can use it to check the results
    return str(len(members_now))

@app.route("/collections/delete", methods=('POST',))
def delete_collection():
    params = request.form
    uc_id = ""

    uc_id = params['uc_id']
    if len(uc_id.split(":")) > 1:
        for this_uc_id in uc_id.split(":"):
            collection_name = g.user_session.delete_collection(this_uc_id)
    else:
        collection_name = g.user_session.delete_collection(uc_id)

    if uc_id != "":
        if len(uc_id.split(":")) > 1:
            flash("We've deleted the selected collections.", "alert-info")
        else:
            flash("We've deleted the collection: {}.".format(uc_id), "alert-info")
    else:
        flash("We've deleted the collection: {}.".format(collection_name), "alert-info")

    return redirect(url_for('list_collections'))

@app.route("/collections/view")
def view_collection():
    params = request.args

    uc_id = params['uc_id']
    uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next()
    traits = uc["members"]

    trait_obs = []
    json_version = []

    for atrait in traits:
        name, dataset_name = atrait.split(':')
        if dataset_name == "Temp":
            group = name.split("_")[2]
            dataset = create_dataset(dataset_name, dataset_type = "Temp", group_name = group)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
        else:
            dataset = create_dataset(dataset_name)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
            trait_ob = trait.retrieve_trait_info(trait_ob, dataset, get_qtl_info=True)
        trait_obs.append(trait_ob)

        json_version.append(trait.jsonable(trait_ob))

    collection_info = dict(trait_obs=trait_obs,
                           uc = uc)

    if "json" in params:
        return json.dumps(json_version)
    else:
        return render_template("collections/view.html",
                           **collection_info
                           )

@app.route("/collections/change_name", methods=('POST',))
def change_collection_name():
    params = request.form

    collection_id = params['collection_id']
    new_name = params['new_name']

    g.user_session.change_collection_name(collection_id, new_name)

    return new_name

<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import package
import files
import shutil
import pwd
import grp
import files

def doUpdate(reppath, root, srcPath, pkgName, binaries = 1, sources = 0):
    if root == "/":
	print "using srs to update to your actual system is dumb."
	import sys
	sys.exit(0)

    pkgSet = package.PackageSet(reppath, pkgName)

    if (not len(pkgSet.versionList())):
	raise KeyError, "no versions exist of %s" % pkgName

    (version, pkg) = pkgSet.getLatest()

    fileList = []
    packageFiles = []

    if binaries:
	packageFiles = packageFiles + pkg.fileList()
    if sources:
	packageFiles = packageFiles + pkg.sourceList()

    for (fileName, version) in packageFiles:
	infoFile = files.FileDB(reppath, reppath + fileName)
	fileList.append(infoFile)

    for infoFile in fileList:
	f = infoFile.getVersion(version)
	f.restore(reppath, srcPath, root)
<commit_msg>1) take full configuration set as a parameter 2) if the package name is relative prepend packagenamespace cfg value<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import package
import files
import shutil
import pwd
import grp
import files

def doUpdate(cfg, root, pkgName, binaries = 1, sources = 0):
    if root == "/":
	print "using srs to update to your actual system is dumb."
	import sys
	sys.exit(0)

    if pkgName[0] != "/":
	pkgName = cfg.packagenamespace + "/" + pkgName

    pkgSet = package.PackageSet(cfg.reppath, pkgName)

    if (not len(pkgSet.versionList())):
	raise KeyError, "no versions exist of %s" % pkgName

    (version, pkg) = pkgSet.getLatest()

    fileList = []
    packageFiles = []

    if binaries:
	packageFiles = packageFiles + pkg.fileList()
    if sources:
	packageFiles = packageFiles + pkg.sourceList()

    for (fileName, version) in packageFiles:
	infoFile = files.FileDB(cfg.reppath, cfg.reppath + fileName)
	fileList.append(infoFile)

    for infoFile in fileList:
	f = infoFile.getVersion(version)
	f.restore(cfg.reppath, cfg.sourcepath, root)
<|endoftext|>


<commit_before>"""
models.py
=========

Autoencoder and denoising autoencoder

by Maxim Ziatdinov (email: ziatdinovmax@gmail.com)
"""

from typing import List, Tuple, Union

import numpy as np
import torch
import torch.nn as nn
import torch.tensor as tt

from .nnblocks import (ConvBlock, UpsampleBlock, features_to_latent,
                       latent_to_features)

class FeatureExtractor(nn.Sequential):
    """
    Convolutional feature extractor

    Parameters
    ----------
    ndim
        Data dimensionality (1, 2, or 3).
    input_channels
        Number of input feature channels
        (Defaults to a greyscale image with a single channel)
    layers_per_block
        Number of layers in each block (Default: [1, 2, 2]).
    nfilters
        Number of convolutional filters in the first convolutional block.
        The number of filters in each consecutive block is computed as
        :math:`block_i = nfilters * (i+1)` (Default: 32).
    batchnorm
        Add batch normalization to each layer in the block (Default: False).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    pool
        Applies max-pooling operation at the end of the block (Default: True).

    Examples
    --------

    Create a feature extractor for RGB-like 2D data

    >>> layers_per_block = [2, 2, 3]
    >>> feature_extractor = FeatureExtractor(
    >>>     ndim=2, input_channels=3, layers_per_block=layers_per_block)

    Create a feature extractor for greyscale 2D data

    >>> layers_per_block = [2, 2, 3]
    >>> feature_extractor = FeatureExtractor(
    >>>     ndim=2, input_channels=1, layers_per_block=layers_per_block)
    """
    def __init__(self,
                 ndim: int,
                 input_channels: int = 1,
                 layers_per_block: List[int] = None,
                 nfilters: int = 32,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 pool: bool = True,
                 ) -> None:
        """
        Initializes feature extractor module
        """
        super(FeatureExtractor, self).__init__()
        if layers_per_block is None:
            layers_per_block = [1, 2, 2]
        for i, layers in enumerate(layers_per_block):
            in_filters = input_channels if i == 0 else nfilters * i
            block = ConvBlock(ndim, layers, in_filters, nfilters * (i+1),
                              batchnorm=batchnorm, activation=activation,
                              pool=pool)
            self.add_module("c{}".format(i), block)

class Upsampler(nn.Sequential):
    """
    Convolutional upsampler (aka 'decoder')

    Parameters
    ----------
    ndim
        Data dimensionality (1, 2, or 3).
    input_channels
        Number of input channels (convolutional filters) for the input layer.
        The number of filters in each consecutive block is computed as
        :math:`block_i = nfilters // (i+1)` (Default: 96).
    layers_per_block
        Number of layers in each block (Default: [2, 2, 1]).
    output_channels
        Number of the output channels (Deafult: 1)
    batchnorm
        Add batch normalization to each layer in the block (Default: False).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    activation_out:
        Applies sigmoid (output_channels=1) or softmax (output_channels>1)
        activation to the final convolutional layer (Default: True)
    upsampling_mode
        Upsampling mode. Select between "bilinear" and "nearest"
        (Default: bilinear for 2D, nearest for 1D and 3D).
    """
    def __init__(self,
                 ndim: int,
                 input_channels: int = 96,
                 layers_per_block: List[int] = None,
                 output_channels: int = 1,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 activation_out: bool = True,
                 upsampling_mode: str = "bilinear",
                 ) -> None:
        """
        Initializes upsampler module
        """
        super(Upsampler, self).__init__()
        if layers_per_block is None:
            layers_per_block = [2, 2, 1]
        if activation_out:
            a_out = nn.Sigmoid() if output_channels == 1 else nn.Softmax(-1)

        nfilters = input_channels
        for i, layers in enumerate(layers_per_block):
            in_filters = nfilters if i == 0 else nfilters // i
            block = ConvBlock(ndim, layers, in_filters, nfilters // (i+1),
                              batchnorm=batchnorm, activation=activation,
                              pool=False)
            self.add_module("conv_block_{}".format(i), block)
            up = UpsampleBlock(ndim, nfilters // (i+1), nfilters // (i+1),
                               mode=upsampling_mode)
            self.add_module("up_{}".format(i), up)

        out = ConvBlock(ndim, 1, nfilters // (i+1), output_channels,
                        1, 1, 0, activation=None)
        self.add_module("output_layer", out)
        if activation_out:
            self.add_module("output_activation", a_out)

class AutoEncoder(nn.Module):
    """
    Convolutional autoencoder with latent space

    Parameters
    ----------
    input_dim
        Input dimensions: (channels, length), (channels, height, width)
        or (height, width, depth).
    latent_dim
        Latent sapce dimensionality (Default: 2).
    layers_per_block
        List with the number of layers for each block of the encoder.
        The number of layers for the decoder is computed by reversing
        this list (Default: [1, 2, 2]).
    nfilters
        Number of convolutional filters in the first convolutional block
        of the encoder. The number of filters in each consecutive block
        is computed as :math:`block_i = nfilters * (i+1)`. The number of
        filters in the first layer of the decoder is equal to the number of
        filters in the last layer of the encoder and the number of filters
        in each consecutive block is computed as :math:`block_i = nfilters // (i+1)`
        (Default: 32).
    batchnorm
        Add batch normalization to each layer (Default: True).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    activation_out:
        Applies sigmoid (output_channels=1) or softmax (output_channels>1)
        activation to the final convolutional layer (Default: True).
    upsampling_mode
        Upsampling mode. Select between "bilinear" and "nearest"
        (Default: bilinear for 2D, nearest for 1D and 3D).

    Examples
    --------

    Create autoencoder for dataset with 64 x 64 greyscale images

    >>> input_dim = (1, 64, 64)  # (channles, height, width)
    >>> ae = AutoEncoder(input_dim, latent_dim=2)
    """
    def __init__(self,
                 input_dim: Tuple[int],
                 latent_dim: int = 2,
                 layers_per_block: List[int] = [1, 2, 2],
                 nfilters: int = 32,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 activation_out: bool = True,
                 upsampling_mode: str = "bilinear"
                 ) -> None:
        """
        Initializes encoder, decoder, and latent parts of the model
        """
        super(AutoEncoder, self).__init__()
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        self.input_dim = input_dim
        ndim = len(input_dim) - 1
        layers_per_block_e = layers_per_block
        layers_per_block_d = layers_per_block[::-1]
        encoder_channels_out = nfilters * len(layers_per_block)
        encoder_size_out = (tt(input_dim[1:]) // 2**len(layers_per_block)).tolist()

        self.encoder = FeatureExtractor(
            ndim, input_dim[0], layers_per_block_e,
            nfilters, batchnorm, activation, pool=True)
        self.features2latent = features_to_latent(
            [encoder_channels_out, *encoder_size_out], latent_dim)
        self.latent2features = latent_to_features(
            latent_dim, [encoder_channels_out, *encoder_size_out])
        self.decoder = Upsampler(
            ndim, encoder_channels_out, layers_per_block_d, input_dim[0],
            batchnorm, activation, activation_out, upsampling_mode)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Forward pass
        """
        x = self.encoder(x)
        x = self.features2latent(x)
        x = self.latent2features(x)
        x = self.decoder(x)
        return x

    def encode(self,
               x: Union[torch.Tensor, np.ndarray],
               **kwargs: int) -> np.ndarray:
        """
        Encodes new data
        """
        def _encode(xi) -> torch.Tensor:
            xi = self._2torch(xi).to(self.device)
            with torch.no_grad():
                xi = self.encoder(xi)
                xi = self.features2latent(xi)
            return xi.cpu()

        num_batches = kwargs.get("num_batches", 10)
        batch_size = len(x) // num_batches if len(x) > num_batches else len(x)
        z_all = []
        for i in range(num_batches):
            xi = x[i*batch_size:(i+1)*batch_size]
            z_i = _encode(xi)
            z_all.append(z_i)
        xi = x[(i+1)*batch_size:]
        if len(xi) > 0:
            z_i = _encode(xi)
            z_all.append(z_i)

        return torch.cat(z_all).numpy()

    def decode(self, x: Union[torch.Tensor, np.ndarray, List]) -> np.ndarray:
        """
        Decodes latent coordinate(s) to data space
        """
        x = self._2torch(x).to(self.device)
        with torch.no_grad():
            x = self.latent2features(x)
            x = self.decoder(x)
        return x.cpu().numpy()

    def decode_grid(self, d: int = 12, z1: Tuple[int] = None,
                    z2: Tuple[int] = None) -> np.ndarray:
        """
        Decodes a grid of latent coordinates to data sapce
        """
        if z1 is None:
            z1 = [-1.65, 1.65]
        if z2 is None:
            z2 = [-1.65, 1.65]
        grid_x = torch.linspace(z1[1], z1[0], d)
        grid_y = torch.linspace(z2[0], z2[1], d)
        decoded_grid = []
        for xi in grid_x:
            for yi in grid_y:
                decoded_grid.append(self.decode([xi, yi]))
        decoded_grid = np.concatenate(decoded_grid)
        return decoded_grid.reshape(-1, *self.input_dim[1:])

    @classmethod
    def _2torch(cls, x: Union[np.ndarray, List]) -> torch.Tensor:
        if isinstance(x, np.ndarray):
            x = torch.from_numpy(x).float()
        elif isinstance(x, list):
            x = tt(x).float()
        x = x.view(1, -1) if x.ndim == 1 else x
        return x
<commit_msg>Add denoising autoencoder<commit_after>"""
models.py
=========

Autoencoder and denoising autoencoder

by Maxim Ziatdinov (email: ziatdinovmax@gmail.com)
"""

from typing import List, Tuple, Union

import numpy as np
import torch
import torch.nn as nn
import torch.tensor as tt

from .nnblocks import (ConvBlock, UpsampleBlock, features_to_latent,
                       latent_to_features)

class FeatureExtractor(nn.Sequential):
    """
    Convolutional feature extractor

    Parameters
    ----------
    ndim
        Data dimensionality (1, 2, or 3).
    input_channels
        Number of input feature channels
        (Defaults to a greyscale image with a single channel)
    layers_per_block
        Number of layers in each block (Default: [1, 2, 2]).
    nfilters
        Number of convolutional filters in the first convolutional block.
        The number of filters in each consecutive block is computed as
        :math:`block_i = nfilters * (i+1)` (Default: 32).
    batchnorm
        Add batch normalization to each layer in the block (Default: False).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    pool
        Applies max-pooling operation at the end of the block (Default: True).

    Examples
    --------

    Create a feature extractor for RGB-like 2D data

    >>> layers_per_block = [2, 2, 3]
    >>> feature_extractor = FeatureExtractor(
    >>>     ndim=2, input_channels=3, layers_per_block=layers_per_block)

    Create a feature extractor for greyscale 2D data

    >>> layers_per_block = [2, 2, 3]
    >>> feature_extractor = FeatureExtractor(
    >>>     ndim=2, input_channels=1, layers_per_block=layers_per_block)
    """
    def __init__(self,
                 ndim: int,
                 input_channels: int = 1,
                 layers_per_block: List[int] = None,
                 nfilters: int = 32,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 pool: bool = True,
                 ) -> None:
        """
        Initializes feature extractor module
        """
        super(FeatureExtractor, self).__init__()
        if layers_per_block is None:
            layers_per_block = [1, 2, 2]
        for i, layers in enumerate(layers_per_block):
            in_filters = input_channels if i == 0 else nfilters * i
            block = ConvBlock(ndim, layers, in_filters, nfilters * (i+1),
                              batchnorm=batchnorm, activation=activation,
                              pool=pool)
            self.add_module("c{}".format(i), block)

class Upsampler(nn.Sequential):
    """
    Convolutional upsampler (aka 'decoder')

    Parameters
    ----------
    ndim
        Data dimensionality (1, 2, or 3).
    input_channels
        Number of input channels (convolutional filters) for the input layer.
        The number of filters in each consecutive block is computed as
        :math:`block_i = nfilters // (i+1)` (Default: 96).
    layers_per_block
        Number of layers in each block (Default: [2, 2, 1]).
    output_channels
        Number of the output channels (Deafult: 1)
    batchnorm
        Add batch normalization to each layer in the block (Default: False).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    activation_out:
        Applies sigmoid (output_channels=1) or softmax (output_channels>1)
        activation to the final convolutional layer (Default: True)
    upsampling_mode
        Upsampling mode. Select between "bilinear" and "nearest"
        (Default: bilinear for 2D, nearest for 1D and 3D).
    """
    def __init__(self,
                 ndim: int,
                 input_channels: int = 96,
                 layers_per_block: List[int] = None,
                 output_channels: int = 1,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 activation_out: bool = True,
                 upsampling_mode: str = "bilinear",
                 ) -> None:
        """
        Initializes upsampler module
        """
        super(Upsampler, self).__init__()
        if layers_per_block is None:
            layers_per_block = [2, 2, 1]
        if activation_out:
            a_out = nn.Sigmoid() if output_channels == 1 else nn.Softmax(-1)

        nfilters = input_channels
        for i, layers in enumerate(layers_per_block):
            in_filters = nfilters if i == 0 else nfilters // i
            block = ConvBlock(ndim, layers, in_filters, nfilters // (i+1),
                              batchnorm=batchnorm, activation=activation,
                              pool=False)
            self.add_module("conv_block_{}".format(i), block)
            up = UpsampleBlock(ndim, nfilters // (i+1), nfilters // (i+1),
                               mode=upsampling_mode)
            self.add_module("up_{}".format(i), up)

        out = ConvBlock(ndim, 1, nfilters // (i+1), output_channels,
                        1, 1, 0, activation=None)
        self.add_module("output_layer", out)
        if activation_out:
            self.add_module("output_activation", a_out)

class AutoEncoder(nn.Module):
    """
    Convolutional autoencoder with latent space

    Parameters
    ----------
    input_dim
        Input dimensions: (channels, length), (channels, height, width)
        or (height, width, depth).
    latent_dim
        Latent sapce dimensionality (Default: 2).
    layers_per_block
        List with the number of layers for each block of the encoder.
        The number of layers for the decoder is computed by reversing
        this list (Default: [1, 2, 2]).
    nfilters
        Number of convolutional filters in the first convolutional block
        of the encoder. The number of filters in each consecutive block
        is computed as :math:`block_i = nfilters * (i+1)`. The number of
        filters in the first layer of the decoder is equal to the number of
        filters in the last layer of the encoder and the number of filters
        in each consecutive block is computed as :math:`block_i = nfilters // (i+1)`
        (Default: 32).
    batchnorm
        Add batch normalization to each layer (Default: True).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    activation_out:
        Applies sigmoid (output_channels=1) or softmax (output_channels>1)
        activation to the final convolutional layer (Default: True).
    upsampling_mode
        Upsampling mode. Select between "bilinear" and "nearest"
        (Default: bilinear for 2D, nearest for 1D and 3D).

    Examples
    --------

    Create autoencoder for dataset with 64 x 64 greyscale images

    >>> input_dim = (1, 64, 64)  # (channles, height, width)
    >>> ae = AutoEncoder(input_dim, latent_dim=2)
    """
    def __init__(self,
                 input_dim: Tuple[int],
                 latent_dim: int = 2,
                 layers_per_block: List[int] = [1, 2, 2],
                 nfilters: int = 32,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 activation_out: bool = True,
                 upsampling_mode: str = "bilinear"
                 ) -> None:
        """
        Initializes encoder, decoder, and latent parts of the model
        """
        super(AutoEncoder, self).__init__()
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        self.input_dim = input_dim
        ndim = len(input_dim) - 1
        layers_per_block_e = layers_per_block
        layers_per_block_d = layers_per_block[::-1]
        encoder_channels_out = nfilters * len(layers_per_block)
        encoder_size_out = (tt(input_dim[1:]) // 2**len(layers_per_block)).tolist()

        self.encoder = FeatureExtractor(
            ndim, input_dim[0], layers_per_block_e,
            nfilters, batchnorm, activation, pool=True)
        self.features2latent = features_to_latent(
            [encoder_channels_out, *encoder_size_out], latent_dim)
        self.latent2features = latent_to_features(
            latent_dim, [encoder_channels_out, *encoder_size_out])
        self.decoder = Upsampler(
            ndim, encoder_channels_out, layers_per_block_d, input_dim[0],
            batchnorm, activation, activation_out, upsampling_mode)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Forward pass
        """
        x = self.encoder(x)
        x = self.features2latent(x)
        x = self.latent2features(x)
        x = self.decoder(x)
        return x

    def encode(self,
               x: Union[torch.Tensor, np.ndarray],
               **kwargs: int) -> np.ndarray:
        """
        Encodes new data
        """
        def _encode(xi) -> torch.Tensor:
            xi = self._2torch(xi).to(self.device)
            with torch.no_grad():
                xi = self.encoder(xi)
                xi = self.features2latent(xi)
            return xi.cpu()

        num_batches = kwargs.get("num_batches", 10)
        batch_size = len(x) // num_batches if len(x) > num_batches else len(x)
        z_all = []
        for i in range(num_batches):
            xi = x[i*batch_size:(i+1)*batch_size]
            z_i = _encode(xi)
            z_all.append(z_i)
        xi = x[(i+1)*batch_size:]
        if len(xi) > 0:
            z_i = _encode(xi)
            z_all.append(z_i)

        return torch.cat(z_all).numpy()

    def decode(self, x: Union[torch.Tensor, np.ndarray, List]) -> np.ndarray:
        """
        Decodes latent coordinate(s) to data space
        """
        x = self._2torch(x).to(self.device)
        with torch.no_grad():
            x = self.latent2features(x)
            x = self.decoder(x)
        return x.cpu().numpy()

    def decode_grid(self, d: int = 12, z1: Tuple[int] = None,
                    z2: Tuple[int] = None) -> np.ndarray:
        """
        Decodes a grid of latent coordinates to data sapce
        """
        if z1 is None:
            z1 = [-1.65, 1.65]
        if z2 is None:
            z2 = [-1.65, 1.65]
        grid_x = torch.linspace(z1[1], z1[0], d)
        grid_y = torch.linspace(z2[0], z2[1], d)
        decoded_grid = []
        for xi in grid_x:
            for yi in grid_y:
                decoded_grid.append(self.decode([xi, yi]))
        decoded_grid = np.concatenate(decoded_grid)
        return decoded_grid.reshape(-1, *self.input_dim[1:])

    @classmethod
    def _2torch(cls, x: Union[np.ndarray, List]) -> torch.Tensor:
        if isinstance(x, np.ndarray):
            x = torch.from_numpy(x).float()
        elif isinstance(x, list):
            x = tt(x).float()
        x = x.view(1, -1) if x.ndim == 1 else x
        return x

class DenoisingAutoEncoder(nn.Module):
    """
    Fully convolutional convolutional denosing autoencoder

    Parameters
    ----------
    n_dim
        Dimensionality (1, 2 or 3)
    layers_per_block
        List with the number of layers for each block of the encoder.
        The number of layers for the decoder is computed by reversing
        this list (Default: [1, 2, 2]).
    nfilters
        Number of convolutional filters in the first convolutional block
        of the encoder. The number of filters in each consecutive block
        is computed as :math:`block_i = nfilters * (i+1)`. The number of
        filters in the first layer of the decoder is equal to the number of
        filters in the last layer of the encoder and the number of filters
        in each consecutive block is computed as :math:`block_i = nfilters // (i+1)`
        (Default: 32).
    batchnorm
        Add batch normalization to each layer (Default: True).
    activation
        Non-linear activation: "relu", "lrelu", "tanh", "softplus", or None.
        (Default: "lrelu").
    activation_out:
        Applies sigmoid (output_channels=1) or softmax (output_channels>1)
        activation to the final convolutional layer (Default: True).
    upsampling_mode
        Upsampling mode. Select between "bilinear" and "nearest"
        (Default: bilinear for 2D, nearest for 1D and 3D).
    """
    def __init__(self,
                 ndim: Tuple[int],
                 input_channels: int = 1,
                 layers_per_block: List[int] = [1, 2, 2],
                 nfilters: int = 32,
                 batchnorm: bool = True,
                 activation: str = "lrelu",
                 activation_out: bool = True,
                 upsampling_mode: str = "bilinear"
                 ) -> None:
        """
        Initializes encoder, decoder, and latent parts of the model
        """
        super(DenoisingAutoEncoder, self).__init__()
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        layers_per_block_e = layers_per_block
        layers_per_block_d = layers_per_block[::-1]
        encoder_channels_out = nfilters * len(layers_per_block)

        self.encoder = FeatureExtractor(
            ndim, input_channels, layers_per_block_e,
            nfilters, batchnorm, activation, pool=True)
        self.decoder = Upsampler(
            ndim, encoder_channels_out, layers_per_block_d, input_channels,
            batchnorm, activation, activation_out, upsampling_mode)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Forward pass
        """
        x = self.encoder(x)
        x = self.decoder(x)
        return x

    def predict(self,
                x: Union[torch.Tensor, np.ndarray],
                **kwargs: int) -> np.ndarray:
        """
        Predicts on new data
        """
        def _predict(xi) -> torch.Tensor:
            xi = self._2torch(xi).to(self.device)
            with torch.no_grad():
                xi = self.encoder(xi)
                xi = self.decoder(xi)
            return xi.cpu()

        num_batches = kwargs.get("num_batches", 10)
        batch_size = len(x) // num_batches if len(x) > num_batches else len(x)
        z_all = []
        for i in range(num_batches):
            xi = x[i*batch_size:(i+1)*batch_size]
            z_i = _predict(xi)
            z_all.append(z_i)
        xi = x[(i+1)*batch_size:]
        if len(xi) > 0:
            z_i = _predict(xi)
            z_all.append(z_i)

        return torch.cat(z_all).numpy()

    @classmethod
    def _2torch(cls, x: Union[np.ndarray, List]) -> torch.Tensor:
        if isinstance(x, np.ndarray):
            x = torch.from_numpy(x).float()
        elif isinstance(x, list):
            x = tt(x).float()
        x = x.view(1, -1) if x.ndim == 1 else x
        return x
<|endoftext|>


<commit_before>from nevow import page, loaders, tags

from pydoctor import epydoc2stan, model
from pydoctor.nevowhtml.util import fillSlots, taglink, templatefile

def moduleSummary(modorpack):
    r = tags.li[taglink(modorpack), ' - ', epydoc2stan.doc2html(modorpack, summary=True)]
    if isinstance(modorpack, model.Package) and len(modorpack.orderedcontents) > 1:
        ul = tags.ul()
        for m in sorted(modorpack.orderedcontents,
                        key=lambda m:m.fullName()):
            if m.name != '__init__':
                ul[moduleSummary(m)]
        r[ul]
    return r

def _lckey(x):
    return x.fullName().lower()

class ModuleIndexPage(page.Element):
    filename = 'moduleIndex.html'
    docFactory = loaders.xmlfile(templatefile('summary.html'))
    def __init__(self, system):
        self.system = system
    @page.renderer
    def title(self, request, tag):
        return tag.clear()["Module Index"]
    @page.renderer
    def stuff(self, request, tag):
        r = []
        for o in self.system.rootobjects:
            r.append(moduleSummary(o))
        return tag.clear()[r]
    @page.renderer
    def heading(self, request, tag):
        return tag().clear()["Module Index"]

def findRootClasses(system):
    roots = {}
    for cls in system.objectsOfType(model.Class):
        if ' ' in cls.name:
            continue
        if cls.bases:
            for n, b in zip(cls.bases, cls.baseobjects):
                if b is None:
                    roots.setdefault(n, []).append(cls)
                elif b.system is not system:
                    roots[b.fullName()] = b
        else:
            roots[cls.fullName()] = cls
    return sorted(roots.items(), key=lambda x:x[0].lower())

def subclassesFrom(hostsystem, cls, anchors):
    r = tags.li()
    name = cls.fullName()
    if name not in anchors:
        r[tags.a(name=name)]
        anchors.add(name)
    r[taglink(cls), ' - ', epydoc2stan.doc2html(cls, summary=True)]
    scs = [sc for sc in cls.subclasses if sc.system is hostsystem and ' ' not in sc.fullName()]
    if len(scs) > 0:
        ul = tags.ul()
        for sc in sorted(scs, key=_lckey):
            ul[subclassesFrom(hostsystem, sc, anchors)]
        r[ul]
    return r

class ClassIndexPage(page.Element):
    filename = 'classIndex.html'
    docFactory = loaders.xmlfile(templatefile('summary.html'))
    def __init__(self, system):
        self.system = system
    @page.renderer
    def title(self, request, tag):
        return tag.clear()["Class Hierarchy"]
    @page.renderer
    def stuff(self, request, tag):
        t = tag
        anchors = set()
        for b, o in findRootClasses(self.system):
            if isinstance(o, model.Class):
                t[subclassesFrom(self.system, o, anchors)]
            else:
                item = tags.li[b]
                if o:
                    ul = tags.ul()
                    for sc in sorted(o, key=_lckey):
                        ul[subclassesFrom(self.system, sc, anchors)]
                    item[ul]
                t[item]
        return t
    @page.renderer
    def heading(self, request, tag):
        return tag.clear()["Class Hierarchy"]

class NameIndexPage(page.Element):
    filename = 'nameIndex.html'
    docFactory = loaders.xmlfile(templatefile('nameIndex.html'))
    def __init__(self, system):
        self.system = system

    @page.renderer
    def title(self, request, tag):
        return tag.clear()["Index Of Names"]

    @page.renderer
    def heading(self, request, tag):
        return tag.clear()["Index Of Names"]

    @page.renderer
    def index(self, request, tag):
        letter = tag.patternGenerator('letter')
        singleName = tag.patternGenerator('singleName')
        manyNames = tag.patternGenerator('manyNames')
        initials = {}
        for ob in self.system.orderedallobjects:
            initials.setdefault(ob.name[0].upper(), []).append(ob)
        for initial in sorted(initials):
            letterlinks = []
            for initial2 in sorted(initials):
                if initial == initial2:
                    letterlinks.append(initial2)
                else:
                    letterlinks.append(tags.a(href='#'+initial2)[initial2])
                letterlinks.append(' - ')
            if letterlinks:
                del letterlinks[-1]
            name2obs = {}
            for obj in initials[initial]:
                name2obs.setdefault(obj.name, []).append(obj)
            lettercontents = []
            for name in sorted(name2obs, key=lambda x:x.lower()):
                obs = sorted(name2obs[name], key=lambda x:x.fullName().lower())
                if len(obs) == 1:
                    ob, = obs
                    lettercontents.append(fillSlots(singleName,
                                                    name=ob.name,
                                                    link=taglink(ob)))
                else:
                    lettercontents.append(fillSlots(manyNames,
                                                    name=obs[0].name,
                                                    manyNames=[tags.li[taglink(ob)] for ob in obs]))

            tag[fillSlots(letter,
                          letter=initial,
                          letterlinks=letterlinks,
                          lettercontents=lettercontents)]
        return tag

class IndexPage(page.Element):
    filename = 'index.html'
    docFactory = loaders.xmlfile(templatefile('index.html'))
    def __init__(self, system):
        self.system = system
    @page.renderer
    def project_link(self, request, tag):
        if self.system.options.projecturl:
            return tags.a(href=self.system.options.projecturl)[self.system.options.projectname]
        elif self.system.options.projectname:
            return self.system.options.projectname
        else:
            return self.system.guessedprojectname
    @page.renderer
    def project(self, request, tag):
        if self.system.options.projectname:
            return self.system.options.projectname
        else:
            return self.system.guessedprojectname
    @page.renderer
    def recentChanges(self, request, tag):
        return ()
    @page.renderer
    def onlyIfOneRoot(self, request, tag):
        if len(self.system.rootobjects) != 1:
            return []
        else:
            root, = self.system.rootobjects
            return tag.clear()[
                "Start at ", taglink(root),
                ", the root ", root.kind.lower(), "."]
    @page.renderer
    def onlyIfMultipleRoots(self, request, tag):
        if len(self.system.rootobjects) == 1:
            return []
        else:
            return tag
    @page.renderer
    def roots(self, request, tag):
        item = tag.patternGenerator("item")
        r = []
        for o in self.system.rootobjects:
            r.append(fillSlots(item, root=taglink(o)))
        return tag[r]

summarypages = [ModuleIndexPage, ClassIndexPage, IndexPage, NameIndexPage]

<commit_msg>make an undocumented public objects summary page<commit_after>from nevow import page, loaders, tags

from pydoctor import epydoc2stan, model
from pydoctor.nevowhtml.util import fillSlots, taglink, templatefile

def moduleSummary(modorpack):
    r = tags.li[taglink(modorpack), ' - ', epydoc2stan.doc2html(modorpack, summary=True)]
    if isinstance(modorpack, model.Package) and len(modorpack.orderedcontents) > 1:
        ul = tags.ul()
        for m in sorted(modorpack.orderedcontents,
                        key=lambda m:m.fullName()):
            if m.name != '__init__':
                ul[moduleSummary(m)]
        r[ul]
    return r

def _lckey(x):
    return x.fullName().lower()

class ModuleIndexPage(page.Element):
    filename = 'moduleIndex.html'
    docFactory = loaders.xmlfile(templatefile('summary.html'))
    def __init__(self, system):
        self.system = system
    @page.renderer
    def title(self, request, tag):
        return tag.clear()["Module Index"]
    @page.renderer
    def stuff(self, request, tag):
        r = []
        for o in self.system.rootobjects:
            r.append(moduleSummary(o))
        return tag.clear()[r]
    @page.renderer
    def heading(self, request, tag):
        return tag().clear()["Module Index"]

def findRootClasses(system):
    roots = {}
    for cls in system.objectsOfType(model.Class):
        if ' ' in cls.name:
            continue
        if cls.bases:
            for n, b in zip(cls.bases, cls.baseobjects):
                if b is None:
                    roots.setdefault(n, []).append(cls)
                elif b.system is not system:
                    roots[b.fullName()] = b
        else:
            roots[cls.fullName()] = cls
    return sorted(roots.items(), key=lambda x:x[0].lower())

def subclassesFrom(hostsystem, cls, anchors):
    r = tags.li()
    name = cls.fullName()
    if name not in anchors:
        r[tags.a(name=name)]
        anchors.add(name)
    r[taglink(cls), ' - ', epydoc2stan.doc2html(cls, summary=True)]
    scs = [sc for sc in cls.subclasses if sc.system is hostsystem and ' ' not in sc.fullName()]
    if len(scs) > 0:
        ul = tags.ul()
        for sc in sorted(scs, key=_lckey):
            ul[subclassesFrom(hostsystem, sc, anchors)]
        r[ul]
    return r

class ClassIndexPage(page.Element):
    filename = 'classIndex.html'
    docFactory = loaders.xmlfile(templatefile('summary.html'))
    def __init__(self, system):
        self.system = system
    @page.renderer
    def title(self, request, tag):
        return tag.clear()["Class Hierarchy"]
    @page.renderer
    def stuff(self, request, tag):
        t = tag
        anchors = set()
        for b, o in findRootClasses(self.system):
            if isinstance(o, model.Class):
                t[subclassesFrom(self.system, o, anchors)]
            else:
                item = tags.li[b]
                if o:
                    ul = tags.ul()
                    for sc in sorted(o, key=_lckey):
                        ul[subclassesFrom(self.system, sc, anchors)]
                    item[ul]
                t[item]
        return t
    @page.renderer
    def heading(self, request, tag):
        return tag.clear()["Class Hierarchy"]

class NameIndexPage(page.Element):
    filename = 'nameIndex.html'
    docFactory = loaders.xmlfile(templatefile('nameIndex.html'))
    def __init__(self, system):
        self.system = system

    @page.renderer
    def title(self, request, tag):
        return tag.clear()["Index Of Names"]

    @page.renderer
    def heading(self, request, tag):
        return tag.clear()["Index Of Names"]

    @page.renderer
    def index(self, request, tag):
        letter = tag.patternGenerator('letter')
        singleName = tag.patternGenerator('singleName')
        manyNames = tag.patternGenerator('manyNames')
        initials = {}
        for ob in self.system.orderedallobjects:
            initials.setdefault(ob.name[0].upper(), []).append(ob)
        for initial in sorted(initials):
            letterlinks = []
            for initial2 in sorted(initials):
                if initial == initial2:
                    letterlinks.append(initial2)
                else:
                    letterlinks.append(tags.a(href='#'+initial2)[initial2])
                letterlinks.append(' - ')
            if letterlinks:
                del letterlinks[-1]
            name2obs = {}
            for obj in initials[initial]:
                name2obs.setdefault(obj.name, []).append(obj)
            lettercontents = []
            for name in sorted(name2obs, key=lambda x:x.lower()):
                obs = sorted(name2obs[name], key=lambda x:x.fullName().lower())
                if len(obs) == 1:
                    ob, = obs
                    lettercontents.append(fillSlots(singleName,
                                                    name=ob.name,
                                                    link=taglink(ob)))
                else:
                    lettercontents.append(fillSlots(manyNames,
                                                    name=obs[0].name,
                                                    manyNames=[tags.li[taglink(ob)] for ob in obs]))

            tag[fillSlots(letter,
                          letter=initial,
                          letterlinks=letterlinks,
                          lettercontents=lettercontents)]
        return tag

class IndexPage(page.Element):
    filename = 'index.html'
    docFactory = loaders.xmlfile(templatefile('index.html'))
    def __init__(self, system):
        self.system = system
    @page.renderer
    def project_link(self, request, tag):
        if self.system.options.projecturl:
            return tags.a(href=self.system.options.projecturl)[self.system.options.projectname]
        elif self.system.options.projectname:
            return self.system.options.projectname
        else:
            return self.system.guessedprojectname
    @page.renderer
    def project(self, request, tag):
        if self.system.options.projectname:
            return self.system.options.projectname
        else:
            return self.system.guessedprojectname
    @page.renderer
    def recentChanges(self, request, tag):
        return ()
    @page.renderer
    def onlyIfOneRoot(self, request, tag):
        if len(self.system.rootobjects) != 1:
            return []
        else:
            root, = self.system.rootobjects
            return tag.clear()[
                "Start at ", taglink(root),
                ", the root ", root.kind.lower(), "."]
    @page.renderer
    def onlyIfMultipleRoots(self, request, tag):
        if len(self.system.rootobjects) == 1:
            return []
        else:
            return tag
    @page.renderer
    def roots(self, request, tag):
        item = tag.patternGenerator("item")
        r = []
        for o in self.system.rootobjects:
            r.append(fillSlots(item, root=taglink(o)))
        return tag[r]

class UndocumentedSummaryPage(page.Element):
    filename = 'undoccedSummary.html'
    docFactory = loaders.xmlfile(templatefile('summary.html'))
    def __init__(self, system):
        self.system = system

    @page.renderer
    def title(self, request, tag):
        return tag["Summary of Undocumented Objects"]

    @page.renderer
    def heading(self, request, tag):
        return tag["Summary of Undocumented Objects"]

    @page.renderer
    def stuff(self, request, tag):
        def public(ob):
            for part in ob.fullName().split('.'):
                if part[0] == '_':
                    return False
            return True
        undoccedpublic = [o for o in self.system.orderedallobjects
                          if public(o) and o.docstring is None]
        undoccedpublic.sort(key=lambda o:o.fullName())
        for o in undoccedpublic:
            tag[tags.li[o.kind, " - ", taglink(o)]]
        return tag

summarypages = [ModuleIndexPage, ClassIndexPage, IndexPage, NameIndexPage, UndocumentedSummaryPage]

<|endoftext|>


<commit_before>import numpy as np
from numpy.testing import run_module_suite, assert_equal
from dipy.reconst.odf import (OdfFit, OdfModel, minmax_normalize)

from dipy.core.subdivide_octahedron import create_unit_hemisphere
from dipy.sims.voxel import multi_tensor, multi_tensor_odf
from dipy.data import get_sphere
from dipy.core.gradients import gradient_table, GradientTable

_sphere = create_unit_hemisphere(4)
_odf = (_sphere.vertices * [1, 2, 3]).sum(-1)
_gtab = GradientTable(np.ones((64, 3)))

class SimpleOdfModel(OdfModel):
    sphere = _sphere

    def fit(self, data):
        fit = SimpleOdfFit(self, data)
        return fit

class SimpleOdfFit(OdfFit):

    def odf(self, sphere=None):
        if sphere is None:
            sphere = self.model.sphere

        # Use ascontiguousarray to work around a bug in NumPy
        return np.ascontiguousarray((sphere.vertices * [1, 2, 3]).sum(-1))

def test_OdfFit():
    m = SimpleOdfModel(_gtab)
    f = m.fit(None)
    odf = f.odf(_sphere)
    assert_equal(len(odf), len(_sphere.theta))

def test_minmax_normalize():

    bvalue = 3000
    S0 = 1
    SNR = 100

    sphere = get_sphere('symmetric362')
    bvecs = np.concatenate(([[0, 0, 0]], sphere.vertices))
    bvals = np.zeros(len(bvecs)) + bvalue
    bvals[0] = 0
    gtab = gradient_table(bvals, bvecs)

    evals = np.array(([0.0017, 0.0003, 0.0003], [0.0017, 0.0003, 0.0003]))
    evecs = [np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
             np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]])]

    S, sticks = multi_tensor(gtab, evals, S0, angles=[(0, 0), (90, 0)],
                             fractions=[50, 50], snr=SNR)
    odf = multi_tensor_odf(sphere.vertices, [0.5, 0.5], evals, evecs)

    odf2 = minmax_normalize(odf)
    assert_equal(odf2.max(), 1)
    assert_equal(odf2.min(), 0)

    odf3 = np.empty(odf.shape)
    odf3 = minmax_normalize(odf, odf3)
    assert_equal(odf3.max(), 1)
    assert_equal(odf3.min(), 0)

def test_peaks_shm_coeff():

    SNR = 100
    S0 = 100

    _, fbvals, fbvecs = get_data('small_64D')

    from dipy.data import get_sphere

    sphere = get_sphere('symmetric724')

    bvals = np.load(fbvals)
    bvecs = np.load(fbvecs)

    gtab = gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0003],
                       [0.0015, 0.0003, 0.0003]))

    data, _ = multi_tensor(gtab, mevals, S0, angles=[(0, 0), (60, 0)],
                             fractions=[50, 50], snr=SNR)

    from dipy.reconst.shm import CsaOdfModel

    model = CsaOdfModel(gtab, 4)

    pam = peaks_from_model(model, data[None,:], sphere, .5, 45,
                           return_odf=True, return_sh=True)
    # Test that spherical harmonic coefficients return back correctly
    B = np.linalg.pinv(pam.invB)
    odf2 = np.dot(pam.shm_coeff, B)
    assert_array_almost_equal(pam.odf, odf2)
    assert_equal(pam.shm_coeff.shape[-1], 45)

    pam = peaks_from_model(model, data[None,:], sphere, .5, 45,
                           return_odf=True, return_sh=False)
    assert_equal(pam.shm_coeff, None)

    pam = peaks_from_model(model, data[None,:], sphere, .5, 45,
                           return_odf=True, return_sh=True, sh_basis_type='mrtrix')

    B = np.linalg.pinv(pam.invB)
    odf2 = np.dot(pam.shm_coeff, B)
    assert_array_almost_equal(pam.odf, odf2)

if __name__ == '__main__':

    run_module_suite()

<commit_msg>Readded good, unbroken test file<commit_after>import numpy as np
from numpy.testing import run_module_suite, assert_equal
from dipy.reconst.odf import (OdfFit, OdfModel, minmax_normalize)

from dipy.core.subdivide_octahedron import create_unit_hemisphere
from dipy.sims.voxel import multi_tensor, multi_tensor_odf
from dipy.data import get_sphere
from dipy.core.gradients import gradient_table, GradientTable

_sphere = create_unit_hemisphere(4)
_odf = (_sphere.vertices * [1, 2, 3]).sum(-1)
_gtab = GradientTable(np.ones((64, 3)))

class SimpleOdfModel(OdfModel):
    sphere = _sphere

    def fit(self, data):
        fit = SimpleOdfFit(self, data)
        return fit

class SimpleOdfFit(OdfFit):

    def odf(self, sphere=None):
        if sphere is None:
            sphere = self.model.sphere

        # Use ascontiguousarray to work around a bug in NumPy
        return np.ascontiguousarray((sphere.vertices * [1, 2, 3]).sum(-1))

def test_OdfFit():
    m = SimpleOdfModel(_gtab)
    f = m.fit(None)
    odf = f.odf(_sphere)
    assert_equal(len(odf), len(_sphere.theta))

def test_minmax_normalize():

    bvalue = 3000
    S0 = 1
    SNR = 100

    sphere = get_sphere('symmetric362')
    bvecs = np.concatenate(([[0, 0, 0]], sphere.vertices))
    bvals = np.zeros(len(bvecs)) + bvalue
    bvals[0] = 0
    gtab = gradient_table(bvals, bvecs)

    evals = np.array(([0.0017, 0.0003, 0.0003], [0.0017, 0.0003, 0.0003]))
    evecs = [np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
             np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]])]

    S, sticks = multi_tensor(gtab, evals, S0, angles=[(0, 0), (90, 0)],
                             fractions=[50, 50], snr=SNR)
    odf = multi_tensor_odf(sphere.vertices, [0.5, 0.5], evals, evecs)

    odf2 = minmax_normalize(odf)
    assert_equal(odf2.max(), 1)
    assert_equal(odf2.min(), 0)

    odf3 = np.empty(odf.shape)
    odf3 = minmax_normalize(odf, odf3)
    assert_equal(odf3.max(), 1)
    assert_equal(odf3.min(), 0)

def test_peaks_shm_coeff():

    SNR = 100
    S0 = 100

    _, fbvals, fbvecs = get_data('small_64D')

    from dipy.data import get_sphere

    sphere = get_sphere('symmetric724')

    bvals = np.load(fbvals)
    bvecs = np.load(fbvecs)

    gtab = gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0003],
                       [0.0015, 0.0003, 0.0003]))

    data, _ = multi_tensor(gtab, mevals, S0, angles=[(0, 0), (60, 0)],
                             fractions=[50, 50], snr=SNR)

    from dipy.reconst.shm import CsaOdfModel

    model = CsaOdfModel(gtab, 4)

    pam = peaks_from_model(model, data[None,:], sphere, .5, 45,
                           return_odf=True, return_sh=True)
    # Test that spherical harmonic coefficients return back correctly
    B = np.linalg.pinv(pam.invB)
    odf2 = np.dot(pam.shm_coeff, B)
    assert_array_almost_equal(pam.odf, odf2)
    assert_equal(pam.shm_coeff.shape[-1], 45)

    pam = peaks_from_model(model, data[None,:], sphere, .5, 45,
                           return_odf=True, return_sh=False)
    assert_equal(pam.shm_coeff, None)

    pam = peaks_from_model(model, data[None,:], sphere, .5, 45,
                           return_odf=True, return_sh=True, sh_basis_type='mrtrix')

    B = np.linalg.pinv(pam.invB)
    odf2 = np.dot(pam.shm_coeff, B)
    assert_array_almost_equal(pam.odf, odf2)

if __name__ == '__main__':

    run_module_suite()<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c)2012 Rackspace US, Inc.

# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import pyrax

# Used to indicate values that are lazy-loaded
class Fault(object):
    def __nonzero__(self):
        return False

FAULT = Fault()

class Container(object):
    """Represents a CloudFiles container."""
    def __init__(self, client, name, object_count=None, total_bytes=None):
        self.client = client
        self.name = name
        self.object_count = int(object_count)
        self.total_bytes = int(total_bytes)
        self._cdn_uri = FAULT
        self._cdn_ttl = FAULT
        self._cdn_ssl_uri = FAULT
        self._cdn_streaming_uri = FAULT
        self._cdn_ios_uri = FAULT
        self._cdn_log_retention = FAULT
        self._object_cache = {}

    def _set_cdn_defaults(self):
        """Sets all the CDN-related attributes to default values."""
        self._cdn_uri = None
        self._cdn_ttl = self.client.default_cdn_ttl
        self._cdn_ssl_uri = None
        self._cdn_streaming_uri = None
        self._cdn_ios_uri = None
        self._cdn_log_retention = False

    def _fetch_cdn_data(self):
        """Fetches the object's CDN data from the CDN service"""
        if not self.client.cdn_enabled:
            # Not CDN enabled; set the defaults.
            self._set_cdn_defaults()
            return
        response = self.client.connection.cdn_request("HEAD", [self.name])
        if 200 <= response.status < 300:
            # Set defaults in case not all headers are present.
            self._set_cdn_defaults()
            for hdr in response.getheaders():
                low_hdr = hdr[0].lower()
                if low_hdr == "x-cdn-uri":
                    self._cdn_uri = hdr[1]
                elif low_hdr == "x-ttl":
                    self._cdn_ttl = int(hdr[1])
                elif low_hdr == "x-cdn-ssl-uri":
                    self._cdn_ssl_uri = hdr[1]
                elif low_hdr == "x-cdn-streaming-uri":
                    self._cdn_streaming_uri = hdr[1]
                elif low_hdr == "x-cdn-ios-uri":
                    self._cdn_ios_uri = hdr[1]
                elif low_hdr == "x-log-retention":
                    self._cdn_log_retention = (hdr[1] == "True")
        elif response.status == 404:
            # Not CDN enabled; set the defaults.
            self._set_cdn_defaults()
        # We need to read the response in order to clear it for
        # the next call
        response.read()

    def get_objects(self, marker=None, limit=None, prefix=None, delimiter=None,
            full_listing=False):
        """
        Returns a list of StorageObjects representing the objects in the
        container. You can use the marker and limit params to handle pagination,
        and the prefix and delimiter params to filter the objects returned.
        Also, by default only the first 10,000 objects are returned; if you set
        full_listing to True, all objects in the container are returned.
        """
        objs = self.client.get_container_objects(self.name, marker=marker,
                limit=limit, prefix=prefix, delimiter=delimiter,
                full_listing=full_listing)
        return objs
    list = get_objects

    def get_object(self, name, cached=True):
        """
        Return the StorageObject in this container with the specified name. By
        default, if a reference to that object has already been retrieved, a
        cached reference will be returned. If you need to get an updated
        version of the object, pass `cached=False` to the method call.
        """
        if isinstance(name, str):
            name = name.decode(pyrax.get_encoding())
        ret = None
        if cached:
            ret = self._object_cache.get(name)
        if not ret:
            ret = self.client.get_object(self, name)
            self._object_cache[name] = ret
        return ret
    get = get_object

    def get_object_names(self, marker=None, limit=None, prefix=None,
            delimiter=None, full_listing=False):
        """
        Returns a list of the names of all the objects in this container. The
        same pagination parameters apply as in self.get_objects().
        """
        return self.client.get_container_object_names(self.name, marker=marker,
                limit=limit, prefix=prefix, delimiter=delimiter,
                full_listing=full_listing)
    list_object_names = get_object_names

    def list_subdirs(self, marker=None, limit=None, prefix=None, delimiter=None,
            full_listing=False):
        """
        Return a list of StorageObjects representing the pseudo-subdirectories
        in this container. You can use the marker and limit params to handle
        pagination, and the prefix and delimiter params to filter the objects
        returned.
        """
        subdirs = self.client.list_container_subdirs(self.name, marker=marker,
                limit=limit, prefix=prefix, delimiter=delimiter,
                full_listing=full_listing)
        return subdirs

    def store_object(self, obj_name, data, content_type=None, etag=None,
            content_encoding=None, ttl=None, return_none=False,
            extra_info=None):
        """
        Creates a new object in this container, and populates it with
        the given data.
        """
        return self.client.store_object(self, obj_name, data,
                content_type=content_type, etag=etag,
                content_encoding=content_encoding, ttl=ttl,
                return_none=return_none, extra_info=extra_info)

    def upload_file(self, file_or_path, obj_name=None, content_type=None,
            etag=None, return_none=False, content_encoding=None, ttl=None,
            content_length=None):
        """
        Uploads the specified file to this container. If no name is supplied,
        the file's name will be used. Either a file path or an open file-like
        object may be supplied. A StorageObject reference to the uploaded file
        will be returned, unless 'return_none' is set to True.
        """
        return self.client.upload_file(self, file_or_path, obj_name=obj_name,
                content_type=content_type, etag=etag, return_none=return_none,
                content_encoding=content_encoding, ttl=ttl,
                content_length=content_length)

    def delete_object(self, obj):
        """Deletes the specified object from this container."""
        self.remove_from_cache(obj)
        return self.client.delete_object(self, obj)

    def delete_all_objects(self, async=False):
        """
        Deletes all objects from this container.

        By default the call will block until all objects have been deleted. By
        passing True for the 'async' parameter, this method will not block, and
        instead return an object that can be used to follow the progress of the
        deletion. When deletion is complete the bulk deletion object's
        'results' attribute will be populated with the information returned
        from the API call. In synchronous mode this is the value that is
        returned when the call completes. It is a dictionary with the following
        keys:

            deleted - the number of objects deleted
            not_found - the number of objects not found
            status - the HTTP return status code. '200 OK' indicates success
            errors - a list of any errors returned by the bulk delete call
        """
        nms = self.get_object_names(full_listing=True)
        self.client.bulk_delete(self, nms, async=False)

    def remove_from_cache(self, obj):
        """Removes the object from the cache."""
        nm = self.client._resolve_name(obj)
        self._object_cache.pop(nm, None)

    def delete(self, del_objects=False):
        """
        Deletes this Container. If the container contains objects, the
        command will fail unless 'del_objects' is passed as True. In that
        case, each object will be deleted first, and then the container.
        """
        return self.client.delete_container(self.name, del_objects=del_objects)

    def fetch_object(self, obj_name, include_meta=False, chunk_size=None):
        """
        Fetches the object from storage.

        If 'include_meta' is False, only the bytes representing the
        file is returned.

        Note: if 'chunk_size' is defined, you must fully read the object's
        contents before making another request.

        When 'include_meta' is True, what is returned from this method is
        a 2-tuple:
            Element 0: a dictionary containing metadata about the file.
            Element 1: a stream of bytes representing the object's contents.
        """
        return self.client.fetch_object(self, obj_name,
                include_meta=include_meta, chunk_size=chunk_size)

    def download_object(self, obj_name, directory, structure=True):
        """
        Fetches the object from storage, and writes it to the specified
        directory. The directory must exist before calling this method.

        If the object name represents a nested folder structure, such as
        "foo/bar/baz.txt", that folder structure will be created in the target
        directory by default. If you do not want the nested folders to be
        created, pass `structure=False` in the parameters.
        """
        return self.client.download_object(self, obj_name, directory,
                structure=structure)

    def get_metadata(self, prefix=None):
        """
        Returns a dictionary containing the metadata for the container.
        """
        return self.client.get_container_metadata(self, prefix=prefix)

    def set_metadata(self, metadata, clear=False, prefix=None):
        """
        Accepts a dictionary of metadata key/value pairs and updates the
        specified container metadata with them.

        If 'clear' is True, any existing metadata is deleted and only the
        passed metadata is retained. Otherwise, the values passed here update
        the container's metadata.

        'extra_info' is an optional dictionary which will be populated with
        'status', 'reason', and 'headers' keys from the underlying swiftclient
        call.

        By default, the standard container metadata prefix
        ('X-Container-Meta-') is prepended to the header name if it isn't
        present. For non-standard headers, you must include a non-None prefix,
        such as an empty string.
        """
        return self.client.set_container_metadata(self, metadata, clear=clear,
                prefix=prefix)

    def remove_metadata_key(self, key, prefix=None):
        """
        Removes the specified key from the container's metadata. If the key
        does not exist in the metadata, nothing is done.
        """
        return self.client.remove_container_metadata_key(self, key,
                prefix=prefix)

    def set_web_index_page(self, page):
        """
        Sets the header indicating the index page for this container
        when creating a static website.

        Note: the container must be CDN-enabled for this to have
        any effect.
        """
        return self.client.set_container_web_index_page(self, page)

    def set_web_error_page(self, page):
        """
        Sets the header indicating the error page for this container
        when creating a static website.

        Note: the container must be CDN-enabled for this to have
        any effect.
        """
        return self.client.set_container_web_error_page(self, page)

    def make_public(self, ttl=None):
        """Enables CDN access for the specified container."""
        return self.client.make_container_public(self, ttl)

    def make_private(self):
        """
        Disables CDN access to this container. It may still appear public until
        its TTL expires.
        """
        return self.client.make_container_private(self)

    def copy_object(self, obj, new_container, new_obj_name=None,
            extra_info=None):
        """
        Copies the object to the new container, optionally giving it a new name.
        If you copy to the same container, you must supply a different name.
        """
        return self.client.copy_object(self, obj, new_container,
                new_obj_name=new_obj_name, extra_info=extra_info)

    def move_object(self, obj, new_container, new_obj_name=None,
            extra_info=None):
        """
        Works just like copy_object, except that the source object is deleted
        after a successful copy.
        """
        return self.client.move_object(self, obj, new_container,
                new_obj_name=new_obj_name, extra_info=extra_info)

    def change_object_content_type(self, obj, new_ctype, guess=False):
        """
        Copies object to itself, but applies a new content-type. The guess
        feature requires the container to be CDN-enabled. If not then the
        content-type must be supplied. If using guess with a CDN-enabled
        container, new_ctype can be set to None.
        Failure during the put will result in a swift exception.
        """
        self.client.change_object_content_type(self, obj, new_ctype=new_ctype,
                guess=guess)

    def get_temp_url(self, obj, seconds, method="GET"):
        """
        Returns a URL that can be used to access the specified object in this
        container. The URL will expire after `seconds` seconds.

        The only methods supported are GET and PUT. Anything else will raise
        an InvalidTemporaryURLMethod exception.
        """
        return self.client.get_temp_url(self, obj, seconds=seconds,
                method=method)

    def delete_object_in_seconds(self, obj, seconds):
        """
        Sets the object to be deleted after the specified number of seconds.
        """
        self.client.delete_object_in_seconds(self, obj, seconds)

    def __repr__(self):
        return "<Container '%s'>" % self.name

    # BEGIN - CDN property definitions ##
    @property
    def cdn_enabled(self):
        return bool(self.cdn_uri)

    def _get_cdn_log_retention(self):
        if self._cdn_log_retention is FAULT:
            self._fetch_cdn_data()
        return self._cdn_log_retention

    def _set_cdn_log_retention(self, val):
        self.client._set_cdn_log_retention(self, val)
        self._cdn_log_retention = val

    def _get_cdn_uri(self):
        if self._cdn_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_uri

    def _set_cdn_uri(self, val):
        self._cdn_uri = val

    def _get_cdn_ttl(self):
        if self._cdn_ttl is FAULT:
            self._fetch_cdn_data()
        return self._cdn_ttl

    def _set_cdn_ttl(self, val):
        self._cdn_ttl = val

    def _get_cdn_ssl_uri(self):
        if self._cdn_ssl_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_ssl_uri

    def _set_cdn_ssl_uri(self, val):
        self._cdn_ssl_uri = val

    def _get_cdn_streaming_uri(self):
        if self._cdn_streaming_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_streaming_uri

    def _set_cdn_streaming_uri(self, val):
        self._cdn_streaming_uri = val

    def _get_cdn_ios_uri(self):
        if self._cdn_ios_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_ios_uri

    def _set_cdn_ios_uri(self, val):
        self._cdn_ios_uri = val

    cdn_log_retention = property(_get_cdn_log_retention, _set_cdn_log_retention)
    cdn_uri = property(_get_cdn_uri, _set_cdn_uri)
    cdn_ttl = property(_get_cdn_ttl, _set_cdn_ttl)
    cdn_ssl_uri = property(_get_cdn_ssl_uri, _set_cdn_ssl_uri)
    cdn_streaming_uri = property(_get_cdn_streaming_uri, _set_cdn_streaming_uri)
    cdn_ios_uri = property(_get_cdn_ios_uri, _set_cdn_ios_uri)
    # END - CDN property definitions ##
<commit_msg>bulk_delete async support<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c)2012 Rackspace US, Inc.

# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import pyrax

# Used to indicate values that are lazy-loaded
class Fault(object):
    def __nonzero__(self):
        return False

FAULT = Fault()

class Container(object):
    """Represents a CloudFiles container."""
    def __init__(self, client, name, object_count=None, total_bytes=None):
        self.client = client
        self.name = name
        self.object_count = int(object_count)
        self.total_bytes = int(total_bytes)
        self._cdn_uri = FAULT
        self._cdn_ttl = FAULT
        self._cdn_ssl_uri = FAULT
        self._cdn_streaming_uri = FAULT
        self._cdn_ios_uri = FAULT
        self._cdn_log_retention = FAULT
        self._object_cache = {}

    def _set_cdn_defaults(self):
        """Sets all the CDN-related attributes to default values."""
        self._cdn_uri = None
        self._cdn_ttl = self.client.default_cdn_ttl
        self._cdn_ssl_uri = None
        self._cdn_streaming_uri = None
        self._cdn_ios_uri = None
        self._cdn_log_retention = False

    def _fetch_cdn_data(self):
        """Fetches the object's CDN data from the CDN service"""
        if not self.client.cdn_enabled:
            # Not CDN enabled; set the defaults.
            self._set_cdn_defaults()
            return
        response = self.client.connection.cdn_request("HEAD", [self.name])
        if 200 <= response.status < 300:
            # Set defaults in case not all headers are present.
            self._set_cdn_defaults()
            for hdr in response.getheaders():
                low_hdr = hdr[0].lower()
                if low_hdr == "x-cdn-uri":
                    self._cdn_uri = hdr[1]
                elif low_hdr == "x-ttl":
                    self._cdn_ttl = int(hdr[1])
                elif low_hdr == "x-cdn-ssl-uri":
                    self._cdn_ssl_uri = hdr[1]
                elif low_hdr == "x-cdn-streaming-uri":
                    self._cdn_streaming_uri = hdr[1]
                elif low_hdr == "x-cdn-ios-uri":
                    self._cdn_ios_uri = hdr[1]
                elif low_hdr == "x-log-retention":
                    self._cdn_log_retention = (hdr[1] == "True")
        elif response.status == 404:
            # Not CDN enabled; set the defaults.
            self._set_cdn_defaults()
        # We need to read the response in order to clear it for
        # the next call
        response.read()

    def get_objects(self, marker=None, limit=None, prefix=None, delimiter=None,
            full_listing=False):
        """
        Returns a list of StorageObjects representing the objects in the
        container. You can use the marker and limit params to handle pagination,
        and the prefix and delimiter params to filter the objects returned.
        Also, by default only the first 10,000 objects are returned; if you set
        full_listing to True, all objects in the container are returned.
        """
        objs = self.client.get_container_objects(self.name, marker=marker,
                limit=limit, prefix=prefix, delimiter=delimiter,
                full_listing=full_listing)
        return objs
    list = get_objects

    def get_object(self, name, cached=True):
        """
        Return the StorageObject in this container with the specified name. By
        default, if a reference to that object has already been retrieved, a
        cached reference will be returned. If you need to get an updated
        version of the object, pass `cached=False` to the method call.
        """
        if isinstance(name, str):
            name = name.decode(pyrax.get_encoding())
        ret = None
        if cached:
            ret = self._object_cache.get(name)
        if not ret:
            ret = self.client.get_object(self, name)
            self._object_cache[name] = ret
        return ret
    get = get_object

    def get_object_names(self, marker=None, limit=None, prefix=None,
            delimiter=None, full_listing=False):
        """
        Returns a list of the names of all the objects in this container. The
        same pagination parameters apply as in self.get_objects().
        """
        return self.client.get_container_object_names(self.name, marker=marker,
                limit=limit, prefix=prefix, delimiter=delimiter,
                full_listing=full_listing)
    list_object_names = get_object_names

    def list_subdirs(self, marker=None, limit=None, prefix=None, delimiter=None,
            full_listing=False):
        """
        Return a list of StorageObjects representing the pseudo-subdirectories
        in this container. You can use the marker and limit params to handle
        pagination, and the prefix and delimiter params to filter the objects
        returned.
        """
        subdirs = self.client.list_container_subdirs(self.name, marker=marker,
                limit=limit, prefix=prefix, delimiter=delimiter,
                full_listing=full_listing)
        return subdirs

    def store_object(self, obj_name, data, content_type=None, etag=None,
            content_encoding=None, ttl=None, return_none=False,
            extra_info=None):
        """
        Creates a new object in this container, and populates it with
        the given data.
        """
        return self.client.store_object(self, obj_name, data,
                content_type=content_type, etag=etag,
                content_encoding=content_encoding, ttl=ttl,
                return_none=return_none, extra_info=extra_info)

    def upload_file(self, file_or_path, obj_name=None, content_type=None,
            etag=None, return_none=False, content_encoding=None, ttl=None,
            content_length=None):
        """
        Uploads the specified file to this container. If no name is supplied,
        the file's name will be used. Either a file path or an open file-like
        object may be supplied. A StorageObject reference to the uploaded file
        will be returned, unless 'return_none' is set to True.
        """
        return self.client.upload_file(self, file_or_path, obj_name=obj_name,
                content_type=content_type, etag=etag, return_none=return_none,
                content_encoding=content_encoding, ttl=ttl,
                content_length=content_length)

    def delete_object(self, obj):
        """Deletes the specified object from this container."""
        self.remove_from_cache(obj)
        return self.client.delete_object(self, obj)

    def delete_all_objects(self, async=False):
        """
        Deletes all objects from this container.

        By default the call will block until all objects have been deleted. By
        passing True for the 'async' parameter, this method will not block, and
        instead return an object that can be used to follow the progress of the
        deletion. When deletion is complete the bulk deletion object's
        'results' attribute will be populated with the information returned
        from the API call. In synchronous mode this is the value that is
        returned when the call completes. It is a dictionary with the following
        keys:

            deleted - the number of objects deleted
            not_found - the number of objects not found
            status - the HTTP return status code. '200 OK' indicates success
            errors - a list of any errors returned by the bulk delete call
        """
        nms = self.get_object_names(full_listing=True)
        return self.client.bulk_delete(self, nms, async=async)

    def remove_from_cache(self, obj):
        """Removes the object from the cache."""
        nm = self.client._resolve_name(obj)
        self._object_cache.pop(nm, None)

    def delete(self, del_objects=False):
        """
        Deletes this Container. If the container contains objects, the
        command will fail unless 'del_objects' is passed as True. In that
        case, each object will be deleted first, and then the container.
        """
        return self.client.delete_container(self.name, del_objects=del_objects)

    def fetch_object(self, obj_name, include_meta=False, chunk_size=None):
        """
        Fetches the object from storage.

        If 'include_meta' is False, only the bytes representing the
        file is returned.

        Note: if 'chunk_size' is defined, you must fully read the object's
        contents before making another request.

        When 'include_meta' is True, what is returned from this method is
        a 2-tuple:
            Element 0: a dictionary containing metadata about the file.
            Element 1: a stream of bytes representing the object's contents.
        """
        return self.client.fetch_object(self, obj_name,
                include_meta=include_meta, chunk_size=chunk_size)

    def download_object(self, obj_name, directory, structure=True):
        """
        Fetches the object from storage, and writes it to the specified
        directory. The directory must exist before calling this method.

        If the object name represents a nested folder structure, such as
        "foo/bar/baz.txt", that folder structure will be created in the target
        directory by default. If you do not want the nested folders to be
        created, pass `structure=False` in the parameters.
        """
        return self.client.download_object(self, obj_name, directory,
                structure=structure)

    def get_metadata(self, prefix=None):
        """
        Returns a dictionary containing the metadata for the container.
        """
        return self.client.get_container_metadata(self, prefix=prefix)

    def set_metadata(self, metadata, clear=False, prefix=None):
        """
        Accepts a dictionary of metadata key/value pairs and updates the
        specified container metadata with them.

        If 'clear' is True, any existing metadata is deleted and only the
        passed metadata is retained. Otherwise, the values passed here update
        the container's metadata.

        'extra_info' is an optional dictionary which will be populated with
        'status', 'reason', and 'headers' keys from the underlying swiftclient
        call.

        By default, the standard container metadata prefix
        ('X-Container-Meta-') is prepended to the header name if it isn't
        present. For non-standard headers, you must include a non-None prefix,
        such as an empty string.
        """
        return self.client.set_container_metadata(self, metadata, clear=clear,
                prefix=prefix)

    def remove_metadata_key(self, key, prefix=None):
        """
        Removes the specified key from the container's metadata. If the key
        does not exist in the metadata, nothing is done.
        """
        return self.client.remove_container_metadata_key(self, key,
                prefix=prefix)

    def set_web_index_page(self, page):
        """
        Sets the header indicating the index page for this container
        when creating a static website.

        Note: the container must be CDN-enabled for this to have
        any effect.
        """
        return self.client.set_container_web_index_page(self, page)

    def set_web_error_page(self, page):
        """
        Sets the header indicating the error page for this container
        when creating a static website.

        Note: the container must be CDN-enabled for this to have
        any effect.
        """
        return self.client.set_container_web_error_page(self, page)

    def make_public(self, ttl=None):
        """Enables CDN access for the specified container."""
        return self.client.make_container_public(self, ttl)

    def make_private(self):
        """
        Disables CDN access to this container. It may still appear public until
        its TTL expires.
        """
        return self.client.make_container_private(self)

    def copy_object(self, obj, new_container, new_obj_name=None,
            extra_info=None):
        """
        Copies the object to the new container, optionally giving it a new name.
        If you copy to the same container, you must supply a different name.
        """
        return self.client.copy_object(self, obj, new_container,
                new_obj_name=new_obj_name, extra_info=extra_info)

    def move_object(self, obj, new_container, new_obj_name=None,
            extra_info=None):
        """
        Works just like copy_object, except that the source object is deleted
        after a successful copy.
        """
        return self.client.move_object(self, obj, new_container,
                new_obj_name=new_obj_name, extra_info=extra_info)

    def change_object_content_type(self, obj, new_ctype, guess=False):
        """
        Copies object to itself, but applies a new content-type. The guess
        feature requires the container to be CDN-enabled. If not then the
        content-type must be supplied. If using guess with a CDN-enabled
        container, new_ctype can be set to None.
        Failure during the put will result in a swift exception.
        """
        self.client.change_object_content_type(self, obj, new_ctype=new_ctype,
                guess=guess)

    def get_temp_url(self, obj, seconds, method="GET"):
        """
        Returns a URL that can be used to access the specified object in this
        container. The URL will expire after `seconds` seconds.

        The only methods supported are GET and PUT. Anything else will raise
        an InvalidTemporaryURLMethod exception.
        """
        return self.client.get_temp_url(self, obj, seconds=seconds,
                method=method)

    def delete_object_in_seconds(self, obj, seconds):
        """
        Sets the object to be deleted after the specified number of seconds.
        """
        self.client.delete_object_in_seconds(self, obj, seconds)

    def __repr__(self):
        return "<Container '%s'>" % self.name

    # BEGIN - CDN property definitions ##
    @property
    def cdn_enabled(self):
        return bool(self.cdn_uri)

    def _get_cdn_log_retention(self):
        if self._cdn_log_retention is FAULT:
            self._fetch_cdn_data()
        return self._cdn_log_retention

    def _set_cdn_log_retention(self, val):
        self.client._set_cdn_log_retention(self, val)
        self._cdn_log_retention = val

    def _get_cdn_uri(self):
        if self._cdn_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_uri

    def _set_cdn_uri(self, val):
        self._cdn_uri = val

    def _get_cdn_ttl(self):
        if self._cdn_ttl is FAULT:
            self._fetch_cdn_data()
        return self._cdn_ttl

    def _set_cdn_ttl(self, val):
        self._cdn_ttl = val

    def _get_cdn_ssl_uri(self):
        if self._cdn_ssl_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_ssl_uri

    def _set_cdn_ssl_uri(self, val):
        self._cdn_ssl_uri = val

    def _get_cdn_streaming_uri(self):
        if self._cdn_streaming_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_streaming_uri

    def _set_cdn_streaming_uri(self, val):
        self._cdn_streaming_uri = val

    def _get_cdn_ios_uri(self):
        if self._cdn_ios_uri is FAULT:
            self._fetch_cdn_data()
        return self._cdn_ios_uri

    def _set_cdn_ios_uri(self, val):
        self._cdn_ios_uri = val

    cdn_log_retention = property(_get_cdn_log_retention, _set_cdn_log_retention)
    cdn_uri = property(_get_cdn_uri, _set_cdn_uri)
    cdn_ttl = property(_get_cdn_ttl, _set_cdn_ttl)
    cdn_ssl_uri = property(_get_cdn_ssl_uri, _set_cdn_ssl_uri)
    cdn_streaming_uri = property(_get_cdn_streaming_uri, _set_cdn_streaming_uri)
    cdn_ios_uri = property(_get_cdn_ios_uri, _set_cdn_ios_uri)
    # END - CDN property definitions ##
<|endoftext|>