<commit_before>from urlparse import urlparse
from functools import wraps
import tempfile
import logging

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig
from zinc.catalog import ZincAbstractCatalog, ZincCatalogPathHelper
from zinc.formats import Formats

from zinc.defaults import defaults
from zinc.utils import *
from zinc.helpers import *

log = logging.getLogger(__name__)

################################################################################

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, coordinator=None, storage=None,
            path_helper=None, **kwargs):
        assert coordinator
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self._reload()

        self.lock_timeout = defaults['catalog_lock_timeout']

    ### Properties ###

    @property
    def url(self):
        return self._coordinator.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    ### General Internal Methods ###

    def _reload(self):

        ## TODO: check format, just assume 1 for now
        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        #log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        #config_path = pjoin(self.path, defaults['catalog_config_name'])
        #self.config = load_config(config_path)

    def _lock_index(func):
        @wraps(func)
        def with_lock_index(self, *args, **kwargs):

            #lock = self._coordinator.get_index_lock_index()
            #lock.acquire(timeout=self.lock_timeout)
            #output = func(*args, **kwargs)
            #lock.release()

            with self._coordinator.get_index_lock(domain=self.id):
                self._reload()
                output = func(self, *args, **kwargs)
                self._save()

            return output
        return with_lock_index

    def _save(self):
        self._write_index(self.index)

    ### I/O Helpers ###

    def _read(self, rel_path):
        f = self._storage.get(rel_path)
        return f.read() if f is not None else None

    def _write(self, subpath, bytes, raw=True, gzip=True):
        if raw:
            self._storage.puts(subpath, bytes)
        if gzip:
            self._storage.puts(subpath + '.gz', gzip_bytes(bytes))

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath)
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(
                bundle_name, version)
        bytes = self._read(subpath)
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                        'sha' : sha,
                        'size' : meta['size'],
                        'format' : format
                        }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        format = format or Formats.RAW # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                version, flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        return self._storage.get(subpath)

    ### "Public" Methods

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name, version):
        return self._read_manifest(bundle_name, version)

    def update_bundle(self, new_manifest):

        assert new_manifest

        ## verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        ## TODO: verify archives?

        ## write manifest

        self._write_manifest(new_manifest)

        ## update catalog index

        self._add_version_for_bundle(new_manifest.bundle_name,
                                     new_manifest.version)

    def import_path(self, src_path):

        sha = sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path) as src_file:
                tmp_file.write(gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(
                    sha, final_src_path, format=format)

        file_info = {
                'sha' :  sha,
                'size' : final_src_size,
                'format' : format
                }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return  file_info

    @_lock_index
    def _add_version_for_bundle(self, bundle_name, version):
        self.index.add_version_for_bundle(bundle_name, version)

    @_lock_index
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    @_lock_index
    def delete_bundle_version(self, bundle_name, version):
        self.index.delete_bundle_version(bundle_name, version)

    @_lock_index
    def update_distribution(self, distribution_name, bundle_name, bundle_version):
        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_lock_index
    def delete_distribution(self, distribution_name, bundle_name):
        self.index.delete_distribution(distribution_name, bundle_name)

    @_lock_index
    def clean(self, dry_run=False):
        bundle_descriptors = self.bundle_descriptors()
        verb = 'Would remove' if dry_run else 'Removing'

        ### 1. scan manifests for ones that aren't in index
        for root, dirs, files in os.walk(self._manifests_url()):
            for f in files:
                remove = False
                if not (f.endswith(".json") or f.endswith(".json.gz")):
                    # remove stray files
                    remove = True
                else:
                    bundle_descr = f.split(".")[0]
                    if bundle_descr not in bundle_descriptors:
                        remove = True
                if remove:
                    log.info("%s %s" % (verb, pjoin(root, f)))
                    if not dry_run: os.remove(pjoin(root, f))

        ### 2. scan archives for ones that aren't in index
        for root, dirs, files in os.walk(self._archives_dir()):
            for f in files:
                remove = False
                if not (f.endswith(".tar")):
                    # remove stray files
                    remove = True
                else:
                    bundle_descr = f.split(".")[0]
                    if bundle_descr not in bundle_descriptors:
                        remove = True
                if remove:
                    log.info("%s %s" % (verb, pjoin(root, f)))
                    if not dry_run: os.remove(pjoin(root, f))

        ### 3. clean objects
        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            for f, meta in manifest.files.iteritems():
                all_objects.add(meta['sha'])
        for root, dirs, files in os.walk(self._files_dir()):
            for f in files:
                basename = os.path.splitext(f)[0]
                if basename not in all_objects:
                    log.info("%s %s" % (verb, pjoin(root, f)))
                    if not dry_run: os.remove(pjoin(root, f))

################################################################################

class ZincServiceProvider(object):
    pass

class ZincServiceConsumer(object):

    def __init__(self, **kwargs):
        pass

    def create_catalog(self, id=None, loc=None):
        raise NotImplementedError()

    def get_catalog(self, loc=None, id=None):
        raise NotImplementedError()

class CustomServiceConsumer(ZincServiceConsumer):

    def __init__(self, coordinator=None, storage=None, **kwargs):
        assert coordinator
        assert storage

        self._coordinator = coordinator
        self._storage = storage

    def get_catalog(self, loc=None, id=None):
        cat_storage = self._storage.bind_to_catalog(loc=loc, id=id)
        # TODO: bind to coordinator?
        return ZincCatalog(coordinator=self._coordinator, storage=cat_storage)
<commit_msg>Locking on update_bundle and verifying manifest versions<commit_after>from urlparse import urlparse
from functools import wraps
import tempfile
import logging

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig
from zinc.catalog import ZincAbstractCatalog, ZincCatalogPathHelper
from zinc.formats import Formats

from zinc.defaults import defaults
from zinc.utils import *
from zinc.helpers import *

log = logging.getLogger(__name__)

################################################################################

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, coordinator=None, storage=None,
            path_helper=None, **kwargs):
        assert coordinator
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self._reload()

        self.lock_timeout = defaults['catalog_lock_timeout']

    ### Properties ###

    @property
    def url(self):
        return self._coordinator.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    ### General Internal Methods ###

    def _reload(self):

        ## TODO: check format, just assume 1 for now
        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        #log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        #config_path = pjoin(self.path, defaults['catalog_config_name'])
        #self.config = load_config(config_path)

    def _lock_index(func):
        @wraps(func)
        def with_lock_index(self, *args, **kwargs):

            #lock = self._coordinator.get_index_lock_index()
            #lock.acquire(timeout=self.lock_timeout)
            #output = func(*args, **kwargs)
            #lock.release()

            with self._coordinator.get_index_lock(domain=self.id):
                self._reload()
                output = func(self, *args, **kwargs)
                self._save()

            return output
        return with_lock_index

    def _save(self):
        self._write_index(self.index)

    ### I/O Helpers ###

    def _read(self, rel_path):
        f = self._storage.get(rel_path)
        return f.read() if f is not None else None

    def _write(self, subpath, bytes, raw=True, gzip=True):
        if raw:
            self._storage.puts(subpath, bytes)
        if gzip:
            self._storage.puts(subpath + '.gz', gzip_bytes(bytes))

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath)
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(
                bundle_name, version)
        bytes = self._read(subpath)
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                        'sha' : sha,
                        'size' : meta['size'],
                        'format' : format
                        }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        format = format or Formats.RAW # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                version, flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        return self._storage.get(subpath)

    ### "Public" Methods

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name, version):
        return self._read_manifest(bundle_name, version)

    @_lock_index
    def update_bundle(self, new_manifest):

        assert new_manifest

        existing_versions = self.index.versions_for_bundle(new_manifest.bundle_name)
        if new_manifest.version in existing_versions:
            raise ValueError("Bundle version already exists.")

        next_version = self.index.next_version_for_bundle(new_manifest.bundle_name)
        if new_manifest.version > next_version:
            raise ValueError("Unexpected manifest version.")

        ## verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        ## TODO: verify archives?

        ## write manifest

        self._write_manifest(new_manifest)

        ## update catalog index

        self.index.add_version_for_bundle(new_manifest.bundle_name,
                                          new_manifest.version)

    def import_path(self, src_path):

        sha = sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path) as src_file:
                tmp_file.write(gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(
                    sha, final_src_path, format=format)

        file_info = {
                'sha' :  sha,
                'size' : final_src_size,
                'format' : format
                }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return  file_info

    @_lock_index
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    @_lock_index
    def delete_bundle_version(self, bundle_name, version):
        self.index.delete_bundle_version(bundle_name, version)

    @_lock_index
    def update_distribution(self, distribution_name, bundle_name, bundle_version):
        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_lock_index
    def delete_distribution(self, distribution_name, bundle_name):
        self.index.delete_distribution(distribution_name, bundle_name)

    @_lock_index
    def clean(self, dry_run=False):
        bundle_descriptors = self.bundle_descriptors()
        verb = 'Would remove' if dry_run else 'Removing'

        ### 1. scan manifests for ones that aren't in index
        for root, dirs, files in os.walk(self._manifests_url()):
            for f in files:
                remove = False
                if not (f.endswith(".json") or f.endswith(".json.gz")):
                    # remove stray files
                    remove = True
                else:
                    bundle_descr = f.split(".")[0]
                    if bundle_descr not in bundle_descriptors:
                        remove = True
                if remove:
                    log.info("%s %s" % (verb, pjoin(root, f)))
                    if not dry_run: os.remove(pjoin(root, f))

        ### 2. scan archives for ones that aren't in index
        for root, dirs, files in os.walk(self._archives_dir()):
            for f in files:
                remove = False
                if not (f.endswith(".tar")):
                    # remove stray files
                    remove = True
                else:
                    bundle_descr = f.split(".")[0]
                    if bundle_descr not in bundle_descriptors:
                        remove = True
                if remove:
                    log.info("%s %s" % (verb, pjoin(root, f)))
                    if not dry_run: os.remove(pjoin(root, f))

        ### 3. clean objects
        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            for f, meta in manifest.files.iteritems():
                all_objects.add(meta['sha'])
        for root, dirs, files in os.walk(self._files_dir()):
            for f in files:
                basename = os.path.splitext(f)[0]
                if basename not in all_objects:
                    log.info("%s %s" % (verb, pjoin(root, f)))
                    if not dry_run: os.remove(pjoin(root, f))

################################################################################

class ZincServiceProvider(object):
    pass

class ZincServiceConsumer(object):

    def __init__(self, **kwargs):
        pass

    def create_catalog(self, id=None, loc=None):
        raise NotImplementedError()

    def get_catalog(self, loc=None, id=None):
        raise NotImplementedError()

class CustomServiceConsumer(ZincServiceConsumer):

    def __init__(self, coordinator=None, storage=None, **kwargs):
        assert coordinator
        assert storage

        self._coordinator = coordinator
        self._storage = storage

    def get_catalog(self, loc=None, id=None):
        cat_storage = self._storage.bind_to_catalog(loc=loc, id=id)
        # TODO: bind to coordinator?
        return ZincCatalog(coordinator=self._coordinator, storage=cat_storage)
<|endoftext|>


<commit_before>import numpy as np
from statsmodels.base import model
import statsmodels.base.wrapper as wrap
import pandas as pd

class DimReductionRegression(model.Model):

    def __init__(self, endog, exog, **kwargs):
        super(DimReductionRegression, self).__init__(endog, exog, **kwargs)

    def _prep(self, n_slice):

        # Sort the data by endog
        ii = np.argsort(self.endog)
        y = self.endog[ii]
        x = self.exog[ii, :]

        # Whiten the data
        x -= x.mean(0)
        covx = np.cov(x.T)
        covxr = np.linalg.cholesky(covx)
        x = np.linalg.solve(covxr, x.T).T
        self.wexog = x
        self._covxr = covxr

        # Split the data into slices
        self._split_wexog = np.array_split(x, n_slice)

class SIR(DimReductionRegression):
    """
    Sliced Inverse Regression (SIR)

    Parameters
    ----------
    endog : array-like (1d)
        The dependent variable
    exog : array-like (2d)
        The covariates

    References
    ----------
    KC Li (1991).  Sliced inverse regression for dimension reduction.
    JASA 86, 316-342.
    """

    def __init__(self, endog, exog, **kwargs):
        super(SIR, self).__init__(endog, exog, **kwargs)

    def fit(self, slice_n=20):
        """
        Estimate the EDR space.

        Parameters
        ----------
        slice_n : int
            Number of observations per slice
        """

        # Number of slices
        n_slice = self.exog.shape[0] // slice_n

        self._prep(n_slice)

        mn = [z.mean(0) for z in self._split_wexog]
        n = [z.shape[0] for z in self._split_wexog]
        mn = np.asarray(mn)
        n = np.asarray(n)
        mnc = np.cov(mn.T, fweights=n)

        a, b = np.linalg.eigh(mnc)
        jj = np.argsort(-a)
        a = a[jj]
        b = b[:, jj]
        params = np.linalg.solve(self._covxr.T, b)

        results = DimReductionResults(self, params, eigs=a)
        return DimReductionResultsWrapper(results)

class PHD(DimReductionRegression):
    """
    Principal Hessian Directions

    Parameters
    ----------
    endog : array-like (1d)
        The dependent variable
    exog : array-like (2d)
        The covariates

    References
    ----------
    KC Li (1992).  On Principal Hessian Directions for Data
    Visualization and Dimension Reduction: Another application
    of Stein's lemma. JASA 87:420.
    """

    def __init__(self, endog, exog, **kwargs):
        super(PHD, self).__init__(endog, exog, **kwargs)

    def fit(self, resid=False):
        """
        Estimate the EDR space using PHD.

        Parameters
        ----------
        resid : bool
            If True, use least squares regression to remove the
            linear relationship between each covariate and the
            response, before conducting PHD.
        """

        y = self.endog - self.endog.mean()
        x = self.exog - self.exog.mean(0)

        if resid:
            from statsmodels.regression.linear_model import OLS
            r = OLS(y, x).fit()
            y = r.resid

        cm = np.einsum('i,ij,ik->jk', y, x, x)
        cm /= len(y)

        cx = np.cov(x.T)
        cb = np.linalg.solve(cx, cm)

        a, b = np.linalg.eig(cb)
        jj = np.argsort(-np.abs(a))
        a = a[jj]
        params = b[:, jj]

        results = DimReductionResults(self, params, eigs=a)
        return DimReductionResultsWrapper(results)

class SAVE(DimReductionRegression):
    """
    Sliced Average Variance Estimation (SAVE)

    Parameters
    ----------
    endog : array-like (1d)
        The dependent variable
    exog : array-like (2d)
        The covariates

    Keyword parameters
    ------------------
    bc : bool
        If True, use the bias-correctedCSAVE method of Li and Zhu.

    References
    ----------
    RD Cook.  SAVE: A method for dimension reduction and graphics
    in regression.
    http://www.stat.umn.edu/RegGraph/RecentDev/save.pdf

    Y Li, L-X Zhu (2007). Asymptotics for sliced average
    variance estimation.  The Annals of Statistics.
    https://arxiv.org/pdf/0708.0462.pdf
    """

    def __init__(self, endog, exog, **kwargs):
        super(SAVE, self).__init__(endog, exog, **kwargs)

        self.bc = False
        if "bc" in kwargs.keys() and kwargs["bc"] == True:
            self.bc = True

    def fit(self, slice_n=50):
        """
        Estimate the EDR space.

        Parameters
        ----------
        slice_n : int
            Number of observations per slice
        """

        # Number of slices
        n_slice = self.exog.shape[0] // slice_n

        self._prep(n_slice)

        cv = [np.cov(z.T) for z in self._split_wexog]
        n = [z.shape[0] for z in self._split_wexog]

        p = self.wexog.shape[1]

        if not self.bc:
            # Cook's original approach
            vm = 0
            for i in range(len(cv)):
                icv = np.eye(p) - cv[i]
                vm += n[i] * np.dot(icv, icv)
            vm /= len(cv)
        else:
            # The bias-corrected approach of Li and Zhu

            # \Lambda_n in Li, Zhu
            av = 0
            for c in cv:
                av += np.dot(c, c)
            av /= len(cv)

            # V_n in Li, Zhu
            vn = 0
            for j, x in enumerate(self._split_wexog):
                r = x - x.mean(0)
                for i in range(r.shape[0]):
                    u = r[i, :]
                    m = np.outer(u, u)
                    vn += np.dot(m, m)
            vn /= self.exog.shape[0]

            c = np.mean(n)
            k1 = c * (c - 1) / ((c - 1)**2 + 1)
            k2 = (c - 1) / ((c - 1)**2 + 1)
            av2 = k1 * av - k2 * vn

            vm = np.eye(p) - 2 * sum(cv) / len(cv) + av2

        a, b = np.linalg.eigh(vm)
        jj = np.argsort(-a)
        a = a[jj]
        b = b[:, jj]
        params = np.linalg.solve(self._covxr.T, b)

        results = DimReductionResults(self, params, eigs=a)
        return DimReductionResultsWrapper(results)

class DimReductionResults(model.Results):

    def __init__(self, model, params, eigs):
        super(DimReductionResults, self).__init__(
              model, params)
        self.eigs = eigs

class DimReductionResultsWrapper(wrap.ResultsWrapper):
    _attrs = {
        'params': 'columns',
    }
    _wrap_attrs = _attrs

wrap.populate_wrapper(DimReductionResultsWrapper,
                      DimReductionResults)
<commit_msg>Fix linting errors<commit_after>import numpy as np
from statsmodels.base import model
import statsmodels.base.wrapper as wrap

class DimReductionRegression(model.Model):

    def __init__(self, endog, exog, **kwargs):
        super(DimReductionRegression, self).__init__(endog, exog, **kwargs)

    def _prep(self, n_slice):

        # Sort the data by endog
        ii = np.argsort(self.endog)
        x = self.exog[ii, :]

        # Whiten the data
        x -= x.mean(0)
        covx = np.cov(x.T)
        covxr = np.linalg.cholesky(covx)
        x = np.linalg.solve(covxr, x.T).T
        self.wexog = x
        self._covxr = covxr

        # Split the data into slices
        self._split_wexog = np.array_split(x, n_slice)

class SIR(DimReductionRegression):
    """
    Sliced Inverse Regression (SIR)

    Parameters
    ----------
    endog : array-like (1d)
        The dependent variable
    exog : array-like (2d)
        The covariates

    References
    ----------
    KC Li (1991).  Sliced inverse regression for dimension reduction.
    JASA 86, 316-342.
    """

    def __init__(self, endog, exog, **kwargs):
        super(SIR, self).__init__(endog, exog, **kwargs)

    def fit(self, **kwargs):
        """
        Estimate the EDR space.

        Optional keyword parameters
        ---------------------------
        slice_n : int
            Number of observations per slice
        """

        # Sample size per slice
        slice_n = kwargs.get("slice_n", 20)

        # Number of slices
        n_slice = self.exog.shape[0] // slice_n

        self._prep(n_slice)

        mn = [z.mean(0) for z in self._split_wexog]
        n = [z.shape[0] for z in self._split_wexog]
        mn = np.asarray(mn)
        n = np.asarray(n)
        mnc = np.cov(mn.T, fweights=n)

        a, b = np.linalg.eigh(mnc)
        jj = np.argsort(-a)
        a = a[jj]
        b = b[:, jj]
        params = np.linalg.solve(self._covxr.T, b)

        results = DimReductionResults(self, params, eigs=a)
        return DimReductionResultsWrapper(results)

class PHD(DimReductionRegression):
    """
    Principal Hessian Directions

    Parameters
    ----------
    endog : array-like (1d)
        The dependent variable
    exog : array-like (2d)
        The covariates

    References
    ----------
    KC Li (1992).  On Principal Hessian Directions for Data
    Visualization and Dimension Reduction: Another application
    of Stein's lemma. JASA 87:420.
    """

    def __init__(self, endog, exog, **kwargs):
        super(PHD, self).__init__(endog, exog, **kwargs)

    def fit(self, **kwargs):
        """
        Estimate the EDR space using PHD.

        Optional keyword parameters
        ---------------------------
        resid : bool
            If True, use least squares regression to remove the
            linear relationship between each covariate and the
            response, before conducting PHD.
        """

        resid = kwargs.get("resid", False)

        y = self.endog - self.endog.mean()
        x = self.exog - self.exog.mean(0)

        if resid:
            from statsmodels.regression.linear_model import OLS
            r = OLS(y, x).fit()
            y = r.resid

        cm = np.einsum('i,ij,ik->jk', y, x, x)
        cm /= len(y)

        cx = np.cov(x.T)
        cb = np.linalg.solve(cx, cm)

        a, b = np.linalg.eig(cb)
        jj = np.argsort(-np.abs(a))
        a = a[jj]
        params = b[:, jj]

        results = DimReductionResults(self, params, eigs=a)
        return DimReductionResultsWrapper(results)

class SAVE(DimReductionRegression):
    """
    Sliced Average Variance Estimation (SAVE)

    Parameters
    ----------
    endog : array-like (1d)
        The dependent variable
    exog : array-like (2d)
        The covariates

    Keyword parameters
    ------------------
    bc : bool
        If True, use the bias-correctedCSAVE method of Li and Zhu.

    References
    ----------
    RD Cook.  SAVE: A method for dimension reduction and graphics
    in regression.
    http://www.stat.umn.edu/RegGraph/RecentDev/save.pdf

    Y Li, L-X Zhu (2007). Asymptotics for sliced average
    variance estimation.  The Annals of Statistics.
    https://arxiv.org/pdf/0708.0462.pdf
    """

    def __init__(self, endog, exog, **kwargs):
        super(SAVE, self).__init__(endog, exog, **kwargs)

        self.bc = False
        if "bc" in kwargs.keys() and kwargs["bc"] == True:
            self.bc = True

    def fit(self, **kwargs):
        """
        Estimate the EDR space.

        Optional keyword arguments
        --------------------------
        slice_n : int
            Number of observations per slice
        """

        # Sample size per slice
        slice_n = kwargs.get("slice_n", 50)

        # Number of slices
        n_slice = self.exog.shape[0] // slice_n

        self._prep(n_slice)

        cv = [np.cov(z.T) for z in self._split_wexog]
        ns = [z.shape[0] for z in self._split_wexog]

        p = self.wexog.shape[1]

        if not self.bc:
            # Cook's original approach
            vm = 0
            for w, cvx in zip(ns, cv):
                icv = np.eye(p) - cvx
                vm += w * np.dot(icv, icv)
            vm /= len(cv)
        else:
            # The bias-corrected approach of Li and Zhu

            # \Lambda_n in Li, Zhu
            av = 0
            for c in cv:
                av += np.dot(c, c)
            av /= len(cv)

            # V_n in Li, Zhu
            vn = 0
            for x in self._split_wexog:
                r = x - x.mean(0)
                for i in range(r.shape[0]):
                    u = r[i, :]
                    m = np.outer(u, u)
                    vn += np.dot(m, m)
            vn /= self.exog.shape[0]

            c = np.mean(ns)
            k1 = c * (c - 1) / ((c - 1)**2 + 1)
            k2 = (c - 1) / ((c - 1)**2 + 1)
            av2 = k1 * av - k2 * vn

            vm = np.eye(p) - 2 * sum(cv) / len(cv) + av2

        a, b = np.linalg.eigh(vm)
        jj = np.argsort(-a)
        a = a[jj]
        b = b[:, jj]
        params = np.linalg.solve(self._covxr.T, b)

        results = DimReductionResults(self, params, eigs=a)
        return DimReductionResultsWrapper(results)

class DimReductionResults(model.Results):

    def __init__(self, model, params, eigs):
        super(DimReductionResults, self).__init__(
              model, params)
        self.eigs = eigs

class DimReductionResultsWrapper(wrap.ResultsWrapper):
    _attrs = {
        'params': 'columns',
    }
    _wrap_attrs = _attrs

wrap.populate_wrapper(DimReductionResultsWrapper,
                      DimReductionResults)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse, os,sys
from numpy import *
import numpy as np
from scipy.special import gamma
from scipy.special import beta as f_beta
import platform, time
import csv
from scipy.special import gdtr, gdtrix
from scipy.special import betainc
import scipy.stats
np.set_printoptions(suppress=True)
np.set_printoptions(precision=3)
self_path=os.getcwd()

# make up data
s = np.random.uniform(1, 25, 10) # random speciation times
e = np.random.uniform(0, s,  10) # random extinction times

#OH# leaving the function below as a ref. but changed the calls below to the "new" BDwelik ('w' stands for weibull and 'e' extinction)
# BD likelihood (constant rate)
def BDlik (l, m):
	sp_events = np.ones(len(s))  # define speciation events
	ex_events = np.zeros(len(e)) # define extinction events
	ex_events[e>0] = 1           # ex_events = 0 for extant taxa, ex_events=1 for extinct taxa
	birth_lik = log(l)*sp_events - l*(s-e) # vector likelihoods for each species
	death_lik = log(m)*ex_events - m*(s-e) #OH# why do you still have the second part for extant taxa here?
	species_lik = birth_lik + death_lik
	lik = sum(species_lik)
	return lik

#OH# Weibull log PDF and CDF functions
def log_wei_pdf(x,scale,shape):
	# Log of Weibull pdf
	log_wei_pdf = log(shape/scale) + (k-1)*log(x/scale) - (x/scale)**shape
	return log_wei_pdf

def wei_cdf(x,scale,shape):
	# Weibull cdf
	log_wei_cdf = 1 - exp(-(x/scale)**shape)
	return wei_cdf

# BDwe likelihood (constant speciation rate and age dependent weibull extinction)
def BDwelik (l, m, shape, scale):
	sp_events = np.ones(len(s))  # define speciation events
	ex_events = np.zeros(len(e)) # define extinction events
	ex_events[e>0] = 1           # ex_events = 0 for extant taxa, ex_events=1 for extinct taxa
	birth_lik = log(l)*sp_events - l*(s-e) # vector likelihoods for each species
	#OH# now following the log of PDF for Weibull, when x>=0, which is our case...
	death_lik = log(m) + log_wei_pdf(e,scale,shape) - m*wei_cdf(s-e,scale,shape)
	#OH# the above should be a already the sum of all species... given we take a plot of a prod log(pod(...) = sum(log(...)) see equation on tex file.
	#OH# Daniele, I am not sure if I am doing this right... I am just guessing from what I studied from your code.... maybe there is
	#also a way to do this without basic arithmetic operators? I am doing weibull first so that you can see if you think I did It
	#correcly before I start changing much (p.s. I preserved the BDlik above)
	#also I am not use of multipying the first part of the operation by the ex_events... I see the point of excluding this first part
	#for the extant taxa on the exponential case.. but here?
	#sorry to bombard with comments, you can reply by e-mail. I am not sure if I am confusing PDF with CDF here too
	species_lik = birth_lik + death_lik
	lik = sum(species_lik)
	return lik

# prior
#OH# should this also be changed?
def prior_gamma(L,a=2,b=2): return sum(scipy.stats.gamma.logpdf(L, a, scale=1./b,loc=0))

# function to update parameters
def update_multiplier_proposal(i,d=1.2,f=1):
	S=shape(i)
	u = np.random.uniform(0,1,S)
	l = 2*log(d)
	m = exp(l*(u-.5))
 	ii = i * m
	U=sum(log(m))
	return ii, U

# create log file
logfile = open("mcmc.log" , "wb")
wlog=csv.writer(logfile, delimiter='\t')

#OH# head = ["it","post","lik","prior","l","m"]
head = ["it", "post", "lik", "prior", "l", "m", "shape", "scale"]
wlog.writerow(head)
logfile.flush()

iteration =0
sampling_freq =10
# init parameters
lA = 0.5
mA = 0.1
shapeA = 0.1 #OH# proposition of parameter, here starting with strong age-dependency, mainly high likelyhood for younger species
scaleA = 0.1 #OH# proposition of parameter

while True:
	# update parameters
	if np.random.random() >0.5:
		l, hastings = update_multiplier_proposal(lA)
		m = mA
		shape = shapeA
		scale = scaleA
	else:
		m, hastings = update_multiplier_proposal(mA)
		shape, hastings = update_multiplier_proposal(shapeA)
		scale, hastings = update_multiplier_proposal(scaleA)
		l = lA

	# calc lik
	#OH# lik = BDlik(l, m)
	lik = BDwelik(l, m, shape, scale)

	# calc priors
	#OH# prior = prior_gamma(l) + prior_gamma(m)
	prior = prior_gamma(l) + prior_gamma(shape) + prior_gamma(scale)

	if iteration ==0:
		likA = lik
		priorA = prior

	posterior_ratio = (prior+lik) - (priorA+likA)

	if posterior_ratio + hastings > log(np.random.random()):
		# accept state
		likA = lik
		priorA = prior
		lA = l
		mA = m
		shapeA = shape
		scaleA = scale

	if iteration % 100 ==0:
		#OH# print likA, priorA, lA, mA
		print likA, priorA, lA, mA, shapeA, scaleA

	if iteration % sampling_freq ==0:
		#OH# log_state=[iteration,likA+priorA,likA,priorA,lA,mA]
		log_state=[iteration,likA+priorA,likA,priorA,lA,mA,shapeA,scaleA]
		wlog.writerow(log_state)
		logfile.flush()

	iteration +=1
	if iteration==10000: break

<commit_msg>Update TimeDepExtinction.py<commit_after>#!/usr/bin/env python
import argparse, os,sys
from numpy import *
import numpy as np
from scipy.special import gamma
from scipy.special import beta as f_beta
import platform, time
import csv
from scipy.special import gdtr, gdtrix
from scipy.special import betainc
import scipy.stats
np.set_printoptions(suppress=True)
np.set_printoptions(precision=3)
self_path=os.getcwd()

# make up data
s = np.random.uniform(1, 25, 10) # random speciation times
e = np.random.uniform(0, s,  10) # random extinction times

#OH# leaving the function below as a reference but changed the calls below to the "new" BDwelik ('w' stands for weibull and 'e' extinction)
# BD likelihood (constant rate)
def BDlik (l, m):
	sp_events = np.ones(len(s))  # define speciation events
	ex_events = np.zeros(len(e)) # define extinction events
	ex_events[e>0] = 1           # ex_events = 0 for extant taxa, ex_events=1 for extinct taxa
	birth_lik = log(l)*sp_events - l*(s-e) # vector likelihoods for each species
	death_lik = log(m)*ex_events - m*(s-e) #OH# why do you still have the second part for extant taxa here?
	species_lik = birth_lik + death_lik
	lik = sum(species_lik)
	return lik

#OH# Weibull log PDF and CDF functions
def log_wei_pdf(x,scale,shape):
	# Log of Weibull pdf
	log_wei_pdf = log(shape/scale) + (k-1)*log(x/scale) - (x/scale)**shape
	return log_wei_pdf

def wei_cdf(x,scale,shape):
	# Weibull cdf
	log_wei_cdf = 1 - exp(-(x/scale)**shape)
	return wei_cdf

#OH# BDwe likelihood (constant speciation rate and age dependent weibull extinction)
def BDwelik (l, m, shape, scale):
	sp_events = np.ones(len(s))  # define speciation events
	ex_events = np.zeros(len(e)) # define extinction events
	ex_events[e>0] = 1           # ex_events = 0 for extant taxa, ex_events=1 for extinct taxa
	birth_lik = log(l)*sp_events - l*(s-e) # vector likelihoods for each species
	#OH# now following the log of PDF for Weibull, when x>=0, which is our case...
	death_lik = log(m)*ex_events + log_wei_pdf(e,scale,shape) - m*wei_cdf(s-e,scale,shape)
	species_lik = birth_lik + death_lik
	lik = sum(species_lik)
	return lik

# prior
#OH# should this also be changed?
def prior_gamma(L,a=2,b=2): return sum(scipy.stats.gamma.logpdf(L, a, scale=1./b,loc=0))

# function to update parameters
def update_multiplier_proposal(i,d=1.2,f=1):
	S=shape(i)
	u = np.random.uniform(0,1,S)
	l = 2*log(d)
	m = exp(l*(u-.5))
 	ii = i * m
	U=sum(log(m))
	return ii, U

# create log file
logfile = open("mcmc.log" , "wb")
wlog=csv.writer(logfile, delimiter='\t')
head = ["it","post","lik","prior","l","m","shape","scale"]
wlog.writerow(head)
logfile.flush()

iteration =0
sampling_freq =10
# init parameters
lA = 0.5
mA = 0.1
shapeA = 0.1 #OH# proposition of parameter, here starting with strong age-dependency, mainly high likelyhood for younger species
scaleA = 0.1 #OH# proposition of parameter

while True:
	# update parameters
	if np.random.random() >0.5:
		l, hastings = update_multiplier_proposal(lA)
		m = mA
		shape = shapeA
		scale = scaleA
	else:
		m, hastings = update_multiplier_proposal(mA)
		shape, hastings = update_multiplier_proposal(shapeA)
		scale, hastings = update_multiplier_proposal(scaleA)
		l = lA

	# calc lik
	lik = BDwelik(l, m, shape, scale)

	# calc priors
	prior = prior_gamma(l) + prior_gamma(m) + prior_gamma(shape) + prior_gamma(scale)

	if iteration ==0:
		likA = lik
		priorA = prior

	posterior_ratio = (prior+lik) - (priorA+likA)

	if posterior_ratio + hastings > log(np.random.random()):
		# accept state
		likA = lik
		priorA = prior
		lA = l
		mA = m
		shapeA = shape
		scaleA = scale

	if iteration % 100 ==0:
		print likA, priorA, lA, mA, shapeA, scaleA

	if iteration % sampling_freq ==0:
		log_state=[iteration,likA+priorA,likA,priorA,lA,mA,shapeA,scaleA]
		wlog.writerow(log_state)
		logfile.flush()

	iteration +=1
	if iteration==10000: break

<|endoftext|>


<commit_before><commit_msg>[FIX] core: only check table existance in public schema<commit_after><|endoftext|>


<commit_before>#
# Copyright (c) 2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
from conary.repository import netclient

# this returns the same server for any server name or label
# requested; because a shim can only refer to one server.
class FakeServerCache:
    def __init__(self, server):
        self._server = server

    def __getitem__(self, item):
        return self._server

class ShimNetClient(netclient.NetworkRepositoryClient):
    """
    A subclass of NetworkRepositoryClient which can take a NetworkRepositoryServer
    instance (plus a few other pieces of information) and expose the netclient
    interface without the overhead of XMLRPC.
    """
    def __init__(self, server, protocol, port, authToken, repMap):
        netclient.NetworkRepositoryClient.__init__(self, repMap)
        proxy = ShimServerProxy(server, protocol, port, authToken)
        self.c = FakeServerCache(proxy)

class _ShimMethod(netclient._Method):
    def __init__(self, server, protocol, port, authToken, name):
        self._server = server
        self._authToken = authToken
        self._name = name
        self._protocol = protocol
        self._port = port

    def __repr__(self):
        return "<server._ShimMethod(%r)>" % (self._name)

    def __call__(self, *args):
        args = [netclient.CLIENT_VERSIONS[-1]] + list(args)
        isException, result = self._server.callWrapper(
            self._protocol, self._port,
            self._name, self._authToken, args)

        if not isException:
            return result
        else:
            self.handleError(result)

class ShimServerProxy(netclient.ServerProxy):
    def __init__(self, server, protocol, port, authToken):
        self._authToken = authToken
        self._server = server
        self._protocol = protocol
        self._port = port

    def __getattr__(self, name):
        return _ShimMethod(self._server,
            self._protocol, self._port,
            self._authToken, name)
<commit_msg>revert accidental commit<commit_after>#
# Copyright (c) 2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
from conary.repository import netclient

# this returns the same server for any server name or label
# requested; because a shim can only refer to one server.
class FakeServerCache:
    def __init__(self, server):
        self._server = server

    def __getitem__(self, item):
        return self._server

class ShimNetClient(netclient.NetworkRepositoryClient):
    """
    A subclass of NetworkRepositoryClient which can take a NetworkRepositoryServer
    instance (plus a few other pieces of information) and expose the netclient
    interface without the overhead of XMLRPC.
    """
    def __init__(self, server, protocol, port, authToken, repMap, userMap):
        netclient.NetworkRepositoryClient.__init__(self, repMap, userMap)
        proxy = ShimServerProxy(server, protocol, port, authToken)
        self.c = FakeServerCache(proxy)

class _ShimMethod(netclient._Method):
    def __init__(self, server, protocol, port, authToken, name):
        self._server = server
        self._authToken = authToken
        self._name = name
        self._protocol = protocol
        self._port = port

    def __repr__(self):
        return "<server._ShimMethod(%r)>" % (self._name)

    def __call__(self, *args):
        args = [netclient.CLIENT_VERSIONS[-1]] + list(args)
        isException, result = self._server.callWrapper(
            self._protocol, self._port,
            self._name, self._authToken, args)

        if not isException:
            return result
        else:
            self.handleError(result)

class ShimServerProxy(netclient.ServerProxy):
    def __init__(self, server, protocol, port, authToken):
        self._authToken = authToken
        self._server = server
        self._protocol = protocol
        self._port = port

    def __getattr__(self, name):
        return _ShimMethod(self._server,
            self._protocol, self._port,
            self._authToken, name)
<|endoftext|>


<commit_before>"""Usage of HDLModule as decorator."""

from hdltools.abshdl.module import HDLModule, input_port, output_port
from hdltools.hdllib.patterns import ParallelBlock, FSM
from hdltools.abshdl.signal import HDLSignal
from hdltools.abshdl.highlvl import HDLBlock
from hdltools.verilog.codegen import VerilogCodeGenerator

class TestFSM(FSM):
    """Test FSM."""

    def __state_zero(self, start):
        if start == 1:
            done_reg = 0
            self.state = "one"

    def __state_one(self, counter):
        counter = counter + 1
        if counter == 4:
            self.state = "two"

    def __state_two(self):
        done_reg = 1
        self.state = "zero"

if __name__ == "__main__":

    HDLBlock.add_custom_block(TestFSM)

    @HDLModule(
        "fsm",
        ports=[
            input_port("clk"),
            input_port("rst"),
            input_port("start"),
            output_port("done"),
        ],
    )
    def fsm_module(mod, counter_size=8):
        """FSM Module."""
        # signals
        mod.add(
            [
                HDLSignal("reg", "state", size="defer"),
                HDLSignal("reg", "done_reg"),
                HDLSignal("reg", "counter", size=counter_size),
            ]
        )

        @HDLBlock(mod)
        @ParallelBlock()
        def fsm_body(clk, rst, done_reg, done, state):
            done = done_reg
            # sequential block generation

            @TestFSM(clk, rst, state, initial="zero")
            def myfsm():
                pass

        # add generated body to module
        mod.extend(*fsm_body())

    # generate module
    # FIXME doesnt work with keyword arguments
    fsm = fsm_module(16)

    print(fsm.dumps())

    # generate verilog code
    gen = VerilogCodeGenerator(indent=True)
    print(gen.dump_element(fsm))
<commit_msg>pass inputs to FSM into scope<commit_after>"""Usage of HDLModule as decorator."""

from hdltools.abshdl.module import HDLModule, input_port, output_port
from hdltools.hdllib.patterns import ParallelBlock, FSM
from hdltools.abshdl.signal import HDLSignal
from hdltools.abshdl.highlvl import HDLBlock
from hdltools.verilog.codegen import VerilogCodeGenerator

class TestFSM(FSM):
    """Test FSM."""

    def __state_zero(self, start):
        if start == 1:
            done_reg = 0
            self.state = "one"

    def __state_one(self, counter):
        counter = counter + 1
        if counter == 4:
            self.state = "two"

    def __state_two(self):
        done_reg = 1
        self.state = "zero"

if __name__ == "__main__":

    HDLBlock.add_custom_block(TestFSM)

    @HDLModule(
        "fsm",
        ports=[
            input_port("clk"),
            input_port("rst"),
            input_port("start"),
            output_port("done"),
        ],
    )
    def fsm_module(mod, counter_size=8):
        """FSM Module."""
        # signals
        mod.add(
            [
                HDLSignal("reg", "state", size="defer"),
                HDLSignal("reg", "done_reg"),
                HDLSignal("reg", "counter", size=counter_size),
            ]
        )

        @HDLBlock(mod)
        @ParallelBlock()
        def fsm_body(clk, rst, done_reg, done, state, start, counter):
            done = done_reg
            # sequential block generation

            @TestFSM(clk, rst, state, initial="zero")
            def myfsm():
                pass

        # add generated body to module
        mod.extend(*fsm_body())

    # generate module
    # FIXME doesnt work with keyword arguments
    fsm = fsm_module(16)

    print(fsm.dumps())

    # generate verilog code
    gen = VerilogCodeGenerator(indent=True)
    print(gen.dump_element(fsm))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

# from django.core.urlresolvers import reverse

from django.contrib.auth.mixins import LoginRequiredMixin

from django.shortcuts import redirect, render, render_to_response
from django.views.generic import DetailView, TemplateView, ListView, CreateView, UpdateView
from django.urls import reverse_lazy
from django.http import HttpResponseForbidden, Http404
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.template.context_processors import csrf

from .models import Listing, HousingUser, User
from .forms import ListingForm, HousingUserCreationForm

# index View
def index(request):
    listings = Listing.objects.filter(**get_queryset(request)).order_by(
        "-datetime_modified")

    # pagination
    paginator = Paginator(listings, 2) #listing per page
    page = request.GET.get("page")

    try:
        listings = paginator.page(page)
    except PageNotAnInteger:
        listings = paginator.page(1)
    except EmptyPage:
        listings = paginator.page(1)

    # store queryset, pass for pagination
    queries_without_page = request.GET.copy()
    if "page" in queries_without_page:
        del queries_without_page["page"]

    return render(request, "listings/index.html",
        {"listings": listings, "queries_without_page": queries_without_page})

# get querysets using GET
def get_queryset(request):
    queryset = {"is_active": True}

    if request.GET.get("price"):
        queryset["lease_monthly_cost__gte"] = request.GET.get("price")

    if request.GET.get("type") in ["house", "apartment", "homestay"]:
        queryset["property_type"] = request.GET.get("type")

    if request.GET.get("beds"):
        queryset["bedroom_count__gte"] = request.GET.get("beds")

    return queryset

class ListingDetail(LoginRequiredMixin, DetailView):
    model = Listing
    template_name = "listings/detail.html"
    context_object_name = "listing"
    pk_url_kwarg = "listing_id"

    def user_passes_test(self, request):
        if request.user.is_authenticated():
            self.object = self.get_object()
            return self.object.listing_owner == HousingUser.objects.get(
                id=self.request.user.pk)
        return False

    def dispatch(self, request, *args, **kwargs):
        if not self.user_passes_test(request):
            return HttpResponseForbidden()
        else:
            return super(ListingDetail, self).dispatch(request, *args, **kwargs)

# registration view
def register(request):
    if request.method == "POST":
        form = HousingUserCreationForm(request.POST)
        if form.is_valid():
            user = form.save()
            user.email = user.username
            user.set_password(request.POST.get("password"))
            form.save()
            return redirect("/accounts/register/complete")
    # if a GET (or any other method), create a blank form
    else:
        form = HousingUserCreationForm()
    token = {}
    token.update(csrf(request))
    token["form"] = form

    return render_to_response("accounts/registration_form.html", token)

class RegistrationCompleteView(LoginRequiredMixin, TemplateView):
    template_name = "accounts/registration_complete.html"

class ProfileView(LoginRequiredMixin, ListView):
    model = HousingUser
    template_name = "accounts/profile.html"
    context_object_name = "listings"

    # listings belong to the user
    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(ProfileView, self).get_context_data(**kwargs)

        current_user = self.request.user
        context['listings'] = Listing.objects.filter(
            listing_owner=HousingUser.objects.get(id=self.request.user.pk)).order_by(
            "-is_active", "-datetime_created")

        return context

class PreferenceView(LoginRequiredMixin, TemplateView):
    template_name = "accounts/preference.html"

class ListingCreate(LoginRequiredMixin, CreateView):
    model = Listing
    form_class = ListingForm
    template_name = "listings/listing_edit.html"
    success_url = reverse_lazy("listings:profile")

    def form_valid(self, form):
        form.instance.listing_owner = HousingUser.objects.get(
            id=self.request.user.pk)

        return super(ListingCreate, self).form_valid(form)

class ListingEdit(LoginRequiredMixin, UpdateView):
    model = Listing
    form_class = ListingForm
    template_name = "listings/listing_edit.html"
    pk_url_kwarg = "listing_id"
    success_url = reverse_lazy("listings:profile")

    def user_passes_test(self, request):
        if request.user.is_authenticated():
            self.object = self.get_object()
            return self.object.listing_owner == HousingUser.objects.get(
                id=self.request.user.pk)
        return False

    def dispatch(self, request, *args, **kwargs):
        if not self.user_passes_test(request):
            return HttpResponseForbidden()
        else:
            return super(ListingEdit, self).dispatch(request, *args, **kwargs)
    # def get_queryset(self):
    #     base_qs = super(ListingEdit, self).get_queryset()
    #
    #     return base_qs.filter(listing_owner=self.request.user.pk)

def listing_status_toggle(request, listing_id):
    try:
        listing = Listing.objects.get(pk=listing_id)
    except Listing.DoesNotExist:
        raise Http404
    else:
        if listing.listing_owner.pk == request.user.pk:
            if listing.is_active:
                listing.is_active = False
            else:
                listing.is_active = True

            listing.save()

            return redirect("/accounts/profile")
        else:
            return HttpResponseForbidden()
<commit_msg>Update pagination to 5 listings per page<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

# from django.core.urlresolvers import reverse

from django.contrib.auth.mixins import LoginRequiredMixin

from django.shortcuts import redirect, render, render_to_response
from django.views.generic import DetailView, TemplateView, ListView, CreateView, UpdateView
from django.urls import reverse_lazy
from django.http import HttpResponseForbidden, Http404
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.template.context_processors import csrf

from .models import Listing, HousingUser, User
from .forms import ListingForm, HousingUserCreationForm

# index View
def index(request):
    listings = Listing.objects.filter(**get_queryset(request)).order_by(
        "-datetime_modified")

    # pagination
    paginator = Paginator(listings, 5) #listing per page
    page = request.GET.get("page")

    try:
        listings = paginator.page(page)
    except PageNotAnInteger:
        listings = paginator.page(1)
    except EmptyPage:
        listings = paginator.page(1)

    # store queryset, pass for pagination
    queries_without_page = request.GET.copy()
    if "page" in queries_without_page:
        del queries_without_page["page"]

    return render(request, "listings/index.html",
        {"listings": listings, "queries_without_page": queries_without_page})

# get querysets using GET
def get_queryset(request):
    queryset = {"is_active": True}

    if request.GET.get("price"):
        queryset["lease_monthly_cost__gte"] = request.GET.get("price")

    if request.GET.get("type") in ["house", "apartment", "homestay"]:
        queryset["property_type"] = request.GET.get("type")

    if request.GET.get("beds"):
        queryset["bedroom_count__gte"] = request.GET.get("beds")

    return queryset

class ListingDetail(LoginRequiredMixin, DetailView):
    model = Listing
    template_name = "listings/detail.html"
    context_object_name = "listing"
    pk_url_kwarg = "listing_id"

    def user_passes_test(self, request):
        if request.user.is_authenticated():
            self.object = self.get_object()
            return self.object.listing_owner == HousingUser.objects.get(
                id=self.request.user.pk)
        return False

    def dispatch(self, request, *args, **kwargs):
        if not self.user_passes_test(request):
            return HttpResponseForbidden()
        else:
            return super(ListingDetail, self).dispatch(request, *args, **kwargs)

# registration view
def register(request):
    if request.method == "POST":
        form = HousingUserCreationForm(request.POST)
        if form.is_valid():
            user = form.save()
            user.email = user.username
            user.set_password(request.POST.get("password"))
            form.save()
            return redirect("/accounts/register/complete")
    # if a GET (or any other method), create a blank form
    else:
        form = HousingUserCreationForm()
    token = {}
    token.update(csrf(request))
    token["form"] = form

    return render_to_response("accounts/registration_form.html", token)

class RegistrationCompleteView(LoginRequiredMixin, TemplateView):
    template_name = "accounts/registration_complete.html"

class ProfileView(LoginRequiredMixin, ListView):
    model = HousingUser
    template_name = "accounts/profile.html"
    context_object_name = "listings"

    # listings belong to the user
    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(ProfileView, self).get_context_data(**kwargs)

        current_user = self.request.user
        context['listings'] = Listing.objects.filter(
            listing_owner=HousingUser.objects.get(id=self.request.user.pk)).order_by(
            "-is_active", "-datetime_created")

        return context

class PreferenceView(LoginRequiredMixin, TemplateView):
    template_name = "accounts/preference.html"

class ListingCreate(LoginRequiredMixin, CreateView):
    model = Listing
    form_class = ListingForm
    template_name = "listings/listing_edit.html"
    success_url = reverse_lazy("listings:profile")

    def form_valid(self, form):
        form.instance.listing_owner = HousingUser.objects.get(
            id=self.request.user.pk)

        return super(ListingCreate, self).form_valid(form)

class ListingEdit(LoginRequiredMixin, UpdateView):
    model = Listing
    form_class = ListingForm
    template_name = "listings/listing_edit.html"
    pk_url_kwarg = "listing_id"
    success_url = reverse_lazy("listings:profile")

    def user_passes_test(self, request):
        if request.user.is_authenticated():
            self.object = self.get_object()
            return self.object.listing_owner == HousingUser.objects.get(
                id=self.request.user.pk)
        return False

    def dispatch(self, request, *args, **kwargs):
        if not self.user_passes_test(request):
            return HttpResponseForbidden()
        else:
            return super(ListingEdit, self).dispatch(request, *args, **kwargs)
    # def get_queryset(self):
    #     base_qs = super(ListingEdit, self).get_queryset()
    #
    #     return base_qs.filter(listing_owner=self.request.user.pk)

def listing_status_toggle(request, listing_id):
    try:
        listing = Listing.objects.get(pk=listing_id)
    except Listing.DoesNotExist:
        raise Http404
    else:
        if listing.listing_owner.pk == request.user.pk:
            if listing.is_active:
                listing.is_active = False
            else:
                listing.is_active = True

            listing.save()

            return redirect("/accounts/profile")
        else:
            return HttpResponseForbidden()
<|endoftext|>


<commit_before>from __future__ import absolute_import

from dateutil import parser
from time import sleep
from mapproxy.script.conf.app import config_command
from mapproxy.seed.seeder import seed
from mapproxy.seed.config import SeedingConfiguration, SeedConfigurationError, ConfigurationError
from mapproxy.seed.spec import validate_seed_conf
from mapproxy.config.loader import ProxyConfiguration
from mapproxy.config.spec import validate_options
from mapproxy.config.config import load_config, base_config
from mapproxy.seed import seeder
from mapproxy.seed import util
import yaml
from django.core.files.temp import NamedTemporaryFile
import logging
import sys
from django.db import IntegrityError
from django.conf import settings
from billiard import Process

logger = logging.getLogger(__name__)

class WMSToGeopackage():
    """
    Convert a WMS services to a geopackage.
    """

    def __init__(self, config=None, gpkgfile=None, bbox=None, wms_url=None, layer=None, debug=None, name=None, level_from=None, level_to=None):
        """
        Initialize the SQliteToKml utility.

        Args:
            gpkgfile: where to write the gpkg output
            debug: turn debugging on / off
        """
        self.gpkgfile = gpkgfile
        self.bbox = bbox
        self.wms_url = wms_url
        self.debug = debug
        self.name = name
        self.level_from = level_from
        self.level_to = level_to
        self.layer = layer
        self.config = config

    def convert(self, ):
        """
        Convert sqlite to gpkg.
        """
        if self.config:
            conf_dict = yaml.load(self.config)
        else:
            conf_dict = create_conf_from_wms(self.wms_url)
        sources = []
        # for source in conf_dict.get('sources'):
        #     sources.append(source)
        conf_dict['caches'] = get_cache_template(["{}_wms".format(self.layer)], self.gpkgfile)

        #disable SSL cert checks
        conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}

        # Add autoconfiguration to base_config
        mapproxy_config = base_config()
        load_config(mapproxy_config, config_dict=conf_dict)

        logger.error('CONF:')
        logger.error('{}'.format(conf_dict))
        errors, informal_only = validate_options(mapproxy_config)
        if not informal_only:
            raise ConfigurationError('Mapproxy configuration error - {}'.format(', '.join(errors)))

        #Create a configuration object
        mapproxy_configuration = ProxyConfiguration(mapproxy_config, seed=seed, renderd=None)

        seed_dict = get_seed_template(bbox=self.bbox, level_from=self.level_from, level_to=self.level_to)
        logger.error('SEED:')
        logger.error('{}'.format(seed_dict))
        errors, informal_only = validate_seed_conf(seed_dict)
        if not informal_only:
            raise SeedConfigurationError('Mapproxy seed configuration error  - {}'.format(', '.join(errors)))

        # Create a seed configuration object
        seed_configuration = SeedingConfiguration(seed_dict, mapproxy_conf=mapproxy_configuration)
        logger.error("Beginning seeding to {}".format(self.gpkgfile))

        # Call seeder using billiard without daemon, because of limitations of running child processes in python.
        try:
            p = Process(target=seeder.seed, daemon=False, kwargs={"tasks": seed_configuration.seeds(['seed']),
                                                                  "concurrency": 1})
            p.start()
            p.join()
        except Exception as e:
            logger.error("wms failed.")
            raise e
        return self.gpkgfile

def get_cache_template(sources, geopackage):
    return {'cache': {
            "sources": sources,
            "cache": {
                "type": "geopackage",
                "filename": str(geopackage)
            },
            "grids": ["GLOBAL_WEBMERCATOR"]
        }}

def get_seed_template(bbox=[-180,-89,180,89], level_from=None, level_to=None):
    return {
            'coverages': {
                'geom': {
                    'srs': 'EPSG:4326',
                    'bbox': bbox
                }
            },
            'seeds': {
                'seed': {
                    'coverages': ['geom'],
                    'refresh_before': {
                        'minutes': 0
                    },
                    'levels': {
                        'from': level_from or 0
                    },
                    'caches': ['cache']
                }
            }
        }

def create_conf_from_wms(wms_url):
    temp_file = NamedTemporaryFile()
    # wms_url = wms_url.replace('"','')
    params = ['--capabilities', wms_url, '--output', temp_file.name, '--force']
    config_command(params)

    conf_dict = None
    try:
        conf_dict = yaml.load(temp_file)
    except yaml.YAMLError as exc:
        logger.error(exc)
    return conf_dict
<commit_msg>changed default cache level<commit_after>from __future__ import absolute_import

from dateutil import parser
from time import sleep
from mapproxy.script.conf.app import config_command
from mapproxy.seed.seeder import seed
from mapproxy.seed.config import SeedingConfiguration, SeedConfigurationError, ConfigurationError
from mapproxy.seed.spec import validate_seed_conf
from mapproxy.config.loader import ProxyConfiguration
from mapproxy.config.spec import validate_options
from mapproxy.config.config import load_config, base_config
from mapproxy.seed import seeder
from mapproxy.seed import util
import yaml
from django.core.files.temp import NamedTemporaryFile
import logging
import sys
from django.db import IntegrityError
from django.conf import settings
from billiard import Process

logger = logging.getLogger(__name__)

class WMSToGeopackage():
    """
    Convert a WMS services to a geopackage.
    """

    def __init__(self, config=None, gpkgfile=None, bbox=None, wms_url=None, layer=None, debug=None, name=None, level_from=None, level_to=None):
        """
        Initialize the SQliteToKml utility.

        Args:
            gpkgfile: where to write the gpkg output
            debug: turn debugging on / off
        """
        self.gpkgfile = gpkgfile
        self.bbox = bbox
        self.wms_url = wms_url
        self.debug = debug
        self.name = name
        self.level_from = level_from
        self.level_to = level_to
        self.layer = layer
        self.config = config

    def convert(self, ):
        """
        Convert sqlite to gpkg.
        """
        if self.config:
            conf_dict = yaml.load(self.config)
        else:
            conf_dict = create_conf_from_wms(self.wms_url)
        sources = []
        # for source in conf_dict.get('sources'):
        #     sources.append(source)
        conf_dict['caches'] = get_cache_template(["{}_wms".format(self.layer)], self.gpkgfile)

        #disable SSL cert checks
        conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}

        # Add autoconfiguration to base_config
        mapproxy_config = base_config()
        load_config(mapproxy_config, config_dict=conf_dict)

        logger.error('CONF:')
        logger.error('{}'.format(conf_dict))
        errors, informal_only = validate_options(mapproxy_config)
        if not informal_only:
            raise ConfigurationError('Mapproxy configuration error - {}'.format(', '.join(errors)))

        #Create a configuration object
        mapproxy_configuration = ProxyConfiguration(mapproxy_config, seed=seed, renderd=None)

        seed_dict = get_seed_template(bbox=self.bbox, level_from=self.level_from, level_to=self.level_to)
        logger.error('SEED:')
        logger.error('{}'.format(seed_dict))
        errors, informal_only = validate_seed_conf(seed_dict)
        if not informal_only:
            raise SeedConfigurationError('Mapproxy seed configuration error  - {}'.format(', '.join(errors)))

        # Create a seed configuration object
        seed_configuration = SeedingConfiguration(seed_dict, mapproxy_conf=mapproxy_configuration)
        logger.error("Beginning seeding to {}".format(self.gpkgfile))

        # Call seeder using billiard without daemon, because of limitations of running child processes in python.
        try:
            p = Process(target=seeder.seed, daemon=False, kwargs={"tasks": seed_configuration.seeds(['seed']),
                                                                  "concurrency": 1})
            p.start()
            p.join()
        except Exception as e:
            logger.error("wms failed.")
            raise e
        return self.gpkgfile

def get_cache_template(sources, geopackage):
    return {'cache': {
            "sources": sources,
            "cache": {
                "type": "geopackage",
                "filename": str(geopackage)
            },
            "grids": ["GLOBAL_WEBMERCATOR"]
        }}

def get_seed_template(bbox=[-180,-89,180,89], level_from=None, level_to=None):
    return {
            'coverages': {
                'geom': {
                    'srs': 'EPSG:4326',
                    'bbox': bbox
                }
            },
            'seeds': {
                'seed': {
                    'coverages': ['geom'],
                    'refresh_before': {
                        'minutes': 0
                    },
                    'levels': {
                        'to': level_to or 12,
                        'from': level_from or 0
                    },
                    'caches': ['cache']
                }
            }
        }

def create_conf_from_wms(wms_url):
    temp_file = NamedTemporaryFile()
    # wms_url = wms_url.replace('"','')
    params = ['--capabilities', wms_url, '--output', temp_file.name, '--force']
    config_command(params)

    conf_dict = None
    try:
        conf_dict = yaml.load(temp_file)
    except yaml.YAMLError as exc:
        logger.error(exc)
    return conf_dict
<|endoftext|>