<commit_before>from smashbox.utilities import *

class Response:
    def __init__(self):
        self.rc = None
        self.headers = []
        self.body = None

import pycurl, cStringIO

class Client:

    def __init__(self):
        c = pycurl.Curl()

        c.setopt(c.SSL_VERIFYPEER, 0)
        c.setopt(c.CONNECTTIMEOUT, 60)
        c.setopt(c.TIMEOUT, 60)
        c.setopt(c.COOKIEFILE, '')
        c.setopt(c.FOLLOWLOCATION, 0) # by default owncloud sync client does not follow redirects

        c.setopt(c.VERBOSE, 0)
        #c.setopt(c.VERBOSE, True)

        self.c = c

    def PUT(self,fn,url,headers={},offset=0,size=0):
        logger.debug('PUT %s %s %s',fn,url,headers)

        c = self.c

        c.setopt(c.CUSTOMREQUEST, "PUT")
        c.setopt(c.UPLOAD,1)

        f = open(fn,'rb')

        if offset:
            f.seek(offset)

        if not size:
            size = os.path.getsize(fn) - offset

        c.setopt(c.INFILE,f)
        c.setopt(c.INFILESIZE,size)

        r = Response()

        r.body_stream = cStringIO.StringIO()
        c.setopt(c.WRITEFUNCTION,r.body_stream.write)

        return self._perform_request(url,headers)

    def GET(self,url,fn,headers={}):
        logger.debug('GET %s %s %s',url,fn,headers)

        c = self.c

        f = open(fn,'w')
        c.setopt(c.WRITEFUNCTION,f.write)
        return self._perform_request(url,headers)

    def _perform_request(self,url,headers,response_obj=None):
        c = self.c

        c.setopt(c.URL, url)
        c.setopt(pycurl.HTTPHEADER,[str(x)+':'+str(y) for x,y in zip(headers.keys(),headers.values())])

        ret_headers=[]
        c.setopt(pycurl.HEADERFUNCTION, ret_headers.append)

        c.perform()

        if response_obj is None:
            response_obj = Response()

        response_obj.rc=c.getinfo(c.HTTP_CODE)

        response_obj.headers = {}
        for h in ret_headers:
            h = h.strip()
            if not h:
                continue
            fields = h.split(':',1)
            assert(len(fields) <= 2)
            if len(fields)==2:
                response_obj.headers[fields[0]] = fields[1]

        logger.debug('_perform_request url=%s header=%s rc=%s reply_headers=%s',url,headers,response_obj.rc,response_obj.headers)

        return response_obj
<commit_msg>add pycurl_USERAGENT to 'fake' Android-ownCloud client and pycurl_VERBOSE variables for --debug<commit_after>from smashbox.utilities import *

class Response:
    def __init__(self):
        self.rc = None
        self.headers = []
        self.body = None

import pycurl, cStringIO

class Client:

    def __init__(self):
        c = pycurl.Curl()

        c.setopt(c.SSL_VERIFYPEER, 0)
        c.setopt(c.CONNECTTIMEOUT, 60)
        c.setopt(c.TIMEOUT, 60)
        c.setopt(c.COOKIEFILE, '')
        c.setopt(c.FOLLOWLOCATION, 0) # by default owncloud sync client does not follow redirects

        if config.get('pycurl_USERAGENT',None):
            c.setopt(c.USERAGENT, config.pycurl_USERAGENT)

        if config.get('pycurl_VERBOSE',None) is not None:
            self.verbose = config.pycurl_VERBOSE
        else:
            from logging import DEBUG
            self.verbose = config._loglevel <= DEBUG

        c.setopt(c.VERBOSE, self.verbose)

        self.c = c

    def PUT(self,fn,url,headers={},offset=0,size=0):
        logger.debug('PUT %s %s %s',fn,url,headers)

        c = self.c

        c.setopt(c.CUSTOMREQUEST, "PUT")
        c.setopt(c.UPLOAD,1)

        f = open(fn,'rb')

        if offset:
            f.seek(offset)

        if not size:
            size = os.path.getsize(fn) - offset

        c.setopt(c.INFILE,f)
        c.setopt(c.INFILESIZE,size)

        r = Response()

        r.body_stream = cStringIO.StringIO()
        c.setopt(c.WRITEFUNCTION,r.body_stream.write)

        x = self._perform_request(url,headers)

        if self.verbose:
            logger.info('PUT response body: %s',r.body_stream.getvalue())

        return x

    def GET(self,url,fn,headers={}):
        logger.debug('GET %s %s %s',url,fn,headers)

        c = self.c

        f = open(fn,'w')
        c.setopt(c.WRITEFUNCTION,f.write)
        return self._perform_request(url,headers)

    def _perform_request(self,url,headers,response_obj=None):
        c = self.c

        c.setopt(c.URL, url)
        c.setopt(pycurl.HTTPHEADER,[str(x)+':'+str(y) for x,y in zip(headers.keys(),headers.values())])

        ret_headers=[]
        c.setopt(pycurl.HEADERFUNCTION, ret_headers.append)

        c.perform()

        if response_obj is None:
            response_obj = Response()

        response_obj.rc=c.getinfo(c.HTTP_CODE)

        response_obj.headers = {}
        for h in ret_headers:
            h = h.strip()
            if not h:
                continue
            fields = h.split(':',1)
            assert(len(fields) <= 2)
            if len(fields)==2:
                response_obj.headers[fields[0]] = fields[1]

        logger.debug('_perform_request url=%s header=%s rc=%s reply_headers=%s',url,headers,response_obj.rc,response_obj.headers)

        return response_obj
<|endoftext|>


<commit_before>import numpy as np
import bct
from sklearn.externals import joblib
from my_settings import (bands, source_folder)

from sklearn.cross_validation import (StratifiedKFold, cross_val_score)
from sklearn.grid_search import GridSearchCV
from sklearn.ensemble import AdaBoostClassifier

subjects = [
    "0008", "0009", "0010", "0012", "0014", "0015", "0016", "0017", "0018",
    "0019", "0020", "0021", "0022"
]

cls_all = []
pln_all = []

for subject in subjects:
    cls = np.load(source_folder + "graph_data/%s_cls_pow_sliding.npy" %
                  subject).item()

    pln = np.load(source_folder + "graph_data/%s_pln_pow_sliding.npy" %
                  subject).item()

    cls_all.append(cls["trans_alpha"])
    pln_all.append(pln["trans_alpha"])

data_cls = np.asarray(cls_all)
data_pln = np.asarray(pln_all)

X = np.vstack([data_cls, data_pln])
y = np.concatenate([np.zeros(len(data_cls)), np.ones(len(data_pln))])

cv = StratifiedKFold(y, n_folds=6, shuffle=True)

cv_params = {
    "learning_rate": np.arange(0.1, 1.1, 0.1),
    'n_estimators': np.arange(1, 80, 2)
}

grid = GridSearchCV(
    AdaBoostClassifier(),
    cv_params,
    scoring='accuracy',
    cv=cv,
    n_jobs=1,
    verbose=1)
grid.fit(X, y)
ada_cv = grid.best_estimator_

scores = cross_val_score(ada_cv, X, y, cv=cv)
<commit_msg>just a quick paste, DOES NOT WORK<commit_after>import numpy as np
import bct
from sklearn.externals import joblib
from my_settings import (bands, source_folder)

from sklearn.cross_validation import (StratifiedKFold, cross_val_score)
from sklearn.grid_search import GridSearchCV
from sklearn.ensemble import AdaBoostClassifier

subjects = [
    "0008", "0009", "0010", "0012", "0014", "0015", "0016", "0017", "0018",
    "0019", "0020", "0021", "0022"
]

cls_all = []
pln_all = []

for subject in subjects:
    cls = np.load(source_folder + "graph_data/%s_cls_pow_sliding.npy" %
                  subject).item()

    pln = np.load(source_folder + "graph_data/%s_pln_pow_sliding.npy" %
                  subject).item()

    cls_all.append(cls["trans_alpha"])
    pln_all.append(pln["trans_alpha"])

data_cls = np.asarray(cls_all)
data_pln = np.asarray(pln_all)

X = np.vstack([data_cls, data_pln])
y = np.concatenate([np.zeros(len(data_cls)), np.ones(len(data_pln))])

cv = StratifiedKFold(y, n_folds=6, shuffle=True)

cv_params = {
    "learning_rate": np.arange(0.1, 1.1, 0.1),
    'n_estimators': np.arange(1, 80, 2)
}

grid = GridSearchCV(
    AdaBoostClassifier(),
    cv_params,
    scoring='accuracy',
    cv=cv,
    n_jobs=1,
    verbose=1)
grid.fit(X, y)
ada_cv = grid.best_estimator_

scores = cross_val_score(ada_cv, X, y, cv=cv)
# Logistic Regression with cross validation for C
scores = []
coefs = []
Cs = []
LRs = []

for train, test in cv.split(X, y):
    # clf = LogisticRegression(C=1)
    clf = LogisticRegressionCV()
    clf.fit(X[train], y[train])
    y_pred = clf.predict(X[test])

    scores.append(roc_auc_score(y[test], y_pred))
    coefs.append(clf.coef_)
    Cs.append(clf.C_)
    LRs.append(clf)

lr_mean = LogisticRegression()
lr_mean.coef_ = np.asarray(coefs).mean(axis=0)
lr_mean.C = np.asarray(Cs).mean()
lr_mean.intercept_ = np.asarray([est.intercept_ for est in LRs]).mean()

lr_coef_mean = np.asarray(coefs).mean(axis=0)
lr_coef_std = np.asarray(coefs).std(axis=0)

cv_scores = cross_val_score(
    lr_mean, X, y, scoring="roc_auc", cv=StratifiedKFold(9))

score_full_X, perm_scores_full_X, pvalue_full_X = permutation_test_score(
    lr_mean,
    X,
    y,
    scoring="roc_auc",
    cv=StratifiedKFold(9),
    n_permutations=2000,
    n_jobs=2)

# RandomizedLogisticRegression feature selection
for i in range(20):
    rlr = RandomizedLogisticRegression(
        n_resampling=5000, C=lr_mean.C, selection_threshold=0.6)
    rlr.fit(X, y)
    print(sum(rlr.get_support()))

X_rlr = rlr.transform(X)

cv_scores_rlr = cross_val_score(lr_mean, X_rlr, y,
                                scoring="roc_auc", cv=StratifiedKFold(9))

score_rlr, perm_scores_rlr, pvalue_rlr = permutation_test_score(
    lr_mean,
    X_rlr,
    y,
    scoring="roc_auc",
    cv=StratifiedKFold(9),
    n_permutations=2000,
    n_jobs=2)

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Application logic for Mimic.

The Mimic application must serve two distinct types of requests:

* Control requests are used to interact with Mimc and are handled internally.
* Target requests represent the user's application and processed according
  to the files retrieved from the provided tree.

The target application may include Python code which will expect to run
in a pristine environment, so measures are taken not to invoke any sort of
WSGI framework until it is determined that the request is a control request.
"""

import httplib
import logging
import mimetypes
import os
import rfc822
import sys
import time
import urlparse

from __mimic import common
from __mimic import control
from __mimic import shell
from __mimic import target_env
from __mimic import target_info
import yaml

from google.appengine.api import app_identity
# pylint: disable-msg=g-import-not-at-top
try:
  from google.appengine.api import appinfo
except ImportError:
  # import parts of the google.appengine.api.appinfo package (extracted from
  # the local SDK by sdkapi.sh), since the appinfo package is currently
  # unavailable in the App Engine production environment
  from sdkapi import appinfo
from google.appengine.api import namespace_manager
from google.appengine.api import users
from google.appengine.ext.webapp.util import run_wsgi_app
# pylint: enable-msg=g-import-not-at-top

# os.environ key representing 'X-AppEngine-QueueName' HTTP header,
# which should only be present for 'offline' task queue requests, see
# https://developers.google.com/appengine/docs/python/taskqueue/overview-push#Task_Execution
_HTTP_X_APPENGINE_QUEUENAME = 'HTTP_X_APPENGINE_QUEUENAME'

# os.environ key representing 'X-AppEngine-Cron' HTTP header,
# which should only be present for 'offline' cron requests, see
# https://developers.google.com/appengine/docs/python/config/cron#Securing_URLs_for_Cron
_HTTP_X_APPENGINE_CRON = 'HTTP_X_APPENGINE_CRON'

# os.environ key representing 'X-AppEngine-Current-Namespace' HTTP header,
# which identifies the effective namespace when a task was created
_HTTP_X_APPENGINE_CURRENT_NAMESPACE = 'HTTP_X_APPENGINE_CURRENT_NAMESPACE'

# TODO: see if "app.yaml" can be made into a link to the actual
# app.yaml file in the user's workspace.
_NOT_FOUND_PAGE = """

404 Not Found

    Your application's app.yaml file does not have a handler for
    the requested path: %s

    See
    https://developers.google.com/appengine/docs/python/config/appconfig

"""

# most recently seen query string project_id (dev_appserver only)
_most_recent_query_string_project_id = None

def RespondWithStatus(status_code, expiration_s=0, content_type='text/plain',
                      data=None, headers=None):
  """Respond with a status code and optional text/plain data."""
  print 'Content-Type: %s' % content_type
  print 'Status: %d' % status_code
  if expiration_s:
    print 'Expires: %s' % rfc822.formatdate(time.time() + expiration_s)
    print 'Cache-Control: public, max-age=%s' % expiration_s
  if headers:
    for k, v in headers:
      print '{0}: {1}'.format(k, v)
  print ''
  if data is not None:
    print data,

def ServeStaticPage(tree, page):
  """Respond by serving a single static file.

  Args:
    tree: A tree object to use to retrieve files.
    page: A StaticPage object describing the file to be served.
  """
  file_path = page.file_path
  logging.info('Serving static page %s', file_path)
  file_data = tree.GetFileContents(file_path)
  if file_data is None:
    RespondWithStatus(httplib.NOT_FOUND, content_type='text/html',
                      data=_NOT_FOUND_PAGE % file_path)
    return
  if page.mime_type is not None:
    content_type = page.mime_type
  else:
    content_type, _ = mimetypes.guess_type(file_path)
  if content_type is None:
    if file_path.lower().endswith('.ico'):
      content_type = 'image/x-icon'
    else:
      content_type = 'text/plain'
  # should not raise ConfigurationError, but even that would be ok
  expiration_s = appinfo.ParseExpiration(page.expiration)
  RespondWithStatus(httplib.OK, content_type=content_type,
                    data=file_data, expiration_s=expiration_s)

def ServeScriptPage(tree, config, page, namespace):
  """Respond by invoking a python cgi script.

  Args:
    tree: A tree object to use to retrieve files.
    config: The app's config loaded from the app's app.yaml.
    page: A ScriptPage object describing the file to be served.
    namespace: The datastore and memcache namespace used for metadata.
  """
  logging.info('Serving script page %s', page.script_path)
  env = target_env.TargetEnvironment(tree, config, namespace)
  try:
    env.RunScript(page.script_path, control.LoggingHandler())
  except target_env.ScriptNotFoundError:
    RespondWithStatus(httplib.NOT_FOUND,
                      data='Error: could not find script %s' % page.script_path)

def _IsAuthorized(page, users_mod):
  # page does not require login
  if page.login == target_info.LOGIN_NONE:
    return True

  # admins are always allowed in
  # call get_current_user even though checking is_current_user_admin suffices
  if (users_mod.get_current_user() is not None and
      users_mod.is_current_user_admin()):
    return True

  # treat task queue and cron requests as admin equivalents
  # note: mimic currently does not actually provide cron support
  if (os.environ.get(_HTTP_X_APPENGINE_QUEUENAME) or
      os.environ.get(_HTTP_X_APPENGINE_CRON)):
    return True

  # login required and user is currently logged in
  if (page.login == target_info.LOGIN_REQUIRED and
      users_mod.get_current_user() is not None):
    return True

  return False

def _GetUrl(force_https=False):
  if force_https:
    scheme = 'https'
  else:
    scheme = os.environ['wsgi.url_scheme']
  url = '{0}://{1}{2}'.format(scheme, os.environ['HTTP_HOST'],
                              os.environ['PATH_INFO'])
  query = os.environ['QUERY_STRING']
  if query:
    url = '{0}?{1}'.format(url, query)
  return url

def RunTargetApp(tree, path_info, namespace, users_mod):
  """Top level handling of target application requests.

  Args:
    tree: A tree object to use to retrieve files.
    path_info: The path to be served.
    namespace: The datastore and memcache namespace used for metadata.
    users_mod: A users module to use for authentication.
  """
  app_yaml = tree.GetFileContents('app.yaml')
  if app_yaml is None:
    RespondWithStatus(httplib.NOT_FOUND, data='Error: no app.yaml file.')
    return
  try:
    config = yaml.load(app_yaml)
  except yaml.YAMLError:
    errmsg = ('Error: app.yaml configuration is missing or invalid: {0}'
              .format(sys.exc_info()[1]))
    RespondWithStatus(httplib.NOT_FOUND, data=errmsg)
    return
  # bail if yaml.load fails to return dict due to malformed yaml input
  if not isinstance(config, dict):
    errmsg = 'Error: app.yaml configuration is missing or invalid.'
    RespondWithStatus(httplib.NOT_FOUND, data=errmsg)
    return
  page = target_info.FindPage(config, path_info)

  if not page:
    RespondWithStatus(httplib.NOT_FOUND, content_type='text/html',
                      data=_NOT_FOUND_PAGE % path_info)
    return

  # in production redirect to https for handlers specifying 'secure: always'
  if (page.secure == target_info.SECURE_ALWAYS
      and not common.IsDevMode()
      and os.environ['wsgi.url_scheme'] != 'https'):
    https_url = _GetUrl(force_https=True)
    RespondWithStatus(httplib.FOUND, headers=[('Location', https_url)])
    return

  if not _IsAuthorized(page, users_mod):
    user = users_mod.get_current_user()
    if user:
      url = users_mod.create_logout_url(_GetUrl())
      message = ('User {0} is not authorized to view this page.
'
                 'Please logout and then login as an '
                 'authorized user.'.format(user.nickname(), url))
    else:
      url = users_mod.create_login_url(_GetUrl())
      message = ('You are not authorized to view this page. '
                 'You may need to login.'.format(url))
    RespondWithStatus(httplib.FORBIDDEN, data=message,
                      headers=[('Content-Type', 'text/html')])
    return
  # dispatch the page
  if isinstance(page, target_info.StaticPage):
    ServeStaticPage(tree, page)
  elif isinstance(page, target_info.ScriptPage):
    ServeScriptPage(tree, config, page, namespace)
  else:
    raise NotImplementedError('Unrecognized page {0!r}'.format(page))

def GetProjectIdFromServerName():
  """Returns the project id from the SERVER_NAME env var.

  For appspot.com domains, a project id is extracted from the left most
  portion of the subdomain. If no subdomain is specified, or if the project
  id cannot be determined, '' is returned. Finally, when the server name
  is 'localhost', '' is also returned.

  For custom domains, it's not possible to determine with certainty the
  subdomain vs. the default version hostname. In this case we end up using
  the left most component of the server name.

  Example mapping of server name to project id:

  Server Name                              Project Id
  -----------                              ------------
  proj1.your-app-id.appspot.com        ->  'proj1'
  proj1-dot-your-app-id.appspot.com    ->  'proj1'
  your-app-id.appspot.com              ->  None
  www.mydomain.com                     ->  'www'
  proj2.www.mydomain.com               ->  'proj2'
  localhost                            ->  None

  Returns:
    The project id or None.
  """
  # The project id is sent as a "subdomain" of the app, e.g.
  # 'project-id-dot-your-app-id.appspot.com' or
  # 'project-id.your-app-id.appspot.com'

  server_name = os.environ['SERVER_NAME']
  # use a consistent delimiter
  server_name = server_name.replace('-dot-', '.')

  if (server_name == 'localhost' or
      server_name == app_identity.get_default_version_hostname()):
    return None

  return server_name.split('.')[0]

def GetProjectIdFromDevAppserverQueryParam():
  """Returns the project id from the (or a recent) project id query param.

  In the dev_appserver, we need to support running any one of the (potentially)
  many cloud playground projects. Because subdomains would be extremely painful
  to use in the localhost environment, we extract the project id from the
  current query string, or the most recently provided query string project id.
  In production, we use subdomains, so always return None.

  Returns:
    The project id or None.
  """
  global _most_recent_query_string_project_id  # pylint: disable-msg=W0603
  if not common.IsDevMode():
    return None
  qs = os.environ.get('QUERY_STRING')
  if qs:
    params = dict(urlparse.parse_qsl(qs, strict_parsing=True))
    _most_recent_query_string_project_id = params.get(
        common.config.PROJECT_ID_QUERY_PARAM,
        _most_recent_query_string_project_id)
  return _most_recent_query_string_project_id

def GetProjectIdFromPathInfo(path_info):
  """Returns the project id from the request path."""
  m = common.config.PROJECT_ID_FROM_PATH_INFO_RE.match(path_info)
  if not m:
    return None
  return m.group(1)

def GetProjectId():
  """Returns the project id from the HTTP request.

  A number of sources for project id are tried in order. See implementation
  details.

  Returns:
    The project id or None.
  """
  # for task queues, use persisted namespace as the project id
  project_id = os.environ.get(_HTTP_X_APPENGINE_CURRENT_NAMESPACE)
  if project_id:
    return project_id
  project_id = GetProjectIdFromPathInfo(os.environ['PATH_INFO'])
  if project_id:
    return project_id
  project_id = GetProjectIdFromServerName()
  if project_id:
    return project_id
  return GetProjectIdFromDevAppserverQueryParam()

def GetNamespace():
  namespace = GetProjectId() or ''
  # throws BadValueError
  namespace_manager.validate_namespace(namespace)
  return namespace

def RunMimic(create_tree_func, users_mod=users):
  """Entry point for mimic.

  Args:
    create_tree_func: A callable that creates a common.Tree.
    users_mod: A users module to use for authentication (default is the
        AppEngine users module).
  """
  # ensures that namespace_manager_default_namespace_for_request is used
  namespace_manager.set_namespace(None)

  # use PATH_INFO to determine if this is a control or target request
  path_info = os.environ['PATH_INFO']

  is_control_request = path_info.startswith(common.CONTROL_PREFIX)
  if is_control_request:
    # some control requests don't require a tree, like /version_id
    requires_tree = control.ControlRequestRequiresTree(path_info)
  else:
    # requests to the target app always require a tree
    requires_tree = True

  if requires_tree:
    namespace = GetNamespace()
    tree = create_tree_func(namespace)
  else:
    tree = None

  if is_control_request:
    run_wsgi_app(control.MakeControlApp(tree))
  elif path_info.startswith(common.SHELL_PREFIX):
    run_wsgi_app(shell.MakeShellApp(tree, namespace))
  else:
    RunTargetApp(tree, path_info, namespace, users_mod)
<commit_msg>Use yaml.safe_load() instead of yaml.load() ------------- Created by MOE: http://code.google.com/p/moe-java MOE_MIGRATED_REVID=42098088<commit_after>#!/usr/bin/env python
#
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Application logic for Mimic.

The Mimic application must serve two distinct types of requests:

* Control requests are used to interact with Mimc and are handled internally.
* Target requests represent the user's application and processed according
  to the files retrieved from the provided tree.

The target application may include Python code which will expect to run
in a pristine environment, so measures are taken not to invoke any sort of
WSGI framework until it is determined that the request is a control request.
"""

import httplib
import logging
import mimetypes
import os
import rfc822
import sys
import time
import urlparse

from __mimic import common
from __mimic import control
from __mimic import shell
from __mimic import target_env
from __mimic import target_info
import yaml

from google.appengine.api import app_identity
# pylint: disable-msg=g-import-not-at-top
try:
  from google.appengine.api import appinfo
except ImportError:
  # import parts of the google.appengine.api.appinfo package (extracted from
  # the local SDK by sdkapi.sh), since the appinfo package is currently
  # unavailable in the App Engine production environment
  from sdkapi import appinfo
from google.appengine.api import namespace_manager
from google.appengine.api import users
from google.appengine.ext.webapp.util import run_wsgi_app
# pylint: enable-msg=g-import-not-at-top

# os.environ key representing 'X-AppEngine-QueueName' HTTP header,
# which should only be present for 'offline' task queue requests, see
# https://developers.google.com/appengine/docs/python/taskqueue/overview-push#Task_Execution
_HTTP_X_APPENGINE_QUEUENAME = 'HTTP_X_APPENGINE_QUEUENAME'

# os.environ key representing 'X-AppEngine-Cron' HTTP header,
# which should only be present for 'offline' cron requests, see
# https://developers.google.com/appengine/docs/python/config/cron#Securing_URLs_for_Cron
_HTTP_X_APPENGINE_CRON = 'HTTP_X_APPENGINE_CRON'

# os.environ key representing 'X-AppEngine-Current-Namespace' HTTP header,
# which identifies the effective namespace when a task was created
_HTTP_X_APPENGINE_CURRENT_NAMESPACE = 'HTTP_X_APPENGINE_CURRENT_NAMESPACE'

# TODO: see if "app.yaml" can be made into a link to the actual
# app.yaml file in the user's workspace.
_NOT_FOUND_PAGE = """

404 Not Found

    Your application's app.yaml file does not have a handler for
    the requested path: %s

    See
    https://developers.google.com/appengine/docs/python/config/appconfig

"""

# most recently seen query string project_id (dev_appserver only)
_most_recent_query_string_project_id = None

def RespondWithStatus(status_code, expiration_s=0, content_type='text/plain',
                      data=None, headers=None):
  """Respond with a status code and optional text/plain data."""
  print 'Content-Type: %s' % content_type
  print 'Status: %d' % status_code
  if expiration_s:
    print 'Expires: %s' % rfc822.formatdate(time.time() + expiration_s)
    print 'Cache-Control: public, max-age=%s' % expiration_s
  if headers:
    for k, v in headers:
      print '{0}: {1}'.format(k, v)
  print ''
  if data is not None:
    print data,

def ServeStaticPage(tree, page):
  """Respond by serving a single static file.

  Args:
    tree: A tree object to use to retrieve files.
    page: A StaticPage object describing the file to be served.
  """
  file_path = page.file_path
  logging.info('Serving static page %s', file_path)
  file_data = tree.GetFileContents(file_path)
  if file_data is None:
    RespondWithStatus(httplib.NOT_FOUND, content_type='text/html',
                      data=_NOT_FOUND_PAGE % file_path)
    return
  if page.mime_type is not None:
    content_type = page.mime_type
  else:
    content_type, _ = mimetypes.guess_type(file_path)
  if content_type is None:
    if file_path.lower().endswith('.ico'):
      content_type = 'image/x-icon'
    else:
      content_type = 'text/plain'
  # should not raise ConfigurationError, but even that would be ok
  expiration_s = appinfo.ParseExpiration(page.expiration)
  RespondWithStatus(httplib.OK, content_type=content_type,
                    data=file_data, expiration_s=expiration_s)

def ServeScriptPage(tree, config, page, namespace):
  """Respond by invoking a python cgi script.

  Args:
    tree: A tree object to use to retrieve files.
    config: The app's config loaded from the app's app.yaml.
    page: A ScriptPage object describing the file to be served.
    namespace: The datastore and memcache namespace used for metadata.
  """
  logging.info('Serving script page %s', page.script_path)
  env = target_env.TargetEnvironment(tree, config, namespace)
  try:
    env.RunScript(page.script_path, control.LoggingHandler())
  except target_env.ScriptNotFoundError:
    RespondWithStatus(httplib.NOT_FOUND,
                      data='Error: could not find script %s' % page.script_path)

def _IsAuthorized(page, users_mod):
  # page does not require login
  if page.login == target_info.LOGIN_NONE:
    return True

  # admins are always allowed in
  # call get_current_user even though checking is_current_user_admin suffices
  if (users_mod.get_current_user() is not None and
      users_mod.is_current_user_admin()):
    return True

  # treat task queue and cron requests as admin equivalents
  # note: mimic currently does not actually provide cron support
  if (os.environ.get(_HTTP_X_APPENGINE_QUEUENAME) or
      os.environ.get(_HTTP_X_APPENGINE_CRON)):
    return True

  # login required and user is currently logged in
  if (page.login == target_info.LOGIN_REQUIRED and
      users_mod.get_current_user() is not None):
    return True

  return False

def _GetUrl(force_https=False):
  if force_https:
    scheme = 'https'
  else:
    scheme = os.environ['wsgi.url_scheme']
  url = '{0}://{1}{2}'.format(scheme, os.environ['HTTP_HOST'],
                              os.environ['PATH_INFO'])
  query = os.environ['QUERY_STRING']
  if query:
    url = '{0}?{1}'.format(url, query)
  return url

def RunTargetApp(tree, path_info, namespace, users_mod):
  """Top level handling of target application requests.

  Args:
    tree: A tree object to use to retrieve files.
    path_info: The path to be served.
    namespace: The datastore and memcache namespace used for metadata.
    users_mod: A users module to use for authentication.
  """
  app_yaml = tree.GetFileContents('app.yaml')
  if app_yaml is None:
    RespondWithStatus(httplib.NOT_FOUND, data='Error: no app.yaml file.')
    return
  try:
    config = yaml.safe_load(app_yaml)
  except yaml.YAMLError:
    errmsg = ('Error: app.yaml configuration is missing or invalid: {0}'
              .format(sys.exc_info()[1]))
    RespondWithStatus(httplib.NOT_FOUND, data=errmsg)
    return
  # bail if yaml.safe_load fails to return dict due to malformed yaml input
  if not isinstance(config, dict):
    errmsg = 'Error: app.yaml configuration is missing or invalid.'
    RespondWithStatus(httplib.NOT_FOUND, data=errmsg)
    return
  page = target_info.FindPage(config, path_info)

  if not page:
    RespondWithStatus(httplib.NOT_FOUND, content_type='text/html',
                      data=_NOT_FOUND_PAGE % path_info)
    return

  # in production redirect to https for handlers specifying 'secure: always'
  if (page.secure == target_info.SECURE_ALWAYS
      and not common.IsDevMode()
      and os.environ['wsgi.url_scheme'] != 'https'):
    https_url = _GetUrl(force_https=True)
    RespondWithStatus(httplib.FOUND, headers=[('Location', https_url)])
    return

  if not _IsAuthorized(page, users_mod):
    user = users_mod.get_current_user()
    if user:
      url = users_mod.create_logout_url(_GetUrl())
      message = ('User {0} is not authorized to view this page.
'
                 'Please logout and then login as an '
                 'authorized user.'.format(user.nickname(), url))
    else:
      url = users_mod.create_login_url(_GetUrl())
      message = ('You are not authorized to view this page. '
                 'You may need to login.'.format(url))
    RespondWithStatus(httplib.FORBIDDEN, data=message,
                      headers=[('Content-Type', 'text/html')])
    return
  # dispatch the page
  if isinstance(page, target_info.StaticPage):
    ServeStaticPage(tree, page)
  elif isinstance(page, target_info.ScriptPage):
    ServeScriptPage(tree, config, page, namespace)
  else:
    raise NotImplementedError('Unrecognized page {0!r}'.format(page))

def GetProjectIdFromServerName():
  """Returns the project id from the SERVER_NAME env var.

  For appspot.com domains, a project id is extracted from the left most
  portion of the subdomain. If no subdomain is specified, or if the project
  id cannot be determined, '' is returned. Finally, when the server name
  is 'localhost', '' is also returned.

  For custom domains, it's not possible to determine with certainty the
  subdomain vs. the default version hostname. In this case we end up using
  the left most component of the server name.

  Example mapping of server name to project id:

  Server Name                              Project Id
  -----------                              ------------
  proj1.your-app-id.appspot.com        ->  'proj1'
  proj1-dot-your-app-id.appspot.com    ->  'proj1'
  your-app-id.appspot.com              ->  None
  www.mydomain.com                     ->  'www'
  proj2.www.mydomain.com               ->  'proj2'
  localhost                            ->  None

  Returns:
    The project id or None.
  """
  # The project id is sent as a "subdomain" of the app, e.g.
  # 'project-id-dot-your-app-id.appspot.com' or
  # 'project-id.your-app-id.appspot.com'

  server_name = os.environ['SERVER_NAME']
  # use a consistent delimiter
  server_name = server_name.replace('-dot-', '.')

  if (server_name == 'localhost' or
      server_name == app_identity.get_default_version_hostname()):
    return None

  return server_name.split('.')[0]

def GetProjectIdFromDevAppserverQueryParam():
  """Returns the project id from the (or a recent) project id query param.

  In the dev_appserver, we need to support running any one of the (potentially)
  many cloud playground projects. Because subdomains would be extremely painful
  to use in the localhost environment, we extract the project id from the
  current query string, or the most recently provided query string project id.
  In production, we use subdomains, so always return None.

  Returns:
    The project id or None.
  """
  global _most_recent_query_string_project_id  # pylint: disable-msg=W0603
  if not common.IsDevMode():
    return None
  qs = os.environ.get('QUERY_STRING')
  if qs:
    params = dict(urlparse.parse_qsl(qs, strict_parsing=True))
    _most_recent_query_string_project_id = params.get(
        common.config.PROJECT_ID_QUERY_PARAM,
        _most_recent_query_string_project_id)
  return _most_recent_query_string_project_id

def GetProjectIdFromPathInfo(path_info):
  """Returns the project id from the request path."""
  m = common.config.PROJECT_ID_FROM_PATH_INFO_RE.match(path_info)
  if not m:
    return None
  return m.group(1)

def GetProjectId():
  """Returns the project id from the HTTP request.

  A number of sources for project id are tried in order. See implementation
  details.

  Returns:
    The project id or None.
  """
  # for task queues, use persisted namespace as the project id
  project_id = os.environ.get(_HTTP_X_APPENGINE_CURRENT_NAMESPACE)
  if project_id:
    return project_id
  project_id = GetProjectIdFromPathInfo(os.environ['PATH_INFO'])
  if project_id:
    return project_id
  project_id = GetProjectIdFromServerName()
  if project_id:
    return project_id
  return GetProjectIdFromDevAppserverQueryParam()

def GetNamespace():
  namespace = GetProjectId() or ''
  # throws BadValueError
  namespace_manager.validate_namespace(namespace)
  return namespace

def RunMimic(create_tree_func, users_mod=users):
  """Entry point for mimic.

  Args:
    create_tree_func: A callable that creates a common.Tree.
    users_mod: A users module to use for authentication (default is the
        AppEngine users module).
  """
  # ensures that namespace_manager_default_namespace_for_request is used
  namespace_manager.set_namespace(None)

  # use PATH_INFO to determine if this is a control or target request
  path_info = os.environ['PATH_INFO']

  is_control_request = path_info.startswith(common.CONTROL_PREFIX)
  if is_control_request:
    # some control requests don't require a tree, like /version_id
    requires_tree = control.ControlRequestRequiresTree(path_info)
  else:
    # requests to the target app always require a tree
    requires_tree = True

  if requires_tree:
    namespace = GetNamespace()
    tree = create_tree_func(namespace)
  else:
    tree = None

  if is_control_request:
    run_wsgi_app(control.MakeControlApp(tree))
  elif path_info.startswith(common.SHELL_PREFIX):
    run_wsgi_app(shell.MakeShellApp(tree, namespace))
  else:
    RunTargetApp(tree, path_info, namespace, users_mod)
<|endoftext|>


<commit_before>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2013:
#    Gabes Jean, naparuba@gmail.com
#    Gerhard Lausser, Gerhard.Lausser@consol.de
#    Gregory Starck, g.starck@gmail.com
#    Hartmut Goebel, h.goebel@goebel-consult.de
#    Thomas Cellerier, thomascellerier@gmail.com
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

# This Class is a plugin for the Shinken Broker.
# Its job is to publish monitoring updates to the
# specified ZeroMQ endpoint point by use of a
# ZeroMQ broker.
#
# A ZeroMQ endpoint can be tcp, ipc, or inproc.
# E.g:
# tcp://127.0.0.1:22777
# ipc:///tmp/shinken_pub
# inproc://shinken_pub
#
# The monitoring data itself is serialized
# using either json or msgpack depending on
# the module configuration.
#

from shinken.basemodule import BaseModule
from shinken.log import logger
import zmq
import json

properties = {
    'daemons': ['broker'],
    'type': 'zmq_broker',
    'external': False,
    }

# called by the plugin manager to get a broker
def get_instance(mod_conf):
    logger.info("[Zmq Broker] Get a Zmq broker module for plugin %s" % mod_conf.get_name())
    pub_endpoint = getattr(mod_conf, 'pub_endpoint', "tcp://127.0.0.1:22777")
    serialize_to = getattr(mod_conf, 'serialize_to', "json")
    instance = Zmq_broker(mod_conf, pub_endpoint, serialize_to)
    return instance

# Json custom encoder, encodes sets to lists
class SetEncoder(json.JSONEncoder):
    def default(self, obj):
	if isinstance(obj, set):
	    return list(obj)
	return json.JSONEncoder.default(self, obj)

# Msgpack custom encoder, encodes set to lists
def encode_monitoring_data(obj):
    if isinstance(obj, set):
        return list(obj)
    return obj

# Class for the ZeroMQ broker
class Zmq_broker(BaseModule):
    context = None
    s_pub = None
    pub_endpoint = None
    serialize_to = None
    serialize = None

    def __init__(self, mod_conf, pub_endpoint, serialize_to):
        BaseModule.__init__(self, mod_conf)
        self.pub_endpoint = pub_endpoint
        self.serialize_to = serialize_to
        logger.info("[Zmq Broker] Binding to endpoint " + self.pub_endpoint)

        # This doesn't work properly in init()
        # sometimes it ends up beings called several
        # times and the address becomes already in use.
        self.context = zmq.Context()
        self.s_pub = self.context.socket(zmq.PUB)
        self.s_pub.bind(self.pub_endpoint)

        # Load the correct serialization function
        # depending on the serialization method
        # chosen in the configuration.
        if self.serialize_to == "msgpack":
            import msgpack
            packer = msgpack.Packer(default = encode_monitoring_data)
            self.serialize = lambda msg: packer.pack(msg)
        elif self.serialize_to == "json":
            self.serialize = lambda msg: json.dumps(msg, cls=SetEncoder)
        else:
            raise Exception("[Zmq Broker] No valid serialization method defined (Got "+str(self.serializ_to)+")!")

    # Called by Broker to say 'let's prepare yourself guy'
    def init(self):
        logger.info("[Zmq Broker] Initialization of the Zmq broker module")

    # Publish to the ZeroMQ socket
    # using the chosen serialization method
    def publish(self, msg):
        data = self.serialize(msg)
        self.s_pub.send(data)

    # An host check have just arrived, we UPDATE data info with this
    def manage_brok(self, b):
        logger.debug("[Zmq Broker] Got broker update: " + str(b.data))

        #Publish update data to the ZeroMQ endpoint.
        msg = b.data
        self.publish(msg)

    # Properly close down this thing.
    def do_stop(self):
        self.s_pub.close()
        self.context.term()

<commit_msg>Update shinken/modules/zmq_broker.py<commit_after>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2013:
#    Gabes Jean, naparuba@gmail.com
#    Gerhard Lausser, Gerhard.Lausser@consol.de
#    Gregory Starck, g.starck@gmail.com
#    Hartmut Goebel, h.goebel@goebel-consult.de
#    Thomas Cellerier, thomascellerier@gmail.com
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

# This Class is a plugin for the Shinken Broker.
# Its job is to publish monitoring updates to the
# specified ZeroMQ endpoint point by use of a
# ZeroMQ broker.
#
# A ZeroMQ endpoint can be tcp, ipc, or inproc.
# E.g:
# tcp://127.0.0.1:22777
# ipc:///tmp/shinken_pub
# inproc://shinken_pub
#
# The monitoring data itself is serialized
# using either json or msgpack depending on
# the module configuration.
#

from shinken.basemodule import BaseModule
from shinken.log import logger
import zmq
import json

properties = {
    'daemons': ['broker'],
    'type': 'zmq_broker',
    'external': False,
    }

# called by the plugin manager to get a broker
def get_instance(mod_conf):
    logger.info("[Zmq Broker] Get a Zmq broker module for plugin %s" % mod_conf.get_name())
    pub_endpoint = getattr(mod_conf, 'pub_endpoint', "tcp://127.0.0.1:22777")
    serialize_to = getattr(mod_conf, 'serialize_to', "json")
    instance = Zmq_broker(mod_conf, pub_endpoint, serialize_to)
    return instance

# Json custom encoder, encodes sets to lists
class SetEncoder(json.JSONEncoder):
    def default(self, obj):
	if isinstance(obj, set):
	    return list(obj)
	return json.JSONEncoder.default(self, obj)

# Msgpack custom encoder, encodes set to lists
def encode_monitoring_data(obj):
    if isinstance(obj, set):
        return list(obj)
    return obj

# Class for the ZeroMQ broker
class Zmq_broker(BaseModule):
    context = None
    s_pub = None
    pub_endpoint = None
    serialize_to = None
    serialize = None

    def __init__(self, mod_conf, pub_endpoint, serialize_to):
        BaseModule.__init__(self, mod_conf)
        self.pub_endpoint = pub_endpoint
        self.serialize_to = serialize_to
        logger.info("[Zmq Broker] Binding to endpoint " + self.pub_endpoint)

        # This doesn't work properly in init()
        # sometimes it ends up beings called several
        # times and the address becomes already in use.
        self.context = zmq.Context()
        self.s_pub = self.context.socket(zmq.PUB)
        self.s_pub.bind(self.pub_endpoint)

        # Load the correct serialization function
        # depending on the serialization method
        # chosen in the configuration.
        if self.serialize_to == "msgpack":
            import msgpack
            packer = msgpack.Packer(default = encode_monitoring_data)
            self.serialize = lambda msg: packer.pack(msg)
        elif self.serialize_to == "json":
            self.serialize = lambda msg: json.dumps(msg, cls=SetEncoder)
        else:
            raise Exception("[Zmq Broker] No valid serialization method defined (Got "+str(self.serializ_to)+")!")

    # Called by Broker to say 'let's prepare yourself guy'
    def init(self):
        logger.info("[Zmq Broker] Initialization of the Zmq broker module")

    # Publish to the ZeroMQ socket
    # using the chosen serialization method
    def publish(self, msg, topic=""):
        data = self.serialize(msg)
        self.s_pub.send(topic, zmq.SNDMORE)
        self.s_pub.send(data)

    # An host check have just arrived, we UPDATE data info with this
    def manage_brok(self, b):
        logger.debug("[Zmq Broker] Got broker update: " + str(b.data))

        #Publish update data to the ZeroMQ endpoint.
        msg = b.data
        self.publish(msg, b.type)

    # Properly close down this thing.
    def do_stop(self):
        self.s_pub.close()
        self.context.term()

<|endoftext|>


<commit_before># ==============================================================================
# License info here?
# ==============================================================================

from __future__ import print_function

# import monitor
import os
import pandas as pd
import numpy as np
import sncosmo
# import astropy.units as u
from astropy.io import fits
from astropy.table import Table
from lsst.utils import getPackageDir

# ==============================================================================

class Monitor(object):
    '''
    Simple class for extracting DM forced photometry light curves.
    '''
    def __init__(self):
        return

    def read(self, photfile):
        print("No IO enabled yet.")
        return

    def run(self, algorithm=None):
        if algorithm is None:
            print("No algorithms coded yet.")
        return

# =============================================================================

class LightCurve(object):
    '''
    Fetch ForcedSource data with Butler and package for use with accompanying visualization
    routines.
    '''
    def __init__(self, fp_table_dir=None, bandpasses=None, visitLists=None, mjdFile=None):

        self.fp_table_dir = fp_table_dir
        self.visitMap = {}
        self.bandpasses = bandpasses
        self.lightcurve = None
        self.visitMjd = {}

        # Associate mjd with visits (just a hack for now, need to think about this more):
        mjdList = np.genfromtxt(mjdFile, delimiter=',')
        for visitDate in mjdList:
            self.visitMjd[str(int(visitDate[0]))] = visitDate[1]

        for bandpass, visitList in zip(self.bandpasses, visitLists):
            # Associate correct visits with bandpass:
            self.visitMap[bandpass] = visitList
            # Register required lsst bandpass in sncosmo registry:
            bandpassFile = os.path.join(str(getPackageDir('throughputs') + '/baseline/total_' +
                                            bandpass + '.dat'))
            bandpassInfo = np.genfromtxt(bandpassFile, names=['wavelen', 'transmission'])
            band = sncosmo.Bandpass(bandpassInfo['wavelen'], bandpassInfo['transmission'],
                                    name=str('lsst' + bandpass), wave_unit=u.nm)
            sncosmo.registry.register(band)

    def build_lightcurve(self, objid):
        """
        Assemble a light curve data table from available files.
        """
        lightcurve = {}
        lightcurve['bandpass'] = []
        lightcurve['mjd'] = []
        lightcurve['ra'] = []
        lightcurve['dec'] = []
        lightcurve['flux'] = []
        lightcurve['flux_error'] = []
        lightcurve['zp'] = []
        lightcurve['zpsys'] = []

        for bandpass in self.bandpasses:
            for visit in self.visitMap[bandpass]:
                # NB. Hard-coded filename conventions:
                hdulist = fits.open(str(self.fp_table_dir + '/0/v' + str(visit) + '-fr/R22/S11.fits'))
                objData = hdulist[1].data[np.where(hdulist[1].data['objectId'] == objid)]
                if len(objData) > 0:
                    lightcurve['bandpass'].append(str('lsst' + bandpass))
                    lightcurve['mjd'].append(self.visitMjd[str(visit)])
                    lightcurve['ra'].append(objData['coord_ra'][0])
                    lightcurve['dec'].append(objData['coord_dec'][0])
                    lightcurve['flux'].append(objData['base_PsfFlux_flux'][0])
                    lightcurve['flux_error'].append(objData['base_PsfFlux_fluxSigma'][0])
                    lightcurve['zp'].append(25.0)
                    lightcurve['zpsys'].append('ab')
        self.lightcurve = lightcurve

    def visualize_lightcurve(self):
        """
        Make a simple light curve plot.
        """
        if self.lightcurve is None:
            raise ValueError('No lightcurve yet. Use build_lightcurve first.')

        lcTable = Table(data=self.lightcurve)
        fig = sncosmo.plot_lc(lcTable)

        return fig

# ==============================================================================
# Need better tests...

if __name__ == '__main__':

    newMonitor = Monitor()

# ==============================================================================
<commit_msg>Add license header<commit_after># Copyright (C) 2016, LSST Dark Energy Science Collaboration (DESC)
# All rights reserved.
#
# This software may be modified and distributed under the terms
# of the modified BSD license.  See the LICENSE file for details.

from __future__ import print_function

# import monitor
import os
import pandas as pd
import numpy as np
import sncosmo
# import astropy.units as u
from astropy.io import fits
from astropy.table import Table
from lsst.utils import getPackageDir

# ==============================================================================

class Monitor(object):
    '''
    Simple class for extracting DM forced photometry light curves.
    '''
    def __init__(self):
        return

    def read(self, photfile):
        print("No IO enabled yet.")
        return

    def run(self, algorithm=None):
        if algorithm is None:
            print("No algorithms coded yet.")
        return

# =============================================================================

class LightCurve(object):
    '''
    Fetch ForcedSource data with Butler and package for use with accompanying visualization
    routines.
    '''
    def __init__(self, fp_table_dir=None, bandpasses=None, visitLists=None, mjdFile=None):

        self.fp_table_dir = fp_table_dir
        self.visitMap = {}
        self.bandpasses = bandpasses
        self.lightcurve = None
        self.visitMjd = {}

        # Associate mjd with visits (just a hack for now, need to think about this more):
        mjdList = np.genfromtxt(mjdFile, delimiter=',')
        for visitDate in mjdList:
            self.visitMjd[str(int(visitDate[0]))] = visitDate[1]

        for bandpass, visitList in zip(self.bandpasses, visitLists):
            # Associate correct visits with bandpass:
            self.visitMap[bandpass] = visitList
            # Register required lsst bandpass in sncosmo registry:
            bandpassFile = os.path.join(str(getPackageDir('throughputs') + '/baseline/total_' +
                                            bandpass + '.dat'))
            bandpassInfo = np.genfromtxt(bandpassFile, names=['wavelen', 'transmission'])
            band = sncosmo.Bandpass(bandpassInfo['wavelen'], bandpassInfo['transmission'],
                                    name=str('lsst' + bandpass), wave_unit=u.nm)
            sncosmo.registry.register(band)

    def build_lightcurve(self, objid):
        """
        Assemble a light curve data table from available files.
        """
        lightcurve = {}
        lightcurve['bandpass'] = []
        lightcurve['mjd'] = []
        lightcurve['ra'] = []
        lightcurve['dec'] = []
        lightcurve['flux'] = []
        lightcurve['flux_error'] = []
        lightcurve['zp'] = []
        lightcurve['zpsys'] = []

        for bandpass in self.bandpasses:
            for visit in self.visitMap[bandpass]:
                # NB. Hard-coded filename conventions:
                hdulist = fits.open(str(self.fp_table_dir + '/0/v' + str(visit) + '-fr/R22/S11.fits'))
                objData = hdulist[1].data[np.where(hdulist[1].data['objectId'] == objid)]
                if len(objData) > 0:
                    lightcurve['bandpass'].append(str('lsst' + bandpass))
                    lightcurve['mjd'].append(self.visitMjd[str(visit)])
                    lightcurve['ra'].append(objData['coord_ra'][0])
                    lightcurve['dec'].append(objData['coord_dec'][0])
                    lightcurve['flux'].append(objData['base_PsfFlux_flux'][0])
                    lightcurve['flux_error'].append(objData['base_PsfFlux_fluxSigma'][0])
                    lightcurve['zp'].append(25.0)
                    lightcurve['zpsys'].append('ab')
        self.lightcurve = lightcurve

    def visualize_lightcurve(self):
        """
        Make a simple light curve plot.
        """
        if self.lightcurve is None:
            raise ValueError('No lightcurve yet. Use build_lightcurve first.')

        lcTable = Table(data=self.lightcurve)
        fig = sncosmo.plot_lc(lcTable)

        return fig

# ==============================================================================
# Need better tests...

if __name__ == '__main__':

    newMonitor = Monitor()

# ==============================================================================
<|endoftext|>


<commit_before>import pytest
from gevent import sleep
from copper.errors import ConnectionClosedError
from copper.client import CopperClient

def test_client():
    with CopperClient(('tcp', 'localhost:5323')) as client:
        def handler(stream):
            stream.write('Hello, world!')
        with client.publish('test:helloworld', handler):
            with client.subscribe('test:helloworld') as sub:
                assert sub.endpoints
                with sub.open() as stream:
                    assert stream.read(128) == 'Hello, world!'
                oldconn, client._conn = client._conn, None
                oldconn.close()
                sleep(0.001)
                with sub.open() as stream:
                    assert stream.read(128) == 'Hello, world!'
<commit_msg>Use wait instead of sleep in test_client<commit_after>import pytest
from gevent import sleep
from copper.errors import ConnectionClosedError
from copper.client import CopperClient

def test_client():
    with CopperClient(('tcp', 'localhost:5323')) as client:
        def handler(stream):
            stream.write('Hello, world!')
        with client.publish('test:helloworld', handler):
            with client.subscribe('test:helloworld') as sub:
                assert sub.endpoints
                with sub.open() as stream:
                    assert stream.read(128) == 'Hello, world!'
                oldconn, client._conn = client._conn, None
                oldconn.close()
                with pytest.raises(ConnectionClosedError):
                    oldconn.wait()
                with sub.open() as stream:
                    assert stream.read(128) == 'Hello, world!'
<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import absolute_import, division, print_function
from future.standard_library import hooks

import hashlib
from os import remove, makedirs
from os.path import exists, isdir
from functools import partial
from warnings import warn
from types import FunctionType

with hooks():
    from itertools import zip_longest

class MiniRegistry(dict):
    def __call__(self, name):
        def decorator(func):
            self[name] = func
            return func
        return decorator

    def copy(self):
        return self.__class__(super(MiniRegistry, self).copy())

    def formatted_listing(self):
        if len(self) == 0:
            return "\tNone"
        else:
            return "\n".join(["\t%r\n\t  %s" %
                             (name, self[name].__doc__.split("\n")[0])
                              for name in self])

    def interpolate(self, obj, name):
        f = getattr(obj, name).__func__
        # Deep magic happens here. I *think* the closure is interpereted in a
        # new context for each subclass making the whole thing work.
        f2 = FunctionType(f.func_code, f.func_globals, name=f.func_name,
                          argdefs=f.func_defaults, closure=f.func_closure)
        # Conveniently the original docstring is on f2, not the new ones if
        # inheritence is happening.
        t = f2.__doc__.split("\n\n")
        t.insert(2, self.formatted_listing())
        f2.__doc__ = "\n\n".join(t)

        setattr(obj, name, f2)

def cardinal_to_ordinal(n):
    """Return ordinal string version of cardinal int `n`.

    Parameters
    ----------
    n : int
        Cardinal to convert to ordinal. Must be >= 0.

    Returns
    -------
    str
        Ordinal version of cardinal `n`.

    Raises
    ------
    ValueError
        If `n` is less than 0.

    Notes
    -----
    This function can be useful when writing human-readable error messages.

    Examples
    --------
    >>> from skbio.util import cardinal_to_ordinal
    >>> cardinal_to_ordinal(0)
    '0th'
    >>> cardinal_to_ordinal(1)
    '1st'
    >>> cardinal_to_ordinal(2)
    '2nd'
    >>> cardinal_to_ordinal(3)
    '3rd'

    """
    # Taken and modified from http://stackoverflow.com/a/20007730/3776794
    # Originally from http://codegolf.stackexchange.com/a/4712 by Gareth
    if n < 0:
        raise ValueError("Cannot convert negative integer %d to ordinal "
                         "string." % n)
    return "%d%s" % (n, "tsnrhtdd"[(n//10 % 10 != 1)*(n % 10 < 4)*n % 10::4])

def is_casava_v180_or_later(header_line):
    """Check if the header looks like it is Illumina software post-casava v1.8

    Parameters
    ----------
    header_line : bytes
        A header line

    Returns
    -------
    bool
        ``True`` for if casava v1.8+, otherwise ``False``

    Examples
    --------
    >>> from skbio.util import is_casava_v180_or_later
    >>> print(is_casava_v180_or_later('@foo'))
    False
    >>> id_ = '@M00176:17:000000000-A0CNA:1:1:15487:1773 1:N:0:0'
    >>> print(is_casava_v180_or_later(id_))
    True

    """
    if not header_line.startswith(b'@'):
        raise ValueError("Non-header line passed in!")
    fields = header_line.split(b':')

    return len(fields) == 10 and fields[7] in b'YN'

def safe_md5(open_file, block_size=2 ** 20):
    """Computes an md5 sum without loading the file into memory

    Parameters
    ----------
    open_file : file object
        open file handle to the archive to compute the checksum. It
        must be open as a binary file
    block_size : int, optional
        size of the block taken per iteration

    Returns
    -------
    md5 : md5 object from the hashlib module
        object with the loaded file

    Notes
    -----
    This method is based on the answers given in:
    http://stackoverflow.com/a/1131255/379593

    Examples
    --------
    >>> from StringIO import StringIO
    >>> from skbio.util import safe_md5
    >>> fd = StringIO("foo bar baz") # open file like object
    >>> x = safe_md5(fd)
    >>> x.hexdigest()
    'ab07acbb1e496801937adfa772424bf7'
    >>> fd.close()

    """
    md5 = hashlib.md5()
    data = True
    while data:
        data = open_file.read(block_size)
        if data:
            md5.update(data)
    return md5

def remove_files(list_of_filepaths, error_on_missing=True):
    """Remove list of filepaths, optionally raising an error if any are missing

    Parameters
    ----------
    list_of_filepaths : list of strings
        list with filepaths to remove
    error_on_missing : bool, optional
        whether or not the function should raise an ``OSError`` if a file is
        not found

    Raises
    ------
    OSError
        If a filepath in the list does not exist

    Examples
    --------
    >>> from tempfile import NamedTemporaryFile
    >>> from os.path import exists
    >>> from skbio.util import remove_files
    >>> h = NamedTemporaryFile(delete=False)
    >>> exists(h.name) # it exists
    True
    >>> remove_files([h.name])
    >>> exists(h.name) # and now it's gone
    False

    """
    missing = []
    for fp in list_of_filepaths:
        try:
            remove(fp)
        except OSError:
            missing.append(fp)

    if error_on_missing and missing:
        raise OSError("Some filepaths were not accessible: %s" %
                      '\t'.join(missing))

def create_dir(dir_name, fail_on_exist=False, handle_errors_externally=False):
    """Create a directory safely and fail meaningfully

    Parameters
    ----------
    dir_name: string
        name of directory to create

    fail_on_exist: bool, optional
        if true raise an error if ``dir_name`` already exists

    handle_errors_externally: bool, optional
        if True do not raise Errors, but return failure codes. This allows to
        handle errors locally and e.g. hint the user at a --force_overwrite
        options.

    Returns
    -------
    return_value : int
        These values are only returned if no error is raised:

        - ``0``:  directory was safely created
        - ``1``:  directory already existed
        - ``2``:  a file with the same name exists
        - ``3``:  any other unspecified ``OSError``

    Notes
    -----
    Depending  of how thorough we want to be we could add tests, e.g. for
    testing actual write permission in an existing dir.

    Examples
    --------
    >>> from skbio.util import create_dir
    >>> from os.path import exists, join
    >>> from tempfile import gettempdir
    >>> from os import rmdir
    >>> new_dir = join(gettempdir(), 'scikitbio')
    >>> create_dir(new_dir)
    0
    >>> exists(new_dir)
    True
    >>> rmdir(new_dir)

    """
    error_code_lookup = _get_create_dir_error_codes()
    # pre-instanciate function with
    ror = partial(_handle_error_codes, dir_name, handle_errors_externally)

    if exists(dir_name):
        if isdir(dir_name):
            # dir is there
            if fail_on_exist:
                return ror(error_code_lookup['DIR_EXISTS'])
            else:
                return error_code_lookup['DIR_EXISTS']
        else:
            # must be file with same name
            return ror(error_code_lookup['FILE_EXISTS'])
    else:
        # no dir there, try making it
        try:
            makedirs(dir_name)
        except OSError:
            return ror(error_code_lookup['OTHER_OS_ERROR'])

    return error_code_lookup['NO_ERROR']

def find_duplicates(iterable):
    """Find duplicate elements in an iterable.

    Parameters
    ----------
    iterable : iterable
        Iterable to be searched for duplicates (i.e., elements that are
        repeated).

    Returns
    -------
    set
        Repeated elements in `iterable`.

    """
    # modified from qiita.qiita_db.util.find_repeated
    # https://github.com/biocore/qiita
    # see licenses/qiita.txt
    seen, repeated = set(), set()
    for e in iterable:
        if e in seen:
            repeated.add(e)
        else:
            seen.add(e)
    return repeated

def flatten(items):
    """Removes one level of nesting from items

    .. note:: Deprecated in scikit-bio 0.2.3-dev
       ``skbio.util.flatten`` will be removed in scikit-bio 0.3.1
       it is being deprecated in favor of solutions present
       in the standard python library.
       Please refer to the following links for good alternatives:
       http://stackoverflow.com/a/952952/3639023
       http://stackoverflow.com/a/406199/3639023.

    Parameters
    ----------
    items : iterable
        list of items to flatten one level

    Returns
    -------
    flattened_items : list
        list of flattened items, items can be any sequence, but flatten always
        returns a list.

    Examples
    --------
    >>> from skbio.util import flatten
    >>> h = [['a', 'b', 'c', 'd'], [1, 2, 3, 4, 5], ['x', 'y'], ['foo']]
    >>> print(flatten(h))
    ['a', 'b', 'c', 'd', 1, 2, 3, 4, 5, 'x', 'y', 'foo']

    """
    warn("skbio.util.flatten is deprecated. Please refer to the following "
         "links for solutions from the standard python library: "
         "http://stackoverflow.com/a/952952/3639023 "
         "http://stackoverflow.com/a/406199/3639023", DeprecationWarning)

    result = []
    for i in items:
        try:
            result.extend(i)
        except TypeError:
            result.append(i)
    return result

def reprnator(start, tokens, end, separator=', '):
    """Live your dreams"""
    offset = len(start)
    indent = '\n%s' % (' ' * offset)
    remaining = 79 - offset

    repr_ = [start]

    for token, sep in zip_longest(tokens, [separator] * (len(tokens) - 1),
                                  fillvalue=''):
        token_len = len(token)
        sep_len = len(sep)

        if remaining - token_len - sep_len >= 0:
            repr_.append(token + sep)
            remaining -= token_len + sep_len
        else:
            repr_.append('%s%s' % (indent, token + sep))
            remaining = 79 - offset - token_len - sep_len

    if remaining - len(end) >= 0:
        repr_.append(end)
    else:
        repr_.append('%s%s' % (indent, end))

    return ''.join(repr_)

def _get_create_dir_error_codes():
    return {'NO_ERROR':      0,
            'DIR_EXISTS':    1,
            'FILE_EXISTS':   2,
            'OTHER_OS_ERROR': 3}

def _handle_error_codes(dir_name, suppress_errors=False,
                        error_code=None):
    """Wrapper function for error_handling.

    dir_name: name of directory that raised the error
    suppress_errors: if True raise Errors, otherwise return error_code
    error_code: the code for the error

    """
    error_code_lookup = _get_create_dir_error_codes()

    if error_code is None:
        error_code = error_code_lookup['NO_ERROR']

    error_strings = \
        {error_code_lookup['DIR_EXISTS']:
         "Directory already exists: %s" % dir_name,
         error_code_lookup['FILE_EXISTS']:
         "File with same name exists: %s" % dir_name,
         error_code_lookup['OTHER_OS_ERROR']:
         "Could not create output directory: %s. " % dir_name +
         "Check the permissions."}

    if error_code == error_code_lookup['NO_ERROR']:
        return error_code_lookup['NO_ERROR']
    if suppress_errors:
        return error_code
    else:
        raise OSError(error_strings[error_code])
<commit_msg>PY3: MiniRegistry works in py3 now<commit_after># ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import absolute_import, division, print_function
from future.standard_library import hooks

import hashlib
from os import remove, makedirs
from os.path import exists, isdir
from functools import partial
from warnings import warn
from types import FunctionType

with hooks():
    from itertools import zip_longest

class MiniRegistry(dict):
    def __call__(self, name):
        def decorator(func):
            self[name] = func
            return func
        return decorator

    def copy(self):
        return self.__class__(super(MiniRegistry, self).copy())

    def formatted_listing(self):
        if len(self) == 0:
            return "\tNone"
        else:
            return "\n".join(["\t%r\n\t  %s" %
                             (name, self[name].__doc__.split("\n")[0])
                              for name in self])

    def interpolate(self, obj, name):
        # Py2/3 compatible way of calling getattr(obj, name).__func__
        f = getattr(obj, name).__get__(None, type(None))
        # Deep magic happens here. I *think* the closure is interpereted in a
        # new context for each subclass making the whole thing work.
        if hasattr(f, 'func_code'):
            f2 = FunctionType(f.func_code, f.func_globals, name=f.func_name,
                              argdefs=f.func_defaults, closure=f.func_closure)
        else:
            f2 = FunctionType(f.__code__, f.__globals__, name=f.__name__,
                              argdefs=f.__defaults__, closure=f.__closure__)
        # Conveniently the original docstring is on f2, not the new ones if
        # inheritence is happening. I have no idea why.
        t = f2.__doc__.split("\n\n")
        t.insert(2, self.formatted_listing())
        f2.__doc__ = "\n\n".join(t)

        setattr(obj, name, f2)

def cardinal_to_ordinal(n):
    """Return ordinal string version of cardinal int `n`.

    Parameters
    ----------
    n : int
        Cardinal to convert to ordinal. Must be >= 0.

    Returns
    -------
    str
        Ordinal version of cardinal `n`.

    Raises
    ------
    ValueError
        If `n` is less than 0.

    Notes
    -----
    This function can be useful when writing human-readable error messages.

    Examples
    --------
    >>> from skbio.util import cardinal_to_ordinal
    >>> cardinal_to_ordinal(0)
    '0th'
    >>> cardinal_to_ordinal(1)
    '1st'
    >>> cardinal_to_ordinal(2)
    '2nd'
    >>> cardinal_to_ordinal(3)
    '3rd'

    """
    # Taken and modified from http://stackoverflow.com/a/20007730/3776794
    # Originally from http://codegolf.stackexchange.com/a/4712 by Gareth
    if n < 0:
        raise ValueError("Cannot convert negative integer %d to ordinal "
                         "string." % n)
    return "%d%s" % (n, "tsnrhtdd"[(n//10 % 10 != 1)*(n % 10 < 4)*n % 10::4])

def is_casava_v180_or_later(header_line):
    """Check if the header looks like it is Illumina software post-casava v1.8

    Parameters
    ----------
    header_line : bytes
        A header line

    Returns
    -------
    bool
        ``True`` for if casava v1.8+, otherwise ``False``

    Examples
    --------
    >>> from skbio.util import is_casava_v180_or_later
    >>> print(is_casava_v180_or_later('@foo'))
    False
    >>> id_ = '@M00176:17:000000000-A0CNA:1:1:15487:1773 1:N:0:0'
    >>> print(is_casava_v180_or_later(id_))
    True

    """
    if not header_line.startswith(b'@'):
        raise ValueError("Non-header line passed in!")
    fields = header_line.split(b':')

    return len(fields) == 10 and fields[7] in b'YN'

def safe_md5(open_file, block_size=2 ** 20):
    """Computes an md5 sum without loading the file into memory

    Parameters
    ----------
    open_file : file object
        open file handle to the archive to compute the checksum. It
        must be open as a binary file
    block_size : int, optional
        size of the block taken per iteration

    Returns
    -------
    md5 : md5 object from the hashlib module
        object with the loaded file

    Notes
    -----
    This method is based on the answers given in:
    http://stackoverflow.com/a/1131255/379593

    Examples
    --------
    >>> from StringIO import StringIO
    >>> from skbio.util import safe_md5
    >>> fd = StringIO("foo bar baz") # open file like object
    >>> x = safe_md5(fd)
    >>> x.hexdigest()
    'ab07acbb1e496801937adfa772424bf7'
    >>> fd.close()

    """
    md5 = hashlib.md5()
    data = True
    while data:
        data = open_file.read(block_size)
        if data:
            md5.update(data)
    return md5

def remove_files(list_of_filepaths, error_on_missing=True):
    """Remove list of filepaths, optionally raising an error if any are missing

    Parameters
    ----------
    list_of_filepaths : list of strings
        list with filepaths to remove
    error_on_missing : bool, optional
        whether or not the function should raise an ``OSError`` if a file is
        not found

    Raises
    ------
    OSError
        If a filepath in the list does not exist

    Examples
    --------
    >>> from tempfile import NamedTemporaryFile
    >>> from os.path import exists
    >>> from skbio.util import remove_files
    >>> h = NamedTemporaryFile(delete=False)
    >>> exists(h.name) # it exists
    True
    >>> remove_files([h.name])
    >>> exists(h.name) # and now it's gone
    False

    """
    missing = []
    for fp in list_of_filepaths:
        try:
            remove(fp)
        except OSError:
            missing.append(fp)

    if error_on_missing and missing:
        raise OSError("Some filepaths were not accessible: %s" %
                      '\t'.join(missing))

def create_dir(dir_name, fail_on_exist=False, handle_errors_externally=False):
    """Create a directory safely and fail meaningfully

    Parameters
    ----------
    dir_name: string
        name of directory to create

    fail_on_exist: bool, optional
        if true raise an error if ``dir_name`` already exists

    handle_errors_externally: bool, optional
        if True do not raise Errors, but return failure codes. This allows to
        handle errors locally and e.g. hint the user at a --force_overwrite
        options.

    Returns
    -------
    return_value : int
        These values are only returned if no error is raised:

        - ``0``:  directory was safely created
        - ``1``:  directory already existed
        - ``2``:  a file with the same name exists
        - ``3``:  any other unspecified ``OSError``

    Notes
    -----
    Depending  of how thorough we want to be we could add tests, e.g. for
    testing actual write permission in an existing dir.

    Examples
    --------
    >>> from skbio.util import create_dir
    >>> from os.path import exists, join
    >>> from tempfile import gettempdir
    >>> from os import rmdir
    >>> new_dir = join(gettempdir(), 'scikitbio')
    >>> create_dir(new_dir)
    0
    >>> exists(new_dir)
    True
    >>> rmdir(new_dir)

    """
    error_code_lookup = _get_create_dir_error_codes()
    # pre-instanciate function with
    ror = partial(_handle_error_codes, dir_name, handle_errors_externally)

    if exists(dir_name):
        if isdir(dir_name):
            # dir is there
            if fail_on_exist:
                return ror(error_code_lookup['DIR_EXISTS'])
            else:
                return error_code_lookup['DIR_EXISTS']
        else:
            # must be file with same name
            return ror(error_code_lookup['FILE_EXISTS'])
    else:
        # no dir there, try making it
        try:
            makedirs(dir_name)
        except OSError:
            return ror(error_code_lookup['OTHER_OS_ERROR'])

    return error_code_lookup['NO_ERROR']

def find_duplicates(iterable):
    """Find duplicate elements in an iterable.

    Parameters
    ----------
    iterable : iterable
        Iterable to be searched for duplicates (i.e., elements that are
        repeated).

    Returns
    -------
    set
        Repeated elements in `iterable`.

    """
    # modified from qiita.qiita_db.util.find_repeated
    # https://github.com/biocore/qiita
    # see licenses/qiita.txt
    seen, repeated = set(), set()
    for e in iterable:
        if e in seen:
            repeated.add(e)
        else:
            seen.add(e)
    return repeated

def flatten(items):
    """Removes one level of nesting from items

    .. note:: Deprecated in scikit-bio 0.2.3-dev
       ``skbio.util.flatten`` will be removed in scikit-bio 0.3.1
       it is being deprecated in favor of solutions present
       in the standard python library.
       Please refer to the following links for good alternatives:
       http://stackoverflow.com/a/952952/3639023
       http://stackoverflow.com/a/406199/3639023.

    Parameters
    ----------
    items : iterable
        list of items to flatten one level

    Returns
    -------
    flattened_items : list
        list of flattened items, items can be any sequence, but flatten always
        returns a list.

    Examples
    --------
    >>> from skbio.util import flatten
    >>> h = [['a', 'b', 'c', 'd'], [1, 2, 3, 4, 5], ['x', 'y'], ['foo']]
    >>> print(flatten(h))
    ['a', 'b', 'c', 'd', 1, 2, 3, 4, 5, 'x', 'y', 'foo']

    """
    warn("skbio.util.flatten is deprecated. Please refer to the following "
         "links for solutions from the standard python library: "
         "http://stackoverflow.com/a/952952/3639023 "
         "http://stackoverflow.com/a/406199/3639023", DeprecationWarning)

    result = []
    for i in items:
        try:
            result.extend(i)
        except TypeError:
            result.append(i)
    return result

def reprnator(start, tokens, end, separator=', '):
    """Live your dreams"""
    offset = len(start)
    indent = '\n%s' % (' ' * offset)
    remaining = 79 - offset

    repr_ = [start]

    for token, sep in zip_longest(tokens, [separator] * (len(tokens) - 1),
                                  fillvalue=''):
        token_len = len(token)
        sep_len = len(sep)

        if remaining - token_len - sep_len >= 0:
            repr_.append(token + sep)
            remaining -= token_len + sep_len
        else:
            repr_.append('%s%s' % (indent, token + sep))
            remaining = 79 - offset - token_len - sep_len

    if remaining - len(end) >= 0:
        repr_.append(end)
    else:
        repr_.append('%s%s' % (indent, end))

    return ''.join(repr_)

def _get_create_dir_error_codes():
    return {'NO_ERROR':      0,
            'DIR_EXISTS':    1,
            'FILE_EXISTS':   2,
            'OTHER_OS_ERROR': 3}

def _handle_error_codes(dir_name, suppress_errors=False,
                        error_code=None):
    """Wrapper function for error_handling.

    dir_name: name of directory that raised the error
    suppress_errors: if True raise Errors, otherwise return error_code
    error_code: the code for the error

    """
    error_code_lookup = _get_create_dir_error_codes()

    if error_code is None:
        error_code = error_code_lookup['NO_ERROR']

    error_strings = \
        {error_code_lookup['DIR_EXISTS']:
         "Directory already exists: %s" % dir_name,
         error_code_lookup['FILE_EXISTS']:
         "File with same name exists: %s" % dir_name,
         error_code_lookup['OTHER_OS_ERROR']:
         "Could not create output directory: %s. " % dir_name +
         "Check the permissions."}

    if error_code == error_code_lookup['NO_ERROR']:
        return error_code_lookup['NO_ERROR']
    if suppress_errors:
        return error_code
    else:
        raise OSError(error_strings[error_code])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.utils.datetime_safe import datetime
from django.utils.translation import ugettext_lazy as _, pgettext_lazy
from shop.models import order

class Order(order.BaseOrder):
    """Default materialized model for Order"""
    number = models.PositiveIntegerField(_("Order Number"), null=True, default=None, unique=True)
    shipping_address_text = models.TextField(_("Shipping Address"), blank=True, null=True,
        help_text=_("Shipping address at the moment of purchase."))
    billing_address_text = models.TextField(_("Billing Address"), blank=True, null=True,
        help_text=_("Billing address at the moment of purchase."))

    class Meta:
        verbose_name = pgettext_lazy('order_models', "Order")
        verbose_name_plural = pgettext_lazy('order_models', "Orders")

    def get_or_assign_number(self):
        """
        Set a unique number to identify this Order object. The first 4 digits represent the
        current year. The last five digits represent a zero-padded incremental counter.
        """
        if self.number is None:
            epoch = datetime.now().date()
            epoch = epoch.replace(epoch.year, 1, 1)
            aggr = Order.objects.filter(number__isnull=False, created_at__gt=epoch).aggregate(models.Max('number'))
            try:
                epoc_number = int(str(aggr['number__max'])[4:]) + 1
                self.number = int('{0}{1:05d}'.format(epoch.year, epoc_number))
            except (KeyError, ValueError):
                # the first order this year
                self.number = int('{0}00001'.format(epoch.year))
        return self.get_number()

    def get_number(self):
        return '{0}-{1}'.format(str(self.number)[:4], str(self.number)[4:])

    def populate_from_cart(self, cart, request):
        self.shipping_address_text = cart.shipping_address.as_text()
        self.billing_address_text = cart.billing_address.as_text() if cart.billing_address else self.shipping_address_text
        super(Order, self).populate_from_cart(cart, request)
<commit_msg>shipping and billing address shall not be editable<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.utils.datetime_safe import datetime
from django.utils.translation import ugettext_lazy as _, pgettext_lazy
from shop.models import order

class Order(order.BaseOrder):
    """Default materialized model for Order"""
    number = models.PositiveIntegerField(_("Order Number"), null=True, default=None, unique=True)
    shipping_address_text = models.TextField(_("Shipping Address"), blank=True, null=True,
        editable=False, help_text=_("Shipping address at the moment of purchase."))
    billing_address_text = models.TextField(_("Billing Address"), blank=True, null=True,
        editable=False, help_text=_("Billing address at the moment of purchase."))

    class Meta:
        verbose_name = pgettext_lazy('order_models', "Order")
        verbose_name_plural = pgettext_lazy('order_models', "Orders")

    def get_or_assign_number(self):
        """
        Set a unique number to identify this Order object. The first 4 digits represent the
        current year. The last five digits represent a zero-padded incremental counter.
        """
        if self.number is None:
            epoch = datetime.now().date()
            epoch = epoch.replace(epoch.year, 1, 1)
            aggr = Order.objects.filter(number__isnull=False, created_at__gt=epoch).aggregate(models.Max('number'))
            try:
                epoc_number = int(str(aggr['number__max'])[4:]) + 1
                self.number = int('{0}{1:05d}'.format(epoch.year, epoc_number))
            except (KeyError, ValueError):
                # the first order this year
                self.number = int('{0}00001'.format(epoch.year))
        return self.get_number()

    def get_number(self):
        return '{0}-{1}'.format(str(self.number)[:4], str(self.number)[4:])

    def populate_from_cart(self, cart, request):
        self.shipping_address_text = cart.shipping_address.as_text()
        self.billing_address_text = cart.billing_address.as_text() if cart.billing_address else self.shipping_address_text
        super(Order, self).populate_from_cart(cart, request)
<|endoftext|>


<commit_before>"""Data file utilities for dFab motion capture package.

Copyright (c) 2014, Garth Zeglin.  All rights reserved. Licensed under the terms
of the BSD 3-clause license as included in LICENSE.

This module produces and parses data files in several ad hoc plain text formats
convenient for readability and portability.

This deliberately *does not* import numpy, as it is intended to work with either
Rhino IronPython, 32-bit IronPython, or 32-or-64-bit CPython.

"""

################################################################
def write_point_trajectory_file( filename, times, points ):
    """Write a plain ASCII data matrix for plotting a trajectory of moving points.

    The units are assumed to be millimeters (and noted in the file comments).

    filename -- the full path to the output file to create
    times    -- N-element list of timestamps, one per frame
    points   -- N-element list of vectors with 3 or more elements (first three used)
    """

    plot = open( filename, "w")
    plot.write("""# point trajectory file
# Each line represents one Cartesian points for plotting the moving point, e.g.
# the origin of a moving coordinate frame.
# format: timestamp x, y, z
# units: seconds, millimeters
""")

    for i,pt in enumerate( points ):
        plot.write( "%f  " % times[i] )
        plot.write( "%f %f %f\n" % tuple( pt[0:3] ) )
    plot.close()
    return

################################################################
def write_trajectory_plot_file( filename, times, frames ):
    """Write a plain ASCII data matrix for plotting a trajectory of moving frames expressed as homogeneous transforms.

    The units are assumed to be millimeters (and noted in the file comments).

    filename -- the full path to the output file to create
    times    -- N-element list of timestamps, one per frame
    frames   -- N-element list of 4x4 homeogeneous transform matrices
    """

    plot = open( filename, "w")
    plot.write("""# frame plot file
# Each line represents four Cartesian points for plotting the origin and axis
# vectors of a moving coordinate frame.  A <> in the format represents an (x,y,z)
# triple.  Each axis vector has magnitude 20mm for visibility.
# format: timestamp <origin> <end of X axis> <end of Y axis> <end of Z axis>
# units: seconds, millimeters
""")

    for i,tool in enumerate( frames ):
        origin = tool[0:3,3] # origin vector, expressed in ground frame
        xaxis  = tool[0:3,0]
        yaxis  = tool[0:3,1]
        zaxis  = tool[0:3,2]
        plot.write( "%f   " % times[i] )
        plot.write( "%f %f %f   " % tuple(origin) )
        plot.write( "%f %f %f   " % tuple(origin+20*xaxis) )
        plot.write( "%f %f %f   " % tuple(origin+20*yaxis) )
        plot.write( "%f %f %f\n" % tuple( origin+20*zaxis) )
    plot.close()
    return

################################################################
def write_frame_trajectory_file( filename, times, frames ):
    """Write a plain ASCII data matrix for a trajectory of moving frames expressed as homogeneous transforms.

    The units are assumed to be millimeters (and noted in the file comments).

    filename -- the full path to the output file to create
    times    -- N-element list of timestamps, one per frame
    frames   -- N-element list of 4x4 homeogeneous transform matrices
    """

    plot = open( filename, "w")
    plot.write("""# frame trajectory file
# Each line represents the origin and axis vectors of a moving coordinate frame.
# A <> in the format represents an (x,y,z) triple.
# format: timestamp <origin> <X axis> <Y axis> <Z axis>
# units: seconds, millimeters
""")

    for i,tool in enumerate( frames ):
        xaxis  = tool[0:3,0]   # unit X axis basis vector
        yaxis  = tool[0:3,1]   # unit Y axis basis vector
        zaxis  = tool[0:3,2]   # unit Z axis basis vector
        origin = tool[0:3,3]   # origin vector, expressed in ground frame
        plot.write( "%f   " % times[i] )
        plot.write( "%f %f %f   " % tuple(origin) )
        plot.write( "%f %f %f   " % tuple(xaxis) )
        plot.write( "%f %f %f   " % tuple(yaxis) )
        plot.write( "%f %f %f\n"  % tuple( zaxis) )
    plot.close()
    return

################################################################
def read_frame_trajectory_file( filename ):
    """Read a plain ASCII frame trajectory file specifying a sequence of coordinate frames.

    Returns path, timestamps:

    path --- 3-dim list representing a list of coordinate frames : [ frame0, frame1...]
    timestamps --- list of scalars [ time0, time1, ...]

    Each frame is a 2-dim list in the following form:
    frame: [[ origin_x, origin_y, origin_z,],   origin vector of frame in parent frame
            [ unitx_x,  unitx_y,  unitx_z, ],   unit basis X vector expressed in parent frame
            [ unity_x,  unity_y,  unity_z, ],   unit basis Y vector expressed in parent frame
            [ unitz_x,  unitz_y,  unitz_z, ]]   unit basis Z vector expressed in parent frame
    """
    file = open(filename, "r")

    timestamps = list()
    path = list()

    for line in file:
        # eliminate leading spaces
        line = line.strip()

        # ignore comments and empty lines
        if len(line) == 0 or line[0] == '#':
            continue

        # divide on whitespace and convert to numbers
        nums = [float(x) for x in line.split()]

        # separate out components and build lists

        timestamps.append( nums[0] )

        origin = list( nums[1:4] )
        unitx  = list( nums[4:7] )
        unity  = list( nums[7:10] )
        unitz  = list( nums[10:13] )

        path.append( list( (origin, unitx, unity, unitz ) ) )

    return path, timestamps

################################################################
<commit_msg>Adding some simple JSON parameter file I/O.<commit_after>"""Data file utilities for dFab motion capture package.

Copyright (c) 2014, Garth Zeglin.  All rights reserved. Licensed under the terms
of the BSD 3-clause license as included in LICENSE.

This module produces and parses data files in several ad hoc plain text formats
convenient for readability and portability.

This deliberately *does not* import numpy, as it is intended to work with either
Rhino IronPython, 32-bit IronPython, or 32-or-64-bit CPython.

"""
import os
import json
import time

################################################################
def write_point_trajectory_file( filename, times, points ):
    """Write a plain ASCII data matrix for plotting a trajectory of moving points.

    The units are assumed to be millimeters (and noted in the file comments).

    filename -- the full path to the output file to create
    times    -- N-element list of timestamps, one per frame
    points   -- N-element list of vectors with 3 or more elements (first three used)
    """

    plot = open( filename, "w")
    plot.write("""# point trajectory file
# Each line represents one Cartesian points for plotting the moving point, e.g.
# the origin of a moving coordinate frame.
# format: timestamp x, y, z
# units: seconds, millimeters
""")

    for i,pt in enumerate( points ):
        plot.write( "%f  " % times[i] )
        plot.write( "%f %f %f\n" % tuple( pt[0:3] ) )
    plot.close()
    return

################################################################
def write_trajectory_plot_file( filename, times, frames ):
    """Write a plain ASCII data matrix for plotting a trajectory of moving frames expressed as homogeneous transforms.

    The units are assumed to be millimeters (and noted in the file comments).

    filename -- the full path to the output file to create
    times    -- N-element list of timestamps, one per frame
    frames   -- N-element list of 4x4 homeogeneous transform matrices
    """

    plot = open( filename, "w")
    plot.write("""# frame plot file
# Each line represents four Cartesian points for plotting the origin and axis
# vectors of a moving coordinate frame.  A <> in the format represents an (x,y,z)
# triple.  Each axis vector has magnitude 20mm for visibility.
# format: timestamp <origin> <end of X axis> <end of Y axis> <end of Z axis>
# units: seconds, millimeters
""")

    for i,tool in enumerate( frames ):
        origin = tool[0:3,3] # origin vector, expressed in ground frame
        xaxis  = tool[0:3,0]
        yaxis  = tool[0:3,1]
        zaxis  = tool[0:3,2]
        plot.write( "%f   " % times[i] )
        plot.write( "%f %f %f   " % tuple(origin) )
        plot.write( "%f %f %f   " % tuple(origin+20*xaxis) )
        plot.write( "%f %f %f   " % tuple(origin+20*yaxis) )
        plot.write( "%f %f %f\n" % tuple( origin+20*zaxis) )
    plot.close()
    return

################################################################
def write_frame_trajectory_file( filename, times, frames ):
    """Write a plain ASCII data matrix for a trajectory of moving frames expressed as homogeneous transforms.

    The units are assumed to be millimeters (and noted in the file comments).

    filename -- the full path to the output file to create
    times    -- N-element list of timestamps, one per frame
    frames   -- N-element list of 4x4 homeogeneous transform matrices
    """

    plot = open( filename, "w")
    plot.write("""# frame trajectory file
# Each line represents the origin and axis vectors of a moving coordinate frame.
# A <> in the format represents an (x,y,z) triple.
# format: timestamp <origin> <X axis> <Y axis> <Z axis>
# units: seconds, millimeters
""")

    for i,tool in enumerate( frames ):
        xaxis  = tool[0:3,0]   # unit X axis basis vector
        yaxis  = tool[0:3,1]   # unit Y axis basis vector
        zaxis  = tool[0:3,2]   # unit Z axis basis vector
        origin = tool[0:3,3]   # origin vector, expressed in ground frame
        plot.write( "%f   " % times[i] )
        plot.write( "%f %f %f   " % tuple(origin) )
        plot.write( "%f %f %f   " % tuple(xaxis) )
        plot.write( "%f %f %f   " % tuple(yaxis) )
        plot.write( "%f %f %f\n"  % tuple( zaxis) )
    plot.close()
    return

################################################################
def read_frame_trajectory_file( filename ):
    """Read a plain ASCII frame trajectory file specifying a sequence of coordinate frames.

    Returns path, timestamps:

    path --- 3-dim list representing a list of coordinate frames : [ frame0, frame1...]
    timestamps --- list of scalars [ time0, time1, ...]

    Each frame is a 2-dim list in the following form:
    frame: [[ origin_x, origin_y, origin_z,],   origin vector of frame in parent frame
            [ unitx_x,  unitx_y,  unitx_z, ],   unit basis X vector expressed in parent frame
            [ unity_x,  unity_y,  unity_z, ],   unit basis Y vector expressed in parent frame
            [ unitz_x,  unitz_y,  unitz_z, ]]   unit basis Z vector expressed in parent frame
    """
    file = open(filename, "r")

    timestamps = list()
    path = list()

    for line in file:
        # eliminate leading spaces
        line = line.strip()

        # ignore comments and empty lines
        if len(line) == 0 or line[0] == '#':
            continue

        # divide on whitespace and convert to numbers
        nums = [float(x) for x in line.split()]

        # separate out components and build lists

        timestamps.append( nums[0] )

        origin = list( nums[1:4] )
        unitx  = list( nums[4:7] )
        unity  = list( nums[7:10] )
        unitz  = list( nums[10:13] )

        path.append( list( (origin, unitx, unity, unitz ) ) )

    return path, timestamps

################################################################
def read_parameter_file( param_file_name ):
    """Read a JSON parameter file into a plain Python dictionary."""
    return json.loads( open( param_file_name).read() )

################################################################
def write_parameter_file( param_file_name, dictionary ):
    """Write a dictionary of plain Python types into a JSON parameter file."""
    # if the file exists, rename it as a backup
    if os.path.exists( param_file_name ):
        os.rename( param_file_name, param_file_name + '.bak.' + str(int(time.time())))

    # write a new JSON file
    param_file = open( param_file_name, 'w' )
    param_file.write( json.dumps( dictionary, indent = 1) )
    param_file.close()
    return

################################################################

<|endoftext|>