<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Params related methods.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from django import forms
from django.utils.translation import ugettext

from soc.logic import cleaning
from soc.logic import dicts
from soc.models import linkable
from soc.views import helper
from soc.views.helper import access
from soc.views.helper import dynaform
from soc.views.helper import redirects

DEF_LIST_DESCRIPTION_FMT = ugettext(
    'List of %(name_plural)s in Google Open Source Programs.')

DEF_CREATE_INSTRUCTION_MSG_FMT = ugettext(
      'Please use this form to select a %(name).')

DEF_SUBMIT_MSG_PARAM_NAME = 's'
DEF_SUBMIT_MSG_PROFILE_SAVED = 0

def constructParams(params):
  """Constructs a new params dictionary based on params.

  Params usage:
    The params dictionary is passed to getCreateForm and getEditForm,
    see their docstring on how they use it.

    rights: The rights value is merged with a default rights
      dictionary and then used as rights value.
    url_name: The url_name value is used in constructing several
      redirects as the first part of the url.
    module_name: The module_name value is used in constructing the
      location of several templates. It is expected that it matches
      the part after "/templates/soc/" for this View.
    name_plural: The name_plural argument is provided to the
      LIST_DESCRIPTION when constructing the list_description field.
    extra_dynainclude: The extra_dynainclude value is used when
      constructing the create_dynainclude value.
    extra_dynaexclude: The extra_dynaexclude value is used when
      constructing the create_dynaexclude value.
    logic: The logic value is used as argument to save the scope_logic
      and create a create form.
  """

  logic = params['logic']

  rights = access.Checker(params)
  rights['unspecified'] = []
  rights['any_access'] = ['checkIsLoggedIn']
  rights['show'] = ['checkIsUser']
  rights['create'] = ['checkIsDeveloper']
  rights['edit'] = ['checkIsDeveloper']
  rights['delete'] = ['checkIsDeveloper']
  rights['list'] = ['checkIsDeveloper']
  rights['pick'] = ['checkIsUser'] # TODO(SRabbelier): proper check

  new_params = {}
  new_params['scope_logic'] = logic.getScopeLogic()

  if 'name_short' not in params:
    params['name_short'] = params['name']

  if 'name_plural' not in params:
    params['name_plural'] = params['name'] + 's'

  if 'module_name' not in params:
    params['module_name'] = params['name_short'].replace(' ', '_').lower()

  if 'url_name' not in params:
    params['url_name'] = params['module_name']

  if 'document_prefix' not in params:
    params['document_prefix'] = params['name_short'].lower()

  # Do not expand edit_redirect to allow it to be overriden without suffix
  new_params['edit_redirect'] = '/%(url_name)s/edit/%(suffix)s'
  new_params['missing_redirect'] = '/%(url_name)s/create' % params
  new_params['delete_redirect'] = '/%(url_name)s/list' % params
  new_params['invite_redirect'] = '/request/list'
  new_params['edit_cancel_redirect'] = '/%(url_name)s/list' % params

  new_params['sidebar'] = None
  new_params['sidebar_grouping'] = 'main'
  new_params['sidebar_defaults'] = [
      ('/%s/create', 'New %(name)s', 'create'),
      ('/%s/list', 'List %(name_plural)s', 'list'),
      ]
  new_params['sidebar_additional'] = []

  new_params['link_id_arg_pattern'] = linkable.LINK_ID_ARG_PATTERN
  new_params['link_id_pattern_core'] = linkable.LINK_ID_PATTERN_CORE
  new_params['scope_path_pattern'] = getScopePattern(params)

  new_params['django_patterns'] = None
  new_params['extra_django_patterns'] = []
  new_params['django_patterns_defaults'] = [
      (r'^%(url_name)s/(?P<access_type>show)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.public', 'Show %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>export)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.export', 'Export %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>create)$',
          'soc.views.models.%(module_name)s.create', 'Create %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>delete)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.delete', 'Delete %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>edit)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.edit', 'Edit %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>list)$',
          'soc.views.models.%(module_name)s.list', 'List %(name_plural)s'),
      (r'^%(url_name)s/(?P<access_type>pick)$',
          'soc.views.models.%(module_name)s.pick', 'Pick %(name_short)s'),
      ]

  if not params.get('no_create_with_scope'):
    new_params['django_patterns_defaults'] += [
        (r'^%(url_name)s/(?P<access_type>create)/%(scope)s$',
        'soc.views.models.%(module_name)s.create', 'Create %(name_short)s')]

  if not params.get('no_create_with_key_fields'):
    new_params['django_patterns_defaults'] += [
        (r'^%(url_name)s/(?P<access_type>create)/%(key_fields)s$',
        'soc.views.models.%(module_name)s.create', 'Create %(name_short)s')]

  new_params['public_template'] = 'soc/%(module_name)s/public.html' % params
  new_params['export_template'] = 'soc/%(module_name)s/export.html' % params
  new_params['create_template'] = 'soc/models/edit.html'
  new_params['edit_template'] = 'soc/models/edit.html'
  new_params['list_template'] = 'soc/models/list.html'
  new_params['invite_template'] = 'soc/models/invite.html'

  new_params['export_content_type'] = None

  new_params['error_public'] = 'soc/%(module_name)s/error.html' % params
  new_params['error_export'] = new_params['error_public']
  new_params['error_edit'] = new_params['error_public']

  new_params['list_main'] = 'soc/list/main.html'
  new_params['list_pagination'] = 'soc/list/pagination.html'
  new_params['list_row'] = 'soc/%(module_name)s/list/row.html' % params
  new_params['list_heading'] = 'soc/%(module_name)s/list/heading.html' % params

  new_params['list_action'] = (redirects.getEditRedirect, params)
  new_params['list_params'] = {
      'list_action': 'action',
      'list_description': 'description',
      'list_main': 'main',
      'list_pagination': 'pagination',
      'list_row': 'row',
      'list_heading': 'heading',
      }

  new_params['list_description'] = DEF_LIST_DESCRIPTION_FMT % params
  new_params['save_message'] = [ugettext('%(name)s saved.' % params)]
  new_params['submit_msg_param_name'] = DEF_SUBMIT_MSG_PARAM_NAME
  new_params['edit_params'] = {
      DEF_SUBMIT_MSG_PARAM_NAME: DEF_SUBMIT_MSG_PROFILE_SAVED,
      }

  new_params['dynabase'] = helper.forms.BaseForm

  create_dynafields = {
      'clean_link_id': cleaning.clean_link_id('link_id'),
      'clean_feed_url': cleaning.clean_feed_url,
      }
  create_dynafields.update(params.get('create_extra_dynafields', {}))

  new_params['references'] = []
  new_params['create_dynainclude'] = [] + params.get('extra_dynainclude', [])
  new_params['create_dynaexclude'] = ['scope', 'scope_path'] + \
      params.get('extra_dynaexclude', [])
  new_params['create_dynafields'] = create_dynafields

  edit_dynafields = {
      'clean_link_id': cleaning.clean_link_id('link_id'),
      'link_id': forms.CharField(widget=helper.widgets.ReadOnlyInput()),
      }
  edit_dynafields.update(params.get('edit_extra_dynafields', {}))

  new_params['edit_dynainclude'] = None
  new_params['edit_dynaexclude'] = None
  new_params['edit_dynafields'] = edit_dynafields

  params = dicts.merge(params, new_params)

  # These need to be constructed separately, because they require
  # parameters that can be defined either in params, or new_params.
  if not 'create_form' in params:
    params['create_form'] = getCreateForm(params, logic.getModel())

  if not 'edit_form' in params:
    params['edit_form'] = getEditForm(params, params['create_form'])

  if not 'key_fields_pattern' in params:
    params['key_fields_pattern'] = getKeyFieldsPattern(params)

  # merge already done by access.Checker
  params['rights'] = rights

  return params

def getCreateForm(params, model):
  """Constructs a new CreateForm using params.

  Params usage:
    dynabase: The dynabase value is used as the base argument to
      dynaform.newDynaForm.
    logic: The logic value is used to get the model argument to newDynaForm.
    create_dynainclude: same as dynabase, but as dynainclude argument
    create_dynaexclude: same as dynabase, but as dynaexclude argument
    create_dynafields: same as dynabase, but as dynafields argument
  """

  create_form = dynaform.newDynaForm(
    dynabase = params['dynabase'],
    dynamodel = model,
    dynainclude = params['create_dynainclude'],
    dynaexclude = params['create_dynaexclude'],
    dynafields = params['create_dynafields'],
    )

  if 'extra_key_order' in params:
    for field in params['extra_key_order']:
      create_form.base_fields.keyOrder.remove(field)
    create_form.base_fields.keyOrder.extend(params['extra_key_order'])

  return create_form

def getEditForm(params, base_form):
  """Constructs a new EditForm using params.

  Params usage:
    create_form: The dynabase value is used as the dynaform argument
      to dyanform.extendDynaForm.
    edit_dynainclude: same as create_form, but as dynainclude argument
    edit_dynaexclude: same as create_form, but as dynaexclude argument
    edit_dynafields: same as create_form, but as dynafields argument
  """

  edit_form = dynaform.extendDynaForm(
    dynaform = base_form,
    dynainclude = params['edit_dynainclude'],
    dynaexclude = params['edit_dynaexclude'],
    dynafields = params['edit_dynafields'],
    )

  return edit_form

def getKeyFieldsPattern(params):
  """Returns the Django pattern for this View's entity.
  """

  names = params['logic'].getKeyFieldNames()
  scope_path_pattern = params['scope_path_pattern']

  return getPattern(names, scope_path_pattern)

def getPattern(names, scope_path_pattern):
  """Returns the Django patterns for the specified names.

  Args:
    names: the field names that should be included in the pattern
    scope_path_pattern: the pattern to use if the name is 'scope_path'
  """

  patterns = []

  for name in names:
    if name == 'scope_path':
      pattern = scope_path_pattern
    else:
      pattern = r'(?P<%s>%s)' % (name, linkable.LINK_ID_PATTERN_CORE)
    patterns.append(pattern)

  result = '/'.join(patterns)
  return result

def getScopePattern(params):
  """Returns the Scope pattern for this entity.
  """

  logic = params['logic']
  depth = logic.getScopeDepth()
  if depth is None:
    return linkable.SCOPE_PATH_ARG_PATTERN

  regexps = [linkable.LINK_ID_PATTERN_CORE for i in range(depth)]
  regexp = '/'.join(regexps)
  return r'(?P<scope_path>%s)' % regexp
<commit_msg>Document prefix in params now defaults to url_name.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Params related methods.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from django import forms
from django.utils.translation import ugettext

from soc.logic import cleaning
from soc.logic import dicts
from soc.models import linkable
from soc.views import helper
from soc.views.helper import access
from soc.views.helper import dynaform
from soc.views.helper import redirects

DEF_LIST_DESCRIPTION_FMT = ugettext(
    'List of %(name_plural)s in Google Open Source Programs.')

DEF_CREATE_INSTRUCTION_MSG_FMT = ugettext(
      'Please use this form to select a %(name).')

DEF_SUBMIT_MSG_PARAM_NAME = 's'
DEF_SUBMIT_MSG_PROFILE_SAVED = 0

def constructParams(params):
  """Constructs a new params dictionary based on params.

  Params usage:
    The params dictionary is passed to getCreateForm and getEditForm,
    see their docstring on how they use it.

    rights: The rights value is merged with a default rights
      dictionary and then used as rights value.
    url_name: The url_name value is used in constructing several
      redirects as the first part of the url.
    module_name: The module_name value is used in constructing the
      location of several templates. It is expected that it matches
      the part after "/templates/soc/" for this View.
    name_plural: The name_plural argument is provided to the
      LIST_DESCRIPTION when constructing the list_description field.
    extra_dynainclude: The extra_dynainclude value is used when
      constructing the create_dynainclude value.
    extra_dynaexclude: The extra_dynaexclude value is used when
      constructing the create_dynaexclude value.
    logic: The logic value is used as argument to save the scope_logic
      and create a create form.
  """

  logic = params['logic']

  rights = access.Checker(params)
  rights['unspecified'] = []
  rights['any_access'] = ['checkIsLoggedIn']
  rights['show'] = ['checkIsUser']
  rights['create'] = ['checkIsDeveloper']
  rights['edit'] = ['checkIsDeveloper']
  rights['delete'] = ['checkIsDeveloper']
  rights['list'] = ['checkIsDeveloper']
  rights['pick'] = ['checkIsUser'] # TODO(SRabbelier): proper check

  new_params = {}
  new_params['scope_logic'] = logic.getScopeLogic()

  if 'name_short' not in params:
    params['name_short'] = params['name']

  if 'name_plural' not in params:
    params['name_plural'] = params['name'] + 's'

  if 'module_name' not in params:
    params['module_name'] = params['name_short'].replace(' ', '_').lower()

  if 'url_name' not in params:
    params['url_name'] = params['module_name']

  if 'document_prefix' not in params:
    params['document_prefix'] = params['url_name']

  # Do not expand edit_redirect to allow it to be overriden without suffix
  new_params['edit_redirect'] = '/%(url_name)s/edit/%(suffix)s'
  new_params['missing_redirect'] = '/%(url_name)s/create' % params
  new_params['delete_redirect'] = '/%(url_name)s/list' % params
  new_params['invite_redirect'] = '/request/list'
  new_params['edit_cancel_redirect'] = '/%(url_name)s/list' % params

  new_params['sidebar'] = None
  new_params['sidebar_grouping'] = 'main'
  new_params['sidebar_defaults'] = [
      ('/%s/create', 'New %(name)s', 'create'),
      ('/%s/list', 'List %(name_plural)s', 'list'),
      ]
  new_params['sidebar_additional'] = []

  new_params['link_id_arg_pattern'] = linkable.LINK_ID_ARG_PATTERN
  new_params['link_id_pattern_core'] = linkable.LINK_ID_PATTERN_CORE
  new_params['scope_path_pattern'] = getScopePattern(params)

  new_params['django_patterns'] = None
  new_params['extra_django_patterns'] = []
  new_params['django_patterns_defaults'] = [
      (r'^%(url_name)s/(?P<access_type>show)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.public', 'Show %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>export)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.export', 'Export %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>create)$',
          'soc.views.models.%(module_name)s.create', 'Create %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>delete)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.delete', 'Delete %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>edit)/%(key_fields)s$',
          'soc.views.models.%(module_name)s.edit', 'Edit %(name_short)s'),
      (r'^%(url_name)s/(?P<access_type>list)$',
          'soc.views.models.%(module_name)s.list', 'List %(name_plural)s'),
      (r'^%(url_name)s/(?P<access_type>pick)$',
          'soc.views.models.%(module_name)s.pick', 'Pick %(name_short)s'),
      ]

  if not params.get('no_create_with_scope'):
    new_params['django_patterns_defaults'] += [
        (r'^%(url_name)s/(?P<access_type>create)/%(scope)s$',
        'soc.views.models.%(module_name)s.create', 'Create %(name_short)s')]

  if not params.get('no_create_with_key_fields'):
    new_params['django_patterns_defaults'] += [
        (r'^%(url_name)s/(?P<access_type>create)/%(key_fields)s$',
        'soc.views.models.%(module_name)s.create', 'Create %(name_short)s')]

  new_params['public_template'] = 'soc/%(module_name)s/public.html' % params
  new_params['export_template'] = 'soc/%(module_name)s/export.html' % params
  new_params['create_template'] = 'soc/models/edit.html'
  new_params['edit_template'] = 'soc/models/edit.html'
  new_params['list_template'] = 'soc/models/list.html'
  new_params['invite_template'] = 'soc/models/invite.html'

  new_params['export_content_type'] = None

  new_params['error_public'] = 'soc/%(module_name)s/error.html' % params
  new_params['error_export'] = new_params['error_public']
  new_params['error_edit'] = new_params['error_public']

  new_params['list_main'] = 'soc/list/main.html'
  new_params['list_pagination'] = 'soc/list/pagination.html'
  new_params['list_row'] = 'soc/%(module_name)s/list/row.html' % params
  new_params['list_heading'] = 'soc/%(module_name)s/list/heading.html' % params

  new_params['list_action'] = (redirects.getEditRedirect, params)
  new_params['list_params'] = {
      'list_action': 'action',
      'list_description': 'description',
      'list_main': 'main',
      'list_pagination': 'pagination',
      'list_row': 'row',
      'list_heading': 'heading',
      }

  new_params['list_description'] = DEF_LIST_DESCRIPTION_FMT % params
  new_params['save_message'] = [ugettext('%(name)s saved.' % params)]
  new_params['submit_msg_param_name'] = DEF_SUBMIT_MSG_PARAM_NAME
  new_params['edit_params'] = {
      DEF_SUBMIT_MSG_PARAM_NAME: DEF_SUBMIT_MSG_PROFILE_SAVED,
      }

  new_params['dynabase'] = helper.forms.BaseForm

  create_dynafields = {
      'clean_link_id': cleaning.clean_link_id('link_id'),
      'clean_feed_url': cleaning.clean_feed_url,
      }
  create_dynafields.update(params.get('create_extra_dynafields', {}))

  new_params['references'] = []
  new_params['create_dynainclude'] = [] + params.get('extra_dynainclude', [])
  new_params['create_dynaexclude'] = ['scope', 'scope_path'] + \
      params.get('extra_dynaexclude', [])
  new_params['create_dynafields'] = create_dynafields

  edit_dynafields = {
      'clean_link_id': cleaning.clean_link_id('link_id'),
      'link_id': forms.CharField(widget=helper.widgets.ReadOnlyInput()),
      }
  edit_dynafields.update(params.get('edit_extra_dynafields', {}))

  new_params['edit_dynainclude'] = None
  new_params['edit_dynaexclude'] = None
  new_params['edit_dynafields'] = edit_dynafields

  params = dicts.merge(params, new_params)

  # These need to be constructed separately, because they require
  # parameters that can be defined either in params, or new_params.
  if not 'create_form' in params:
    params['create_form'] = getCreateForm(params, logic.getModel())

  if not 'edit_form' in params:
    params['edit_form'] = getEditForm(params, params['create_form'])

  if not 'key_fields_pattern' in params:
    params['key_fields_pattern'] = getKeyFieldsPattern(params)

  # merge already done by access.Checker
  params['rights'] = rights

  return params

def getCreateForm(params, model):
  """Constructs a new CreateForm using params.

  Params usage:
    dynabase: The dynabase value is used as the base argument to
      dynaform.newDynaForm.
    logic: The logic value is used to get the model argument to newDynaForm.
    create_dynainclude: same as dynabase, but as dynainclude argument
    create_dynaexclude: same as dynabase, but as dynaexclude argument
    create_dynafields: same as dynabase, but as dynafields argument
  """

  create_form = dynaform.newDynaForm(
    dynabase = params['dynabase'],
    dynamodel = model,
    dynainclude = params['create_dynainclude'],
    dynaexclude = params['create_dynaexclude'],
    dynafields = params['create_dynafields'],
    )

  if 'extra_key_order' in params:
    for field in params['extra_key_order']:
      create_form.base_fields.keyOrder.remove(field)
    create_form.base_fields.keyOrder.extend(params['extra_key_order'])

  return create_form

def getEditForm(params, base_form):
  """Constructs a new EditForm using params.

  Params usage:
    create_form: The dynabase value is used as the dynaform argument
      to dyanform.extendDynaForm.
    edit_dynainclude: same as create_form, but as dynainclude argument
    edit_dynaexclude: same as create_form, but as dynaexclude argument
    edit_dynafields: same as create_form, but as dynafields argument
  """

  edit_form = dynaform.extendDynaForm(
    dynaform = base_form,
    dynainclude = params['edit_dynainclude'],
    dynaexclude = params['edit_dynaexclude'],
    dynafields = params['edit_dynafields'],
    )

  return edit_form

def getKeyFieldsPattern(params):
  """Returns the Django pattern for this View's entity.
  """

  names = params['logic'].getKeyFieldNames()
  scope_path_pattern = params['scope_path_pattern']

  return getPattern(names, scope_path_pattern)

def getPattern(names, scope_path_pattern):
  """Returns the Django patterns for the specified names.

  Args:
    names: the field names that should be included in the pattern
    scope_path_pattern: the pattern to use if the name is 'scope_path'
  """

  patterns = []

  for name in names:
    if name == 'scope_path':
      pattern = scope_path_pattern
    else:
      pattern = r'(?P<%s>%s)' % (name, linkable.LINK_ID_PATTERN_CORE)
    patterns.append(pattern)

  result = '/'.join(patterns)
  return result

def getScopePattern(params):
  """Returns the Scope pattern for this entity.
  """

  logic = params['logic']
  depth = logic.getScopeDepth()
  if depth is None:
    return linkable.SCOPE_PATH_ARG_PATTERN

  regexps = [linkable.LINK_ID_PATTERN_CORE for i in range(depth)]
  regexp = '/'.join(regexps)
  return r'(?P<scope_path>%s)' % regexp
<|endoftext|>


<commit_before># Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
General constants.
"""

SERVICE = 'org.storage.stratis1'
TOP_OBJECT = '/org/storage/stratis1'
MANAGER_INTERFACE = 'org.storage.stratis1.Manager'

SERVICE_UNKNOWN_ERROR = 'org.freedesktop.DBus.Error.ServiceUnknown'

REDUNDANCY = {
   'none' : 'STRATIS_RAID_TYPE_SINGLE'
}

SECTOR_SIZE = 512
<commit_msg>Delete some obsoleted constants<commit_after># Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
General constants.
"""

SERVICE = 'org.storage.stratis1'
TOP_OBJECT = '/org/storage/stratis1'
MANAGER_INTERFACE = 'org.storage.stratis1.Manager'

SECTOR_SIZE = 512
<|endoftext|>


<commit_before># Copyright (C) Ivan Kravets <me@ikravets.com>
# See LICENSE for details.

"""
mbed

The mbed framework The mbed SDK has been designed to provide enough
hardware abstraction to be intuitive and concise, yet powerful enough to
build complex projects. It is built on the low-level ARM CMSIS APIs,
allowing you to code down to the metal if needed. In addition to RTOS,
USB and Networking libraries, a cookbook of hundreds of reusable
peripheral and module libraries have been built on top of the SDK by
the mbed Developer Community.

http://mbed.org/
"""

import re
import xml.etree.ElementTree as ElementTree
from binascii import crc32
from os import walk
from os.path import basename, isfile, join, normpath

from SCons.Script import DefaultEnvironment, Exit

env = DefaultEnvironment()

BOARD_OPTS = env.get("BOARD_OPTIONS", {}).get("build", {})

env.Replace(
    PLATFORMFW_DIR=join("$PIOPACKAGES_DIR", "framework-mbed")
)

MBED_VARIANTS = {
    "stm32f3discovery": "DISCO_F303VC",
    "stm32f4discovery": "DISCO_F407VG",
    "stm32f429discovery": "DISCO_F429ZI",
    "blueboard_lpc11u24": "LPC11U24",
    "dipcortexm0": "LPC11U24",
    "seeeduinoArchPro": "ARCH_PRO",
    "ubloxc027": "UBLOX_C027",
    "lpc1114fn28": "LPC1114",
    "lpc11u35": "LPC11U35_401",
    "mbuino": "LPC11U24",
    "nrf51_mkit": "NRF51822",
    "seeedTinyBLE": "SEEED_TINY_BLE",
    "redBearLab": "RBLAB_NRF51822",
    "nrf51-dt": "NRF51_DK",
    "redBearLabBLENano": "RBLAB_NRF51822",
    "wallBotBLE": "NRF51822",
    "frdm_kl25z": "KL25Z",
    "frdm_kl46z": "KL46Z",
    "frdm_k64f": "K64F",
    "frdm_kl05z": "KL05Z",
    "frdm_k20d50m": "K20D50M",
    "frdm_k22f": "K22F"
}

MBED_LIBS_MAP = {
    "dsp": {"ar": ["dsp", "cmsis_dsp"]},
    "eth": {"ar": ["eth"], "deps": ["rtos"]},
    "fat": {"ar": ["fat"]},
    "rtos": {"ar": ["rtos", "rtx"]},
    "usb": {"ar": ["USBDevice"]},
    "usb_host": {"ar": ["USBHost"]}
}

def get_mbedlib_includes():
    result = []
    for lib in MBED_LIBS_MAP.keys():
        includes = []
        lib_dir = join(env.subst("$PLATFORMFW_DIR"), "libs", lib)
        for _, _, files in walk(lib_dir):
            for libfile in files:
                if libfile.endswith(".h"):
                    includes.append(libfile)
        result.append((lib, set(includes)))
    return result

def get_used_mbedlibs():
    re_includes = re.compile(r"^(#include\s+(?:\<|\")([^\r\n\"]+))",
                             re.M | re.I)
    srcincs = []
    for root, _, files in walk(env.get("PROJECTSRC_DIR")):
        for pfile in files:
            if not any([pfile.endswith(ext) for ext in (".h", ".c", ".cpp")]):
                continue
            with open(join(root, pfile)) as fp:
                srcincs.extend([i[1] for i in re_includes.findall(fp.read())])
    srcincs = set(srcincs)

    result = {}
    for libname, libincs in get_mbedlib_includes():
        if libincs & srcincs and libname not in result:
            result[libname] = MBED_LIBS_MAP[libname]

    return result

def add_mbedlib(libname, libar):
    if libar in env.get("LIBS"):
        return

    lib_dir = join(env.subst("$PLATFORMFW_DIR"), "libs", libname)
    if not isfile(join(lib_dir, "TARGET_%s" % variant,
                       "TOOLCHAIN_GCC_ARM", "lib%s.a" % libar)):
        Exit("Error: %s board doesn't support %s library!" %
             (env.get("BOARD"), libname))

    env.Append(
        LIBPATH=[
            join(env.subst("$PLATFORMFW_DIR"), "libs", libname,
                 "TARGET_%s" % variant, "TOOLCHAIN_GCC_ARM")
        ],
        LIBS=[libar]
    )

    sysincdirs = (
        "eth",
        "include",
        "ipv4",
        "lwip-eth",
        "lwip-sys"
    )

    for root, _, files in walk(lib_dir):
        if (not any(f.endswith(".h") for f in files) and
                basename(root) not in sysincdirs):
            continue
        var_dir = join("$BUILD_DIR", "FrameworkMbed%sInc%d" %
                       (libname.upper(), crc32(root)))
        if var_dir in env.get("CPPPATH"):
            continue
        env.VariantDir(var_dir, root)
        env.Append(CPPPATH=[var_dir])

def parse_eix_file(filename):
    result = {}
    paths = (
        ("CFLAGS", "./Target/Source/CC/Switch"),
        ("CXXFLAGS", "./Target/Source/CPPC/Switch"),
        ("CPPDEFINES", "./Target/Source/Symbols/Symbol"),
        ("FILES", "./Target/Files/File"),
        ("LINKFLAGS", "./Target/Source/LD/Switch"),
        ("OBJFILES", "./Target/Source/Addobjects/Addobject"),
        ("LIBPATH", "./Target/Linker/Librarypaths/Librarypath"),
        ("STDLIBS", "./Target/Source/Syslibs/Library"),
        ("LDSCRIPT_PATH", "./Target/Source/Scriptfile"),
        ("CPPPATH", "./Target/Compiler/Includepaths/Includepath")
    )

    tree = ElementTree.parse(filename)

    for (key, path) in paths:
        if key not in result:
            result[key] = []

        for node in tree.findall(path):
            _nkeys = node.keys()
            result[key].append(
                node.get(_nkeys[0]) if len(_nkeys) == 1 else node.attrib)

    return result

def get_build_flags(data):
    flags = {}
    cflags = set(data.get("CFLAGS", []))
    cxxflags = set(data.get("CXXFLAGS", []))
    cppflags = set(cflags & cxxflags)
    flags['CPPFLAGS'] = list(cppflags)
    flags['CXXFLAGS'] = list(cxxflags - cppflags)
    flags['CFLAGS'] = list(cflags - cppflags)
    return flags

def _mbed_whole_archive_hook(flags):
    if not isinstance(flags, list):
        return flags

    for pos, flag in enumerate(flags[:]):
        if isinstance(flag, basestring):
            continue
        flags.insert(pos, "-Wl,-whole-archive")
        flags.insert(pos + 2, "-Wl,-no-whole-archive")

    return flags

env.Replace(
    _mbed_whole_archive_hook=_mbed_whole_archive_hook,
    _LIBFLAGS="${_mbed_whole_archive_hook(%s)}" % env.get("_LIBFLAGS")[2:-1]
)

board_type = env.subst("$BOARD")
variant = MBED_VARIANTS[
    board_type] if board_type in MBED_VARIANTS else board_type.upper()
eixdata = parse_eix_file(
    join(env.subst("$PLATFORMFW_DIR"), "variant", variant, "%s.eix" % variant))

build_flags = get_build_flags(eixdata)
variant_dir = join("$PLATFORMFW_DIR", "variant", variant)

env.Replace(
    _mbed_whole_archive_hook=_mbed_whole_archive_hook,
    _LIBFLAGS="${_mbed_whole_archive_hook(%s)}" % env.get("_LIBFLAGS")[2:-1],
    CPPFLAGS=build_flags.get("CPPFLAGS", []),
    CFLAGS=build_flags.get("CFLAGS", []),
    CXXFLAGS=build_flags.get("CXXFLAGS", []),
    LINKFLAGS=eixdata.get("LINKFLAGS", []),
    CPPDEFINES=[define for define in eixdata.get("CPPDEFINES", [])],
    LDSCRIPT_PATH=normpath(
        join(variant_dir, eixdata.get("LDSCRIPT_PATH")[0]))
)

# Hook for K64F and K22F
if board_type in ("frdm_k22f", "frdm_k64f"):
    env.Append(
        LINKFLAGS=["-Wl,--start-group"]
    )

for lib_path in eixdata.get("CPPPATH"):
    _vdir = join("$BUILD_DIR", "FrameworkMbedInc%d" % crc32(lib_path))
    env.VariantDir(_vdir, join(variant_dir, lib_path))
    env.Append(CPPPATH=[_vdir])

env.Append(
    LIBPATH=[join(variant_dir, lib_path)
             for lib_path in eixdata.get("LIBPATH", [])
             if lib_path.startswith("mbed")]
)

#
# Target: Build mbed Library
#

libs = [l for l in eixdata.get("STDLIBS", []) if l not in env.get("LIBS")]
libs.extend(["mbed", "c", "gcc"])

libs.append(env.Library(
    join("$BUILD_DIR", "FrameworkMbed"),
    [join(variant_dir, f)
     for f in eixdata.get("OBJFILES", [])]
))

env.Append(LIBS=libs)

for _libname, _libdata in get_used_mbedlibs().iteritems():
    for _libar in _libdata['ar']:
        add_mbedlib(_libname, _libar)
    if "deps" not in _libdata:
        continue
    for libdep in _libdata['deps']:
        for _libar in MBED_LIBS_MAP[libdep]['ar']:
            add_mbedlib(libdep, _libar)
<commit_msg>Remove duplicate code<commit_after># Copyright (C) Ivan Kravets <me@ikravets.com>
# See LICENSE for details.

"""
mbed

The mbed framework The mbed SDK has been designed to provide enough
hardware abstraction to be intuitive and concise, yet powerful enough to
build complex projects. It is built on the low-level ARM CMSIS APIs,
allowing you to code down to the metal if needed. In addition to RTOS,
USB and Networking libraries, a cookbook of hundreds of reusable
peripheral and module libraries have been built on top of the SDK by
the mbed Developer Community.

http://mbed.org/
"""

import re
import xml.etree.ElementTree as ElementTree
from binascii import crc32
from os import walk
from os.path import basename, isfile, join, normpath

from SCons.Script import DefaultEnvironment, Exit

env = DefaultEnvironment()

BOARD_OPTS = env.get("BOARD_OPTIONS", {}).get("build", {})

env.Replace(
    PLATFORMFW_DIR=join("$PIOPACKAGES_DIR", "framework-mbed")
)

MBED_VARIANTS = {
    "stm32f3discovery": "DISCO_F303VC",
    "stm32f4discovery": "DISCO_F407VG",
    "stm32f429discovery": "DISCO_F429ZI",
    "blueboard_lpc11u24": "LPC11U24",
    "dipcortexm0": "LPC11U24",
    "seeeduinoArchPro": "ARCH_PRO",
    "ubloxc027": "UBLOX_C027",
    "lpc1114fn28": "LPC1114",
    "lpc11u35": "LPC11U35_401",
    "mbuino": "LPC11U24",
    "nrf51_mkit": "NRF51822",
    "seeedTinyBLE": "SEEED_TINY_BLE",
    "redBearLab": "RBLAB_NRF51822",
    "nrf51-dt": "NRF51_DK",
    "redBearLabBLENano": "RBLAB_NRF51822",
    "wallBotBLE": "NRF51822",
    "frdm_kl25z": "KL25Z",
    "frdm_kl46z": "KL46Z",
    "frdm_k64f": "K64F",
    "frdm_kl05z": "KL05Z",
    "frdm_k20d50m": "K20D50M",
    "frdm_k22f": "K22F"
}

MBED_LIBS_MAP = {
    "dsp": {"ar": ["dsp", "cmsis_dsp"]},
    "eth": {"ar": ["eth"], "deps": ["rtos"]},
    "fat": {"ar": ["fat"]},
    "rtos": {"ar": ["rtos", "rtx"]},
    "usb": {"ar": ["USBDevice"]},
    "usb_host": {"ar": ["USBHost"]}
}

def get_mbedlib_includes():
    result = []
    for lib in MBED_LIBS_MAP.keys():
        includes = []
        lib_dir = join(env.subst("$PLATFORMFW_DIR"), "libs", lib)
        for _, _, files in walk(lib_dir):
            for libfile in files:
                if libfile.endswith(".h"):
                    includes.append(libfile)
        result.append((lib, set(includes)))
    return result

def get_used_mbedlibs():
    re_includes = re.compile(r"^(#include\s+(?:\<|\")([^\r\n\"]+))",
                             re.M | re.I)
    srcincs = []
    for root, _, files in walk(env.get("PROJECTSRC_DIR")):
        for pfile in files:
            if not any([pfile.endswith(ext) for ext in (".h", ".c", ".cpp")]):
                continue
            with open(join(root, pfile)) as fp:
                srcincs.extend([i[1] for i in re_includes.findall(fp.read())])
    srcincs = set(srcincs)

    result = {}
    for libname, libincs in get_mbedlib_includes():
        if libincs & srcincs and libname not in result:
            result[libname] = MBED_LIBS_MAP[libname]

    return result

def add_mbedlib(libname, libar):
    if libar in env.get("LIBS"):
        return

    lib_dir = join(env.subst("$PLATFORMFW_DIR"), "libs", libname)
    if not isfile(join(lib_dir, "TARGET_%s" % variant,
                       "TOOLCHAIN_GCC_ARM", "lib%s.a" % libar)):
        Exit("Error: %s board doesn't support %s library!" %
             (env.get("BOARD"), libname))

    env.Append(
        LIBPATH=[
            join(env.subst("$PLATFORMFW_DIR"), "libs", libname,
                 "TARGET_%s" % variant, "TOOLCHAIN_GCC_ARM")
        ],
        LIBS=[libar]
    )

    sysincdirs = (
        "eth",
        "include",
        "ipv4",
        "lwip-eth",
        "lwip-sys"
    )

    for root, _, files in walk(lib_dir):
        if (not any(f.endswith(".h") for f in files) and
                basename(root) not in sysincdirs):
            continue
        var_dir = join("$BUILD_DIR", "FrameworkMbed%sInc%d" %
                       (libname.upper(), crc32(root)))
        if var_dir in env.get("CPPPATH"):
            continue
        env.VariantDir(var_dir, root)
        env.Append(CPPPATH=[var_dir])

def parse_eix_file(filename):
    result = {}
    paths = (
        ("CFLAGS", "./Target/Source/CC/Switch"),
        ("CXXFLAGS", "./Target/Source/CPPC/Switch"),
        ("CPPDEFINES", "./Target/Source/Symbols/Symbol"),
        ("FILES", "./Target/Files/File"),
        ("LINKFLAGS", "./Target/Source/LD/Switch"),
        ("OBJFILES", "./Target/Source/Addobjects/Addobject"),
        ("LIBPATH", "./Target/Linker/Librarypaths/Librarypath"),
        ("STDLIBS", "./Target/Source/Syslibs/Library"),
        ("LDSCRIPT_PATH", "./Target/Source/Scriptfile"),
        ("CPPPATH", "./Target/Compiler/Includepaths/Includepath")
    )

    tree = ElementTree.parse(filename)

    for (key, path) in paths:
        if key not in result:
            result[key] = []

        for node in tree.findall(path):
            _nkeys = node.keys()
            result[key].append(
                node.get(_nkeys[0]) if len(_nkeys) == 1 else node.attrib)

    return result

def get_build_flags(data):
    flags = {}
    cflags = set(data.get("CFLAGS", []))
    cxxflags = set(data.get("CXXFLAGS", []))
    cppflags = set(cflags & cxxflags)
    flags['CPPFLAGS'] = list(cppflags)
    flags['CXXFLAGS'] = list(cxxflags - cppflags)
    flags['CFLAGS'] = list(cflags - cppflags)
    return flags

def _mbed_whole_archive_hook(flags):
    if not isinstance(flags, list):
        return flags

    for pos, flag in enumerate(flags[:]):
        if isinstance(flag, basestring):
            continue
        flags.insert(pos, "-Wl,-whole-archive")
        flags.insert(pos + 2, "-Wl,-no-whole-archive")

    return flags

board_type = env.subst("$BOARD")
variant = MBED_VARIANTS[
    board_type] if board_type in MBED_VARIANTS else board_type.upper()
eixdata = parse_eix_file(
    join(env.subst("$PLATFORMFW_DIR"), "variant", variant, "%s.eix" % variant))

build_flags = get_build_flags(eixdata)
variant_dir = join("$PLATFORMFW_DIR", "variant", variant)

env.Replace(
    _mbed_whole_archive_hook=_mbed_whole_archive_hook,
    _LIBFLAGS="${_mbed_whole_archive_hook(%s)}" % env.get("_LIBFLAGS")[2:-1],
    CPPFLAGS=build_flags.get("CPPFLAGS", []),
    CFLAGS=build_flags.get("CFLAGS", []),
    CXXFLAGS=build_flags.get("CXXFLAGS", []),
    LINKFLAGS=eixdata.get("LINKFLAGS", []),
    CPPDEFINES=[define for define in eixdata.get("CPPDEFINES", [])],
    LDSCRIPT_PATH=normpath(
        join(variant_dir, eixdata.get("LDSCRIPT_PATH")[0]))
)

# Hook for K64F and K22F
if board_type in ("frdm_k22f", "frdm_k64f"):
    env.Append(
        LINKFLAGS=["-Wl,--start-group"]
    )

for lib_path in eixdata.get("CPPPATH"):
    _vdir = join("$BUILD_DIR", "FrameworkMbedInc%d" % crc32(lib_path))
    env.VariantDir(_vdir, join(variant_dir, lib_path))
    env.Append(CPPPATH=[_vdir])

env.Append(
    LIBPATH=[join(variant_dir, lib_path)
             for lib_path in eixdata.get("LIBPATH", [])
             if lib_path.startswith("mbed")]
)

#
# Target: Build mbed Library
#

libs = [l for l in eixdata.get("STDLIBS", []) if l not in env.get("LIBS")]
libs.extend(["mbed", "c", "gcc"])

libs.append(env.Library(
    join("$BUILD_DIR", "FrameworkMbed"),
    [join(variant_dir, f)
     for f in eixdata.get("OBJFILES", [])]
))

env.Append(LIBS=libs)

for _libname, _libdata in get_used_mbedlibs().iteritems():
    for _libar in _libdata['ar']:
        add_mbedlib(_libname, _libar)
    if "deps" not in _libdata:
        continue
    for libdep in _libdata['deps']:
        for _libar in MBED_LIBS_MAP[libdep]['ar']:
            add_mbedlib(libdep, _libar)
<|endoftext|>


<commit_before>#!/usr/bin/env python
__version__ = '1.0'
__revision__ = "$Revision$"
__date__     = '$Date$'

import os
from os.path import join

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs
    nxheader = join(get_numpy_include_dirs()[0],'numpy',)
    confgr = Configuration('timeseries',parent_package,top_path)
    sources = join('src', 'cseries.c')
    confgr.add_extension('cseries',
                         sources=[sources,],
                         include_dirs=[nxheader],
                         )
    confgr.add_data_dir('docs')
    confgr.add_subpackage('addons')
    confgr.add_subpackage('io')
    confgr.add_subpackage('plotlib')
    confgr.add_subpackage('tests')
    return confgr

if __name__ == "__main__":
    from numpy.distutils.core import setup
    #setup.update(nmasetup)
    config = configuration(top_path='').todict()
    setup(**config)<commit_msg>fixed typo with "doc" dir<commit_after>#!/usr/bin/env python
__version__ = '1.0'
__revision__ = "$Revision$"
__date__     = '$Date$'

import os
from os.path import join

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs
    nxheader = join(get_numpy_include_dirs()[0],'numpy',)
    confgr = Configuration('timeseries',parent_package,top_path)
    sources = join('src', 'cseries.c')
    confgr.add_extension('cseries',
                         sources=[sources,],
                         include_dirs=[nxheader],
                         )
    confgr.add_data_dir('doc')
    confgr.add_subpackage('addons')
    confgr.add_subpackage('io')
    confgr.add_subpackage('plotlib')
    confgr.add_subpackage('tests')
    return confgr

if __name__ == "__main__":
    from numpy.distutils.core import setup
    #setup.update(nmasetup)
    config = configuration(top_path='').todict()
    setup(**config)<|endoftext|>


<commit_before>import os
import ConfigParser

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from schema import *

def fetch_engine():
    """
    Read from config.ini file and load appropriate database
    """
    echo = False
    config = ConfigParser.ConfigParser()
    config.read('{0}/config.ini'.format(os.path.dirname(os.path.realpath(__file__))))
    if config.get('global', 'database') == "sqlite":
        engine = create_engine('sqlite:///{0}'.format(config.get('sqlite', 'database')), echo=echo)
    else:
        engine = create_engine('mysql+mysqldb://{0}:{1}@{2}/{3}?charset=utf8'.format(
            config.get('mysql', 'user'),
            config.get('mysql', 'password'),
            config.get('mysql', 'host'),
            config.get('mysql', 'database')), echo=echo)
    return engine

def add(obj, override=True):
    """
    PatentGrant Object converting to tables via SQLAlchemy
    Necessary to convert dates to datetime because of SQLite (OK on MySQL)

    Case Sensitivity and Table Reflection
    MySQL has inconsistent support for case-sensitive identifier names,
    basing support on specific details of the underlying operating system.
    However, it has been observed that no matter what case sensitivity
    behavior is present, the names of tables in foreign key declarations
    are always received from the database as all-lower case, making it
    impossible to accurately reflect a schema where inter-related tables
    use mixed-case identifier names.

    Therefore it is strongly advised that table names be declared as all
    lower case both within SQLAlchemy as well as on the MySQL database
    itself, especially if database reflection features are to be used.
    """

    # if a patent exists, remove it so we can replace it
    pat_query = session.query(Patent).filter(Patent.number == obj.patent)
    if pat_query.count():
        if override:
            session.delete(pat_query.one())
        else:
            return

    #add
    # lots of abstracts seem to be missing. why?

    pat = Patent(**obj.pat)
    pat.application = Application(**obj.app)

    #+asg
    for asg, loc in obj.assignee_list():
        asg = Assignee(**asg)
        loc = Location(**loc)
        session.merge(loc)
        asg.location = loc
        pat.assignees.append(asg)

    #+inv
    for inv, loc in obj.inventor_list():
        inv = Inventor(**inv)
        loc = Location(**loc)
        session.merge(loc)
        inv.location = loc
        pat.inventors.append(inv)

    #+cit, +othercit
    cits, refs = obj.citation_list()
    for cit in cits:
        cit = Citation(**cit)
        pat.citations.append(cit)
    for ref in refs:
        ref = OtherReference(**ref)
        pat.otherreferences.append(ref)

    #+law
    for law in obj.lawyer_list():
        law = Lawyer(**law)
        pat.lawyers.append(law)

    #+usreldoc
    for usr in obj.us_relation_list():
        usr = USRelDoc(**usr)
        pat.usreldocs.append(usr)

    # ----------------------------------------

    #+classes (TODO for later)
    for i, cls in enumerate(obj.classes):
        uspc = USPC(i)
        mc = MainClass(cls[0])
        sc = SubClass("/".join(cls))
        session.merge(mc)
        session.merge(sc)
        uspc.mainclass = mc
        uspc.subclass = sc
        pat.classes.append(uspc)

    session.merge(pat)

def commit():
    try:
        session.commit()
    except Exception, e:
        session.rollback()
        print str(e)

engine = fetch_engine()
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
<commit_msg>a bit of error handling?<commit_after>import os
import ConfigParser

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from schema import *

def fetch_engine():
    """
    Read from config.ini file and load appropriate database
    """
    echo = False
    config = ConfigParser.ConfigParser()
    config.read('{0}/config.ini'.format(os.path.dirname(os.path.realpath(__file__))))
    if config.get('global', 'database') == "sqlite":
        engine = create_engine('sqlite:///{0}'.format(config.get('sqlite', 'database')), echo=echo)
    else:
        engine = create_engine('mysql+mysqldb://{0}:{1}@{2}/{3}?charset=utf8'.format(
            config.get('mysql', 'user'),
            config.get('mysql', 'password'),
            config.get('mysql', 'host'),
            config.get('mysql', 'database')), echo=echo)
    return engine

def add(obj, override=True):
    """
    PatentGrant Object converting to tables via SQLAlchemy
    Necessary to convert dates to datetime because of SQLite (OK on MySQL)

    Case Sensitivity and Table Reflection
    MySQL has inconsistent support for case-sensitive identifier names,
    basing support on specific details of the underlying operating system.
    However, it has been observed that no matter what case sensitivity
    behavior is present, the names of tables in foreign key declarations
    are always received from the database as all-lower case, making it
    impossible to accurately reflect a schema where inter-related tables
    use mixed-case identifier names.

    Therefore it is strongly advised that table names be declared as all
    lower case both within SQLAlchemy as well as on the MySQL database
    itself, especially if database reflection features are to be used.
    """

    # if a patent exists, remove it so we can replace it
    pat_query = session.query(Patent).filter(Patent.number == obj.patent)
    if pat_query.count():
        if override:
            session.delete(pat_query.one())
        else:
            return

    #add
    # lots of abstracts seem to be missing. why?

    pat = Patent(**obj.pat)
    pat.application = Application(**obj.app)

    #+asg
    for asg, loc in obj.assignee_list():
        asg = Assignee(**asg)
        loc = Location(**loc)
        session.merge(loc)
        asg.location = loc
        pat.assignees.append(asg)

    #+inv
    for inv, loc in obj.inventor_list():
        inv = Inventor(**inv)
        loc = Location(**loc)
        session.merge(loc)
        inv.location = loc
        pat.inventors.append(inv)

    #+cit, +othercit
    cits, refs = obj.citation_list()
    for cit in cits:
        cit = Citation(**cit)
        pat.citations.append(cit)
    for ref in refs:
        ref = OtherReference(**ref)
        pat.otherreferences.append(ref)

    #+law
    for law in obj.lawyer_list():
        law = Lawyer(**law)
        pat.lawyers.append(law)

    #+usreldoc
    for usr in obj.us_relation_list():
        usr = USRelDoc(**usr)
        pat.usreldocs.append(usr)

    # ----------------------------------------

    #+classes (TODO for later)
    for i, cls in enumerate(obj.classes):
        uspc = USPC(i)
        mc = MainClass(cls[0])
        sc = SubClass("/".join(cls))
        session.merge(mc)
        session.merge(sc)
        uspc.mainclass = mc
        uspc.subclass = sc
        pat.classes.append(uspc)

    session.merge(pat)

def commit():
    try:
        session.commit()
    except:
        pass
    #except Exception, e:
    #    session.rollback()
    #    print str(e)

engine = fetch_engine()
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
<|endoftext|>


<commit_before><commit_msg>taggable-mixin directory removed from the repository.<commit_after><|endoftext|>


<commit_before>import sys, os

import pytest

# Put source dir first in path

here = os.path.dirname(os.path.abspath(__file__))
mc_dir = os.path.join(os.path.dirname(here), 'multiconf')
sys.path.insert(0, here)

os.environ['MULTICONF_WARN_JSON_NESTING'] = "false"

py3 = sys.version_info[0] >= 3

class DummyCollector(pytest.collect.File):
    def collect(self):
        return []

def pytest_pycollect_makemodule(path, parent):
    bn = path.basename
    if "py3" in bn and not py3 or ("py2" in bn and py3):
        return DummyCollector(path, parent=parent)
<commit_msg>Removed py2/py3 test DummyCollector<commit_after>import sys, os

import pytest

# Put source dir first in path

here = os.path.dirname(os.path.abspath(__file__))
mc_dir = os.path.join(os.path.dirname(here), 'multiconf')
sys.path.insert(0, here)

os.environ['MULTICONF_WARN_JSON_NESTING'] = "false"
<|endoftext|>


<commit_before><commit_msg>updated troll count sir<commit_after><|endoftext|>


<commit_before>'''
Created on Sep 24, 2014

@author: Aaron Koeppel
'''

import urllib2
from bs4 import BeautifulSoup
import sys
import re

reload(sys)
sys.setdefaultencoding("utf-8")

class GameData(object):
   def __init__(self, link, headline):
      self.link = link
      self.headline = headline

   def char_convert_link(self, link):
      self.link = re.sub('[&]', '&', self.link)

def extract_game_data():
   games = []
#   w = open("out.txt", 'w')
   response = urllib2.urlopen("http://espn.go.com/nhl/team/schedule/_/name/pit/pittsburgh-penguins")
   page_source = response.read()
   soup = BeautifulSoup(page_source)

   for div in soup.find_all(attrs={"class" : "score"}):
      complete_link = "http://espn.go.com" + str(div.find('a').get('href').encode('utf-8', 'ignore'))
      print complete_link

      new_game = GameData(complete_link, get_game_headline(complete_link))
      new_game.char_convert_link(new_game.link)
      games.append(new_game)

   return games

def get_game_headline(link):
   response = urllib2.urlopen(link)
   page_source = response.read()
   soup = BeautifulSoup(page_source)

   for meta in soup.findAll('meta', {"property":'og:title'}):
      return meta.get('content')

def markup(games):
   xml = open("feed.xml", 'w')
   xml.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
   xml.write("<rss version='2.0'>\n")
   xml.write("<channel>\n")
   xml.write("<title>Penguins scores</title>\n")
   xml.write("<description>Latest Penguins scores</description>\n")
   xml.write("<link>http://espn.go.com/nhl/team/schedule/_/name/pit/pittsburgh-penguins</link>\n")

   for game in games:
      xml.write("<item>\n")
      xml.write("<title>%s</title>\n" % game.headline)
#      xml.write("<description></description>\n")
      xml.write("<link>%s</link>\n" % game.link)
      xml.write("</item>\n")
   xml.write("</channel>\n</rss>")
   xml.close()

def main():
   games = extract_game_data()
   games.reverse()

   for game in games:
      print game.link
      print game.headline
   markup(games)

main()<commit_msg>current script<commit_after>'''
Created on Sep 24, 2014

@author: Aaron Koeppel
'''

import urllib2
from bs4 import BeautifulSoup
import sys
import re

reload(sys)
sys.setdefaultencoding("utf-8")

class GameData(object):
   def __init__(self, link, headline):
      self.link = link
      self.headline = headline

   def char_convert_link(self, link):
      self.link = re.sub('[&]', '&', self.link)

def extract_game_data():
   games = []
#   w = open("out.txt", 'w')
   response = urllib2.urlopen("http://espn.go.com/nhl/team/schedule/_/name/pit/pittsburgh-penguins")
   page_source = response.read()
   soup = BeautifulSoup(page_source)

   for div in soup.find_all(attrs={"class" : "score"}):
      complete_link = "http://espn.go.com" + str(div.find('a').get('href').encode('utf-8', 'ignore'))
#      print complete_link

      new_game = GameData(complete_link, get_game_headline(complete_link))
      new_game.char_convert_link(new_game.link)
      games.append(new_game)

   return games

def get_game_headline(link):
   response = urllib2.urlopen(link)
   page_source = response.read()
   soup = BeautifulSoup(page_source)

   for meta in soup.findAll('meta', {"property":'og:title'}):
      return meta.get('content')

def markup(games):
   xml = open("feed.xml", 'w')
   xml.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
   xml.write("<rss version='2.0'>\n")
   xml.write("<channel>\n")
   xml.write("<title>Penguins scores</title>\n")
   xml.write("<description>Latest Penguins scores</description>\n")
   xml.write("<link>http://espn.go.com/nhl/team/schedule/_/name/pit/pittsburgh-penguins</link>\n")

   for game in games:
      xml.write("<item>\n")
      xml.write("<title>%s</title>\n" % game.headline)
#      xml.write("<description></description>\n")
      xml.write("<link>%s</link>\n" % game.link)
      xml.write("</item>\n")
   xml.write("</channel>\n</rss>")
   xml.close()

def main():
   games = extract_game_data()
   games.reverse()

   for game in games:
      print game.link
      print game.headline
   markup(games)

main()<|endoftext|>


<commit_before>from polyaxon.utils import config

CLI_MIN_VERSION = config.get_string('POLYAXON_CLI_MIN_VERSION',
                                    is_optional=True,
                                    default='0.0.0')
CLI_LATEST_VERSION = config.get_string('POLYAXON_CLI_LATEST_VERSION',
                                       is_optional=True,
                                       default='0.0.0')
PLATFORM_MIN_VERSION = config.get_string('POLYAXON_PLATFORM_MIN_VERSION',
                                         is_optional=True,
                                         default='0.0.0')
PLATFORM_LATEST_VERSION = config.get_string('POLYAXON_PLATFORM_LATEST_VERSION',
                                            is_optional=True,
                                            default='0.0.0')
LIB_MIN_VERSION = config.get_string('POLYAXON_LIB_MIN_VERSION',
                                    is_optional=True,
                                    default='0.0.0')
LIB_LATEST_VERSION = config.get_string('POLYAXON_LIB_LATEST_VERSION',
                                       is_optional=True,
                                       default='0.0.0')
CHART_VERSION = config.get_string('POLYAXON_CHART_VERSION',
                                  is_optional=True,
                                  default='0.0.0')
CHART_IS_UPGRADE = config.get_string('POLYAXON_CHART_IS_UPGRADE')
<commit_msg>Fix type of env variable is chart upgrade<commit_after>from polyaxon.utils import config

CLI_MIN_VERSION = config.get_string('POLYAXON_CLI_MIN_VERSION',
                                    is_optional=True,
                                    default='0.0.0')
CLI_LATEST_VERSION = config.get_string('POLYAXON_CLI_LATEST_VERSION',
                                       is_optional=True,
                                       default='0.0.0')
PLATFORM_MIN_VERSION = config.get_string('POLYAXON_PLATFORM_MIN_VERSION',
                                         is_optional=True,
                                         default='0.0.0')
PLATFORM_LATEST_VERSION = config.get_string('POLYAXON_PLATFORM_LATEST_VERSION',
                                            is_optional=True,
                                            default='0.0.0')
LIB_MIN_VERSION = config.get_string('POLYAXON_LIB_MIN_VERSION',
                                    is_optional=True,
                                    default='0.0.0')
LIB_LATEST_VERSION = config.get_string('POLYAXON_LIB_LATEST_VERSION',
                                       is_optional=True,
                                       default='0.0.0')
CHART_VERSION = config.get_string('POLYAXON_CHART_VERSION',
                                  is_optional=True,
                                  default='0.0.0')
CHART_IS_UPGRADE = config.get_boolean('POLYAXON_CHART_IS_UPGRADE')
<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import numpy as np

from .transform import Transform
from .affine import apply_affine
from ._register import _apply_polyaffine

TINY_SIGMA = 1e-200

class PolyAffine(Transform):

    def __init__(self, centers, affines, sigma, glob_affine=None):
        """
        centers: N times 3 array

        We are given a set of affine transforms T_i with centers x_i,
        all in homogeneous coordinates. The polyaffine transform is
        defined, up to a right composition with a global affine, as:

        T(x) = sum_i w_i(x) T_i x

        where w_i(x) = g(x-x_i)/Z(x) are normalized Gaussian weights
        that sum up to one for every x.
        """

        # Format input arguments
        self.centers = np.asarray(centers, dtype='double', order='C')
        self.sigma = np.zeros(3)
        self.sigma[:] = np.maximum(TINY_SIGMA, sigma)
        if hasattr(affines[0], 'as_affine'):
            affines = np.array([a.as_affine() for a in affines])
        else:
            affines = np.asarray(affines)
        if hasattr(glob_affine, 'as_affine'):
            self.glob_affine = glob_affine.as_affine()
        else:
            self.glob_affine = glob_affine

        # Cache a (N, 12) matrix containing the affines coefficients,
        # should be C-contiguous double.
        self._affines = np.zeros((len(self.centers), 12))
        self._affines[:] = np.reshape(affines[:, 0:3, :],
                                      (len(self.centers), 12))

    def affine(self, i):
        aff = np.eye(4)
        aff[0:3, :] = self._affines[i].reshape(3, 4)
        return aff

    def affines(self):
        return [self.affine(i) for i in range(len(self.centers))]

    def apply(self, xyz):
        """
        xyz is an (N, 3) array
        """
        # txyz should be double C-contiguous for the the cython
        # routine _apply_polyaffine
        if self.glob_affine is None:
            txyz = np.array(xyz, copy=True, dtype='double', order='C')
        else:
            txyz = apply_affine(self.glob_affine, xyz)
        _apply_polyaffine(txyz, self.centers, self._affines, self.sigma)
        return txyz

    def compose(self, other):
        """
        Compose this transform onto another

        Parameters
        ----------
        other : Transform
            transform that we compose onto

        Returns
        -------
        composed_transform : Transform
            a transform implementing the composition of self on `other`
        """
        # If other is not an Affine, use the generic compose method
        if not hasattr(other, 'as_affine'):
            return Transform(self.apply).compose(other)

        # Affine case: the result is a polyaffine transform with same
        # local affines
        if self.glob_affine is None:
            glob_affine = other.as_affine()
        else:
            glob_affine = np.dot(self.glob_affine, other.as_affine())

        return self.__class__(self.centers, self.affines(), self.sigma,
                              glob_affine=glob_affine)

    def left_compose(self, other):

        # If other is not an Affine, use the generic compose method
        if not hasattr(other, 'as_affine'):
            return Transform(other.apply).compose(self)

        # Affine case: the result is a polyaffine transform with same
        # global affine
        other_affine = other.as_affine()
        affines = [np.dot(other_affine, self.affine(i)) \
                       for i in range(len(self.centers))]
        return self.__class__(self.centers, affines, self.sigma,
                              glob_affine=self.glob_affine)
<commit_msg>add docstring for polyaffine object<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import numpy as np

from .transform import Transform
from .affine import apply_affine
from ._register import _apply_polyaffine

TINY_SIGMA = 1e-200

class PolyAffine(Transform):

    def __init__(self, centers, affines, sigma, glob_affine=None):
        """Poly-affine transformation constructor.

        We are given a set of affine transforms T_i with centers x_i,
        all in homogeneous coordinates. The polyaffine transform is
        defined, up to a right composition with a global affine, as:

        T(x) = sum_i w_i(x) T_i x

        where w_i(x) = g(x-x_i)/Z(x) are normalized Gaussian weights
        that sum up to one for every x.

        Parameters
        ----------
        centers : ndarray
          Local affine transform centers. Should be with shape (N, 3),
          where N is the number of local affine transforms.

        affines : sequence
          Defines the local affine transforms associated with
          `centers`. Elements should be array-like objects with shape
          (4, 4) or nireg-like transform objects endowed with an
          `as_affine` method.

        sigma : float
          Isotropic standard deviation used for the spatial weighting
          of local affine transforms.

        glob_affine: array-like or nireg-like affine object
           Optionally defines a global affine transform to be
           right-composed with the local affines defined by
           `affines`. If None, the identity transform is assumed.
        """
        # Format input arguments
        self.centers = np.asarray(centers, dtype='double', order='C')
        self.sigma = np.zeros(3)
        self.sigma[:] = np.maximum(TINY_SIGMA, sigma)
        if hasattr(affines[0], 'as_affine'):
            affines = np.array([a.as_affine() for a in affines])
        else:
            affines = np.asarray(affines)
        if hasattr(glob_affine, 'as_affine'):
            self.glob_affine = glob_affine.as_affine()
        else:
            self.glob_affine = glob_affine

        # Cache a (N, 12) matrix containing the affines coefficients,
        # should be C-contiguous double.
        self._affines = np.zeros((len(self.centers), 12))
        self._affines[:] = np.reshape(affines[:, 0:3, :],
                                      (len(self.centers), 12))

    def affine(self, i):
        aff = np.eye(4)
        aff[0:3, :] = self._affines[i].reshape(3, 4)
        return aff

    def affines(self):
        return [self.affine(i) for i in range(len(self.centers))]

    def apply(self, xyz):
        """
        xyz is an (N, 3) array
        """
        # txyz should be double C-contiguous for the the cython
        # routine _apply_polyaffine
        if self.glob_affine is None:
            txyz = np.array(xyz, copy=True, dtype='double', order='C')
        else:
            txyz = apply_affine(self.glob_affine, xyz)
        _apply_polyaffine(txyz, self.centers, self._affines, self.sigma)
        return txyz

    def compose(self, other):
        """
        Compose this transform onto another

        Parameters
        ----------
        other : Transform
            transform that we compose onto

        Returns
        -------
        composed_transform : Transform
            a transform implementing the composition of self on `other`
        """
        # If other is not an Affine, use the generic compose method
        if not hasattr(other, 'as_affine'):
            return Transform(self.apply).compose(other)

        # Affine case: the result is a polyaffine transform with same
        # local affines
        if self.glob_affine is None:
            glob_affine = other.as_affine()
        else:
            glob_affine = np.dot(self.glob_affine, other.as_affine())

        return self.__class__(self.centers, self.affines(), self.sigma,
                              glob_affine=glob_affine)

    def left_compose(self, other):

        # If other is not an Affine, use the generic compose method
        if not hasattr(other, 'as_affine'):
            return Transform(other.apply).compose(self)

        # Affine case: the result is a polyaffine transform with same
        # global affine
        other_affine = other.as_affine()
        affines = [np.dot(other_affine, self.affine(i)) \
                       for i in range(len(self.centers))]
        return self.__class__(self.centers, affines, self.sigma,
                              glob_affine=self.glob_affine)
<|endoftext|>


<commit_before><commit_msg>significantly more efficient query to retrieve hosts. We were getting uwsgi processes dying from harakiri after 20s.<commit_after><|endoftext|>


<commit_before>from django import forms as newform
from django.forms import ModelForm
from people.models import Beneficiary, Donor
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

class BeneficiaryForm(ModelForm):

    class Meta:
        model = Beneficiary
        exclude = ('user', 'is_approved',)
    ben_name  = newform.CharField(max_length=100)
    password1 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password'))
    password2 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password (again)'))

    def clean_ben_name(self):
        try:
            user = User.objects.get(
                username__iexact=self.cleaned_data['ben_name'])
        except Exception as e:
            return self.cleaned_data['ben_name']
        raise newform.ValidationError(_(
            "This username is already taken. Please choose another."))

    def clean_password2(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise newform.ValidationError(_(
                    u'You must type the same password each time'))
        return self.cleaned_data

class DonorForm(ModelForm):

    class Meta:
        model = Donor
        exclude = ('user',)

    donor_name = newform.CharField(max_length=100)
    password1 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password'))
    password2 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password (again)'))

    def clean_donor_name(self):
        try:
            user = User.objects.get(
                username__iexact=self.cleaned_data['donor_name'])
        except Exception as e:
            return self.cleaned_data['donor_name']
        raise newform.ValidationError(_(
            "This username is already taken. Please choose another."))

    def clean_password2(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise newform.ValidationError(_(
                    u'You must type the same password each time'))
        return self.cleaned_data
<commit_msg>Following django convention, putting Meta after field names<commit_after>from django import forms as newform
from django.forms import ModelForm
from people.models import Beneficiary, Donor
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

class BeneficiaryForm(ModelForm):

    ben_name  = newform.CharField(max_length=100)
    password1 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password'))
    password2 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password (again)'))

    class Meta:
        model = Beneficiary
        exclude = ('user', 'is_approved',)

    def clean_ben_name(self):
        try:
            user = User.objects.get(
                username__iexact=self.cleaned_data['ben_name'])
        except Exception as e:
            return self.cleaned_data['ben_name']
        raise newform.ValidationError(_(
            "This username is already taken. Please choose another."))

    def clean_password2(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise newform.ValidationError(_(
                    u'You must type the same password each time'))
        return self.cleaned_data

class DonorForm(ModelForm):

    donor_name = newform.CharField(max_length=100)
    password1 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password'))
    password2 = newform.CharField(
        widget=newform.PasswordInput(), label=(u'password (again)'))

    class Meta:
        model = Donor
        exclude = ('user',)

    def clean_donor_name(self):
        try:
            user = User.objects.get(
                username__iexact=self.cleaned_data['donor_name'])
        except Exception as e:
            return self.cleaned_data['donor_name']
        raise newform.ValidationError(_(
            "This username is already taken. Please choose another."))

    def clean_password2(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise newform.ValidationError(_(
                    u'You must type the same password each time'))
        return self.cleaned_data
<|endoftext|>


<commit_before>"""
Utilities to download NeuroImaging datasets
"""
# Author: Gael Varoquaux
# License: simplified BSD

import os
import re
import glob
import nibabel
from sklearn.datasets.base import Bunch

from nilearn.datasets.utils import (
    _get_dataset_dir, _fetch_files, _fetch_file, _uncompress_file)

SPM_AUDITORY_DATA_FILES = ["fM00223/fM00223_%03i.img" % index
                           for index in range(4, 100)]
SPM_AUDITORY_DATA_FILES.append("sM00223/sM00223_002.img")

def fetch_localizer_first_level(data_dir=None, verbose=1):
    """ Download a first-level localizer fMRI dataset

    Parameters
    ----------
    data_dir: string
        directory where data should be downloaded and unpacked.

    Returns
    -------
    data: sklearn.datasets.base.Bunch
        dictionary-like object, keys are:
        epi_img: the input 4D image
        paradigm: a csv file describing the paardigm
    """
    url = 'ftp://ftp.cea.fr/pub/dsv/madic/download/nipy'

    dataset_name = "localizer_first_level"
    files = dict(epi_img="s12069_swaloc1_corr.nii.gz",
                 paradigm="localizer_paradigm.csv")
    # The options needed for _fetch_files
    options = [(filename, os.path.join(url, filename), {})
               for _, filename in sorted(files.items())]

    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
                                verbose=verbose)
    sub_files = _fetch_files(data_dir, options, resume=True,
                             verbose=verbose)

    params = dict(zip(sorted(files.keys()), sub_files))

    return Bunch(**params)

def fetch_spm_auditory(data_dir=None, data_name='spm_auditory',
                       subject_id="sub001", verbose=1):
    """Function to fetch SPM auditory single-subject data.

    Parameters
    ----------
    data_dir: string
        Path of the data directory. Used to force data storage in a specified
        location. If the data is already present there, then will simply
        glob it.

    Returns
    -------
    data: sklearn.datasets.base.Bunch
        Dictionary-like object, the interest attributes are:
        - 'func': string list. Paths to functional images
        - 'anat': string list. Path to anat image

    References
    ----------
    :download:
        http://www.fil.ion.ucl.ac.uk/spm/data/auditory/

    """
    data_dir = _get_dataset_dir(data_name, data_dir=data_dir,
                                verbose=verbose)
    subject_dir = os.path.join(data_dir, subject_id)

    def _glob_spm_auditory_data():
        """glob data from subject_dir.

        """

        if not os.path.exists(subject_dir):
            return None

        subject_data = {}
        for file_name in SPM_AUDITORY_DATA_FILES:
            file_path = os.path.join(subject_dir, file_name)
            if os.path.exists(file_path):
                subject_data[file_name] = file_path
            else:
                print("%s missing from filelist!" % file_name)
                return None

        _subject_data = {}
        _subject_data["func"] = sorted(
            [subject_data[x] for x in subject_data.keys()
             if re.match("^fM00223_0\d\d\.img$", os.path.basename(x))])

        # volumes for this dataset of shape (64, 64, 64, 1); let's fix this
        for x in _subject_data["func"]:
            vol = nibabel.load(x)
            if len(vol.shape) == 4:
                vol = nibabel.Nifti1Image(vol.get_data()[:, :, :, 0],
                                          vol.get_affine())
                nibabel.save(vol, x)

        _subject_data["anat"] = [subject_data[x] for x in subject_data.keys()
                                 if re.match("^sM00223_002\.img$",
                                             os.path.basename(x))][0]

        # ... same thing for anat
        vol = nibabel.load(_subject_data["anat"])
        if len(vol.shape) == 4:
            vol = nibabel.Nifti1Image(vol.get_data()[:, :, :, 0],
                                      vol.get_affine())
            nibabel.save(vol, _subject_data["anat"])

        return Bunch(**_subject_data)

    # maybe data_dir already contains the data ?
    data = _glob_spm_auditory_data()
    if data is not None:
        return data

    # No. Download the data
    print("Data absent, downloading...")
    url = ("http://www.fil.ion.ucl.ac.uk/spm/download/data/MoAEpilot/"
           "MoAEpilot.zip")
    archive_path = os.path.join(subject_dir, os.path.basename(url))
    _fetch_file(url, subject_dir)
    try:
        _uncompress_file(archive_path)
    except:
        print("Archive corrupted, trying to download it again.")
        return fetch_spm_auditory(data_dir=data_dir, data_name="",
                                  subject_id=subject_id)

    return _glob_spm_auditory_data()

def fetch_spm_multimodal_fmri(data_dir=None, data_name="spm_multimodal_fmri",
                              subject_id="sub001", verbose=1):
    """Fetcher for Multi-modal Face Dataset.

    Parameters
    ----------
    data_dir: string
        path of the data directory. Used to force data storage in a specified
        location. If the data is already present there, then will simply
        glob it.

    Returns
    -------
    data: sklearn.datasets.base.Bunch
        Dictionary-like object, the interest attributes are:
        - 'func1': string list. Paths to functional images for session 1
        - 'func2': string list. Paths to functional images for session 2
        - 'trials_ses1': string list. Path to onsets file for session 1
        - 'trials_ses2': string list. Path to onsets file for session 2
        - 'anat': string. Path to anat file

    References
    ----------
    :download:
        http://www.fil.ion.ucl.ac.uk/spm/data/mmfaces/

    """

    data_dir = _get_dataset_dir(data_name, data_dir=data_dir,
                                verbose=verbose)
    subject_dir = os.path.join(data_dir, subject_id)

    def _glob_spm_multimodal_fmri_data():
        """glob data from subject_dir."""
        _subject_data = {'slice_order': 'descending'}

        for session in range(2):
            # glob func data for session s + 1
            session_func = sorted(glob.glob(
                    os.path.join(
                        subject_dir,
                        ("fMRI/Session%i/fMETHODS-000%i-*-01.img" % (
                                session + 1, session + 5)))))
            if len(session_func) < 390:
                print("Missing %i functional scans for session %i." % (
                    390 - len(session_func), session))
                return None

            _subject_data['func%i' % (session + 1)] = session_func

            # glob trials .mat file
            sess_trials = os.path.join(
                subject_dir,
                "fMRI/trials_ses%i.mat" % (session + 1))
            if not os.path.isfile(sess_trials):
                print("Missing session file: %s" % sess_trials)
                return None

            _subject_data['trials_ses%i' % (session + 1)] = sess_trials

        # glob for anat data
        anat = os.path.join(subject_dir, "sMRI/smri.img")
        if not os.path.isfile(anat):
            print("Missing structural image.")
            return None

        _subject_data["anat"] = anat

        return Bunch(**_subject_data)

    # maybe data_dir already contains the data ?
    data = _glob_spm_multimodal_fmri_data()
    if data is not None:
        return data

    # No. Download the data
    print("Data absent, downloading...")
    urls = [
        # fmri
        ("http://www.fil.ion.ucl.ac.uk/spm/download/data/mmfaces/"
        "multimodal_fmri.zip"),

        # structural
        ("http://www.fil.ion.ucl.ac.uk/spm/download/data/mmfaces/"
         "multimodal_smri.zip")
        ]

    for url in urls:
        archive_path = os.path.join(subject_dir, os.path.basename(url))
        _fetch_file(url, subject_dir)
        try:
            _uncompress_file(archive_path)
        except:
            print("Archive corrupted, trying to download it again.")
            return fetch_spm_multimodal_fmri(data_dir=data_dir,
                                             data_name="",
                                             subject_id=subject_id)

    return _glob_spm_multimodal_fmri_data()

def fetch_fiac_first_level(data_dir=None, verbose=1):
    """ Download a first-level fiac fMRI dataset (2 sessions)

    Parameters
    ----------
    data_dir: string
        directory where data should be downloaded and unpacked.
    """
    data_dir = _get_dataset_dir('nipy-data-0.2/data', data_dir=data_dir,
                                verbose=verbose)

    def _glob_fiac_data():
        """glob data from subject_dir."""
        _subject_data = {}
        subject_dir = os.path.join(data_dir, 'fiac/fiac0')
        for session in [1, 2]:
            # glob func data for session session + 1
            session_func = os.path.join(subject_dir, 'run%i.nii.gz' % session)
            if not os.path.isfile(session_func):
                print('Missing functional scan for session %i.' % session)
                return None

            _subject_data['func%i' % session] = session_func

            # glob design matrix .npz file
            sess_dmtx = os.path.join(subject_dir, 'run%i_design.npz' % session)
            if not os.path.isfile(sess_dmtx):
                print('Missing session file: %s' % sess_dmtx)
                return None

            _subject_data['design_matrix%i' % session] = sess_dmtx

        # glob for mask data
        mask = os.path.join(subject_dir, 'mask.nii.gz')
        if not os.path.isfile(mask):
            print('Missing mask image.')
            return None

        _subject_data['mask'] = mask
        return Bunch(**_subject_data)

    # maybe data_dir already contains the data ?
    data = _glob_fiac_data()
    if data is not None:
        return data

    # No. Download the data
    print('Data absent, downloading...')
    url = 'http://nipy.sourceforge.net/data-packages/nipy-data-0.2.tar.gz'

    archive_path = os.path.join(data_dir, os.path.basename(url))
    _fetch_file(url, data_dir)
    try:
        _uncompress_file(archive_path)
    except:
        print('Archive corrupted, trying to download it again.')
        return fetch_fiac_first_level(data_dir=data_dir)

    return _glob_fiac_data()
<commit_msg>trying to fix download of fiac<commit_after>"""
Utilities to download NeuroImaging datasets
"""
# Author: Gael Varoquaux
# License: simplified BSD

import os
import re
import glob
import nibabel
from sklearn.datasets.base import Bunch

from nilearn.datasets.utils import (
    _get_dataset_dir, _fetch_files, _fetch_file, _uncompress_file)

SPM_AUDITORY_DATA_FILES = ["fM00223/fM00223_%03i.img" % index
                           for index in range(4, 100)]
SPM_AUDITORY_DATA_FILES.append("sM00223/sM00223_002.img")

def fetch_localizer_first_level(data_dir=None, verbose=1):
    """ Download a first-level localizer fMRI dataset

    Parameters
    ----------
    data_dir: string
        directory where data should be downloaded and unpacked.

    Returns
    -------
    data: sklearn.datasets.base.Bunch
        dictionary-like object, keys are:
        epi_img: the input 4D image
        paradigm: a csv file describing the paardigm
    """
    url = 'ftp://ftp.cea.fr/pub/dsv/madic/download/nipy'

    dataset_name = "localizer_first_level"
    files = dict(epi_img="s12069_swaloc1_corr.nii.gz",
                 paradigm="localizer_paradigm.csv")
    # The options needed for _fetch_files
    options = [(filename, os.path.join(url, filename), {})
               for _, filename in sorted(files.items())]

    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
                                verbose=verbose)
    sub_files = _fetch_files(data_dir, options, resume=True,
                             verbose=verbose)

    params = dict(zip(sorted(files.keys()), sub_files))

    return Bunch(**params)

def fetch_spm_auditory(data_dir=None, data_name='spm_auditory',
                       subject_id="sub001", verbose=1):
    """Function to fetch SPM auditory single-subject data.

    Parameters
    ----------
    data_dir: string
        Path of the data directory. Used to force data storage in a specified
        location. If the data is already present there, then will simply
        glob it.

    Returns
    -------
    data: sklearn.datasets.base.Bunch
        Dictionary-like object, the interest attributes are:
        - 'func': string list. Paths to functional images
        - 'anat': string list. Path to anat image

    References
    ----------
    :download:
        http://www.fil.ion.ucl.ac.uk/spm/data/auditory/

    """
    data_dir = _get_dataset_dir(data_name, data_dir=data_dir,
                                verbose=verbose)
    subject_dir = os.path.join(data_dir, subject_id)

    def _glob_spm_auditory_data():
        """glob data from subject_dir.

        """

        if not os.path.exists(subject_dir):
            return None

        subject_data = {}
        for file_name in SPM_AUDITORY_DATA_FILES:
            file_path = os.path.join(subject_dir, file_name)
            if os.path.exists(file_path):
                subject_data[file_name] = file_path
            else:
                print("%s missing from filelist!" % file_name)
                return None

        _subject_data = {}
        _subject_data["func"] = sorted(
            [subject_data[x] for x in subject_data.keys()
             if re.match("^fM00223_0\d\d\.img$", os.path.basename(x))])

        # volumes for this dataset of shape (64, 64, 64, 1); let's fix this
        for x in _subject_data["func"]:
            vol = nibabel.load(x)
            if len(vol.shape) == 4:
                vol = nibabel.Nifti1Image(vol.get_data()[:, :, :, 0],
                                          vol.get_affine())
                nibabel.save(vol, x)

        _subject_data["anat"] = [subject_data[x] for x in subject_data.keys()
                                 if re.match("^sM00223_002\.img$",
                                             os.path.basename(x))][0]

        # ... same thing for anat
        vol = nibabel.load(_subject_data["anat"])
        if len(vol.shape) == 4:
            vol = nibabel.Nifti1Image(vol.get_data()[:, :, :, 0],
                                      vol.get_affine())
            nibabel.save(vol, _subject_data["anat"])

        return Bunch(**_subject_data)

    # maybe data_dir already contains the data ?
    data = _glob_spm_auditory_data()
    if data is not None:
        return data

    # No. Download the data
    print("Data absent, downloading...")
    url = ("http://www.fil.ion.ucl.ac.uk/spm/download/data/MoAEpilot/"
           "MoAEpilot.zip")
    archive_path = os.path.join(subject_dir, os.path.basename(url))
    _fetch_file(url, subject_dir)
    try:
        _uncompress_file(archive_path)
    except:
        print("Archive corrupted, trying to download it again.")
        return fetch_spm_auditory(data_dir=data_dir, data_name="",
                                  subject_id=subject_id)

    return _glob_spm_auditory_data()

def fetch_spm_multimodal_fmri(data_dir=None, data_name="spm_multimodal_fmri",
                              subject_id="sub001", verbose=1):
    """Fetcher for Multi-modal Face Dataset.

    Parameters
    ----------
    data_dir: string
        path of the data directory. Used to force data storage in a specified
        location. If the data is already present there, then will simply
        glob it.

    Returns
    -------
    data: sklearn.datasets.base.Bunch
        Dictionary-like object, the interest attributes are:
        - 'func1': string list. Paths to functional images for session 1
        - 'func2': string list. Paths to functional images for session 2
        - 'trials_ses1': string list. Path to onsets file for session 1
        - 'trials_ses2': string list. Path to onsets file for session 2
        - 'anat': string. Path to anat file

    References
    ----------
    :download:
        http://www.fil.ion.ucl.ac.uk/spm/data/mmfaces/

    """

    data_dir = _get_dataset_dir(data_name, data_dir=data_dir,
                                verbose=verbose)
    subject_dir = os.path.join(data_dir, subject_id)

    def _glob_spm_multimodal_fmri_data():
        """glob data from subject_dir."""
        _subject_data = {'slice_order': 'descending'}

        for session in range(2):
            # glob func data for session s + 1
            session_func = sorted(glob.glob(
                    os.path.join(
                        subject_dir,
                        ("fMRI/Session%i/fMETHODS-000%i-*-01.img" % (
                                session + 1, session + 5)))))
            if len(session_func) < 390:
                print("Missing %i functional scans for session %i." % (
                    390 - len(session_func), session))
                return None

            _subject_data['func%i' % (session + 1)] = session_func

            # glob trials .mat file
            sess_trials = os.path.join(
                subject_dir,
                "fMRI/trials_ses%i.mat" % (session + 1))
            if not os.path.isfile(sess_trials):
                print("Missing session file: %s" % sess_trials)
                return None

            _subject_data['trials_ses%i' % (session + 1)] = sess_trials

        # glob for anat data
        anat = os.path.join(subject_dir, "sMRI/smri.img")
        if not os.path.isfile(anat):
            print("Missing structural image.")
            return None

        _subject_data["anat"] = anat

        return Bunch(**_subject_data)

    # maybe data_dir already contains the data ?
    data = _glob_spm_multimodal_fmri_data()
    if data is not None:
        return data

    # No. Download the data
    print("Data absent, downloading...")
    urls = [
        # fmri
        ("http://www.fil.ion.ucl.ac.uk/spm/download/data/mmfaces/"
        "multimodal_fmri.zip"),

        # structural
        ("http://www.fil.ion.ucl.ac.uk/spm/download/data/mmfaces/"
         "multimodal_smri.zip")
        ]

    for url in urls:
        archive_path = os.path.join(subject_dir, os.path.basename(url))
        _fetch_file(url, subject_dir)
        try:
            _uncompress_file(archive_path)
        except:
            print("Archive corrupted, trying to download it again.")
            return fetch_spm_multimodal_fmri(data_dir=data_dir,
                                             data_name="",
                                             subject_id=subject_id)

    return _glob_spm_multimodal_fmri_data()

def fetch_fiac_first_level(data_dir=None, verbose=1):
    """ Download a first-level fiac fMRI dataset (2 sessions)

    Parameters
    ----------
    data_dir: string
        directory where data should be downloaded and unpacked.
    """
    data_dir = _get_dataset_dir('', data_dir=data_dir,
                                verbose=verbose)

    def _glob_fiac_data():
        """glob data from subject_dir."""
        _subject_data = {}
        subject_dir = os.path.join(data_dir, 'fiac/fiac0')
        for session in [1, 2]:
            # glob func data for session session + 1
            session_func = os.path.join(subject_dir, 'run%i.nii.gz' % session)
            if not os.path.isfile(session_func):
                print('Missing functional scan for session %i.' % session)
                return None

            _subject_data['func%i' % session] = session_func

            # glob design matrix .npz file
            sess_dmtx = os.path.join(subject_dir, 'run%i_design.npz' % session)
            if not os.path.isfile(sess_dmtx):
                print('Missing session file: %s' % sess_dmtx)
                return None

            _subject_data['design_matrix%i' % session] = sess_dmtx

        # glob for mask data
        mask = os.path.join(subject_dir, 'mask.nii.gz')
        if not os.path.isfile(mask):
            print('Missing mask image.')
            return None

        _subject_data['mask'] = mask
        return Bunch(**_subject_data)

    # maybe data_dir already contains the data ?
    data = _glob_fiac_data()
    if data is not None:
        return data

    # No. Download the data
    print('Data absent, downloading...')
    url = 'http://nipy.sourceforge.net/data-packages/nipy-data-0.2.tar.gz'

    archive_path = os.path.join(data_dir, os.path.basename(url))
    _fetch_file(url, data_dir)
    try:
        _uncompress_file(archive_path)
    except:
        print('Archive corrupted, trying to download it again.')
        return fetch_fiac_first_level(data_dir=data_dir)

    return _glob_fiac_data()
<|endoftext|>