<commit_before>import json

from django.conf import settings
from django.contrib import messages
from django.core.exceptions import ObjectDoesNotExist
from django.urls import reverse
from django.utils import timezone
from django.utils.translation import ugettext
from django.utils.translation import ugettext_lazy as _
from django.views import generic

from adhocracy4.dashboard import mixins as a4dashboard_mixins
from adhocracy4.exports import mixins as export_mixins
from adhocracy4.exports import unescape_and_strip_html
from adhocracy4.exports import views as export_views
from adhocracy4.rules import mixins as rules_mixins
from meinberlin.apps.contrib.views import CanonicalURLDetailView
from meinberlin.apps.maps.models import MapPreset
from meinberlin.apps.plans.forms import PlanForm
from meinberlin.apps.plans.models import Plan

from . import models

class PlanDetailView(rules_mixins.PermissionRequiredMixin,
                     CanonicalURLDetailView):
    model = models.Plan
    template_name = 'meinberlin_plans/plan_detail.html'
    permission_required = 'meinberlin_plans.view_plan'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['berlin_polygon'] = settings.BERLIN_POLYGON
        return context

class PlanListView(rules_mixins.PermissionRequiredMixin,
                   generic.ListView):
    model = models.Plan
    template_name = 'meinberlin_plans/plan_list.html'
    permission_required = 'meinberlin_plans.list_plan'

    def get_districts(self):
        try:
            return MapPreset.objects.filter(
                category__name='Bezirke - Berlin')
        except ObjectDoesNotExist:
            return []

    def _get_status_string(self, projects):

        future_phase = None
        for project in projects:
            phases = project.phases
            if phases.active_phases():
                return ugettext('running')
            if phases.future_phases():
                date = phases.future_phases().first().start_date
                if not future_phase:
                    future_phase = date
                else:
                    if date < future_phase:
                        future_phase = date

        if future_phase:
            return ugettext('starts at {}').format(future_phase.date())

    def _get_participation_status(self, item):
        projects = item.projects.all()\
            .filter(is_draft=False,
                    is_archived=False,
                    is_public=True)
        if not projects:
            return item.get_participation_display(), False
        else:
            status_string = self._get_status_string(projects)
            if status_string:
                return status_string, True
            else:
                return item.get_participation_display(), False

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        districts = self.get_districts()

        district_list = json.dumps([district.polygon
                                    for district in districts])
        district_names = json.dumps([district.name
                                     for district in districts])
        context['districts'] = district_list
        context['district_names'] = district_names

        items = sorted(context['object_list'],
                       key=lambda x: x.modified or x.created,
                       reverse=True)

        result = []

        for item in items:
            participation_string, active = self._get_participation_status(item)
            result.append({
                'title': item.title,
                'url': item.get_absolute_url(),
                'organisation': item.organisation.name,
                'point': item.point,
                'point_label': item.point_label,
                'cost': item.cost,
                'district': item.district.name,
                'category': item.category,
                'status': item.status,
                'status_display': item.get_status_display(),
                'participation_string': participation_string,
                'participation_active': active,
                'participation': item.participation,
                'participation_display': item.get_participation_display(),
            })

        context['items'] = json.dumps(result)
        context['baseurl'] = settings.A4_MAP_BASEURL
        context['attribution'] = settings.A4_MAP_ATTRIBUTION
        context['bounds'] = json.dumps(settings.A4_MAP_BOUNDING_BOX)

        return context

class PlanExportView(rules_mixins.PermissionRequiredMixin,
                     export_mixins.ItemExportWithLinkMixin,
                     export_mixins.ExportModelFieldsMixin,
                     export_mixins.ItemExportWithLocationMixin,
                     export_views.BaseExport,
                     export_views.AbstractXlsxExportView):

    permission_required = 'meinberlin_plans.list_plan'
    model = models.Plan
    fields = ['title', 'organisation', 'contact', 'district', 'cost',
              'description', 'category', 'status', 'participation']
    html_fields = ['description']

    def get_object_list(self):
        return models.Plan.objects.all()

    def get_base_filename(self):
        return 'plans_%s' % timezone.now().strftime('%Y%m%dT%H%M%S')

    def get_virtual_fields(self, virtual):
        virtual = super().get_virtual_fields(virtual)
        virtual['projects'] = ugettext('Projects')
        virtual['projects_links'] = ugettext('Project Links')
        return virtual

    def get_organisation_data(self, item):
        return item.organisation.name

    def get_district_data(self, item):
        return item.district.name

    def get_contact_data(self, item):
        return unescape_and_strip_html(item.contact)

    def get_status_data(self, item):
        return item.get_status_display()

    def get_participation_data(self, item):
        return item.get_participation_display()

    def get_description_data(self, item):
        return unescape_and_strip_html(item.description)

    def get_projects_data(self, item):
        if item.projects.all():
            return ', \n'.join(
                [project.name
                 for project in item.projects.all()]
            )
        return ''

    def get_projects_links_data(self, item):
        if item.projects.all():
            return ', \n'.join(
                [self.request.build_absolute_uri(
                 project.get_absolute_url())
                 for project in item.projects.all()
                 ])
        return ''

class DashboardPlanListView(a4dashboard_mixins.DashboardBaseMixin,
                            generic.ListView):
    model = Plan
    template_name = 'meinberlin_plans/plan_dashboard_list.html'
    permission_required = 'meinberlin_plans.add_plan'
    menu_item = 'project'

    def get_permission_object(self):
        return self.organisation

    def get_queryset(self):
        return super().get_queryset().filter(organisation=self.organisation)

class DashboardPlanCreateView(a4dashboard_mixins.DashboardBaseMixin,
                              generic.CreateView):
    model = Plan
    form_class = PlanForm
    permission_required = 'meinberlin_plans.add_plan'
    template_name = 'meinberlin_plans/plan_create_form.html'
    menu_item = 'project'

    def form_valid(self, form):
        form.instance.creator = self.request.user
        form.instance.organisation = self.organisation
        return super().form_valid(form)

    def get_permission_object(self):
        return self.organisation

    def get_success_url(self):
        return reverse(
            'a4dashboard:plan-list',
            kwargs={'organisation_slug': self.organisation.slug})

class DashboardPlanUpdateView(a4dashboard_mixins.DashboardBaseMixin,
                              generic.UpdateView):
    model = Plan
    form_class = PlanForm
    permission_required = 'meinberlin_plans.change_plan'
    template_name = 'meinberlin_plans/plan_update_form.html'
    menu_item = 'project'

    def get_permission_object(self):
        return self.organisation

    def get_success_url(self):
        return reverse(
            'a4dashboard:plan-list',
            kwargs={'organisation_slug': self.organisation.slug})

class DashboardPlanDeleteView(a4dashboard_mixins.DashboardBaseMixin,
                              generic.DeleteView):
    model = Plan
    success_message = _('The plan has been deleted')
    permission_required = 'meinberlin_plans.change_plan'
    template_name = 'meinberlin_plans/plan_confirm_delete.html'
    menu_item = 'project'

    def delete(self, request, *args, **kwargs):
        messages.success(self.request, self.success_message)
        return super().delete(request, *args, **kwargs)

    def get_permission_object(self):
        return self.organisation

    def get_success_url(self):
        return reverse(
            'a4dashboard:plan-list',
            kwargs={'organisation_slug': self.organisation.slug})
<commit_msg>return list as string  as the urls are not formatted in the right way<commit_after>import json

from django.conf import settings
from django.contrib import messages
from django.core.exceptions import ObjectDoesNotExist
from django.urls import reverse
from django.utils import timezone
from django.utils.translation import ugettext
from django.utils.translation import ugettext_lazy as _
from django.views import generic

from adhocracy4.dashboard import mixins as a4dashboard_mixins
from adhocracy4.exports import mixins as export_mixins
from adhocracy4.exports import unescape_and_strip_html
from adhocracy4.exports import views as export_views
from adhocracy4.rules import mixins as rules_mixins
from meinberlin.apps.contrib.views import CanonicalURLDetailView
from meinberlin.apps.maps.models import MapPreset
from meinberlin.apps.plans.forms import PlanForm
from meinberlin.apps.plans.models import Plan

from . import models

class PlanDetailView(rules_mixins.PermissionRequiredMixin,
                     CanonicalURLDetailView):
    model = models.Plan
    template_name = 'meinberlin_plans/plan_detail.html'
    permission_required = 'meinberlin_plans.view_plan'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['berlin_polygon'] = settings.BERLIN_POLYGON
        return context

class PlanListView(rules_mixins.PermissionRequiredMixin,
                   generic.ListView):
    model = models.Plan
    template_name = 'meinberlin_plans/plan_list.html'
    permission_required = 'meinberlin_plans.list_plan'

    def get_districts(self):
        try:
            return MapPreset.objects.filter(
                category__name='Bezirke - Berlin')
        except ObjectDoesNotExist:
            return []

    def _get_status_string(self, projects):

        future_phase = None
        for project in projects:
            phases = project.phases
            if phases.active_phases():
                return ugettext('running')
            if phases.future_phases():
                date = phases.future_phases().first().start_date
                if not future_phase:
                    future_phase = date
                else:
                    if date < future_phase:
                        future_phase = date

        if future_phase:
            return ugettext('starts at {}').format(future_phase.date())

    def _get_participation_status(self, item):
        projects = item.projects.all()\
            .filter(is_draft=False,
                    is_archived=False,
                    is_public=True)
        if not projects:
            return item.get_participation_display(), False
        else:
            status_string = self._get_status_string(projects)
            if status_string:
                return status_string, True
            else:
                return item.get_participation_display(), False

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        districts = self.get_districts()

        district_list = json.dumps([district.polygon
                                    for district in districts])
        district_names = json.dumps([district.name
                                     for district in districts])
        context['districts'] = district_list
        context['district_names'] = district_names

        items = sorted(context['object_list'],
                       key=lambda x: x.modified or x.created,
                       reverse=True)

        result = []

        for item in items:
            participation_string, active = self._get_participation_status(item)
            result.append({
                'title': item.title,
                'url': item.get_absolute_url(),
                'organisation': item.organisation.name,
                'point': item.point,
                'point_label': item.point_label,
                'cost': item.cost,
                'district': item.district.name,
                'category': item.category,
                'status': item.status,
                'status_display': item.get_status_display(),
                'participation_string': participation_string,
                'participation_active': active,
                'participation': item.participation,
                'participation_display': item.get_participation_display(),
            })

        context['items'] = json.dumps(result)
        context['baseurl'] = settings.A4_MAP_BASEURL
        context['attribution'] = settings.A4_MAP_ATTRIBUTION
        context['bounds'] = json.dumps(settings.A4_MAP_BOUNDING_BOX)

        return context

class PlanExportView(rules_mixins.PermissionRequiredMixin,
                     export_mixins.ItemExportWithLinkMixin,
                     export_mixins.ExportModelFieldsMixin,
                     export_mixins.ItemExportWithLocationMixin,
                     export_views.BaseExport,
                     export_views.AbstractXlsxExportView):

    permission_required = 'meinberlin_plans.list_plan'
    model = models.Plan
    fields = ['title', 'organisation', 'contact', 'district', 'cost',
              'description', 'category', 'status', 'participation']
    html_fields = ['description']

    def get_object_list(self):
        return models.Plan.objects.all()

    def get_base_filename(self):
        return 'plans_%s' % timezone.now().strftime('%Y%m%dT%H%M%S')

    def get_virtual_fields(self, virtual):
        virtual = super().get_virtual_fields(virtual)
        virtual['projects'] = ugettext('Projects')
        virtual['projects_links'] = ugettext('Project Links')
        return virtual

    def get_organisation_data(self, item):
        return item.organisation.name

    def get_district_data(self, item):
        return item.district.name

    def get_contact_data(self, item):
        return unescape_and_strip_html(item.contact)

    def get_status_data(self, item):
        return item.get_status_display()

    def get_participation_data(self, item):
        return item.get_participation_display()

    def get_description_data(self, item):
        return unescape_and_strip_html(item.description)

    def get_projects_data(self, item):
        if item.projects.all():
            return ', \n'.join(
                [project.name
                 for project in item.projects.all()]
            )
        return ''

    def get_projects_links_data(self, item):
        if item.projects.all():
            return str([self.request.build_absolute_uri(
                        project.get_absolute_url())
                        for project in item.projects.all()
                        ])
        return ''

class DashboardPlanListView(a4dashboard_mixins.DashboardBaseMixin,
                            generic.ListView):
    model = Plan
    template_name = 'meinberlin_plans/plan_dashboard_list.html'
    permission_required = 'meinberlin_plans.add_plan'
    menu_item = 'project'

    def get_permission_object(self):
        return self.organisation

    def get_queryset(self):
        return super().get_queryset().filter(organisation=self.organisation)

class DashboardPlanCreateView(a4dashboard_mixins.DashboardBaseMixin,
                              generic.CreateView):
    model = Plan
    form_class = PlanForm
    permission_required = 'meinberlin_plans.add_plan'
    template_name = 'meinberlin_plans/plan_create_form.html'
    menu_item = 'project'

    def form_valid(self, form):
        form.instance.creator = self.request.user
        form.instance.organisation = self.organisation
        return super().form_valid(form)

    def get_permission_object(self):
        return self.organisation

    def get_success_url(self):
        return reverse(
            'a4dashboard:plan-list',
            kwargs={'organisation_slug': self.organisation.slug})

class DashboardPlanUpdateView(a4dashboard_mixins.DashboardBaseMixin,
                              generic.UpdateView):
    model = Plan
    form_class = PlanForm
    permission_required = 'meinberlin_plans.change_plan'
    template_name = 'meinberlin_plans/plan_update_form.html'
    menu_item = 'project'

    def get_permission_object(self):
        return self.organisation

    def get_success_url(self):
        return reverse(
            'a4dashboard:plan-list',
            kwargs={'organisation_slug': self.organisation.slug})

class DashboardPlanDeleteView(a4dashboard_mixins.DashboardBaseMixin,
                              generic.DeleteView):
    model = Plan
    success_message = _('The plan has been deleted')
    permission_required = 'meinberlin_plans.change_plan'
    template_name = 'meinberlin_plans/plan_confirm_delete.html'
    menu_item = 'project'

    def delete(self, request, *args, **kwargs):
        messages.success(self.request, self.success_message)
        return super().delete(request, *args, **kwargs)

    def get_permission_object(self):
        return self.organisation

    def get_success_url(self):
        return reverse(
            'a4dashboard:plan-list',
            kwargs={'organisation_slug': self.organisation.slug})
<|endoftext|>


<commit_before>"""
=================================================
SVM-Anova: SVM with univariate feature selection
=================================================

This example shows how to perform univariate feature selection before running a
SVC (support vector classifier) to improve the classification scores. We use
the iris dataset (4 features) and add 36 non-informative features. We can find
that our model achieves best performance when we select around 10% of features.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.feature_selection import SelectPercentile, chi2
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC

# #############################################################################
# Import some data to play with
X, y = load_iris(return_X_y=True)
# Add non-informative features
np.random.seed(0)
X = np.hstack((X, 2 * np.random.random((X.shape[0], 36))))

# #############################################################################
# Create a feature-selection transform, a scaler and an instance of SVM that we
# combine together to have an full-blown estimator
clf = Pipeline([('anova', SelectPercentile(chi2)),
                ('scaler', StandardScaler()),
                ('svc', SVC(gamma="auto"))])

# #############################################################################
# Plot the cross-validation score as a function of percentile of features
score_means = list()
score_stds = list()
percentiles = (1, 3, 6, 10, 15, 20, 30, 40, 60, 80, 100)

for percentile in percentiles:
    clf.set_params(anova__percentile=percentile)
    this_scores = cross_val_score(clf, X, y)
    score_means.append(this_scores.mean())
    score_stds.append(this_scores.std())

plt.errorbar(percentiles, score_means, np.array(score_stds))
plt.title(
    'Performance of the SVM-Anova varying the percentile of features selected')
plt.xticks(np.linspace(0, 100, 11, endpoint=True))
plt.xlabel('Percentile')
plt.ylabel('Accuracy Score')
plt.axis('tight')
plt.show()
<commit_msg>DOC Fixed typo in a comment (#19284)<commit_after>"""
=================================================
SVM-Anova: SVM with univariate feature selection
=================================================

This example shows how to perform univariate feature selection before running a
SVC (support vector classifier) to improve the classification scores. We use
the iris dataset (4 features) and add 36 non-informative features. We can find
that our model achieves best performance when we select around 10% of features.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.feature_selection import SelectPercentile, chi2
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC

# #############################################################################
# Import some data to play with
X, y = load_iris(return_X_y=True)
# Add non-informative features
np.random.seed(0)
X = np.hstack((X, 2 * np.random.random((X.shape[0], 36))))

# #############################################################################
# Create a feature-selection transform, a scaler and an instance of SVM that we
# combine together to have a full-blown estimator
clf = Pipeline([('anova', SelectPercentile(chi2)),
                ('scaler', StandardScaler()),
                ('svc', SVC(gamma="auto"))])

# #############################################################################
# Plot the cross-validation score as a function of percentile of features
score_means = list()
score_stds = list()
percentiles = (1, 3, 6, 10, 15, 20, 30, 40, 60, 80, 100)

for percentile in percentiles:
    clf.set_params(anova__percentile=percentile)
    this_scores = cross_val_score(clf, X, y)
    score_means.append(this_scores.mean())
    score_stds.append(this_scores.std())

plt.errorbar(percentiles, score_means, np.array(score_stds))
plt.title(
    'Performance of the SVM-Anova varying the percentile of features selected')
plt.xticks(np.linspace(0, 100, 11, endpoint=True))
plt.xlabel('Percentile')
plt.ylabel('Accuracy Score')
plt.axis('tight')
plt.show()
<|endoftext|>


<commit_before>'''
Created on Mar 7, 2013

@author: crisr
'''
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default',DeprecationWarning)

import sys
import xml.etree.ElementTree as ET
import scipy.stats.distributions  as dist
import numpy as np
from BaseType import BaseType
import scipy.special as polys
from scipy.misc import factorial
import Quadrature

class Distribution(BaseType):
  '''
  a general class containing the distributions
  '''
  def __init__(self):
    BaseType.__init__(self)
    self.upperBoundUsed = False
    self.lowerBoundUsed = False
    self.upperBound       = 0.0
    self.lowerBound       = 0.0
    self.adjustmentType   = ''
    self.bestQuad = None

  def readMoreXML(self,xmlNode):
    try:
      type = xmlNode.attrib['ExpansionQuadrature']
      self.bestQuad = Quadrature.returnInstance(type)
    except:
      pass

    if xmlNode.find('upperBound') !=None:
      self.upperBound = float(xmlNode.find('upperBound').text)
      self.upperBoundUsed = True

    if xmlNode.find('lowerBound')!=None:
      self.lowerBound = float(xmlNode.find('lowerBound').text)
      self.lowerBoundUsed = True
    if xmlNode.find('adjustment') !=None:
      self.adjustment = xmlNode.find('adjustment').text
    else:
      self.adjustment = 'scaling'

  def addInitParams(self,tempDict):
    tempDict['upperBoundUsed'] = self.upperBoundUsed
    tempDict['lowerBoundUsed'] = self.lowerBoundUsed
    tempDict['upperBound'    ] = self.upperBound
    tempDict['lowerBound'    ] = self.lowerBound
    tempDict['adjustmentType'] = self.adjustmentType

  def rvsWithinCDFbounds(self,LowerBound,upperBound):
    point = np.random.rand(1)*(upperBound-LowerBound)+LowerBound
    return self.distribution.ppt(point)

  def rvsWithinbounds(self,LowerBound,upperBound):
    CDFupper = self.distribution.cdf(upperBound)
    CDFlower = self.distribution.cdf(LowerBound)
    return self.rvsWithinCDFbounds(CDFlower,CDFupper)

  def setQuad(self,quad,exp_order):
    self.distQuad=quad
    self.exp_order=exp_order
  def quad(self):
    try: return self.distQuad
    except: raise IOError ('No quadrature has been set for this distr. yet.')
  def polyOrder(self):
    try: return self.exp_order
    except: raise IOError ('Quadrature has not been set for this distr. yet.')

#==============================================================\
#    Distributions convenient for stochastic collocation
#==============================================================\

class Uniform(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.type = 'Uniform'
    self.bestQuad = Quadrature.Legendre
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise 'low value needed for uniform distribution'
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise 'hi value needed for uniform distribution'
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    # no other additional parameters required
  def inDistr(self):
    self.range=self.hi-self.low
    self.distribution = dist.uniform(loc=self.low,scale=self.range)
    #assign associated polynomial types
    self.polynomial = polys.legendre
    #define functions locally, then point to them
    def norm(n):
      '''Returns normalization constant for polynomial type, given the poly ordeir'''
      return np.sqrt((2.*n+1.)/2.)
    def standardToActualPoint(x): #standard -> actual
      '''Given a [-1,1] point, converts to parameter value.'''
      return x*self.range/2.+self.distribution.mean()
    def actualToStandardPoint(x): #actual -> standard
      '''Given a parameter value, converts to [-1,1] point.'''
      return (x-self.distribution.mean())/(self.range/2.)
    def standardToActualWeight(x): #standard -> actual
      '''Given normal quadrature weight, returns adjusted weight.'''
      return x/(self.range/2.)
    def probNorm(x): #normalizes probability if total != 1
      '''Returns the poly factor to scale by so that sum(probability)=1.'''
      return self.range
    # point to functions
    self.poly_norm = norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

class Normal(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.mean  = 0.0
    self.sigma = 0.0
    self.type = 'Normal'
    self.bestQuad = Quadrature.StatHermite

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.mean  = float(xmlNode.find('mean' ).text)
    except: raise Exception('mean value needed for normal distribution')
    try: self.sigma = float(xmlNode.find('sigma').text)
    except: raise Exception('sigma value needed for normal distribution')
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['mean' ] = self.mean
    tempDict['sigma'] = self.sigma
  def inDistr(self):
    if (not self.upperBoundUsed) and (not self.lowerBoundUsed):
      self.distribution = dist.norm(loc=self.mean,scale=self.sigma)
      self.polynomial = polys.hermitenorm
      def norm(n):
        return (np.sqrt(np.sqrt(2.*np.pi)*factorial(n)))**(-1)
      def standardToActualPoint(x): #standard -> actual
        return x*self.sigma**2/2.+self.distribution.mean()
      def actualToStandardPoint(x): #actual -> standard
        return (x-self.distribution.mean())/(self.sigma**2/2.)
      def standardToActualWeight(x): #standard -> actual
        return x/(self.sigma**2/2.)
      def probNorm(x): #normalizes if total prob. != 1
        return 1.0
      self.poly_norm = norm
      self.actual_point = standardToActualPoint
      self.std_point = actualToStandardPoint
      self.actual_weight = standardToActualWeight
      self.probability_norm = probNorm
    else:
      #FIXME special case distribution for stoch collocation
      if self.lowerBoundUsed == False: a = -sys.float_info[max]
      else:a = (self.lowerBound - self.mean) / self.sigma
      if self.upperBoundUsed == False: b = sys.float_info[max]
      else:b = (self.upperBound - self.mean) / self.sigma
      self.distribution = dist.truncnorm(a,b,loc=self.mean,scale=self.sigma)

class Gamma(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.alpha = 0.0
    self.beta = 1.0
    self.type = 'Gamma'
    self.bestQuad = Quadrature.Laguerre
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise 'low value needed for Gamma distribution'
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise 'alpha value needed for Gamma distribution'
    try: self.beta = float(xmlNode.find('beta').text)
    except: self.beta=1.0
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta
  def inDistr(self):
    self.distribution = dist.gamma(self.alpha,loc=self.low,scale=self.beta)
    self.polynomial = polys.genlaguerre
    def norm(n):
      return np.sqrt(factorial(n)/polys.gamma(n+self.alpha+1.0))
    def standardToActualPoint(x): #standard -> actual
      return x/self.alpha+self.alpha+self.low #TODO these correct? no beta used
    def actualToStandardPoint(x): #actual -> standard
      return (x-self.low-self.alpha)*self.alpha
    def standardToActualWeight(x): #standard -> actual
      return x
    def probNorm(x): #normalizes probability if total != 1
      return 1.0
    self.poly_norm=norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

class Beta(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.alpha = 0.0
    self.beta = 0.0
    self.type = 'Beta'
    self.bestQuad = Quadrature.Jacobi
    # TODO default to specific Beta distro?
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise 'low value needed for Gamma distribution'
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise 'hi value needed for Gamma distribution'
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise 'alpha value needed for Gamma distribution'
    try: self.beta = float(xmlNode.find('beta').text)
    except: raise 'beta value needed for Gamma distribution'
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta
  def inDistr(self):
    self.distribution = dist.beta(self.alpha,self.beta,scale=self.hi-self.low)

#==========================================================\
#    other distributions
#==========================================================\

# Add polynomials, shifting, zero-to-one to these!

class Triangular(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.apex = 0.0
    self.min  = 0.0
    self.max  = 0.0
    self.type = 'Triangular'
    self.bestQuad = None
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.apex = float(xmlNode.find('apex').text)
    except: raise Exception('apex value needed for normal distribution')
    try: self.min = float(xmlNode.find('min').text)
    except: raise Exception('min value needed for normal distribution')
    try: self.max = float(xmlNode.find('max').text)
    except: raise Exception('max value needed for normal distribution')
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['apex' ] = self.apex
    tempDict['min'  ] = self.min
    tempDict['max'  ] = self.max
  def inDistr(self):
    if self.lowerBoundUsed == False and self.upperBoundUsed == False:
      c = (self.apex-self.min)/(self.max-self.min)
      self.distribution = dist.triang(c,loc=self.min,scale=(self.max-self.min))
    else:
      raise IOError ('Truncated triangular not yet implemented')

def returnInstance(Type):
  base = 'Distribution'
  InterfaceDict = {}
  InterfaceDict['Uniform'  ]  = Uniform
  InterfaceDict['Normal'   ]  = Normal
  InterfaceDict['Gamma'    ]  = Gamma
  InterfaceDict['Beta'     ]  = Beta
  InterfaceDict['Triangular'] = Triangular
  try: return InterfaceDict[Type]()
  except: raise NameError('not known '+base+' type '+Type)
<commit_msg>Python 3 compatiblity: raise exception instead of string.<commit_after>'''
Created on Mar 7, 2013

@author: crisr
'''
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default',DeprecationWarning)

import sys
import xml.etree.ElementTree as ET
import scipy.stats.distributions  as dist
import numpy as np
from BaseType import BaseType
import scipy.special as polys
from scipy.misc import factorial
import Quadrature

class Distribution(BaseType):
  '''
  a general class containing the distributions
  '''
  def __init__(self):
    BaseType.__init__(self)
    self.upperBoundUsed = False
    self.lowerBoundUsed = False
    self.upperBound       = 0.0
    self.lowerBound       = 0.0
    self.adjustmentType   = ''
    self.bestQuad = None

  def readMoreXML(self,xmlNode):
    try:
      type = xmlNode.attrib['ExpansionQuadrature']
      self.bestQuad = Quadrature.returnInstance(type)
    except:
      pass

    if xmlNode.find('upperBound') !=None:
      self.upperBound = float(xmlNode.find('upperBound').text)
      self.upperBoundUsed = True

    if xmlNode.find('lowerBound')!=None:
      self.lowerBound = float(xmlNode.find('lowerBound').text)
      self.lowerBoundUsed = True
    if xmlNode.find('adjustment') !=None:
      self.adjustment = xmlNode.find('adjustment').text
    else:
      self.adjustment = 'scaling'

  def addInitParams(self,tempDict):
    tempDict['upperBoundUsed'] = self.upperBoundUsed
    tempDict['lowerBoundUsed'] = self.lowerBoundUsed
    tempDict['upperBound'    ] = self.upperBound
    tempDict['lowerBound'    ] = self.lowerBound
    tempDict['adjustmentType'] = self.adjustmentType

  def rvsWithinCDFbounds(self,LowerBound,upperBound):
    point = np.random.rand(1)*(upperBound-LowerBound)+LowerBound
    return self.distribution.ppt(point)

  def rvsWithinbounds(self,LowerBound,upperBound):
    CDFupper = self.distribution.cdf(upperBound)
    CDFlower = self.distribution.cdf(LowerBound)
    return self.rvsWithinCDFbounds(CDFlower,CDFupper)

  def setQuad(self,quad,exp_order):
    self.distQuad=quad
    self.exp_order=exp_order
  def quad(self):
    try: return self.distQuad
    except: raise IOError ('No quadrature has been set for this distr. yet.')
  def polyOrder(self):
    try: return self.exp_order
    except: raise IOError ('Quadrature has not been set for this distr. yet.')

#==============================================================\
#    Distributions convenient for stochastic collocation
#==============================================================\

class Uniform(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.type = 'Uniform'
    self.bestQuad = Quadrature.Legendre
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for uniform distribution')
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise Exception('hi value needed for uniform distribution')
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    # no other additional parameters required
  def inDistr(self):
    self.range=self.hi-self.low
    self.distribution = dist.uniform(loc=self.low,scale=self.range)
    #assign associated polynomial types
    self.polynomial = polys.legendre
    #define functions locally, then point to them
    def norm(n):
      '''Returns normalization constant for polynomial type, given the poly ordeir'''
      return np.sqrt((2.*n+1.)/2.)
    def standardToActualPoint(x): #standard -> actual
      '''Given a [-1,1] point, converts to parameter value.'''
      return x*self.range/2.+self.distribution.mean()
    def actualToStandardPoint(x): #actual -> standard
      '''Given a parameter value, converts to [-1,1] point.'''
      return (x-self.distribution.mean())/(self.range/2.)
    def standardToActualWeight(x): #standard -> actual
      '''Given normal quadrature weight, returns adjusted weight.'''
      return x/(self.range/2.)
    def probNorm(x): #normalizes probability if total != 1
      '''Returns the poly factor to scale by so that sum(probability)=1.'''
      return self.range
    # point to functions
    self.poly_norm = norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

class Normal(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.mean  = 0.0
    self.sigma = 0.0
    self.type = 'Normal'
    self.bestQuad = Quadrature.StatHermite

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.mean  = float(xmlNode.find('mean' ).text)
    except: raise Exception('mean value needed for normal distribution')
    try: self.sigma = float(xmlNode.find('sigma').text)
    except: raise Exception('sigma value needed for normal distribution')
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['mean' ] = self.mean
    tempDict['sigma'] = self.sigma
  def inDistr(self):
    if (not self.upperBoundUsed) and (not self.lowerBoundUsed):
      self.distribution = dist.norm(loc=self.mean,scale=self.sigma)
      self.polynomial = polys.hermitenorm
      def norm(n):
        return (np.sqrt(np.sqrt(2.*np.pi)*factorial(n)))**(-1)
      def standardToActualPoint(x): #standard -> actual
        return x*self.sigma**2/2.+self.distribution.mean()
      def actualToStandardPoint(x): #actual -> standard
        return (x-self.distribution.mean())/(self.sigma**2/2.)
      def standardToActualWeight(x): #standard -> actual
        return x/(self.sigma**2/2.)
      def probNorm(x): #normalizes if total prob. != 1
        return 1.0
      self.poly_norm = norm
      self.actual_point = standardToActualPoint
      self.std_point = actualToStandardPoint
      self.actual_weight = standardToActualWeight
      self.probability_norm = probNorm
    else:
      #FIXME special case distribution for stoch collocation
      if self.lowerBoundUsed == False: a = -sys.float_info[max]
      else:a = (self.lowerBound - self.mean) / self.sigma
      if self.upperBoundUsed == False: b = sys.float_info[max]
      else:b = (self.upperBound - self.mean) / self.sigma
      self.distribution = dist.truncnorm(a,b,loc=self.mean,scale=self.sigma)

class Gamma(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.alpha = 0.0
    self.beta = 1.0
    self.type = 'Gamma'
    self.bestQuad = Quadrature.Laguerre
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for Gamma distribution')
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise Exception('alpha value needed for Gamma distribution')
    try: self.beta = float(xmlNode.find('beta').text)
    except: self.beta=1.0
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta
  def inDistr(self):
    self.distribution = dist.gamma(self.alpha,loc=self.low,scale=self.beta)
    self.polynomial = polys.genlaguerre
    def norm(n):
      return np.sqrt(factorial(n)/polys.gamma(n+self.alpha+1.0))
    def standardToActualPoint(x): #standard -> actual
      return x/self.alpha+self.alpha+self.low #TODO these correct? no beta used
    def actualToStandardPoint(x): #actual -> standard
      return (x-self.low-self.alpha)*self.alpha
    def standardToActualWeight(x): #standard -> actual
      return x
    def probNorm(x): #normalizes probability if total != 1
      return 1.0
    self.poly_norm=norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

class Beta(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.alpha = 0.0
    self.beta = 0.0
    self.type = 'Beta'
    self.bestQuad = Quadrature.Jacobi
    # TODO default to specific Beta distro?
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for Gamma distribution')
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise Exception('hi value needed for Gamma distribution')
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise Exception('alpha value needed for Gamma distribution')
    try: self.beta = float(xmlNode.find('beta').text)
    except: raise Exception('beta value needed for Gamma distribution')
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta
  def inDistr(self):
    self.distribution = dist.beta(self.alpha,self.beta,scale=self.hi-self.low)

#==========================================================\
#    other distributions
#==========================================================\

# Add polynomials, shifting, zero-to-one to these!

class Triangular(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.apex = 0.0
    self.min  = 0.0
    self.max  = 0.0
    self.type = 'Triangular'
    self.bestQuad = None
  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.apex = float(xmlNode.find('apex').text)
    except: raise Exception('apex value needed for normal distribution')
    try: self.min = float(xmlNode.find('min').text)
    except: raise Exception('min value needed for normal distribution')
    try: self.max = float(xmlNode.find('max').text)
    except: raise Exception('max value needed for normal distribution')
    self.inDistr()
  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['apex' ] = self.apex
    tempDict['min'  ] = self.min
    tempDict['max'  ] = self.max
  def inDistr(self):
    if self.lowerBoundUsed == False and self.upperBoundUsed == False:
      c = (self.apex-self.min)/(self.max-self.min)
      self.distribution = dist.triang(c,loc=self.min,scale=(self.max-self.min))
    else:
      raise IOError ('Truncated triangular not yet implemented')

def returnInstance(Type):
  base = 'Distribution'
  InterfaceDict = {}
  InterfaceDict['Uniform'  ]  = Uniform
  InterfaceDict['Normal'   ]  = Normal
  InterfaceDict['Gamma'    ]  = Gamma
  InterfaceDict['Beta'     ]  = Beta
  InterfaceDict['Triangular'] = Triangular
  try: return InterfaceDict[Type]()
  except: raise NameError('not known '+base+' type '+Type)
<|endoftext|>


<commit_before>import postProcessingFcVolMatchedModularDynamicPipeline as ppfc
# import preprocessingPipelineImprovedModular as prep
import preprocessingPipelineImprovedModularWOAnatDynamic as prep

import hypothesisTest as ht
import fdrBrainResultsModular as fdres
import os
import json
from os.path import join as opj
import itertools
from bids.grabbids import BIDSLayout
import time

# ------------- Paths ----------------------------------------------------------------------------------------

path_cwd = os.getcwd()
path_split_list = path_cwd.split('/')
s = path_split_list[0:-1] # for getting to the parent dir of pwd
s = opj('/',*s) # *s converts list to path, # very important to add '/' in the begining so it is read as directory later

json_path = 'scripts/json/paths.json'
with open(json_path, 'rt') as fp:
    task_info = json.load(fp)

base_directory = opj(s,task_info["base_directory_for_results"])
motion_correction_bet_directory = task_info["motion_correction_bet_directory"]
parent_wf_directory = task_info["parent_wf_directory"]
functional_connectivity_directory = task_info["functional_connectivity_directory"]
# functional_connectivity_directory = 'temp_fc'
coreg_reg_directory = task_info["coreg_reg_directory"]
atlas_resize_reg_directory = task_info["atlas_resize_reg_directory"]
data_directory = opj(s,task_info["data_directory"])
datasink_name = task_info["datasink_name"]
fc_datasink_name = task_info["fc_datasink_name"]
# fc_datasink_name = 'temp_dataSink'
atlasPath = opj(s,task_info["atlas_path"]) # Standard Brainnetome path

brain_path = opj(base_directory,datasink_name,'preprocessed_brain_paths/brain_file_list.npy')
mask_path = opj(base_directory,datasink_name,'preprocessed_mask_paths/mask_file_list.npy')
atlas_path = opj(base_directory,datasink_name,'atlas_paths/atlas_file_list.npy') # Brainetome atlas in functional space
tr_path = opj(base_directory,datasink_name,'tr_paths/tr_list.npy')
motion_params_path = opj(base_directory,datasink_name,'motion_params_paths/motion_params_file_list.npy')

func2std_mat_path = opj(base_directory, datasink_name,'joint_xformation_matrix_paths/joint_xformation_matrix_file_list.npy')

MNI3mm_path = opj(base_directory,parent_wf_directory,motion_correction_bet_directory,coreg_reg_directory,'resample_mni/MNI152_T1_2mm_brain_resample.nii')

hypothesis_test_dir = opj(base_directory, task_info["hypothesis_test_dir"])

fdr_results_dir = task_info["fdr_results_dir"]

score_corr_dir =  opj(base_directory,task_info["score_corr_dir"])

demographics_file_path = '/home1/varunk/Autism-Connectome-Analysis-brain_connectivity/notebooks/demographics.csv'
phenotype_file_path = '/home1/varunk/data/ABIDE1/RawDataBIDs/composite_phenotypic_file.csv'
# categoryInfo = '/home1/varunk/data/NYU_Cocaine-BIDS/grouping.csv'
categoryInfo = None

# --------------- Creating Log --------------------

# Get time

current_time = time.asctime( time.localtime(time.time()) )

log_path = opj(base_directory,"log.txt")
log = open(log_path, 'a')
print("-------------Starting the analysis at %s--------------\n"%(current_time))
log.write("-------------Starting the analysis at %s--------------\n"%(current_time))
log.flush()

# ----------------------------------------------------------------------------------------------------------------------------------------------------------

vols = 120 # needed for volume matching. Tells how many volumes you need to keep.
number_of_skipped_volumes = 4
#  True volumes will be vols - number_of_skipped_volumes

num_proc = 7

number_of_subjects = -1
# number_of_subjects = 7 # Number of subjects you wish to work with

log.write("Vols for matching: %s\n"%(vols))
log.write("Number_of_skipped_volumes: %s\n"%(number_of_skipped_volumes))
log.flush()
# ----------------------------- Getting Subjects -------------------------------
# ----------------------------------- BIDS -------------------------------------
layout = BIDSLayout(data_directory)

if number_of_subjects == -1:
    number_of_subjects = len(layout.get_subjects())

subject_list = (layout.get_subjects())[0:number_of_subjects]
# subject_list = list(map(int, subject_list))

# -----------------------------------File List----------------------------------
# group1FilesPath = ''
# group2FilesPath = ''
#
# group1FilesList = np.genfromtxt(group1FilesPath,dtype='unicode')
# group2FilesList = np.genfromtxt(group2FilesPath,dtype='unicode')
#
# fileList = group1FilesPath + group2FilesPath

# -----------------------------------------------------------------------------

paths = [json_path,
base_directory,
motion_correction_bet_directory,
parent_wf_directory,
functional_connectivity_directory,
coreg_reg_directory,
atlas_resize_reg_directory,
subject_list,
datasink_name,
fc_datasink_name,
atlasPath,
brain_path,
mask_path,
atlas_path,
tr_path,
motion_params_path,
func2std_mat_path,
MNI3mm_path,
demographics_file_path,
phenotype_file_path,
data_directory,
hypothesis_test_dir,
fdr_results_dir,
score_corr_dir
]

PREPROC = 0
POSTPROC = 0
HYPOTEST = 0
FDRES = 0
SCORE_CORR = 1

match = 1 # Age matching
applyFisher = True

# itr = (list(itertools.product([0, 1], repeat=3)))
itr = [(1,0,1,1,1)]
# itr = [(1,1,1,1,1)]

log.write("Operations:\n")
log.write("Preprocess = %s\n"%(PREPROC))
log.write("Postprocess = %s\n"%(POSTPROC))
log.write("Hypothesis Test = %s\n"%(HYPOTEST))
log.write("FDR correction and Vizualization = %s\n"%(FDRES))
log.write("Score-Connectivity Correlation  = %s\n"%(SCORE_CORR))
log.flush()

# ---------------------- Preprocess --------------------------------------------

ANAT = 1

itr_preproc = [1,1,0,1]
# itr_preproc = [0,0,0]
extract, slicetimer,motionOutliers, mcflirt= list(map(str, itr_preproc))
options_binary_string = extract+slicetimer+motionOutliers+mcflirt

log.write("Preprocessing Params\n")
log.write("Remove begining slices  = %s\n"%(extract))
log.write("Slice time correction  = %s\n"%(slicetimer))
log.write("Calculate motionOutliers  = %s\n"%(motionOutliers))
log.write("Do motion correction using McFLIRT  = %s\n"%(mcflirt))

log.flush()

if PREPROC == 1:
    print('Preprocessing')
    prep.main(paths,options_binary_string, ANAT, num_proc)

# ------------------------PostProcess------------------------------------------
if POSTPROC == 1:
    print('PostProcessing')

    for motion_param_regression, global_signal_regression, smoothing, band_pass_filtering, volCorrect in itr:
        combination = 'motionRegress' + str(int(motion_param_regression)) + \
         'global' + str(int(global_signal_regression)) + 'smoothing' + str(int(smoothing)) +\
         'filt' + str(int(band_pass_filtering))

        print("Combination: ",combination)
        functional_connectivity_directory =  combination
        print(motion_param_regression,  global_signal_regression, smoothing,band_pass_filtering)
        ppfc.main(paths, vols, motion_param_regression, global_signal_regression, band_pass_filtering, smoothing, volCorrect, \
        number_of_skipped_volumes, num_proc)

# ------------------- Hypothesis Test ------------------------------------------

bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

if HYPOTEST == 1:
    print('Hypothesis Test')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    # bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        ht.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)

# -------------------- FDR and results -----------------------------------------

if FDRES == 1:
    print('FDR Correction and computing files for visualization of results')
    # itr = (list(itertools.product([0, 1], repeat=3)))

    # itr = [(1,1,0,1,1)]
    for params in itr:
        fdres.main(paths, params, num_proc = 7)

if SCORE_CORR == 1:
    print('Calculating Correlation-Score correaltions')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        ht.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)
<commit_msg>Added bugs list of ABIDE 2<commit_after>import postProcessingFcVolMatchedModularDynamicPipeline as ppfc
# import preprocessingPipelineImprovedModular as prep
import preprocessingPipelineImprovedModularWOAnatDynamic as prep

import scoreCorrelation as sc

import hypothesisTest as ht
import fdrBrainResultsModular as fdres
import os
import json
from os.path import join as opj
import itertools
from bids.grabbids import BIDSLayout
import time

# ------------- Paths ----------------------------------------------------------------------------------------

path_cwd = os.getcwd()
path_split_list = path_cwd.split('/')
s = path_split_list[0:-1] # for getting to the parent dir of pwd
s = opj('/',*s) # *s converts list to path, # very important to add '/' in the begining so it is read as directory later

# New path to store results:

results_path = '/mnt/project1/home1/varunk/results/'

json_path = 'scripts/json/paths.json'
with open(json_path, 'rt') as fp:
    task_info = json.load(fp)

base_directory = opj(results_path,task_info["base_directory_for_results"])
motion_correction_bet_directory = task_info["motion_correction_bet_directory"]
parent_wf_directory = task_info["parent_wf_directory"]
functional_connectivity_directory = task_info["functional_connectivity_directory"]
# functional_connectivity_directory = 'temp_fc'
coreg_reg_directory = task_info["coreg_reg_directory"]
atlas_resize_reg_directory = task_info["atlas_resize_reg_directory"]
data_directory = opj(s,task_info["data_directory"])
datasink_name = task_info["datasink_name"]
fc_datasink_name = task_info["fc_datasink_name"]
# fc_datasink_name = 'temp_dataSink'
atlasPath = opj(s,task_info["atlas_path"]) # Standard Brainnetome path

brain_path = opj(base_directory,datasink_name,'preprocessed_brain_paths/brain_file_list.npy')
mask_path = opj(base_directory,datasink_name,'preprocessed_mask_paths/mask_file_list.npy')
atlas_path = opj(base_directory,datasink_name,'atlas_paths/atlas_file_list.npy') # Brainetome atlas in functional space
tr_path = opj(base_directory,datasink_name,'tr_paths/tr_list.npy')
motion_params_path = opj(base_directory,datasink_name,'motion_params_paths/motion_params_file_list.npy')

func2std_mat_path = opj(base_directory, datasink_name,'joint_xformation_matrix_paths/joint_xformation_matrix_file_list.npy')

MNI3mm_path = opj(base_directory,parent_wf_directory,motion_correction_bet_directory,coreg_reg_directory,'resample_mni/MNI152_T1_2mm_brain_resample.nii')

hypothesis_test_dir = opj(base_directory, task_info["hypothesis_test_dir"])

fdr_results_dir = task_info["fdr_results_dir"]

score_corr_dir =  opj(base_directory,task_info["score_corr_dir"])

demographics_file_path = '/home1/varunk/Autism-Connectome-Analysis-brain_connectivity/notebooks/demographics.csv'
phenotype_file_path = '/home1/varunk/data/ABIDE1/RawDataBIDs/composite_phenotypic_file.csv'
# categoryInfo = '/home1/varunk/data/NYU_Cocaine-BIDS/grouping.csv'
categoryInfo = None

# --------------- Creating Log --------------------

# Get time

current_time = time.asctime( time.localtime(time.time()) )

if not os.path.exists(base_directory):
    os.makedirs(base_directory)

log_path = opj(base_directory,"log.txt")
log = open(log_path, 'a')
print("-------------Starting the analysis at %s--------------\n"%(current_time))
log.write("-------------Starting the analysis at %s--------------\n"%(current_time))
log.flush()

# ----------------------------------------------------------------------------------------------------------------------------------------------------------

vols = 120 # needed for volume matching. Tells how many volumes you need to keep.
number_of_skipped_volumes = 4
#  True volumes will be vols - number_of_skipped_volumes

num_proc = 7

number_of_subjects = -1
# number_of_subjects = 7 # Number of subjects you wish to work with

log.write("Vols for matching: %s\n"%(vols))
log.write("Number_of_skipped_volumes: %s\n"%(number_of_skipped_volumes))
log.flush()
# ----------------------------- Getting Subjects -------------------------------
# ----------------------------------- BIDS -------------------------------------
layout = BIDSLayout(data_directory)

if number_of_subjects == -1:
    number_of_subjects = len(layout.get_subjects())

# ABIDE II Bugs

bugs_abide2 = ['28093', '28093', '28681',  '28682', '28683',  '28687', '28711', '28712', '28713', '28741', '28745',  '28751', '28755', '28756', '28757', '28758',
'28759', '28761', '28762','28763', '28764','28765','28766','28767','28768','28769','28770','28771','28772','28773','28774','28775','28776','28777','28778','28779',
'28780','28781','28782','28783'
]

bugs = bugs_abide2

subject_list = (layout.get_subjects())[0:number_of_subjects]
# subject_list = list(map(int, subject_list))

# Ignore Bugs
subject_list = list(set(subject_list) - set(bugs))

subject_list.sort()

# -----------------------------------File List----------------------------------
# group1FilesPath = ''
# group2FilesPath = ''
#
# group1FilesList = np.genfromtxt(group1FilesPath,dtype='unicode')
# group2FilesList = np.genfromtxt(group2FilesPath,dtype='unicode')
#
# fileList = group1FilesPath + group2FilesPath

# -----------------------------------------------------------------------------

paths = [json_path,
base_directory,
motion_correction_bet_directory,
parent_wf_directory,
functional_connectivity_directory,
coreg_reg_directory,
atlas_resize_reg_directory,
subject_list,
datasink_name,
fc_datasink_name,
atlasPath,
brain_path,
mask_path,
atlas_path,
tr_path,
motion_params_path,
func2std_mat_path,
MNI3mm_path,
demographics_file_path,
phenotype_file_path,
data_directory,
hypothesis_test_dir,
fdr_results_dir,
score_corr_dir
]

PREPROC = 1
POSTPROC = 1
HYPOTEST = 0
FDRES = 0
SCORE_CORR = 0

match = 0 # Age matching
applyFisher = True

# itr = (list(itertools.product([0, 1], repeat=3)))
# itr = [(1,1,1,1,1)]
# itr = [(1,0,1,1,1)]
itr = [(0,0,1,1,0)]

log.write("Operations:\n")
log.write("Preprocess = %s\n"%(PREPROC))
log.write("Postprocess = %s\n"%(POSTPROC))
log.write("Hypothesis Test = %s\n"%(HYPOTEST))
log.write("FDR correction and Vizualization = %s\n"%(FDRES))
log.write("Score-Connectivity Correlation  = %s\n"%(SCORE_CORR))
log.flush()

# ---------------------- Preprocess --------------------------------------------

ANAT = 1

itr_preproc = [1,1,0,1]
# itr_preproc = [0,0,0]
extract, slicetimer,motionOutliers, mcflirt= list(map(str, itr_preproc))
options_binary_string = extract+slicetimer+motionOutliers+mcflirt

log.write("Preprocessing Params\n")
log.write("Remove begining slices  = %s\n"%(extract))
log.write("Slice time correction  = %s\n"%(slicetimer))
log.write("Calculate motionOutliers  = %s\n"%(motionOutliers))
log.write("Do motion correction using McFLIRT  = %s\n"%(mcflirt))

log.flush()

if PREPROC == 1:
    print('Preprocessing')
    prep.main(paths,options_binary_string, ANAT, num_proc)

# ------------------------PostProcess------------------------------------------
if POSTPROC == 1:
    print('PostProcessing')
    log.write("Postprocessing Params\n")

    for motion_param_regression, global_signal_regression, smoothing, band_pass_filtering, volCorrect in itr:
        log.write("motion_param_regression  = %s\n"%(motion_param_regression))
        log.write("global_signal_regression  = %s\n"%(global_signal_regression))
        log.write("smoothing  = %s\n"%(smoothing))
        log.write("band_pass_filtering  = %s\n"%(band_pass_filtering))
        log.write("volCorrect  = %s\n"%(volCorrect))

        combination = 'motionRegress' + str(int(motion_param_regression)) + \
         'global' + str(int(global_signal_regression)) + 'smoothing' + str(int(smoothing)) +\
         'filt' + str(int(band_pass_filtering))

        print("Combination: ",combination)
        functional_connectivity_directory =  combination
        print(motion_param_regression,  global_signal_regression, smoothing,band_pass_filtering)
        ppfc.main(paths, vols, motion_param_regression, global_signal_regression, band_pass_filtering, smoothing, volCorrect, \
        number_of_skipped_volumes, num_proc)

# ------------------- Hypothesis Test ------------------------------------------
# ABIDE1 BUGS
bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

if HYPOTEST == 1:
    print('Hypothesis Test')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    # bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        ht.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)

# -------------------- FDR and results -----------------------------------------

if FDRES == 1:
    print('FDR Correction and computing files for visualization of results')
    # itr = (list(itertools.product([0, 1], repeat=3)))

    # itr = [(1,1,0,1,1)]
    for params in itr:
        fdres.main(paths, params, num_proc = 7)

if SCORE_CORR == 1:
    print('Calculating Correlation-Score correaltions')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        sc.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)
<|endoftext|>