<commit_before>from decimal import Decimal

from django import forms
from django.db.models import Q
from django.conf import settings

from django.contrib.auth import models as auth_models
from django.contrib.auth import forms as auth_forms
from django.core.urlresolvers import reverse

from timepiece.models import Project, Entry
from timepiece.fields import PendulumDateTimeField
from timepiece.widgets import PendulumDateTimeWidget, SecondsToHoursWidget
from datetime import datetime, timedelta

from timepiece import models as timepiece

from ajax_select.fields import AutoCompleteSelectMultipleField, \
                               AutoCompleteSelectField, \
                               AutoCompleteSelectWidget

class CreatePersonForm(auth_forms.UserCreationForm):
    class Meta:
        model = auth_models.User
        fields = ("username", "first_name", "last_name", "email", "is_active", "is_staff")

class EditPersonForm(auth_forms.UserChangeForm):
    class Meta:
        model = auth_models.User
        fields = ("username", "first_name", "last_name", "email", "is_active", "is_staff")

class CharAutoCompleteSelectWidget(AutoCompleteSelectWidget):
    def value_from_datadict(self, data, files, name):
        return data.get(name, None)

class QuickSearchForm(forms.Form):
    quick_search = AutoCompleteSelectField(
        'quick_search',
        widget=CharAutoCompleteSelectWidget('quick_search'),
    )

    def clean_quick_search(self):
        item = self.cleaned_data['quick_search']
        if isinstance(item, timepiece.Project):
            return reverse('view_project', kwargs={
                'project_id': item.id,
            })
        elif isinstance(item, timepiece.Business,):
            return reverse('view_business', kwargs={
                'business': item.id,
            })
        elif isinstance(item, auth_models.User,):
            return reverse('view_person', kwargs={
                'person_id': item.id,
            })
        raise forms.ValidationError('Must be a Contact or Project')

    def save(self):
        return self.cleaned_data['quick_search']

class SearchForm(forms.Form):
    search = forms.CharField(required=False)

class AddContactToProjectForm(forms.Form):
    contact = AutoCompleteSelectField('user')

    def save(self):
        return self.cleaned_data['contact']

class ClockInForm(forms.ModelForm):
    class Meta:
        model = timepiece.Entry
        fields = ('location', 'project', 'start_time', 'billable')

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop('user')
        default_loc = getattr(
            settings,
            'TIMEPIECE_DEFAULT_LOCATION_SLUG',
            None,
        )
        if default_loc:
            try:
                loc = timepiece.Location.objects.get(slug=default_loc)
            except timepiece.Location.DoesNotExist:
                loc = None
            if loc:
                initial = kwargs.get('initial', {})
                initial['location'] = loc.pk
        super(ClockInForm, self).__init__(*args, **kwargs)
        self.fields['start_time'].required = False
        self.fields['start_time'].initial = datetime.now()
        self.fields['start_time'].widget = forms.SplitDateTimeWidget(
            attrs={'class': 'timepiece-time'},
            date_format='%m/%d/%Y',
        )
        self.fields['project'].queryset = timepiece.Project.objects.filter(
            contacts=self.user,
        ).filter(
            Q(status__enable_timetracking=True) |
            Q(type__enable_timetracking=True)
        )

    def save(self, commit=True):
        entry = super(ClockInForm, self).save(commit=False)
        entry.hours = 0
        entry.clock_in(self.user, self.cleaned_data['project'])
        if commit:
            entry.save()
        return entry

class ClockOutForm(forms.ModelForm):
    class Meta:
        model = timepiece.Entry
        fields = ('location', 'comments')

    def __init__(self, *args, **kwargs):
        super(ClockOutForm, self).__init__(*args, **kwargs)
        self.fields['end_time'] = forms.DateTimeField(
            widget=forms.SplitDateTimeWidget(
                attrs={'class': 'timepiece-time'},
                date_format='%m/%d/%Y',
            ),
            initial=datetime.now(),
        )
        self.fields.keyOrder = ('location', 'end_time', 'comments')

    def save(self, commit=True):
        entry = super(ClockOutForm, self).save(commit=False)
        entry.end_time = self.cleaned_data['end_time']
        entry.unpause(date=self.cleaned_data['end_time'])
        if commit:
            entry.save()
        return entry

class AddUpdateEntryForm(forms.ModelForm):
    """
    This form will provide a way for users to add missed log entries and to
    update existing log entries.
    """

    start_time = forms.DateTimeField(
        widget=forms.SplitDateTimeWidget(
            attrs={'class': 'timepiece-time'},
            date_format='%m/%d/%Y',
        )
    )
    end_time = forms.DateTimeField(
        widget=forms.SplitDateTimeWidget(
            attrs={'class': 'timepiece-time'},
            date_format='%m/%d/%Y',
        )
    )

    class Meta:
        model = Entry
        exclude = ('user', 'pause_time', 'site', 'hours', 'activity')

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop('user')
        super(AddUpdateEntryForm, self).__init__(*args, **kwargs)
        self.fields['project'].queryset = timepiece.Project.objects.filter(
            contacts=self.user,
        )
        if not self.instance.end_time:
            del self.fields['end_time']

    def clean_start_time(self):
        """
        Make sure that the start time is always before the end time
        """
        start = self.cleaned_data['start_time']

        try:
            end = self.cleaned_data['end_time']

            if start >= end:
                raise forms.ValidationError('The entry must start before it ends!')
        except KeyError:
            pass

        if start > datetime.now():
            raise forms.ValidationError('You cannot add entries in the future!')

        return start

    def clean_end_time(self):
        """
        Make sure no one tries to add entries that end in the future
        """
        try:
            start = self.cleaned_data['start_time']
        except KeyError:
            raise forms.ValidationError('Please enter a start time.')

        try:
            end = self.cleaned_data['end_time']
            if not end: raise Exception
        except:
            raise forms.ValidationError('Please enter an end time.')

        if start >= end:
            raise forms.ValidationError('The entry must start before it ends!')

        return end

    def save(self):
        instance = super(AddUpdateEntryForm, self).save(commit=False)
        instance.user = self.user
        instance.save()
        return instance

class DateForm(forms.Form):
    from_date = forms.DateField(label="From", required=False)
    to_date = forms.DateField(label="To", required=False)

    def save(self):
        from_date = self.cleaned_data.get('from_date', '')
        to_date = self.cleaned_data.get('to_date', '')
        if to_date:
            to_date += timedelta(days=1)
        return (from_date, to_date)

class ProjectionForm(DateForm):
    contact = forms.ModelChoiceField(queryset=None)

    def __init__(self, *args, **kwargs):
        contacts = kwargs.pop('contacts')
        super(ProjectionForm, self).__init__(*args, **kwargs)
        self.fields['contact'].queryset = contacts

class BusinessForm(forms.ModelForm):
    class Meta:
        model = timepiece.Business
        fields = ('name', 'email', 'description', 'notes',)

class ProjectForm(forms.ModelForm):
    class Meta:
        model = timepiece.Project
        fields = (
            'name',
            'business',
            'trac_environment',
            'point_person',
            'type',
            'status',
            'description',
        )

    def __init__(self, *args, **kwargs):
        super(ProjectForm, self).__init__(*args, **kwargs)

    def save(self):
        instance = super(ProjectForm, self).save(commit=False)
        instance.save()
        return instance

class ProjectRelationshipForm(forms.ModelForm):
    class Meta:
        model = timepiece.ProjectRelationship
        fields = ('types',)

    def __init__(self, *args, **kwargs):
        super(ProjectRelationshipForm, self).__init__(*args, **kwargs)
        self.fields['types'].widget = forms.CheckboxSelectMultiple(
            choices=self.fields['types'].choices
        )
        self.fields['types'].help_text = ''

class RepeatPeriodForm(forms.ModelForm):
    class Meta:
        model = timepiece.RepeatPeriod
        fields = ('active', 'count', 'interval')

    def __init__(self, *args, **kwargs):
        super(RepeatPeriodForm, self).__init__(*args, **kwargs)
        self.fields['count'].required = False
        self.fields['interval'].required = False
        self.fields['date'] = forms.DateField(required=False)

    def _clean_optional(self, name):
        active = self.cleaned_data.get('active', False)
        value = self.cleaned_data.get(name, '')
        if active and not value:
            raise forms.ValidationError('This field is required.')
        return self.cleaned_data[name]

    def clean_count(self):
        return self._clean_optional('count')

    def clean_interval(self):
        return self._clean_optional('interval')

    def clean_date(self):
        active = self.cleaned_data.get('active', False)
        date = self.cleaned_data.get('date', '')
        if active and not self.instance.id and not date:
            raise forms.ValidationError('Start date is required for new billing periods')
        return date

    def clean(self):
        date = self.cleaned_data.get('date', '')
        if self.instance.id and date:
            latest = self.instance.billing_windows.latest()
            if self.cleaned_data['active'] and date < latest.end_date:
                raise forms.ValidationError('New start date must be after %s' % latest.end_date)
        return self.cleaned_data

    def save(self):
        period = super(RepeatPeriodForm, self).save(commit=False)
        if not self.instance.id and period.active:
            period.save()
            period.billing_windows.create(
                date=self.cleaned_data['date'],
                end_date=self.cleaned_data['date'] + period.delta(),
            )
        elif self.instance.id:
            period.save()
            start_date = self.cleaned_data['date']
            if period.active and start_date:
                latest = period.billing_windows.latest()
                if start_date > latest.end_date:
                    period.billing_windows.create(
                        date=latest.end_date,
                        end_date=start_date + period.delta(),
                    )
        period.update_billing_windows()
        return period

class PersonTimeSheet(forms.ModelForm):
    class Meta:
        model = timepiece.PersonRepeatPeriod
        fields = ('contact',)

    def __init__(self, *args, **kwargs):
        super(PersonTimeSheet, self).__init__(*args, **kwargs)
        self.fields['contact'].queryset = auth_models.User.objects.all().order_by('last_name')
<commit_msg>added password change on edit person page<commit_after>from decimal import Decimal

from django import forms
from django.db.models import Q
from django.conf import settings

from django.contrib.auth import models as auth_models
from django.contrib.auth import forms as auth_forms
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from timepiece.models import Project, Entry
from timepiece.fields import PendulumDateTimeField
from timepiece.widgets import PendulumDateTimeWidget, SecondsToHoursWidget
from datetime import datetime, timedelta

from timepiece import models as timepiece

from ajax_select.fields import AutoCompleteSelectMultipleField, \
                               AutoCompleteSelectField, \
                               AutoCompleteSelectWidget

class CreatePersonForm(auth_forms.UserCreationForm):
    class Meta:
        model = auth_models.User
        fields = ("username", "first_name", "last_name", "email", "is_active", "is_staff")

class EditPersonForm(auth_forms.UserChangeForm):
    password_one = forms.CharField(required=False, max_length=36, label=_(u'Password'),
                                widget=forms.PasswordInput(render_value=False))
    password_two = forms.CharField(required=False, max_length=36, label=_(u'Repeat Password'),
                                widget=forms.PasswordInput(render_value=False))
    class Meta:
        model = auth_models.User
        fields = (
            "username", "first_name", "last_name",
            "email", "is_active", "is_staff"
        )

    def clean(self):
        super(EditPersonForm, self).clean()
        password_one = self.cleaned_data.get('password_one', None)
        password_two = self.cleaned_data.get('password_two', None)
        if password_one and password_one != password_two:
            raise forms.ValidationError(_('Passwords Must Match.'))
        return self.cleaned_data

    def save(self, *args, **kwargs):
        commit = kwargs.get('commit', True)
        kwargs['commit'] = False
        instance = super(EditPersonForm, self).save(*args, **kwargs)
        password_one = self.cleaned_data.get('password_one', None)
        if password_one:
            print 'tests'
            instance.set_password(password_one)
        if commit:
            instance.save()
        return instance

class CharAutoCompleteSelectWidget(AutoCompleteSelectWidget):
    def value_from_datadict(self, data, files, name):
        return data.get(name, None)

class QuickSearchForm(forms.Form):
    quick_search = AutoCompleteSelectField(
        'quick_search',
        widget=CharAutoCompleteSelectWidget('quick_search'),
    )

    def clean_quick_search(self):
        item = self.cleaned_data['quick_search']
        if isinstance(item, timepiece.Project):
            return reverse('view_project', kwargs={
                'project_id': item.id,
            })
        elif isinstance(item, timepiece.Business,):
            return reverse('view_business', kwargs={
                'business': item.id,
            })
        elif isinstance(item, auth_models.User,):
            return reverse('view_person', kwargs={
                'person_id': item.id,
            })
        raise forms.ValidationError('Must be a Contact or Project')

    def save(self):
        return self.cleaned_data['quick_search']

class SearchForm(forms.Form):
    search = forms.CharField(required=False)

class AddContactToProjectForm(forms.Form):
    contact = AutoCompleteSelectField('user')

    def save(self):
        return self.cleaned_data['contact']

class ClockInForm(forms.ModelForm):
    class Meta:
        model = timepiece.Entry
        fields = ('location', 'project', 'start_time', 'billable')

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop('user')
        default_loc = getattr(
            settings,
            'TIMEPIECE_DEFAULT_LOCATION_SLUG',
            None,
        )
        if default_loc:
            try:
                loc = timepiece.Location.objects.get(slug=default_loc)
            except timepiece.Location.DoesNotExist:
                loc = None
            if loc:
                initial = kwargs.get('initial', {})
                initial['location'] = loc.pk
        super(ClockInForm, self).__init__(*args, **kwargs)
        self.fields['start_time'].required = False
        self.fields['start_time'].initial = datetime.now()
        self.fields['start_time'].widget = forms.SplitDateTimeWidget(
            attrs={'class': 'timepiece-time'},
            date_format='%m/%d/%Y',
        )
        self.fields['project'].queryset = timepiece.Project.objects.filter(
            contacts=self.user,
        ).filter(
            Q(status__enable_timetracking=True) |
            Q(type__enable_timetracking=True)
        )

    def save(self, commit=True):
        entry = super(ClockInForm, self).save(commit=False)
        entry.hours = 0
        entry.clock_in(self.user, self.cleaned_data['project'])
        if commit:
            entry.save()
        return entry

class ClockOutForm(forms.ModelForm):
    class Meta:
        model = timepiece.Entry
        fields = ('location', 'comments')

    def __init__(self, *args, **kwargs):
        super(ClockOutForm, self).__init__(*args, **kwargs)
        self.fields['end_time'] = forms.DateTimeField(
            widget=forms.SplitDateTimeWidget(
                attrs={'class': 'timepiece-time'},
                date_format='%m/%d/%Y',
            ),
            initial=datetime.now(),
        )
        self.fields.keyOrder = ('location', 'end_time', 'comments')

    def save(self, commit=True):
        entry = super(ClockOutForm, self).save(commit=False)
        entry.end_time = self.cleaned_data['end_time']
        entry.unpause(date=self.cleaned_data['end_time'])
        if commit:
            entry.save()
        return entry

class AddUpdateEntryForm(forms.ModelForm):
    """
    This form will provide a way for users to add missed log entries and to
    update existing log entries.
    """

    start_time = forms.DateTimeField(
        widget=forms.SplitDateTimeWidget(
            attrs={'class': 'timepiece-time'},
            date_format='%m/%d/%Y',
        )
    )
    end_time = forms.DateTimeField(
        widget=forms.SplitDateTimeWidget(
            attrs={'class': 'timepiece-time'},
            date_format='%m/%d/%Y',
        )
    )

    class Meta:
        model = Entry
        exclude = ('user', 'pause_time', 'site', 'hours', 'activity')

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop('user')
        super(AddUpdateEntryForm, self).__init__(*args, **kwargs)
        self.fields['project'].queryset = timepiece.Project.objects.filter(
            contacts=self.user,
        )
        if not self.instance.end_time:
            del self.fields['end_time']

    def clean_start_time(self):
        """
        Make sure that the start time is always before the end time
        """
        start = self.cleaned_data['start_time']

        try:
            end = self.cleaned_data['end_time']

            if start >= end:
                raise forms.ValidationError('The entry must start before it ends!')
        except KeyError:
            pass

        if start > datetime.now():
            raise forms.ValidationError('You cannot add entries in the future!')

        return start

    def clean_end_time(self):
        """
        Make sure no one tries to add entries that end in the future
        """
        try:
            start = self.cleaned_data['start_time']
        except KeyError:
            raise forms.ValidationError('Please enter a start time.')

        try:
            end = self.cleaned_data['end_time']
            if not end: raise Exception
        except:
            raise forms.ValidationError('Please enter an end time.')

        if start >= end:
            raise forms.ValidationError('The entry must start before it ends!')

        return end

    def save(self):
        instance = super(AddUpdateEntryForm, self).save(commit=False)
        instance.user = self.user
        instance.save()
        return instance

class DateForm(forms.Form):
    from_date = forms.DateField(label="From", required=False)
    to_date = forms.DateField(label="To", required=False)

    def save(self):
        from_date = self.cleaned_data.get('from_date', '')
        to_date = self.cleaned_data.get('to_date', '')
        if to_date:
            to_date += timedelta(days=1)
        return (from_date, to_date)

class ProjectionForm(DateForm):
    contact = forms.ModelChoiceField(queryset=None)

    def __init__(self, *args, **kwargs):
        contacts = kwargs.pop('contacts')
        super(ProjectionForm, self).__init__(*args, **kwargs)
        self.fields['contact'].queryset = contacts

class BusinessForm(forms.ModelForm):
    class Meta:
        model = timepiece.Business
        fields = ('name', 'email', 'description', 'notes',)

class ProjectForm(forms.ModelForm):
    class Meta:
        model = timepiece.Project
        fields = (
            'name',
            'business',
            'trac_environment',
            'point_person',
            'type',
            'status',
            'description',
        )

    def __init__(self, *args, **kwargs):
        super(ProjectForm, self).__init__(*args, **kwargs)

    def save(self):
        instance = super(ProjectForm, self).save(commit=False)
        instance.save()
        return instance

class ProjectRelationshipForm(forms.ModelForm):
    class Meta:
        model = timepiece.ProjectRelationship
        fields = ('types',)

    def __init__(self, *args, **kwargs):
        super(ProjectRelationshipForm, self).__init__(*args, **kwargs)
        self.fields['types'].widget = forms.CheckboxSelectMultiple(
            choices=self.fields['types'].choices
        )
        self.fields['types'].help_text = ''

class RepeatPeriodForm(forms.ModelForm):
    class Meta:
        model = timepiece.RepeatPeriod
        fields = ('active', 'count', 'interval')

    def __init__(self, *args, **kwargs):
        super(RepeatPeriodForm, self).__init__(*args, **kwargs)
        self.fields['count'].required = False
        self.fields['interval'].required = False
        self.fields['date'] = forms.DateField(required=False)

    def _clean_optional(self, name):
        active = self.cleaned_data.get('active', False)
        value = self.cleaned_data.get(name, '')
        if active and not value:
            raise forms.ValidationError('This field is required.')
        return self.cleaned_data[name]

    def clean_count(self):
        return self._clean_optional('count')

    def clean_interval(self):
        return self._clean_optional('interval')

    def clean_date(self):
        active = self.cleaned_data.get('active', False)
        date = self.cleaned_data.get('date', '')
        if active and not self.instance.id and not date:
            raise forms.ValidationError('Start date is required for new billing periods')
        return date

    def clean(self):
        date = self.cleaned_data.get('date', '')
        if self.instance.id and date:
            latest = self.instance.billing_windows.latest()
            if self.cleaned_data['active'] and date < latest.end_date:
                raise forms.ValidationError('New start date must be after %s' % latest.end_date)
        return self.cleaned_data

    def save(self):
        period = super(RepeatPeriodForm, self).save(commit=False)
        if not self.instance.id and period.active:
            period.save()
            period.billing_windows.create(
                date=self.cleaned_data['date'],
                end_date=self.cleaned_data['date'] + period.delta(),
            )
        elif self.instance.id:
            period.save()
            start_date = self.cleaned_data['date']
            if period.active and start_date:
                latest = period.billing_windows.latest()
                if start_date > latest.end_date:
                    period.billing_windows.create(
                        date=latest.end_date,
                        end_date=start_date + period.delta(),
                    )
        period.update_billing_windows()
        return period

class PersonTimeSheet(forms.ModelForm):
    class Meta:
        model = timepiece.PersonRepeatPeriod
        fields = ('contact',)

    def __init__(self, *args, **kwargs):
        super(PersonTimeSheet, self).__init__(*args, **kwargs)
        self.fields['contact'].queryset = auth_models.User.objects.all().order_by('last_name')
<|endoftext|>


<commit_before>VERSION = (0, 3, 0, 'final', 0)

def get_version(version=None):
    """Derives a PEP386-compliant version number from VERSION."""
    if version is None:
        version = VERSION
    assert len(version) == 5
    assert version[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if version[2] == 0 else 3
    main = '.'.join(str(x) for x in version[:parts])

    sub = ''
    if version[3] == 'alpha' and version[4] == 0:
        # At the toplevel, this would cause an import loop.
        from django.utils.version import get_git_changeset
        git_changeset = get_git_changeset()
        if git_changeset != 'unknown':
            sub = '.dev%s' % git_changeset

    elif version[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[version[3]] + str(version[4])

    return main + sub

__version__ = get_version()
<commit_msg>Bump to version 0.3.1<commit_after>VERSION = (0, 3, 1, 'final', 0)

def get_version(version=None):
    """Derives a PEP386-compliant version number from VERSION."""
    if version is None:
        version = VERSION
    assert len(version) == 5
    assert version[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if version[2] == 0 else 3
    main = '.'.join(str(x) for x in version[:parts])

    sub = ''
    if version[3] == 'alpha' and version[4] == 0:
        # At the toplevel, this would cause an import loop.
        from django.utils.version import get_git_changeset
        git_changeset = get_git_changeset()
        if git_changeset != 'unknown':
            sub = '.dev%s' % git_changeset

    elif version[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[version[3]] + str(version[4])

    return main + sub

__version__ = get_version()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Configure the environment for the onramp REST server.

Usage: ./onramp_setup.py

This script sets up a virtual environment for the REST server, installs
dependencies need by the REST server, imports default educational modules into
the environment, and creates a default admin user.
"""

import json
import os
import shutil
import sys
from subprocess import call
from tempfile import mkstemp

if __name__ == '__main__':
    source_dir = 'src'
    env_dir = source_dir + '/env'
    package_name = 'PCE'
    users_dir = 'users'
    modules_dir = 'modules'
    log_dir = 'log'
    prebuilt_dir = '../modules'
    module_state_dir = 'src/state/modules'
    job_state_dir = 'src/state/jobs'

    tmpl_conf  = "bin/onramp_pce_config.ini.tmpl"
    final_conf = "bin/onramp_pce_config.ini"

    # If the PCE service is already deployed/installed
    if os.path.exists(env_dir):
        print "=" * 70
        print 'Warning: PCE Service appears to be already installed.'
        print "=" * 70

        response = raw_input('(R)emove and re-install or (A)bort? ')
        if response != 'R' and response != 'r':
            sys.exit('Aborted')
        shutil.rmtree(env_dir, True)
        shutil.rmtree(users_dir, True)
        shutil.rmtree(modules_dir, True)
        shutil.rmtree(log_dir, True)
        shutil.rmtree(module_state_dir, True)
        shutil.rmtree(job_state_dir, True)

    # Create the log directory
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    # Set up folder structure for modules
    if not os.path.exists(modules_dir):
        os.makedirs(modules_dir)
    if not os.path.exists(module_state_dir):
        os.makedirs(module_state_dir)
    if not os.path.exists(job_state_dir):
        os.makedirs(job_state_dir)

    #
    # Setup the configuration file(s)
    #
    if os.path.exists(final_conf) is True:
        print "=" * 70
        print 'Warning: PCE Service configuration file present.'
        print "=" * 70

        response = raw_input('(R)eplace or (K)eep? ')
        if response == 'R' or response == 'r':
            call(['rm', final_conf])
            call(['cp', tmpl_conf, final_conf])
    else:
        call(['cp', tmpl_conf, final_conf])
    call(['chmod', 'og+rX', final_conf])

    print "==>"
    print "==> NOTICE: Please edit the file: " + final_conf
    print "==>"

    ###################################################
    print "=" * 70
    print "Status: Setup the virtual environment"
    print "        This may take a while..."
    print "=" * 70

    # Setup virtual environment
    call(['virtualenv', '-p', 'python2.7', env_dir])
    call([env_dir + '/bin/pip', 'install', '-r', source_dir + '/requirements.txt'])

    # Set pce_root in PCE packagage
    cwd = os.getcwd()
    pce_file = os.path.join(os.path.join(cwd, source_dir),
                            os.path.join(package_name, '__init__.py'))
    fh, abs_path = mkstemp()
    pce_root_found = False
    with open(abs_path, 'w') as temp_file:
        with open(pce_file, 'r') as f:
            for line in f:
                if line.startswith('pce_root ='):
                    pce_root_found = True
                    temp_file.write("pce_root = '%s'\n" % cwd)
                else:
                    temp_file.write(line)
        if not pce_root_found:
            temp_file.write("pce_root = '%s'\n" % cwd)
    os.close(fh)
    os.remove(pce_file)
    shutil.move(abs_path, pce_file)

    # Link PCE to virtual environment
    call(['cp', '-rs', cwd + '/' + source_dir + '/' + package_name,
          env_dir + '/lib/python2.7/site-packages/' + package_name])

    # Use virtual environment to complete server setup
    call([env_dir + '/bin/python', source_dir + '/stage_two.py'])

    ###################################################
    print "=" * 70
    print "Status: Setup Complete"
    print "=" * 70
<commit_msg>Moved config file edit msg to end of pce install script.<commit_after>#!/usr/bin/env python
"""Configure the environment for the onramp REST server.

Usage: ./onramp_setup.py

This script sets up a virtual environment for the REST server, installs
dependencies need by the REST server, imports default educational modules into
the environment, and creates a default admin user.
"""

import json
import os
import shutil
import sys
from subprocess import call
from tempfile import mkstemp

if __name__ == '__main__':
    source_dir = 'src'
    env_dir = source_dir + '/env'
    package_name = 'PCE'
    users_dir = 'users'
    modules_dir = 'modules'
    log_dir = 'log'
    prebuilt_dir = '../modules'
    module_state_dir = 'src/state/modules'
    job_state_dir = 'src/state/jobs'

    tmpl_conf  = "bin/onramp_pce_config.ini.tmpl"
    final_conf = "bin/onramp_pce_config.ini"

    # If the PCE service is already deployed/installed
    if os.path.exists(env_dir):
        print "=" * 70
        print 'Warning: PCE Service appears to be already installed.'
        print "=" * 70

        response = raw_input('(R)emove and re-install or (A)bort? ')
        if response != 'R' and response != 'r':
            sys.exit('Aborted')
        shutil.rmtree(env_dir, True)
        shutil.rmtree(users_dir, True)
        shutil.rmtree(modules_dir, True)
        shutil.rmtree(log_dir, True)
        shutil.rmtree(module_state_dir, True)
        shutil.rmtree(job_state_dir, True)

    # Create the log directory
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    # Set up folder structure for modules
    if not os.path.exists(modules_dir):
        os.makedirs(modules_dir)
    if not os.path.exists(module_state_dir):
        os.makedirs(module_state_dir)
    if not os.path.exists(job_state_dir):
        os.makedirs(job_state_dir)

    #
    # Setup the configuration file(s)
    #
    show_edit_msg = False
    if os.path.exists(final_conf) is True:
        print "=" * 70
        print 'Warning: PCE Service configuration file present.'
        print "=" * 70

        response = raw_input('(R)eplace or (K)eep? ')
        if response == 'R' or response == 'r':
            call(['rm', final_conf])
            call(['cp', tmpl_conf, final_conf])
            show_edit_msg = True
    else:
        call(['cp', tmpl_conf, final_conf])
        show_edit_msg = True
    call(['chmod', 'og+rX', final_conf])

    ###################################################
    print "=" * 70
    print "Status: Setup the virtual environment"
    print "        This may take a while..."
    print "=" * 70

    # Setup virtual environment
    call(['virtualenv', '-p', 'python2.7', env_dir])
    call([env_dir + '/bin/pip', 'install', '-r', source_dir + '/requirements.txt'])

    # Set pce_root in PCE packagage
    cwd = os.getcwd()
    pce_file = os.path.join(os.path.join(cwd, source_dir),
                            os.path.join(package_name, '__init__.py'))
    fh, abs_path = mkstemp()
    pce_root_found = False
    with open(abs_path, 'w') as temp_file:
        with open(pce_file, 'r') as f:
            for line in f:
                if line.startswith('pce_root ='):
                    pce_root_found = True
                    temp_file.write("pce_root = '%s'\n" % cwd)
                else:
                    temp_file.write(line)
        if not pce_root_found:
            temp_file.write("pce_root = '%s'\n" % cwd)
    os.close(fh)
    os.remove(pce_file)
    shutil.move(abs_path, pce_file)

    # Link PCE to virtual environment
    call(['cp', '-rs', cwd + '/' + source_dir + '/' + package_name,
          env_dir + '/lib/python2.7/site-packages/' + package_name])

    # Use virtual environment to complete server setup
    call([env_dir + '/bin/python', source_dir + '/stage_two.py'])

    ###################################################
    print "=" * 70
    print "Status: Setup Complete"
    print "=" * 70

    if show_edit_msg:
        print "==>"
        print "==> NOTICE: Please edit the file: " + final_conf
        print "==>"
<|endoftext|>


<commit_before>__title__ = 'pelican-flickrtag'
__version__ = '0.4.0'
__author__ = 'Chris Streeter'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014'

from pelican_flickrtag.plugin import register
<commit_msg>Version bump<commit_after>__title__ = 'pelican-flickrtag'
__version__ = '0.4.1'
__author__ = 'Chris Streeter'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014'

from pelican_flickrtag.plugin import register
<|endoftext|>


<commit_before>
# 1. Unzip openings games into data directory (if not there)
# 2. Add a number of games to the openings book db

import os
import shutil
import parse_game as par_m
import pentai.base.pente_exceptions as pe_m
import zodb_dict as z_m
import openings_book as ob_m
import games_mgr as gm_m
import misc_db as m_m

from pentai.base.defines import *
import pentai.base.logger as log

def misc():
    return m_m.get_instance()

def unzip_section(section, user_data_dir):
    log.info("unzipping %s" % section)
    import zipfile as zf_m
    zip_path = os.path.join("openings", "%s.zip" % section)
    try:
        zf = zf_m.ZipFile(zip_path)
    except IOError:
        # HACK!
        return section == 48

    target_directory = os.path.join(user_data_dir, "openings")
    try:
        os.makedirs(target_directory)
    except OSError:
        log.warn("OSError creating %s" % target_directory)
        pass
    zf.extractall(target_directory)
    return True

def remove_unzipped(section, user_data_dir):
    target_directory = os.path.join(user_data_dir, "openings", "%s" % section)

    log.info("removing unzipped from %s" % target_directory)
    shutil.rmtree(target_directory, ignore_errors=True)

def add_games(openings_book, section_dir, start, count=100):

    game_files = os.listdir(section_dir)

    added = 0

    for gf in game_files[start:]:
        gf_path = os.path.join(section_dir, gf)
        gf_str = open(gf_path).readlines()
        try:
            g = par_m.convert_game(gf_str, int(gf))
            add_game(openings_book, g, g.get_won_by())
            added += 1
            if added >= count:
                break
        except Exception, e:
            log.debug("%s: %s" % (gf, e.message))
    z_m.sync()
    return count - added

def add_game(openings_book, g, won_by):
    log.info("Adding: %s" % g.game_id)
    try:
        openings_book.add_game(g, won_by)
    except pe_m.OpeningsBookDuplicateException:
        pass

def is_finished():
    try:
        return misc()["opening_section"] < 34
    except KeyError:
        return False

def build(openings_book, user_data_dir, section=None, start=None, count=100):
    # Extend library
    if not section:
        try:
            section = misc()["opening_section"]
        except:
            section = misc()["opening_section"] = 50

    openings_dir = os.path.join(user_data_dir, "openings")
    section_dir = os.path.join(openings_dir, str(section))

    if not os.path.isdir(section_dir):
        if not unzip_section(section, user_data_dir):
            return True
        else:
            # This is just for the case where there is no zip file for that section
            misc()["opening_section"] -= 1
            return False

    if not start:
        try:
            start = misc()["opening_start"]
        except:
            start = misc()["opening_start"] = 0

    remaining = count
    while remaining:
        try:
            remaining = add_games(openings_book, section_dir, start, count)
        except OSError:
            pass

        if remaining:
            # Not enough games in that section
            # We've finished with the current section so get rid of the
            # unzipped games directory.
            remove_unzipped(section, user_data_dir)
            start = 0
            section -= 1
            count = remaining
            break
        else:
            start += count

    misc()["opening_section"] = section
    misc()["opening_start"] = start
    z_m.sync()
    log.debug("Completed: section: %s; start game: %s" % (section, start))
    return False

<commit_msg>Several fixes, make it run independently<commit_after>
# 1. Unzip openings games into data directory (if not there)
# 2. Add a number of games to the openings book db

import os
import shutil
import parse_game as par_m
import pentai.base.pente_exceptions as pe_m
import pentai.db.zodb_dict as z_m
import pentai.db.openings_book as ob_m
import pentai.db.games_mgr as gm_m
import pentai.db.misc_db as m_m

from pentai.base.defines import *
import pentai.base.logger as log

def misc():
    return m_m.get_instance()

def unzip_section(section, user_data_dir):
    log.info("unzipping %s" % section)
    import zipfile as zf_m
    zip_path = os.path.join("openings", "%s.zip" % section)
    try:
        zf = zf_m.ZipFile(zip_path)
    except IOError:
        # HACK!
        misc()["opening_section"] -= 1
        return section == 48

    target_directory = os.path.join(user_data_dir, "openings")
    try:
        os.makedirs(target_directory)
    except OSError:
        log.warn("OSError creating %s" % target_directory)
        pass
    zf.extractall(target_directory)
    return True

def remove_unzipped(section, user_data_dir):
    target_directory = os.path.join(user_data_dir, "openings", "%s" % section)

    log.info("removing unzipped from %s" % target_directory)
    shutil.rmtree(target_directory, ignore_errors=True)

def add_games(openings_book, section_dir, start, count=100):

    game_files = os.listdir(section_dir)

    added = 0

    for gf in game_files[start:]:
        gf_path = os.path.join(section_dir, gf)
        gf_str = open(gf_path).readlines()
        try:
            g = par_m.convert_game(gf_str, int(gf))
            add_game(openings_book, g, g.get_won_by())
            global total_added
            total_added += 1
            added += 1
            if added >= count:
                break
        except Exception, e:
            log.debug("%s: %s" % (gf, e.message))
    z_m.sync()
    return count - added

def add_game(openings_book, g, won_by):
    log.info("Adding: %s" % g.game_id)
    try:
        openings_book.add_game(g, won_by)
    except pe_m.OpeningsBookDuplicateException:
        pass

def is_finished():
    try:
        return misc()["opening_section"] < 34
    except KeyError:
        return False

def build(openings_book, user_data_dir, section=None, start=None, count=100):
    # Extend library
    if not section:
        try:
            section = misc()["opening_section"]
        except:
            section = misc()["opening_section"] = 50

    openings_dir = os.path.join(user_data_dir, "openings")
    section_dir = os.path.join(openings_dir, str(section))

    if not os.path.isdir(section_dir):
        if not unzip_section(section, user_data_dir):
            return True
        else:
            return False

    if not start:
        try:
            start = misc()["opening_start"]
        except:
            start = misc()["opening_start"] = 0

    remaining = count
    while remaining:
        try:
            remaining = add_games(openings_book, section_dir, start, count)
        except OSError:
            pass

        if remaining:
            # Not enough games in that section
            # We've finished with the current section so get rid of the
            # unzipped games directory.
            remove_unzipped(section, user_data_dir)
            start = 0
            section -= 1
            count = remaining
            break
        else:
            start += count

    misc()["opening_section"] = section
    misc()["opening_start"] = start
    z_m.sync()
    log.debug("Completed: section: %s; start game: %s" % (section, start))
    return False

total_added = 0

if __name__ == "__main__":
    from main import setup_logging
    setup_logging()
    db_path = os.path.join(".", "db.fs")
    z_m.set_db(db_path)
    import pentai.db.openings_book as ob_m
    openings_book = ob_m.OpeningsBook()
    while True:
        if build(openings_book, ".", count=100):
            break
        section = misc()["opening_section"]
        log.debug("built %s games so far, in section %s" % (total_added, section))
    log.debug("Finished building %s games, now packing" % total_added)
    z_m.pack()
<|endoftext|>


<commit_before>
# 1. Unzip openings games into data directory (if not there)
# 2. Add a number of games to the openings book db

import os
import parse_game as par_m
import pentai.base.pente_exceptions as pe_m
import zodb_dict as z_m
import openings_book as ob_m
import games_mgr as gm_m
import misc_db as m_m

def misc():
    return m_m.get_instance()

def unzip_openings(openings_dir):
    import zipfile as zf_m
    zip_path = "openings.zip"
    zf = zf_m.ZipFile(zip_path)
    target_directory = openings_dir
    os.makedirs(target_directory)
    zf.extractall(target_directory)

def add_games(openings_book, section_dir, start, count=100):

    game_files = os.listdir(section_dir)

    added = 0

    for gf in game_files[start:]:
        gf_path = os.path.join(section_dir, gf)
        gf_str = open(gf_path).readlines()
        try:
            g = par_m.convert_game(gf_str, int(gf))
        except pe_m.IllegalMoveException, e:
            print "%s: %s" % (gf, e.message)
        except pe_m.ParseException, e:
            print "%s: %s" % (gf, e.message)
        except Exception, e:
            print "%s: %s" % (gf, e.message)
        print "Adding: %s" % g.game_id
        try:
            #import pdb
            #pdb.set_trace()
            openings_book.add_game(g, update_cache=False, sync=False)
        except pe_m.OpeningsBookDuplicateException:
            pass

        added += 1
        if added >= count:
            break
    z_m.sync()
    return count - added

def build(openings_book, user_data_dir, section=None, start=None, count=100):
    # Extend library
    openings_dir = os.path.join(user_data_dir, "openings")
    if not os.path.isdir(openings_dir):
        unzip_openings(openings_dir)

    if not section:
        try:
            section = misc()["opening_section"]
        except:
            section = misc()["opening_section"] = 60

    if not start:
        try:
            start = misc()["opening_start"]
        except:
            start = misc()["opening_start"] = 0

    # TODO Stop adding games

    par_m.create_ai_players()

    remaining = count
    while remaining:
        section_dir = os.path.join(openings_dir, str(section))

        try:
            remaining = add_games(openings_book, section_dir, start, count)
        except OSError:
            pass

        if remaining:
            # Not enough games in that section
            start = 0
            section -= 1
            count = remaining
            if section < 30:
                break
        else:
            start += count

    misc()["opening_section"] = section
    misc()["opening_start"] = start
    z_m.sync()

def main():
    # TODO Move this to zodb_dict
    db_path = os.path.join(".", "db.fs")
    print "Loading DB from %s" % db_path
    lockfile_path = db_path + ".lock"
    if os.path.isfile(lockfile_path):
        os.unlink(lockfile_path)
        print "Cleared DB lock"

    z_m.set_db(db_path)

    import sys

    section = sys.argv[1]
    start = sys.argv[2]

    # HACK HACK HACK
    user_data_dir = "/Users/cropleyb/Library/Application Support/pentai"

    par_m.create_ai_players()

    games_mgr = gm_m.GamesMgr()
    openings_book = ob_m.OpeningsBook(games_mgr)

    build(openings_book, user_data_dir, section, start, 100)

if __name__ == "__main__":
    main()

<commit_msg>Fix crash from corrupted game<commit_after>
# 1. Unzip openings games into data directory (if not there)
# 2. Add a number of games to the openings book db

import os
import parse_game as par_m
import pentai.base.pente_exceptions as pe_m
import zodb_dict as z_m
import openings_book as ob_m
import games_mgr as gm_m
import misc_db as m_m

def misc():
    return m_m.get_instance()

def unzip_openings(openings_dir):
    import zipfile as zf_m
    zip_path = "openings.zip"
    zf = zf_m.ZipFile(zip_path)
    target_directory = openings_dir
    os.makedirs(target_directory)
    zf.extractall(target_directory)

def add_games(openings_book, section_dir, start, count=100):

    game_files = os.listdir(section_dir)

    added = 0

    for gf in game_files[start:]:
        gf_path = os.path.join(section_dir, gf)
        gf_str = open(gf_path).readlines()
        try:
            g = par_m.convert_game(gf_str, int(gf))
            add_game(openings_book, g)
            added += 1
            if added >= count:
                break
        except pe_m.IllegalMoveException, e:
            print "%s: %s" % (gf, e.message)
        except pe_m.ParseException, e:
            print "%s: %s" % (gf, e.message)
        except Exception, e:
            print "%s: %s" % (gf, e.message)
    z_m.sync()
    return count - added

def add_game(openings_book, g):
    print "Adding: %s" % g.game_id
    try:
        #import pdb
        #pdb.set_trace()
        openings_book.add_game(g, update_cache=False, sync=False)
    except pe_m.OpeningsBookDuplicateException:
        pass

def build(openings_book, user_data_dir, section=None, start=None, count=100):
    # Extend library
    openings_dir = os.path.join(user_data_dir, "openings")
    if not os.path.isdir(openings_dir):
        unzip_openings(openings_dir)

    if not section:
        try:
            section = misc()["opening_section"]
        except:
            section = misc()["opening_section"] = 60

    if not start:
        try:
            start = misc()["opening_start"]
        except:
            start = misc()["opening_start"] = 0

    par_m.create_ai_players()

    remaining = count
    while remaining:
        section_dir = os.path.join(openings_dir, str(section))

        try:
            remaining = add_games(openings_book, section_dir, start, count)
        except OSError:
            pass

        if remaining:
            # Not enough games in that section
            start = 0
            section -= 1
            count = remaining
            if section < 30:
                break
        else:
            start += count

    misc()["opening_section"] = section
    misc()["opening_start"] = start
    z_m.sync()

def main():
    # TODO Move this to zodb_dict
    db_path = os.path.join(".", "db.fs")
    print "Loading DB from %s" % db_path
    lockfile_path = db_path + ".lock"
    if os.path.isfile(lockfile_path):
        os.unlink(lockfile_path)
        print "Cleared DB lock"

    z_m.set_db(db_path)

    import sys

    section = sys.argv[1]
    start = sys.argv[2]

    # HACK HACK HACK
    user_data_dir = "/Users/cropleyb/Library/Application Support/pentai"

    par_m.create_ai_players()

    games_mgr = gm_m.GamesMgr()
    openings_book = ob_m.OpeningsBook(games_mgr)

    build(openings_book, user_data_dir, section, start, 100)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># dependencies: pip install genshi
import sys
import os
import tarfile
from genshi.template import MarkupTemplate
from genshi.core import Markup

""" - Splits Percolator output into decoy and target files.
    - Extracts unique PSM/peptides/proteins out of a Percolator output file.
    - Merges Percolator output files
    Usage: python percolator_output_modifier.py command psm/peptides/proteins [score] infile outfile [outfile2]
    """

try:
    from lxml import etree
except Exception:
    sys.stderr.write('Failed to import lxml module.')

def readPercout(fname):
    doc = None
    try:
        doc = etree.parse(fname)
    except Exception:
        sys.stderr.write('Could not parse XML provided in %s or error reading file. \n' % (fname))
    return doc

def outputTabSep(fn, to_process, outputfn, ns):
    features = etree.iterparse(fn, tag='{%s}%s' % (ns['xmlns'], to_process) )
    header = {'psm':['ID', 'score', 'q-value', 'PEP', 'peptide', 'protein IDs\n'],
          'peptide':['ID', 'score', 'q-value', 'PEP', 'psm_ids', 'protein IDs\n']
          }
    with open(outputfn, 'w') as fp:
        fp.write('\t'.join(header[to_process]))
        for ac,el in features:
            p_id = el.attrib['{%s}%s_id' % (ns['xmlns'], to_process)]
            score = el.xpath('xmlns:svm_score', namespaces=ns)[0].text
            q = el.xpath('xmlns:q_value', namespaces=ns)[0].text
            PEP = el.xpath('xmlns:pep', namespaces=ns)[0].text
            proteins = '; '.join( [x.text for x in el.xpath('xmlns:protein_id',
                                    namespaces=ns) ] )

            if to_process == 'peptide':
                psm_ids = '; '.join([x.text for x in el.xpath('xmlns:psm_id',
                                    namespaces=ns) ])
                fp.write('\t'.join([p_id, score, q, PEP, psm_ids, proteins]))

            elif to_process == 'psm':
                pepseq = el.xpath('xmlns:peptide_seq',
                        namespaces=ns)[0].attrib['seq']
                fp.write('\t'.join([p_id, score, q, PEP, pepseq, proteins]))
            fp.write('\n')

def target_decoy_generator(element_generator, decoy, ns):
    for ev,el in element_generator:
        if el.attrib['{%s}decoy' % ns['xmlns']] == decoy:
            strxml = etree.tostring(el)
            strxml = strxml.replace('xmlns="{0}" '.format(ns['xmlns']), '')
            strxml = strxml.replace('xmlns:p="{0}" '.format(ns['xmlns:p']), '')
            strxml = strxml.replace('xmlns:xsi="{0}" '.format(ns['xmlns:xsi']), '')
            strxml = Markup(strxml)
        else:
            continue
        el.clear()
        while el.getprevious() is not None:
            del el.getparent()[0]
        yield strxml

def splitTargetDecoy(fn, ns):
    """ Splits XML into target/decoy/notspecified elements.
        Usage: splitTargetDecoy('test.xml', namespace)
    """
    split_elements = {'target': {}, 'decoy': {}}
    feats_to_process = ['psm', 'peptide']
    for feat in feats_to_process:
        targetfeatgen = etree.iterparse(fn, tag='{%s}%s' % (ns['xmlns'], feat))
        decoyfeatgen = etree.iterparse(fn, tag='{%s}%s' % (ns['xmlns'], feat))
        split_elements['target'][feat] = target_decoy_generator(targetfeatgen, 'false', ns)
        split_elements['decoy'][feat] = target_decoy_generator(decoyfeatgen, 'true', ns)

    return split_elements

def get_percolator_static_xml(fn, ns):
    rootgen = etree.iterparse(fn, tag='{%s}percolator_output' % ns['xmlns'], events=('start',))
    root = rootgen.next()[1]
    for child in root.getchildren():
        root.remove(child)
    process = etree.iterparse(fn, tag='{%s}process_info' % ns['xmlns'], events=('start',))
    root.append(process.next()[1])
    return root

def write_percolator_xml(fn, staticxml, feats, ns):
    """Given the static percolator xml root and process info nodes, and all
    psms and peptides as iterators in a dict, this generates percolator out data
    into a Genshi template."""

    # First prepare xml template for Genshi containing static percolator xml
    # and a for loop for Genshi to fill in
    etree.register_namespace('py', 'http://genshi.edgewall.org/')
    peptides = etree.SubElement(staticxml, 'peptides', nsmap={'py': 'http://genshi.edgewall.org/'})
    psms = etree.SubElement(staticxml, 'psms', nsmap={'py': 'http://genshi.edgewall.org/'})
    etree.SubElement(peptides, '{http://genshi.edgewall.org/}for', each='pep in peps').text = '${pep}'
    etree.SubElement(psms, '{http://genshi.edgewall.org/}for', each='psm in psms').text = '${psm}'
    root = etree.tostring(staticxml, pretty_print=True, xml_declaration=True)

    # then insert peptides/psms into new formed template
    tpl = MarkupTemplate(root, encoding='utf-8')
    stream = tpl.generate(peps=feats['peptide'], psms=feats['psm'])
    with open(fn, 'w') as fp:
        for chunk in stream.serialize():
            fp.write(chunk)
    del(stream)

def refillTree(fn, ns, elements, tdb, iterfind_used=False):
    """elements: {'psm': {'target': generator, 'decoy':generator, 'both':generator}, etc } tdb='target' or 'decoy' or 'both' """
    # generate percolator_output root element with process_info element from existing file fn
    topgen = etree.iterparse(fn, tag='{%s}percolator_output' % ns['xmlns'])
    for ac, outtree in topgen:
        outtree.clear()
    process_gen = etree.iterparse(fn, tag='{%s}process_info' % ns['xmlns'])
    for ac, proc in process_gen:
        outtree.append(proc)

    # loop through generators and attach to tree
    for feat in elements:
        top_element = etree.Element(feat+'s')
        outtree.append(top_element)
        if iterfind_used:
            for el in elements[feat][tdb]:
                top_element.append(el)
        else:
            for ac,el in elements[feat][tdb]:
                top_element.append(el)

    return etree.ElementTree(outtree)

def filterUniques(input_file, files_dir, score, ns):
    """ Filters unique peptides from (multiple) Percolator output XML files.
        Takes a dir with a set of XML files, a score to filter on and a namespace.
        Outputs an ElementTree.
    """
    # lookup dict/initializing
    scores = {'q':'q_value', 'pep':'pep', 'p':'p_value', 'svm':'svm_score'}
    filtered_peptides = {}
    try:
        members = [os.path.join(files_dir, fn) for fn in os.listdir(files_dir)]
    except OSError: # no extra_files_dir
        members = [input_file]
    if len(members) == 1:
        print >> sys.stdout, 'WARNING: You are attempting to filter unique peptides from a non-fractioned dataset. This may be unneccessary.'

    # parse XML
    peptide_generators_fractions = []
    for fn in members:
        peptide_generators_fractions.append(etree.iterparse(fn, tag='{%s}peptide' % ns['xmlns']))

    # get results
    for peptide_generator in peptide_generators_fractions:
        for ac, peptide_el in peptide_generator:
            # It's actually faster to loop through the feat's children,
            # but this is 2-line code and still readable.
            featscore = float(peptide_el.xpath('xmlns:%s' % scores[score], namespaces=ns)[0].text)
            seq = str(peptide_el.xpath('@xmlns:peptide_id', namespaces=ns)[0])

            if seq not in filtered_peptides:
                filtered_peptides[seq] = peptide_el
            elif score == 'svm': # greater than score is accepted
                if featscore > filtered_peptides[seq]:
                    filtered_peptides[seq] = peptide_el
            else: # lower than score is accepted
                if featscore < filtered_peptides[seq]:
                    filtered_peptides[seq] = peptide_el

    # create top element and append the found unique high scorers
    top_el = etree.iterparse(fn, tag='{%s}peptides' % ns['xmlns'] ).next()[1]
    top_el.clear()

    for seq in filtered_peptides:
        top_el.append(filtered_peptides[seq])
    # convert to generator
    filtered_peptides = {'peptide': {} }
    filtered_peptides['peptide']['both'] = etree.iterwalk(top_el, tag='{%s}peptide' % ns['xmlns'] )

    # output full percolator tree
    return refillTree(fn, ns, filtered_peptides, 'both')

def mergeXML(datasets, ns):
    """ Merges multiple Percolator XML output documents into a single doc.
    Useful for example when a target and decoy set have been created that have to be
    remerged.
    """
    outdoc = readPercout(datasets[0])
    root = outdoc.getroot()

    for parentfeat in ['psms', 'peptides', 'proteins']:
        featname = parentfeat[:-1]
        parent_el = root.xpath('//xmlns:%s' % parentfeat, namespaces=ns)
        print >>sys.stdout, len(parent_el)
        if parent_el: # empty lists are skipped
            parent_el = parent_el[0]
        for fn in datasets[1:]:
            elements = etree.iterparse(fn, tag='{%s}%s' % (ns['xmlns'], featname) )
            for ac,el in elements:
                parent_el.append(el)
        print >>sys.stdout, len(parent_el)

    return etree.ElementTree(root)

def untar(tar):
    try:
        with tarfile.open(tar, 'r') as f:
            members = f.getmembers()
            f.extractall()
            return [x.name for x in members]
    except:
        sys.stdout.write('Could not extract Percolator files from dataset: %s \n' % tar)
        return 1

def writeXML(*args):
    """Takes a filename and _ElementTree arguments in tuples ((fn, tree), (fn2,tree2),...)
    and writes an XML file.
    """
    try:
    # hello! shouldn't these be exceptions instead?
        assert False not in [isinstance(x, tuple) for x in args] # arguments are tuples
        assert False not in [len(x)>2 for x in args] # tuples are 2-length
        assert False not in [isinstance(x[0], str) for x in args] # first arg from tuple is string for fn.
        assert False not in [isinstance(x[1], etree._ElementTree) for x in args] # second arg from tuple is ElementTree.
    except:
        Exception('function writeXML takes arguments in form of tuples (filename, XMLtree)')

    for fn, doc in args:
        try:
            outfile = open(fn, 'w')
        except Exception:
            sys.stderr('Unable to write XML output to file %s \n' % fn)
        doc.write('<?xml version="1.0" encoding="UTF-8"?>')
        doc.write(outfile)

def parseOptions(args):
    ns = {'xmlns':'http://per-colator.com/percolator_out/14',
    'xmlns:p' : 'http://per-colator.com/percolator_out/14',
    'xmlns:xsi':'http://www.w3.org/2001/XMLSchema-instance',
    'xsi:schemaLocation':'http://per-colator.com/percolator_out/14,https://github.com/percolator/percolator/raw/pout-1-4/src/xml/percolator_out.xsd',
    'p:majorVersion':'0',
    'p:minorVersion':'00',
    'p:percolator_version':'Percolator version 0.00'
    }

    if args[0] == 'splittd':
        fname = args[1]
        staticxml = get_percolator_static_xml(fname, ns)
        split_els = splitTargetDecoy(fname, ns)
        write_percolator_xml(args[2], staticxml, split_els['target'], ns)
        write_percolator_xml(args[3], staticxml, split_els['decoy'], ns)

    elif args[0] == 'filter_uni':
        uniques = filterUniques(args[1], args[2], args[3], ns)
        writeXML((args[4], uniques))

    elif args[0] == 'merge':
        with open(args[1]) as fp:
            datasets = fp.read()
        datasets = [opt.strip() for opt in datasets.strip().split('\n')]
        merged = mergeXML(datasets, ns)
        writeXML((args[2], merged))

    elif args[0] == 'untar_merge':
        files = os.listdir(args[1])
        merged = mergeXML(files, ns)
        writeXML((args[2], merged))

    elif args[0] == 'tab':
        outputTabSep(args[1], args[2], args[3], ns)

    else:
        sys.stderr.write('Argument %s not recognized.\n' % args[0])
        return 1

def main():
    parseOptions(sys.argv[1:])

if __name__ == '__main__':
    main()
<commit_msg>Removed old file<commit_after><|endoftext|>


<commit_before>__version__ = '2.2.0'
__all__ = ['Struct', 'FrozenStruct', 'merged']

from ._pystruct import Struct
try:
    from ._cstruct import _Struct as Struct
except ImportError:  # pragma: no cover
    pass

class FrozenStruct(Struct):
    __slots__ = ('_hash', )

    def __hash__(self):
        try:
            _hash = self['_hash']
        except KeyError:
            _hash = hash(tuple((k, self[k]) for k in sorted(self.keys())))
            dict.__setattr__(self, '_hash', _hash)
        return _hash

    def __setitem__(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def __setattr__(self, key, value):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__,))

    def setdefault(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def update(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def clear(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def __delitem__(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def __delattr__(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__,))

    def __reduce__(self):
        return type(self), (), dict(self)

    def __setstate__(self, state):
        dict.update(self, state)

def merged(*dicts, **kwargs):
    if not dicts:
        return Struct()
    result = dict()
    for d in dicts:
        result.update(d)
    result.update(kwargs)
    struct_type = type(dicts[0])
    return struct_type(**result)
<commit_msg> Use mixin to define FrozenStruct (and enable reuse to freeze other Struct sub classes)<commit_after>__version__ = '2.2.0'
__all__ = ['Struct', 'FrozenStruct', 'merged']

from ._pystruct import Struct
try:
    from ._cstruct import _Struct as Struct
except ImportError:  # pragma: no cover
    pass

class Frozen(object):
    __slots__ = ()

    def __hash__(self):
        try:
            _hash = self['_hash']
        except KeyError:
            _hash = hash(tuple((k, self[k]) for k in sorted(self.keys())))
            dict.__setattr__(self, '_hash', _hash)
        return _hash

    def __setitem__(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def __setattr__(self, key, value):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__,))

    def setdefault(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def update(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def clear(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def __delitem__(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__, ))

    def __delattr__(self, *_, **__):
        raise TypeError("'%s' object attributes are read-only" % (type(self).__name__,))

    def __reduce__(self):
        return type(self), (), dict(self)

    def __setstate__(self, state):
        dict.update(self, state)

class FrozenStruct(Frozen, Struct):
    __slots__ = ('_hash', )

def merged(*dicts, **kwargs):
    if not dicts:
        return Struct()
    result = dict()
    for d in dicts:
        result.update(d)
    result.update(kwargs)
    struct_type = type(dicts[0])
    return struct_type(**result)
<|endoftext|>


<commit_before>from argparse import ArgumentParser

from jinja2 import Environment, FileSystemLoader, Template

MEMORY_QUOTAS = {
    'c3.4xlarge': 24576,  # 30GB RAM
    'm4.2xlarge': 26624,  # 32GB RAM
}

OUTPUT_FILE = 'custom.test'

TEMPLATES_DIR = 'templates'

TEMPLATES = {
    'pillowfight': 'pillowfight.j2',
    'ycsb_workload_a': 'ycsb_workload_a.j2',
}

THREADS_PER_CLIENT = {
    'pillowfight': 20,
    'ycsb_workload_a': 20,
}

def render(template: str, instance: str, threads: int):
    loader = FileSystemLoader(searchpath=TEMPLATES_DIR)
    env = Environment(loader=loader, keep_trailing_newline=True)

    t = env.get_template(TEMPLATES[template])

    mem_quota = MEMORY_QUOTAS[instance]
    worker_instances = estimate_num_clients(template, threads)

    content = render_template(t, mem_quota, worker_instances)
    store_cfg(content)

def estimate_num_clients(template: str, threads: int) -> int:
    return max(1, threads // THREADS_PER_CLIENT[template])

def render_template(t: Template, mem_quota: int, worker_instances: int) -> str:
    return t.render(
        mem_quota=mem_quota,
        worker_instances=worker_instances,
    )

def store_cfg(content: str):
    with open(OUTPUT_FILE, 'w') as f:
        f.write(content)

def main():
    parser = ArgumentParser()

    parser.add_argument('--instance', dest='instance', type=str,
                        choices=list(MEMORY_QUOTAS),
                        required=True,
                        help='EC2 instance type')
    parser.add_argument('--template', dest='template', type=str,
                        choices=list(TEMPLATES),
                        required=True,
                        help='Template name')
    parser.add_argument('--threads', dest='threads', type=int,
                        default=1,
                        help='Total number of workload generator threads')

    args = parser.parse_args()

    render(args.template, args.instance, args.threads)

if __name__ == '__main__':
    main()
<commit_msg>Log what file is being created<commit_after>from argparse import ArgumentParser

from jinja2 import Environment, FileSystemLoader, Template

from logger import logger

MEMORY_QUOTAS = {
    'c3.4xlarge': 24576,  # 30GB RAM
    'm4.2xlarge': 26624,  # 32GB RAM
}

OUTPUT_FILE = 'custom.test'

TEMPLATES_DIR = 'templates'

TEMPLATES = {
    'pillowfight': 'pillowfight.j2',
    'ycsb_workload_a': 'ycsb_workload_a.j2',
}

THREADS_PER_CLIENT = {
    'pillowfight': 20,
    'ycsb_workload_a': 20,
}

def render(template: str, instance: str, threads: int):
    loader = FileSystemLoader(searchpath=TEMPLATES_DIR)
    env = Environment(loader=loader, keep_trailing_newline=True)

    t = env.get_template(TEMPLATES[template])

    mem_quota = MEMORY_QUOTAS[instance]
    worker_instances = estimate_num_clients(template, threads)

    content = render_template(t, mem_quota, worker_instances)
    store_cfg(content)

def estimate_num_clients(template: str, threads: int) -> int:
    return max(1, threads // THREADS_PER_CLIENT[template])

def render_template(t: Template, mem_quota: int, worker_instances: int) -> str:
    return t.render(
        mem_quota=mem_quota,
        worker_instances=worker_instances,
    )

def store_cfg(content: str):
    logger.info('Creating a new configuration file: {}'.format(OUTPUT_FILE))
    with open(OUTPUT_FILE, 'w') as f:
        f.write(content)

def main():
    parser = ArgumentParser()

    parser.add_argument('--instance', dest='instance', type=str,
                        choices=list(MEMORY_QUOTAS),
                        required=True,
                        help='EC2 instance type')
    parser.add_argument('--template', dest='template', type=str,
                        choices=list(TEMPLATES),
                        required=True,
                        help='Template name')
    parser.add_argument('--threads', dest='threads', type=int,
                        default=1,
                        help='Total number of workload generator threads')

    args = parser.parse_args()

    render(args.template, args.instance, args.threads)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import urlparse

from netloc import Netloc
from ports import DEFAULT_PORTS
from query_string import QueryString

class URLObject(unicode):

    """
    A URL.

    This class contains properties and methods for accessing and modifying the
    constituent components of a URL. :class:`URLObject` instances are
    immutable, as they derive from the built-in ``unicode``, and therefore all
    methods return *new* objects; you need to consider this when using
    :class:`URLObject` in your own code.
    """

    def __repr__(self):
        return 'URLObject(%r)' % (unicode(self),)

    @property
    def scheme(self):
        return urlparse.urlsplit(self).scheme
    def with_scheme(self, scheme):
        return self.__replace(scheme=scheme)

    @property
    def netloc(self):
        return Netloc(urlparse.urlsplit(self).netloc)
    def with_netloc(self, netloc):
        return self.__replace(netloc=netloc)

    @property
    def username(self):
        return self.netloc.username
    def with_username(self, username):
        return self.with_netloc(self.netloc.with_username(username))
    def without_username(self):
        return self.with_netloc(self.netloc.without_username())

    @property
    def password(self):
        return self.netloc.password
    def with_password(self, password):
        return self.with_netloc(self.netloc.with_password(password))
    def without_password(self):
        return self.with_netloc(self.netloc.without_password())

    @property
    def hostname(self):
        return self.netloc.hostname
    def with_hostname(self, hostname):
        return self.with_netloc(self.netloc.with_hostname(hostname))
    def without_hostname(self):
        return self.with_netloc(self.netloc.without_hostname())

    @property
    def port(self):
        return self.netloc.port
    def with_port(self, port):
        return self.with_netloc(self.netloc.with_port(port))
    def without_port(self):
        return self.with_netloc(self.netloc.without_port())

    @property
    def auth(self):
        return self.netloc.auth
    def with_auth(self, *auth):
        return self.with_netloc(self.netloc.with_auth(*auth))
    def without_auth(self):
        return self.with_netloc(self.netloc.without_auth())

    @property
    def default_port(self):
        """
        The destination port number for this URL.

        If no port number is explicitly given in the URL, this will return the
        default port number for the scheme if one is known, or ``None``. The
        mapping of schemes to default ports is defined in
        :const:`urlobject.ports.DEFAULT_PORTS`.
        """
        port = urlparse.urlsplit(self).port
        if port is not None:
            return port
        return DEFAULT_PORTS.get(self.scheme)

    @property
    def path(self):
        return urlparse.urlsplit(self).path
    def with_path(self, path):
        return self.__replace(path=path)

    @property
    def query(self):
        return QueryString(urlparse.urlsplit(self).query)
    def with_query(self, query):
        return self.__replace(query=query)
    def without_query(self):
        return self.__replace(query='')

    @property
    def query_list(self):
        return self.query.list

    @property
    def query_dict(self):
        return self.query.dict

    @property
    def query_multi_dict(self):
        return self.query.multi_dict

    def add_query_param(self, name, value):
        return self.with_query(self.query.add_param(name, value))
    def add_query_params(self, *args, **kwargs):
        return self.with_query(self.query.add_params(*args, **kwargs))

    def set_query_param(self, name, value):
        return self.with_query(self.query.set_param(name, value))
    def set_query_params(self, *args, **kwargs):
        return self.with_query(self.query.set_params(*args, **kwargs))

    def del_query_param(self, name):
        return self.with_query(self.query.del_param(name))
    def del_query_params(self, params):
        return self.with_query(self.query.del_params(params))

    @property
    def fragment(self):
        return urlparse.urlsplit(self).fragment
    def with_fragment(self, fragment):
        return self.__replace(fragment=fragment)
    def without_fragment(self):
        return self.__replace(fragment='')

    def __replace(self, **replace):
        """Replace a field in the ``urlparse.SplitResult`` for this URL."""
        return type(self)(urlparse.urlunsplit(
            urlparse.urlsplit(self)._replace(**replace)))
<commit_msg>Make URLObject.path an instance of URLPath.<commit_after>import urlparse

from netloc import Netloc
from path import URLPath
from ports import DEFAULT_PORTS
from query_string import QueryString

class URLObject(unicode):

    """
    A URL.

    This class contains properties and methods for accessing and modifying the
    constituent components of a URL. :class:`URLObject` instances are
    immutable, as they derive from the built-in ``unicode``, and therefore all
    methods return *new* objects; you need to consider this when using
    :class:`URLObject` in your own code.
    """

    def __repr__(self):
        return 'URLObject(%r)' % (unicode(self),)

    @property
    def scheme(self):
        return urlparse.urlsplit(self).scheme
    def with_scheme(self, scheme):
        return self.__replace(scheme=scheme)

    @property
    def netloc(self):
        return Netloc(urlparse.urlsplit(self).netloc)
    def with_netloc(self, netloc):
        return self.__replace(netloc=netloc)

    @property
    def username(self):
        return self.netloc.username
    def with_username(self, username):
        return self.with_netloc(self.netloc.with_username(username))
    def without_username(self):
        return self.with_netloc(self.netloc.without_username())

    @property
    def password(self):
        return self.netloc.password
    def with_password(self, password):
        return self.with_netloc(self.netloc.with_password(password))
    def without_password(self):
        return self.with_netloc(self.netloc.without_password())

    @property
    def hostname(self):
        return self.netloc.hostname
    def with_hostname(self, hostname):
        return self.with_netloc(self.netloc.with_hostname(hostname))
    def without_hostname(self):
        return self.with_netloc(self.netloc.without_hostname())

    @property
    def port(self):
        return self.netloc.port
    def with_port(self, port):
        return self.with_netloc(self.netloc.with_port(port))
    def without_port(self):
        return self.with_netloc(self.netloc.without_port())

    @property
    def auth(self):
        return self.netloc.auth
    def with_auth(self, *auth):
        return self.with_netloc(self.netloc.with_auth(*auth))
    def without_auth(self):
        return self.with_netloc(self.netloc.without_auth())

    @property
    def default_port(self):
        """
        The destination port number for this URL.

        If no port number is explicitly given in the URL, this will return the
        default port number for the scheme if one is known, or ``None``. The
        mapping of schemes to default ports is defined in
        :const:`urlobject.ports.DEFAULT_PORTS`.
        """
        port = urlparse.urlsplit(self).port
        if port is not None:
            return port
        return DEFAULT_PORTS.get(self.scheme)

    @property
    def path(self):
        return URLPath(urlparse.urlsplit(self).path)
    def with_path(self, path):
        return self.__replace(path=path)

    @property
    def query(self):
        return QueryString(urlparse.urlsplit(self).query)
    def with_query(self, query):
        return self.__replace(query=query)
    def without_query(self):
        return self.__replace(query='')

    @property
    def query_list(self):
        return self.query.list

    @property
    def query_dict(self):
        return self.query.dict

    @property
    def query_multi_dict(self):
        return self.query.multi_dict

    def add_query_param(self, name, value):
        return self.with_query(self.query.add_param(name, value))
    def add_query_params(self, *args, **kwargs):
        return self.with_query(self.query.add_params(*args, **kwargs))

    def set_query_param(self, name, value):
        return self.with_query(self.query.set_param(name, value))
    def set_query_params(self, *args, **kwargs):
        return self.with_query(self.query.set_params(*args, **kwargs))

    def del_query_param(self, name):
        return self.with_query(self.query.del_param(name))
    def del_query_params(self, params):
        return self.with_query(self.query.del_params(params))

    @property
    def fragment(self):
        return urlparse.urlsplit(self).fragment
    def with_fragment(self, fragment):
        return self.__replace(fragment=fragment)
    def without_fragment(self):
        return self.__replace(fragment='')

    def __replace(self, **replace):
        """Replace a field in the ``urlparse.SplitResult`` for this URL."""
        return type(self)(urlparse.urlunsplit(
            urlparse.urlsplit(self)._replace(**replace)))
<|endoftext|>


<commit_before>import os
import time
import json
import requests
from oti_search import OTISearch
from ConfigParser import SafeConfigParser
from gluon.tools import fetch
from api_utils import read_config

@request.restful()
def v1():
    "The OpenTree API v1"
    response.view = 'generic.json'
    app_name = "api"
    conf = SafeConfigParser(allow_no_value=True)
    if os.path.isfile("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,)):
        conf.read("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,))
    else:
        conf.read("%s/applications/%s/private/config" % (os.path.abspath('.'), app_name,))

    if conf.has_option("apis", "oti_base_url"):
        oti_base_url = conf.get("apis", "oti_base_url")
        api_base_url = "%s/ext/QueryServices/graphdb/" % (oti_base_url,)
    else:
        # fall back to older convention [TODO: remove this]
        host = conf.get("apis","oti_host")
        port = conf.get("apis","oti_port")
        api_base_url = "http://%s:%s/db/data/ext/QueryServices/graphdb/" % (host, port)

    oti = OTISearch(api_base_url)

    def GET(kind, property_name, search_term,jsoncallback=None,callback=None,_=None,**kwargs):
        """"OpenTree API methods relating to searching
Example:

    http://localhost:8000/api/search/v1/tree/ot-ottTaxonName/Carex
    http://localhost:8000/api/search/v1/node/ot-ottId/1000455

When searching for a property name ot:foo, ot-foo must be used
because web2py does not recognize URLs that contain a colon
other than specifying a port, even if URL encoded.

"""

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # colons don't play nicely with GET requests
        property_name = property_name.replace("-",":")

        valid_kinds = ["study", "tree", "node"]
        if kind in valid_kinds:
            return oti.do_search(kind, property_name, search_term)
        else:
            raise HTTP(400,json.dumps({"error":1, "description":"not a valid property name"}))

    return locals()

def nudgeIndexOnUpdates():
    # just diagnostic stuff for now
    from pprint import pprint
    # nothing interesting in these...
    # pprint(request)
    # pprint(request.args)
    # pprint(request.body.read())
    payload = request.vars
    pprint(payload)

    # EXAMPLE of a working curl call to nudge index:
    # curl -X "POST" -d '{"urls": ["https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/10/10.json", "https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/9/9.json"]}' -H "Content-type: application/json" http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti/ext/IndexServices/graphdb/indexNexsons

    # curl http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti/ext/IndexServices/graphdb/indexNexsons -X POST -d '{"urls": ["https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/9/9.json", "https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/10/10.json"] }' -H "Content-type: application/json"

    repo_path, repo_remote, git_ssh, pkey, repo_nexml2json = read_config(request)
    # EXAMPLE VALUES: (
    #   '/Users/jima/projects/nescent/opentree/phylesystem',
    #   'originssh',
    #   '/Users/jima/projects/nescent/opentree/api.opentreeoflife.org/bin/git.sh',
    #   '/Users/jima/.ssh/opentree/opentreeapi-gh.pem',
    #   '0.0.0'
    # )
    #
    # Hm, turns out none of this is useful! Here's what we need to know:
    #   repo's URL or name (munge this to grab raw NexSON)
    #   OTI_base_URL -- MAKE SURE we're pushing to the right OTI service(s)!
    # Both of these should probably "flow through" private/config from the server-config file.
    REPO_URL = "https://github.com/OpenTreeOfLife/phylesystem"  # TODO: fetch from private/config
    OTI_BASE_URL='http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti'  # TODO: fetch from private/config

    try:
        if payload['repository']['url'] != REPO_URL:
            raise HTTP(400,json.dumps({"error":1, "description":"wrong repo for this API instance"}))

        # how we nudge the index depends on which studies are new, changed, or deleted
        added_study_ids = [ ]
        modified_study_ids = [ ]
        removed_study_ids = [ ]
        # TODO: Should any of these lists override another? maybe use commit timestamps to "trump" based on later operations?
        for commit in payload['commits']:
            _harvest_study_ids_from_paths( commit['added'], added_study_ids )
            _harvest_study_ids_from_paths( commit['modified'], modified_study_ids )
            _harvest_study_ids_from_paths( commit['removed'], removed_study_ids )

        # "flatten" each list to remove duplicates
        added_study_ids = list(set(added_study_ids))
        modified_study_ids = list(set(modified_study_ids))
        removed_study_ids = list(set(removed_study_ids))

    except:
        # TODO: raise HTTP(400,json.dumps({"error":1, "description":"malformed GitHub payload"}))
        added_study_ids = ['9']
        modified_study_ids = ['9','10']
        removed_study_ids = ['10']

    nexson_url_template = REPO_URL.replace("github.com", "raw.github.com") + "/master/study/%s/%s.json"

    # for now, let's just add/update new and modified studies using indexNexsons
    study_ids = added_study_ids + modified_study_ids
    # NOTE that passing deleted_study_ids (any non-existent file paths) will fail on oti, with a FileNotFoundException!
    study_ids = list(set(study_ids))  # remove any duplicates

    nudge_url = "%s/ext/IndexServices/graphdb/indexNexsons" % (OTI_BASE_URL,)
    pprint("nudge_url:")
    pprint(nudge_url)

    nexson_urls = [ (nexson_url_template % (study_id, study_id)) for study_id in study_ids ]
    pprint("nexson_urls:")
    pprint(nexson_urls)

    nudge_index_response = fetch(
        nudge_url,
        data={
            "urls": nexson_urls
        },
        headers={
            "Content-type": "application/json"
        }
    )
    pprint(nudge_index_response)
    # TODO: check returned IDs against our original study_ids list... what if something fails?

    ##### TODO: Call removed studies here, once we have a solid method for nudging for removal!

def _harvest_study_ids_from_paths( path_list, target_array ):
    for path in path_list:
        path_parts = path.split('/')
        if path_parts[0] == "study":
            study_id = path_parts[1]
            target_array.append(study_id)

<commit_msg>Fix failing calls to oti service<commit_after>import os
import time
import json
import requests
from oti_search import OTISearch
from ConfigParser import SafeConfigParser
import urllib2
from api_utils import read_config

@request.restful()
def v1():
    "The OpenTree API v1"
    response.view = 'generic.json'
    app_name = "api"
    conf = SafeConfigParser(allow_no_value=True)
    if os.path.isfile("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,)):
        conf.read("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,))
    else:
        conf.read("%s/applications/%s/private/config" % (os.path.abspath('.'), app_name,))

    if conf.has_option("apis", "oti_base_url"):
        oti_base_url = conf.get("apis", "oti_base_url")
        api_base_url = "%s/ext/QueryServices/graphdb/" % (oti_base_url,)
    else:
        # fall back to older convention [TODO: remove this]
        host = conf.get("apis","oti_host")
        port = conf.get("apis","oti_port")
        api_base_url = "http://%s:%s/db/data/ext/QueryServices/graphdb/" % (host, port)

    oti = OTISearch(api_base_url)

    def GET(kind, property_name, search_term,jsoncallback=None,callback=None,_=None,**kwargs):
        """"OpenTree API methods relating to searching
Example:

    http://localhost:8000/api/search/v1/tree/ot-ottTaxonName/Carex
    http://localhost:8000/api/search/v1/node/ot-ottId/1000455

When searching for a property name ot:foo, ot-foo must be used
because web2py does not recognize URLs that contain a colon
other than specifying a port, even if URL encoded.

"""

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # colons don't play nicely with GET requests
        property_name = property_name.replace("-",":")

        valid_kinds = ["study", "tree", "node"]
        if kind in valid_kinds:
            return oti.do_search(kind, property_name, search_term)
        else:
            raise HTTP(400,json.dumps({"error":1, "description":"not a valid property name"}))

    return locals()

def nudgeIndexOnUpdates():
    # just diagnostic stuff for now
    from pprint import pprint
    # nothing interesting in these...
    # pprint(request)
    # pprint(request.args)
    # pprint(request.body.read())
    payload = request.vars
    pprint(payload)

    # EXAMPLE of a working curl call to nudge index:
    # curl -X "POST" -d '{"urls": ["https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/10/10.json", "https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/9/9.json"]}' -H "Content-type: application/json" http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti/ext/IndexServices/graphdb/indexNexsons

    # curl http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti/ext/IndexServices/graphdb/indexNexsons -X POST -d '{"urls": ["https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/9/9.json", "https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/10/10.json"] }' -H "Content-type: application/json"

    repo_path, repo_remote, git_ssh, pkey, repo_nexml2json = read_config(request)
    # EXAMPLE VALUES: (
    #   '/Users/jima/projects/nescent/opentree/phylesystem',
    #   'originssh',
    #   '/Users/jima/projects/nescent/opentree/api.opentreeoflife.org/bin/git.sh',
    #   '/Users/jima/.ssh/opentree/opentreeapi-gh.pem',
    #   '0.0.0'
    # )
    #
    # Hm, turns out none of this is useful! Here's what we need to know:
    #   repo's URL or name (munge this to grab raw NexSON)
    #   OTI_base_URL -- MAKE SURE we're pushing to the right OTI service(s)!
    # Both of these should probably "flow through" private/config from the server-config file.
    REPO_URL = "https://github.com/OpenTreeOfLife/phylesystem"  # TODO: fetch from private/config
    OTI_BASE_URL='http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti'  # TODO: fetch from private/config

    try:
        if payload['repository']['url'] != REPO_URL:
            raise HTTP(400,json.dumps({"error":1, "description":"wrong repo for this API instance"}))

        # how we nudge the index depends on which studies are new, changed, or deleted
        added_study_ids = [ ]
        modified_study_ids = [ ]
        removed_study_ids = [ ]
        # TODO: Should any of these lists override another? maybe use commit timestamps to "trump" based on later operations?
        for commit in payload['commits']:
            _harvest_study_ids_from_paths( commit['added'], added_study_ids )
            _harvest_study_ids_from_paths( commit['modified'], modified_study_ids )
            _harvest_study_ids_from_paths( commit['removed'], removed_study_ids )

        # "flatten" each list to remove duplicates
        added_study_ids = list(set(added_study_ids))
        modified_study_ids = list(set(modified_study_ids))
        removed_study_ids = list(set(removed_study_ids))

    except:
        raise HTTP(400,json.dumps({"error":1, "description":"malformed GitHub payload"}))
        ## # test data
        ## added_study_ids = [ ]
        ## modified_study_ids = ["10"]
        ## removed_study_ids = [ ]

    nexson_url_template = REPO_URL.replace("github.com", "raw.github.com") + "/master/study/%s/%s.json"

    # for now, let's just add/update new and modified studies using indexNexsons
    study_ids = added_study_ids + modified_study_ids
    # NOTE that passing deleted_study_ids (any non-existent file paths) will fail on oti, with a FileNotFoundException!
    study_ids = list(set(study_ids))  # remove any duplicates

    nudge_url = "%s/ext/IndexServices/graphdb/indexNexsons" % (OTI_BASE_URL,)
    nexson_urls = [ (nexson_url_template % (study_id, study_id)) for study_id in study_ids ]

    # N.B. that gluon.tools.fetch() can't be used here, since it won't send "raw" JSON data as treemachine expects
    req = urllib2.Request(
        url=nudge_url,
        data=json.dumps({
            "urls": nexson_urls
        }),
        headers={"Content-Type": "application/json"}
    )
    nudge_response = urllib2.urlopen(req).read()
    updated_study_ids = json.loads( nudge_response )

    # TODO: Call removed studies here, once we have a solid method for nudging for removal!
    # TODO: check returned IDs against our original lists... what if something failed?

def _harvest_study_ids_from_paths( path_list, target_array ):
    for path in path_list:
        path_parts = path.split('/')
        if path_parts[0] == "study":
            study_id = path_parts[1]
            target_array.append(study_id)

<|endoftext|>