<commit_before>#!/usr/bin/python
# -*- coding:utf-8 -*-
# Powered By KK Studio

import tornado
import tornado.httpserver
import tornado.ioloop
import tornado.web
import tornado.netutil
import tornado.process
import tornado.options
import tornado.locale
import platform
import db
from tornado.log import gen_log
from handler.page import Page404Handler
from config.settings import *
from handler import route
from ui_modules import UIModules
#from Template import TemplateLoader

class App(tornado.web.Application):

    def __init__(self,handlers,conf,log):
        self.__version__ = conf['version']
        self.log = log
        settings = conf['app_settings']
        settings['default_handler_class'] = Page404Handler  # 404
        # Don't Support for Jinja2
        settings['ui_modules'] = UIModules
        tornado.web.Application.__init__(self, handlers, **settings)
        # Support for Jinja2
        #tpl_loader = TemplateLoader(settings['template_path'], False)
        #tornado.web.Application.__init__(self, handlers, template_loader=tpl_loader.Loader(), **settings)
        #每10秒执行一次
        #tornado.ioloop.PeriodicCallback(self.test, 1 * 10 * 1000).start()
        # Init Database
        self.db = db.DB(**conf['db'])
        #Init Redis
        R = db.Redis(**conf['redis'])
        self.redis = R.Connect()
        # Load Locale
        self.__load_locale(settings['default_lang'])

    #def test(self):
    #    self.log.info('Test')

    # Load Locale
    def __load_locale(self,default_lang):
        tornado.locale.load_translations('locale')
        tornado.locale.set_default_locale(default_lang)

class Torweb():

    def __init__(self,processes=4):
        self.__version__ = '1.0.2'
        self.host = config['host']
        self.port = config['port']
        self.urls = route
        self.config = config
        self.config['version'] = self.__version__
        self.log = gen_log
        if platform.system() == "Linux":  #根据操作系统类型来确定是否启用多线程
            self.processes = processes # 当processes>1时，PeriodicCallback定时任务会响相应的执行多次
        else:
            self.processes = 1

    #多线程模式
    def run(self):
        self.log.info('Torweb %s' % self.__version__)  # 启动时打印版本号
        self.log.info('Listen Port: %s' % self.port)
        http_sockets = tornado.netutil.bind_sockets(self.port, self.host)
        tornado.process.fork_processes(num_processes=self.processes)
        http_server = tornado.httpserver.HTTPServer(request_callback=App(self.urls,self.config,self.log), xheaders=True)
        http_server.add_sockets(http_sockets)
        tornado.ioloop.IOLoop.instance().start()

    # 单进程模式
    def run_single(self):
        app = App(self.urls, self.config, self.log)
        app.listen(self.port)
        tornado.ioloop.IOLoop.current().start()
<commit_msg>Improve single process model<commit_after>#!/usr/bin/python
# -*- coding:utf-8 -*-
# Powered By KK Studio

import tornado
import tornado.httpserver
import tornado.ioloop
import tornado.web
import tornado.netutil
import tornado.process
import tornado.options
import tornado.locale
import platform
import db
from tornado.log import gen_log
from handler.page import Page404Handler
from config.settings import *
from handler import route
from ui_modules import UIModules
#from Template import TemplateLoader

class App(tornado.web.Application):

    def __init__(self,handlers,conf,log):
        self.__version__ = conf['version']
        self.log = log
        settings = conf['app_settings']
        settings['default_handler_class'] = Page404Handler  # 404
        # Don't Support for Jinja2
        settings['ui_modules'] = UIModules
        tornado.web.Application.__init__(self, handlers, **settings)
        # Support for Jinja2
        #tpl_loader = TemplateLoader(settings['template_path'], False)
        #tornado.web.Application.__init__(self, handlers, template_loader=tpl_loader.Loader(), **settings)
        #每10秒执行一次
        #tornado.ioloop.PeriodicCallback(self.test, 1 * 10 * 1000).start()
        # Init Database
        self.db = db.DB(**conf['db'])
        #Init Redis
        R = db.Redis(**conf['redis'])
        self.redis = R.Connect()
        # Load Locale
        self.__load_locale(settings['default_lang'])

    #def test(self):
    #    self.log.info('Test')

    # Load Locale
    def __load_locale(self,default_lang):
        tornado.locale.load_translations('locale')
        tornado.locale.set_default_locale(default_lang)

class Torweb():

    def __init__(self,processes=4):
        self.__version__ = '1.0.2'
        self.host = config['host']
        self.port = config['port']
        self.urls = route
        self.config = config
        self.config['version'] = self.__version__
        self.log = gen_log
        if platform.system() == "Linux":  #根据操作系统类型来确定是否启用多线程
            self.processes = processes # 当processes>1时，PeriodicCallback定时任务会响相应的执行多次
        else:
            self.processes = 1

    # 单进程模式
    def run(self):
        app = App(self.urls, self.config, self.log)
        app.listen(self.port)
        tornado.ioloop.IOLoop.current().start()

    #多线程模式
    def run_multi(self):
        self.log.info('Torweb %s' % self.__version__)  # 启动时打印版本号
        self.log.info('Listen Port: %s' % self.port)
        http_sockets = tornado.netutil.bind_sockets(self.port, self.host)
        tornado.process.fork_processes(num_processes=self.processes)
        http_server = tornado.httpserver.HTTPServer(request_callback=App(self.urls,self.config,self.log), xheaders=True)
        http_server.add_sockets(http_sockets)
        tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask_assets import Bundle, Environment

css = Bundle(
    "libs/bootstrap/dist/css/bootstrap.css",
    "css/style.css",
    filters="cssmin",
    output="public/css/common.css"
)

js = Bundle(
    "libs/jQuery/dist/jquery.js",
    "libs/bootstrap/dist/js/bootstrap.js",
    "js/plugins.js",
    filters='jsmin',
    output="public/js/common.js"
)

assets = Environment()

assets.register("js_all", js)
assets.register("css_all", css)
<commit_msg>change assets<commit_after># -*- coding: utf-8 -*-
from flask_assets import Bundle, Environment

css = Bundle(
    "bootstrap.css",
    "css/style.css",
    filters="cssmin",
    output="public/css/common.css"
)

js = Bundle(
    "libs/jQuery/dist/jquery.js",
    "libs/bootstrap/dist/js/bootstrap.js",
    "js/plugins.js",
    filters='jsmin',
    output="public/js/common.js"
)

assets = Environment()

assets.register("js_all", js)
assets.register("css_all", css)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import python_2_unicode_compatible
from django.core.exceptions import ValidationError
from isodate.isoerror import ISO8601Error
import isodate
from dateutil.relativedelta import relativedelta
from . import utils
from . import forms

try:
    from south.modelsinspector import add_introspection_rules
except ImportError:
    add_introspection_rules = None

@python_2_unicode_compatible
class ISO8601DurationField(models.Field):
    """Store and retrieve ISO 8601 formatted durations.

    """

    description = _("A duration of time (ISO 8601 format)")

    __metaclass__ = models.SubfieldBase

    default_error_messages = {
        'invalid': _("This value must be in ISO 8601 Duration format."),
        'unknown_type': _("The value's type could not be converted"),
    }

    def __init__(self, *args, **kwargs):
        self.max_length = kwargs['max_length'] = 64
        super(ISO8601DurationField, self).__init__(*args, **kwargs)

    def db_type(self, connection):
        return 'char({0})'.format(self.max_length)

    def get_internal_type(self):
        return 'CharField'

    def to_python(self, value):
        """
        :returns: A duration object parsed from the given value.
        :rtype: :py:class:`~isodate.duration.Duration`

        """
        # DB value is null
        if value is None:
            return None

        # DB value is empty
        if value == '':
            return isodate.duration.Duration()

        if isinstance(value, isodate.duration.Duration):
            return value

        try:
            duration = isodate.parse_duration(value)
        except ISO8601Error:
            raise ValidationError(self.default_error_messages['invalid'])
        return duration

    def get_prep_value(self, value):

        # Value in DB should be null.
        if value is None:
            return None

        if not isinstance(value, isodate.duration.Duration):
            raise ValidationError('Cannot convert objects that are not Durations.')

        return isodate.duration_isoformat(value)

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_prep_value(value)

class RelativeDeltaField(ISO8601DurationField):
    """Store and retrieve :py:class:`~datetime.relativedelta.relativedelta`.

    Stores the relativedelta as a string representation of a
    :py:class:`~isodate.duration.Duration`.
    """

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.RelativeDeltaChoiceField}
        defaults.update(kwargs)
        return super(RelativeDeltaField, self).formfield(**defaults)

    def to_python(self, value):

        # DB value is null
        if value is None:
            return None

        if isinstance(value, relativedelta):
            return value

        duration = super(RelativeDeltaField, self).to_python(value)

        delta = utils.convert_duration_to_relativedelta(duration)

        return delta

    def get_prep_value(self, value):

        # Value in DB should be null
        if value is None:
            return None

        # Build the Duration object from the given relativedelta.
        duration = utils.convert_relativedelta_to_duration(value)
        duration_string = super(RelativeDeltaField, self).get_prep_value(duration)

        return duration_string

    def value_to_string(self, obj):
        val = self._get_val_from_obj(obj)
        s = self.get_prep_value(val)
        return '' if s is None else s

if add_introspection_rules:
    add_introspection_rules([], ['^thecut\.durationfield\.models\.ISO8601DurationField',
                                 '^thecut\.durationfield\.models\.RelativeDeltaField'])
<commit_msg>Removed unneeded @python_2_unicode_compatible decorator from ISO8601DurationField.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ValidationError
from isodate.isoerror import ISO8601Error
import isodate
from dateutil.relativedelta import relativedelta
from . import utils
from . import forms

try:
    from south.modelsinspector import add_introspection_rules
except ImportError:
    add_introspection_rules = None

class ISO8601DurationField(models.Field):
    """Store and retrieve ISO 8601 formatted durations.

    """

    description = _("A duration of time (ISO 8601 format)")

    __metaclass__ = models.SubfieldBase

    default_error_messages = {
        'invalid': _("This value must be in ISO 8601 Duration format."),
        'unknown_type': _("The value's type could not be converted"),
    }

    def __init__(self, *args, **kwargs):
        self.max_length = kwargs['max_length'] = 64
        super(ISO8601DurationField, self).__init__(*args, **kwargs)

    def db_type(self, connection):
        return 'char({0})'.format(self.max_length)

    def get_internal_type(self):
        return 'CharField'

    def to_python(self, value):
        """
        :returns: A duration object parsed from the given value.
        :rtype: :py:class:`~isodate.duration.Duration`

        """
        # DB value is null
        if value is None:
            return None

        # DB value is empty
        if value == '':
            return isodate.duration.Duration()

        if isinstance(value, isodate.duration.Duration):
            return value

        try:
            duration = isodate.parse_duration(value)
        except ISO8601Error:
            raise ValidationError(self.default_error_messages['invalid'])
        return duration

    def get_prep_value(self, value):

        # Value in DB should be null.
        if value is None:
            return None

        if not isinstance(value, isodate.duration.Duration):
            raise ValidationError('Cannot convert objects that are not Durations.')

        return isodate.duration_isoformat(value)

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_prep_value(value)

class RelativeDeltaField(ISO8601DurationField):
    """Store and retrieve :py:class:`~datetime.relativedelta.relativedelta`.

    Stores the relativedelta as a string representation of a
    :py:class:`~isodate.duration.Duration`.
    """

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.RelativeDeltaChoiceField}
        defaults.update(kwargs)
        return super(RelativeDeltaField, self).formfield(**defaults)

    def to_python(self, value):

        # DB value is null
        if value is None:
            return None

        if isinstance(value, relativedelta):
            return value

        duration = super(RelativeDeltaField, self).to_python(value)

        delta = utils.convert_duration_to_relativedelta(duration)

        return delta

    def get_prep_value(self, value):

        # Value in DB should be null
        if value is None:
            return None

        # Build the Duration object from the given relativedelta.
        duration = utils.convert_relativedelta_to_duration(value)
        duration_string = super(RelativeDeltaField, self).get_prep_value(duration)

        return duration_string

    def value_to_string(self, obj):
        val = self._get_val_from_obj(obj)
        s = self.get_prep_value(val)
        return '' if s is None else s

if add_introspection_rules:
    add_introspection_rules([], ['^thecut\.durationfield\.models\.ISO8601DurationField',
                                 '^thecut\.durationfield\.models\.RelativeDeltaField'])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#
# Copyright 2016 edX PDR Lab, National Central University, Taiwan.
#
#     http://edxpdrlab.ncu.cc/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Created By: yrchen@ATCity.org
# Maintained By: yrchen@ATCity.org
#

"""
Form configurations for ELOs package in Common Repository project.
"""

from __future__ import absolute_import, unicode_literals

from django.conf import settings
from django.forms import ModelForm, ModelChoiceField
from django.shortcuts import get_object_or_404, redirect
from django.utils import timezone

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit, HTML
from crispy_forms.bootstrap import FormActions

from commonrepo.users.models import User as User

from .models import ELO

__author__ = 'yrchen@ATCity.org (Xaver Y.R. Chen)'

class ELOForm(ModelForm):
    """
    Form of ELO
    """

    class Meta:
        model = ELO
        fields = [
            'name',
            'author',
            'cover',
            'description',
            'original_type',
            'license',
            'init_file']

    def __init__(self, *args, **kwargs):
        self.request_user = kwargs.pop("request_user")
        super(ELOForm, self).__init__(*args, **kwargs)

        # check if user is not staff
        if not self.request_user.is_staff:
            self.fields["author"].queryset = User.objects.filter(
                username=self.request_user)
            self.fields["author"].widget.attrs['readonly'] = True

        # author can't be empty
        self.fields["author"].empty_label = None

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))

class ELOForkForm(ModelForm):
    """
    Form of ELO Fork
    """

    class Meta:
        model = ELO
        fields = [
            'name',
            'author',
            'description',
            'original_type',
            'license',
            'init_file',
            'version',
            'parent_elo',
            'parent_elo_uuid',
            'parent_elo_version']

    def __init__(self, pk=None, *args, **kwargs):
        self.request_user = kwargs.pop("request_user")
        super(ELOForkForm, self).__init__(*args, **kwargs)

        elo_original = get_object_or_404(ELO, id=pk)
        elo_source = elo_original

        # If original ELO is not public
        if not elo_original.is_public:
            # and request.user is not author or staff, use Root ELO to replace
            if not self.request_user == elo_original.author and not self.request_user.is_staff:
                elo_source = get_object_or_404(ELO, id=settings.ELO_ROOT_ID)

        self.fields["name"].initial = elo_source.name + \
            " (forked from author " + elo_source.author.username + ")"
        self.fields["name"].widget.attrs['readonly'] = True
        self.fields["author"].queryset = User.objects.filter(
            username=self.request_user)
        self.fields["author"].empty_label = None
        self.fields["author"].widget.attrs['readonly'] = True
        self.fields["description"].initial = elo_source.description
        self.fields["description"].widget.attrs['readonly'] = True
        self.fields["original_type"].initial = elo_source.original_type
        self.fields["original_type"].widget.attrs['readonly'] = True
        self.fields["license"].initial = elo_source.license
        self.fields["license"].widget.attrs['readonly'] = True
        self.fields["init_file"].initial = elo_source.init_file
        self.fields["init_file"].widget.attrs['readonly'] = True
        self.fields["version"].initial = 1
        self.fields["version"].widget.attrs['readonly'] = True

        # If original ELO is not public
        if not elo_original.is_public:
            # author and staff still can fork
            if self.request_user == elo_original.author or self.request_user.is_staff:
                self.fields["parent_elo"].queryset = ELO.objects.filter(
                    id=elo_original.id)
                self.fields["parent_elo_uuid"].initial = elo_original.uuid
                self.fields[
                    "parent_elo_version"].initial = elo_original.version
            # others will force to use Root ELO
            else:
                self.fields["parent_elo"].queryset = ELO.objects.filter(
                    id=settings.ELO_ROOT_ID)
                self.fields["parent_elo_uuid"].initial = elo_source.uuid
                self.fields["parent_elo_version"].initial = elo_source.version
        else:
            self.fields["parent_elo"].queryset = ELO.objects.filter(
                id=elo_original.id)
            self.fields["parent_elo_uuid"].initial = elo_original.uuid
            self.fields["parent_elo_version"].initial = elo_original.version

        self.fields["parent_elo"].empty_label = None
        self.fields["parent_elo"].widget.attrs['readonly'] = True
        self.fields["parent_elo_uuid"].widget.attrs['readonly'] = True
        self.fields["parent_elo_version"].widget.attrs['readonly'] = True

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))

class ELOUpdateForm(ModelForm):
    """
    Form of ELO update
    """

    class Meta:
        model = ELO
        fields = [
            'name',
            'cover',
            'description',
            'original_type',
            'license',
            'metadata',
            'is_public',
            'init_file']

    def __init__(self, pk=None, *args, **kwargs):
        super(ELOUpdateForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))
<commit_msg>新增 forms ELOMetadataForm, 用來處理新增 ELO 的 Metadata<commit_after># -*- coding: utf-8 -*-

#
# Copyright 2016 edX PDR Lab, National Central University, Taiwan.
#
#     http://edxpdrlab.ncu.cc/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Created By: yrchen@ATCity.org
# Maintained By: yrchen@ATCity.org
#

"""
Form configurations for ELOs package in Common Repository project.
"""

from __future__ import absolute_import, unicode_literals

from django.conf import settings
from django.forms import ModelForm, ModelChoiceField
from django.shortcuts import get_object_or_404, redirect
from django.utils import timezone

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit, HTML
from crispy_forms.bootstrap import FormActions

from commonrepo.users.models import User as User

from .models import ELO, ELOMetadata

__author__ = 'yrchen@ATCity.org (Xaver Y.R. Chen)'

class ELOMetadataForm(ModelForm):
    """
    Form of ELO Metadata
    """

    class Meta:
        model = ELOMetadata
        fields = [
            #
            # 1. General

            # 1.1 identifier
            General_identifier,
            # 1.2 title
            General_title,
            # 1.3 language
            General_language,
            # 1.4 description
            General_description,
            # 1.5 keyword
            General_keyword,
            # 1.6 coverage
            General_coverage,
            # 1.7 structure
            General_structure,
            # 1.8 aggregationLevel
            General_aggregationLevel,

            #
            # 2. LifeCycle

            # 2.1 version
            LifeCycle_version,
            # 2.2 status
            LifeCycle_status,
            # 2.3 contribute
            LifeCycle_contribute_role,
            LifeCycle_contribute_entity,
            LifeCycle_contribute_date_dateTime,
            LifeCycle_contribute_date_description,

            #
            # 3. Meta-metadata

            # 3.1 Meta_metadata-identifier
            Meta_metadata_identifier_catalog,
            Meta_metadata_identifier_entry,
            # 3.2 Meta_metadata-contribute
            Meta_metadata_contribute_role,
            Meta_metadata_contribute_entity,
            Meta_metadata_contribute_date_dateTime,
            Meta_metadata_contribute_date_description,
            # 3.3 Meta_metadata-metadataSchema
            Meta_metadata_metadataSchema,
            # 3.4 Meta_metadata-language
            Meta_metadata_language,

            #
            # 4. Technical

            # 4.1 format
            Technical_format,
            # 4.2 size
            Technical_size,
            # 4.3 location
            Technical_location,
            # 4.4 requirement
            Technical_requirement_orComposite_type,
            Technical_requirement_orComposite_name,
            Technical_requirement_orComposite_minimumVersion,
            Technical_requirement_orComposite_maximumVersion,
            # 4.5 installationRemarks
            Technical_installationRemarks,
            # 4.6 otherPlatformRequirements
            Technical_otherPlatformRequirements,
            # 4.7 duration
            Technical_duration_duration,
            Technical_duration_description,

            #
            # 5. Educational

            # 5.1 interactivityTyp
            Educational_interactivityType,
            # 5.2 learningResourceType
            Educational_learningResourceType,
            # 5.3 interactivityLevel
            Educational_interactivityLevel,
            # 5.4 semanticDensity
            Educational_semanticDensity,
            # 5.5 intendedEndUserRole
            Educational_intendedEndUserRole,
            # 5.6 context
            Educational_context,
            # 5.7 typicalAgeRange
            Educational_typicalAgeRange,
            # 5.8 difficulty
            Educational_difficulty,
            # 5.9 typicalLearningTime
            Educational_typicalLearningTime_duration,
            Educational_typicalLearningTime_description,
            # 5.10 description
            Educational_description,
            # 5.11 language
            Educational_language,

            #
            # 6. Rights

            # 6.1 cost
            Rights_cost,
            # 6.2 copyrightAndOtherRestrictions
            Rights_copyrightAndOtherRestrictions,
            # 6.3 description
            Rights_description,

            #
            # 7. Relation

            # 7.1 Kind
            Relation_kind,
            # 7.2 Resource
            Relation_resource_identifier_catalog,
            Relation_resource_identifier_entry,
            Relation_resource_description,

            #
            # 8. Annotation

            # 8.1 entity
            Annotation_entity,
            # 8.2 date
            Annotation_date_dateTime,
            Annotation_date_description,
            # 8.3 description
            Annotation_description,

            #
            # 9. Classification

            # 9.1 purpose
            Classification_purpose,
            # 9.2 taxonPath
            Classification_taxonPath_source,
            # 9.2.2 Classification-taxonPath-taxon
            Classification_taxonPath_taxon,
            Classification_taxonPath_taxon_id,
            Classification_taxonPath_taxon_entry,
            # 9.3 description
            Classification_description,
            # 9.4 keyword
            Classification_keyword, ]

    def __init__(self, *args, **kwargs):
        super(ELOMetadataForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))

class ELOForm(ModelForm):
    """
    Form of ELO
    """

    class Meta:
        model = ELO
        fields = [
            'name',
            'author',
            'cover',
            'description',
            'original_type',
            'license',
            'init_file']

    def __init__(self, *args, **kwargs):
        self.request_user = kwargs.pop("request_user")
        super(ELOForm, self).__init__(*args, **kwargs)

        # check if user is not staff
        if not self.request_user.is_staff:
            self.fields["author"].queryset = User.objects.filter(
                username=self.request_user)
            self.fields["author"].widget.attrs['readonly'] = True

        # author can't be empty
        self.fields["author"].empty_label = None

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))

class ELOForkForm(ModelForm):
    """
    Form of ELO Fork
    """

    class Meta:
        model = ELO
        fields = [
            'name',
            'author',
            'description',
            'original_type',
            'license',
            'init_file',
            'version',
            'parent_elo',
            'parent_elo_uuid',
            'parent_elo_version']

    def __init__(self, pk=None, *args, **kwargs):
        self.request_user = kwargs.pop("request_user")
        super(ELOForkForm, self).__init__(*args, **kwargs)

        elo_original = get_object_or_404(ELO, id=pk)
        elo_source = elo_original

        # If original ELO is not public
        if not elo_original.is_public:
            # and request.user is not author or staff, use Root ELO to replace
            if not self.request_user == elo_original.author and not self.request_user.is_staff:
                elo_source = get_object_or_404(ELO, id=settings.ELO_ROOT_ID)

        self.fields["name"].initial = elo_source.name + \
            " (forked from author " + elo_source.author.username + ")"
        self.fields["name"].widget.attrs['readonly'] = True
        self.fields["author"].queryset = User.objects.filter(
            username=self.request_user)
        self.fields["author"].empty_label = None
        self.fields["author"].widget.attrs['readonly'] = True
        self.fields["description"].initial = elo_source.description
        self.fields["description"].widget.attrs['readonly'] = True
        self.fields["original_type"].initial = elo_source.original_type
        self.fields["original_type"].widget.attrs['readonly'] = True
        self.fields["license"].initial = elo_source.license
        self.fields["license"].widget.attrs['readonly'] = True
        self.fields["init_file"].initial = elo_source.init_file
        self.fields["init_file"].widget.attrs['readonly'] = True
        self.fields["version"].initial = 1
        self.fields["version"].widget.attrs['readonly'] = True

        # If original ELO is not public
        if not elo_original.is_public:
            # author and staff still can fork
            if self.request_user == elo_original.author or self.request_user.is_staff:
                self.fields["parent_elo"].queryset = ELO.objects.filter(
                    id=elo_original.id)
                self.fields["parent_elo_uuid"].initial = elo_original.uuid
                self.fields[
                    "parent_elo_version"].initial = elo_original.version
            # others will force to use Root ELO
            else:
                self.fields["parent_elo"].queryset = ELO.objects.filter(
                    id=settings.ELO_ROOT_ID)
                self.fields["parent_elo_uuid"].initial = elo_source.uuid
                self.fields["parent_elo_version"].initial = elo_source.version
        else:
            self.fields["parent_elo"].queryset = ELO.objects.filter(
                id=elo_original.id)
            self.fields["parent_elo_uuid"].initial = elo_original.uuid
            self.fields["parent_elo_version"].initial = elo_original.version

        self.fields["parent_elo"].empty_label = None
        self.fields["parent_elo"].widget.attrs['readonly'] = True
        self.fields["parent_elo_uuid"].widget.attrs['readonly'] = True
        self.fields["parent_elo_version"].widget.attrs['readonly'] = True

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))

class ELOUpdateForm(ModelForm):
    """
    Form of ELO update
    """

    class Meta:
        model = ELO
        fields = [
            'name',
            'cover',
            'description',
            'original_type',
            'license',
            'metadata',
            'is_public',
            'init_file']

    def __init__(self, pk=None, *args, **kwargs):
        super(ELOUpdateForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper(self)
        self.helper.form_action = "."
        self.helper.layout.append(
            FormActions(
                HTML("""Cancel"""),
                Submit('save', 'Submit'),
            ))
<|endoftext|>


<commit_before>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tfrt_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "6697e3354fbe51242729888529d047c646acdab3"
    LLVM_SHA256 = "32e4e9db18aef9801890cca46b243fa4ce43de5740ea1d89d47fb553401803d6"

    tfrt_http_archive(
        name = name,
        build_file = "//third_party/llvm:BUILD",
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@14a89d00c7cc<commit_after>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tfrt_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "14a89d00c7cc852c605be0378701e1d8cbe1fca9"
    LLVM_SHA256 = "c33ec983cea1d9f4250fe069e86cd144dd43e6792e0b2b11a65af98df5d2a56d"

    tfrt_http_archive(
        name = name,
        build_file = "//third_party/llvm:BUILD",
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
    )
<|endoftext|>


<commit_before>import logging

from django.utils.encoding import python_2_unicode_compatible
from django.conf import settings

from treeherder.client import TreeherderRequest

from treeherder.etl.oauth_utils import OAuthCredentials

logger = logging.getLogger(__name__)

def post_treeherder_collections(th_collections):
    errors = []
    for project in th_collections:

        credentials = OAuthCredentials.get_credentials(project)

        th_request = TreeherderRequest(
            protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
            host=settings.TREEHERDER_REQUEST_HOST,
            project=project,
            oauth_key=credentials.get('consumer_key', None),
            oauth_secret=credentials.get('consumer_secret', None)
        )

        logger.info(
            "collection loading request: {0}".format(
                th_request.get_uri(th_collections[project].endpoint_base)))
        response = th_request.post(th_collections[project])

        if not response or response.status != 200:
            errors.append({
                "project": project,
                "url": th_collections[project].endpoint_base,
                "message": response.read()
            })
    if errors:
        raise CollectionNotLoadedException(errors)

@python_2_unicode_compatible
class CollectionNotLoadedException(Exception):

    def __init__(self, error_list, *args, **kwargs):
        """
        error_list contains dictionaries, each containing
        project, url and message
        """
        super(CollectionNotLoadedException, self).__init__(args, kwargs)
        self.error_list = error_list

    def __str__(self):
        return "\n".join(
            ["[{project}] Error posting data to {url}: {message}".format(
                **error) for error in self.error_list]
        )
<commit_msg>Revert "Bug 1159782 - fix Job ingestion error"<commit_after>import logging

from django.utils.encoding import python_2_unicode_compatible
from django.conf import settings

from treeherder.client import TreeherderRequest

from treeherder.etl.oauth_utils import OAuthCredentials

logger = logging.getLogger(__name__)

def post_treeherder_collections(th_collections):
    errors = []
    for project in th_collections:

        credentials = OAuthCredentials.get_credentials(project)

        th_request = TreeherderRequest(
            protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
            host=settings.TREEHERDER_REQUEST_HOST,
            project=project,
            oauth_key=credentials.get('consumer_key', None),
            oauth_secret=credentials.get('consumer_secret', None)
        )

        logger.info(
            "collection loading request: {0}".format(
                th_request.get_uri(th_collections[project].endpoint_base)))
        response = th_request.post(th_collections[project])

        if not response or response.status == 200:
            errors.append({
                "project": project,
                "url": th_collections[project].endpoint_base,
                "message": response.read()
            })
    if errors:
        raise CollectionNotLoadedException(errors)

@python_2_unicode_compatible
class CollectionNotLoadedException(Exception):

    def __init__(self, error_list, *args, **kwargs):
        """
        error_list contains dictionaries, each containing
        project, url and message
        """
        super(CollectionNotLoadedException, self).__init__(args, kwargs)
        self.error_list = error_list

    def __str__(self):
        return "\n".join(
            ["[{project}] Error posting data to {url}: {message}".format(
                **error) for error in self.error_list]
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2014 Alvaro Lopez Garcia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import abc

from oslo.config import cfg
from oslo.log import log
import yaml

from atrope import cache
import atrope.dispatcher.manager
from atrope import exception
import atrope.image_list.hepix

opts = [
    cfg.StrOpt('image_list_sources',
               default='/etc/atrope/lists.yaml',
               help='Where the image list sources are stored.'),
]

CONF = cfg.CONF
CONF.register_opts(opts)

LOG = log.getLogger(__name__)

class BaseImageListManager(object):
    __metaclass__ = abc.ABCMeta

    def __init__(self, dispatcher=None):
        self.cache_manager = cache.CacheManager()
        self._dispatcher = None

        self.lists = {}
        self._load_sources()

    @property
    def dispatcher(self):
        if self._dispatcher is None:
            self._dispatcher = atrope.dispatcher.manager.DispatcherManager()
        return self._dispatcher

    @abc.abstractmethod
    def _load_sources(self):
        """Load the image sources from disk."""

    @abc.abstractmethod
    def add_image_list_source(self, image):
        """Add an image source to the loaded sources."""

    @abc.abstractmethod
    def write_image_list_sources(self):
        """Write image sources to disk."""

    def _fetch_and_verify(self, l):
        """
        Fetch and verify an image list.

        If there are errors loading the list the appropriate attributes won't
        be set, so there is no need to fail here, but rather return the list.
        """
        try:
            l.fetch()
        except exception.AtropeException as e:
            LOG.error("Error loading list '%s', reason: %s" %
                      (l.name, e.message))
            LOG.debug("Exception while downloading list '%s'" % l.name,
                      exc_info=e)
        return l

    def fetch_list(self, image_list):
        """Fetch (and verify) an individual list."""
        l = self.lists.get(image_list)
        if l is None:
            raise exception.InvalidImageList(reason="not found in config")
        return self._fetch_and_verify(l)

    def fetch_lists(self):
        """Fetch (and verify) all the configured lists."""
        all_lists = []
        for l in self.lists.values():
            l = self._fetch_and_verify(l)
            all_lists.append(l)

        return all_lists

    def sync_cache(self):
        self.fetch_lists()
        self.cache_manager.sync(self.lists)

    def dispatch(self, sync):
        if sync:
            fn = self.dispatcher.dispatch_list_and_sync
        else:
            fn = self.dispatcher.dispatch_list

        for l in self.lists.values():
            fn(l)

class YamlImageListManager(BaseImageListManager):
    def __init__(self):
        super(YamlImageListManager, self).__init__()

    def _load_sources(self):
        """Load sources from YAML file."""

        try:
            with open(CONF.image_list_sources, "rb") as f:
                image_lists = yaml.safe_load(f)
        except IOError as e:
            raise exception.CannotOpenFile(file=CONF.image_list_sources,
                                           errno=e.errno)

        for name, list_meta in image_lists.iteritems():
            l = atrope.image_list.hepix.HepixImageListSource(
                name,
                url=list_meta.get("url", ""),
                enabled=list_meta.get("enabled", True),
                endorser=list_meta.get("endorser", {}),
                token=list_meta.get("token", ""),
                subscribed_images=list_meta.get("images", []),
                prefix=list_meta.get("prefix", "")
            )
            self.lists[name] = l

    def add_image_list_source(self, image_list, force=False):
        """Add an image source to the loaded sources."""
        if image_list.name in self.lists and not force:
            raise exception.DuplicatedImageList(id=image_list.name)

        self.lists[image_list.name] = image_list

    def write_image_list_sources(self):
        """Write images into YAML file."""
        lists = {}
        for name, image_list in self.lists.iteritems():
            lists[name] = {"url": image_list.url,
                           "enabled": image_list.enabled,
                           "endorser": image_list.endorser,
                           "token": image_list.token,
                           "prefix": image_list.prefix,
                           "images": image_list.images}
        dump = yaml.dump(lists)
        if not dump:
            raise exception.AtropeException()

        with open(CONF.image_list_sources, "w") as f:
            f.write(dump)
<commit_msg>list manager: move common method to base class<commit_after># -*- coding: utf-8 -*-

# Copyright 2014 Alvaro Lopez Garcia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import abc

from oslo.config import cfg
from oslo.log import log
import yaml

from atrope import cache
import atrope.dispatcher.manager
from atrope import exception
import atrope.image_list.hepix

opts = [
    cfg.StrOpt('image_list_sources',
               default='/etc/atrope/lists.yaml',
               help='Where the image list sources are stored.'),
]

CONF = cfg.CONF
CONF.register_opts(opts)

LOG = log.getLogger(__name__)

class BaseImageListManager(object):
    __metaclass__ = abc.ABCMeta

    def __init__(self, dispatcher=None):
        self.cache_manager = cache.CacheManager()
        self._dispatcher = None

        self.lists = {}
        self._load_sources()

    @property
    def dispatcher(self):
        if self._dispatcher is None:
            self._dispatcher = atrope.dispatcher.manager.DispatcherManager()
        return self._dispatcher

    @abc.abstractmethod
    def _load_sources(self):
        """Load the image sources from disk."""

    @abc.abstractmethod
    def write_image_list_sources(self):
        """Write image sources to disk."""

    def _fetch_and_verify(self, l):
        """
        Fetch and verify an image list.

        If there are errors loading the list the appropriate attributes won't
        be set, so there is no need to fail here, but rather return the list.
        """
        try:
            l.fetch()
        except exception.AtropeException as e:
            LOG.error("Error loading list '%s', reason: %s" %
                      (l.name, e.message))
            LOG.debug("Exception while downloading list '%s'" % l.name,
                      exc_info=e)
        return l

    def add_image_list_source(self, image_list, force=False):
        """Add an image source to the loaded sources."""
        if image_list.name in self.lists and not force:
            raise exception.DuplicatedImageList(id=image_list.name)

        self.lists[image_list.name] = image_list

    def fetch_list(self, image_list):
        """Fetch (and verify) an individual list."""
        l = self.lists.get(image_list)
        if l is None:
            raise exception.InvalidImageList(reason="not found in config")
        return self._fetch_and_verify(l)

    def fetch_lists(self):
        """Fetch (and verify) all the configured lists."""
        all_lists = []
        for l in self.lists.values():
            l = self._fetch_and_verify(l)
            all_lists.append(l)

        return all_lists

    def sync_cache(self):
        self.fetch_lists()
        self.cache_manager.sync(self.lists)

    def dispatch(self, sync):
        if sync:
            fn = self.dispatcher.dispatch_list_and_sync
        else:
            fn = self.dispatcher.dispatch_list

        for l in self.lists.values():
            fn(l)

class YamlImageListManager(BaseImageListManager):
    def __init__(self):
        super(YamlImageListManager, self).__init__()

    def _load_sources(self):
        """Load sources from YAML file."""

        try:
            with open(CONF.image_list_sources, "rb") as f:
                image_lists = yaml.safe_load(f)
        except IOError as e:
            raise exception.CannotOpenFile(file=CONF.image_list_sources,
                                           errno=e.errno)

        for name, list_meta in image_lists.iteritems():
            l = atrope.image_list.hepix.HepixImageListSource(
                name,
                url=list_meta.get("url", ""),
                enabled=list_meta.get("enabled", True),
                endorser=list_meta.get("endorser", {}),
                token=list_meta.get("token", ""),
                subscribed_images=list_meta.get("images", []),
                prefix=list_meta.get("prefix", "")
            )
            self.lists[name] = l

    def write_image_list_sources(self):
        """Write images into YAML file."""
        lists = {}
        for name, image_list in self.lists.iteritems():
            lists[name] = {"url": image_list.url,
                           "enabled": image_list.enabled,
                           "endorser": image_list.endorser,
                           "token": image_list.token,
                           "prefix": image_list.prefix,
                           "images": image_list.images}
        dump = yaml.dump(lists)
        if not dump:
            raise exception.AtropeException()

        with open(CONF.image_list_sources, "w") as f:
            f.write(dump)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os
from optparse import OptionParser
from string import maketrans
from subprocess import check_call

TYPE = ""
TMPDIR = "."
DATAPATH = "."

def idfiletype(fname):
    with open(fname) as p:
       firstline = p.readline()
       if firstline[0] == ">":
           return "FASTA"
       if firstline[0] == "@":
           return "FASTQ"
       sys.exit("Cannot identify sequence file type")

def idvector(fname):
    '''Run bowtie2 on dataset against library of adapters; create intermediate file with sorted list of adapter names'''
    if TYPE == "FASTA":
        OPTIONS = "-f"
    else:
        OPTIONS = "-q"
    if not os.path.exists(DATAPATH+"/vectors-P5.4.bt2"):
        sys.stderr.write("Can't find bowtie2 index in data directory!\n")
        sys.exit(1)
    check_call("bowtie2 -x {}/vectors-P5 {} {} --no-head --local --upto 2000000 -p 4 > {}.P5.tmp 2> {}.P5.err".format(DATAPATH, OPTIONS, fname, fname, fname), shell=True)
    check_call("bowtie2 -x {}/vectors-P7 {} {} --no-head --local --upto 2000000 -p 4 > {}.P7.tmp 2> {}.P7.err".format(DATAPATH, OPTIONS, fname, fname, fname), shell=True)
    check_call("cut -f 3  {}.P5.tmp | grep -v '*' | head -n 100000 | sort | uniq -c | awk '{{print $2 \"\t\" $1}}' | sort -k 2 -n -r > {}.P5.csv".format(fname, fname), shell=True)
    check_call("cut -f 3  {}.P7.tmp | grep -v '*' | head -n 100000 | sort | uniq -c | awk '{{print $2 \"\t\" $1}}' | sort -k 2 -n -r > {}.P7.csv".format(fname, fname), shell=True)
    if not opts.verbose:
        os.remove(filename+".P5.tmp"); os.remove(filename+".P7.tmp"); os.remove(filename+".P5.err"); os.remove(filename+".P7.err")
    return

def revc(s):
    '''reverse complement a string'''
    t = " " * len(s)
    l = len(s)
    intab = "AaCcGgTt"
    outtab = "TtGgCcAa"
    trantab = maketrans(intab, outtab)
    t = s.translate(trantab)[::-1]
    return t

def write_adapter_fasta(filename, namesandadapters):
    '''Writes a fasta file containing a handful of sequence names, sequences'''
    fh = open(filename, 'w')
    assert len(namesandadapters) % 2 == 0
    for i in range(len(namesandadapters)/2):
        if len(namesandadapters[2*i + 1]) > 0:
            fh.write(">"+namesandadapters[2*i]+"\n" +
                     namesandadapters[2*i + 1] + "\n")
    fh.close()

def read_fasta_to_table(filen):
    '''Loads a simple fasta file into a dictionary'''
    table = {}
    try:
        FILE = open(filen)
    except IOError:
        sys.stderr.write("Can't open file "+ filen + "!\n")
        sys.exit(1)
    for line in FILE:
        if line[0] == ">":
            header = line.strip()[1:].split()[0]
        else:
            sequence = line.strip()
            table[header] = sequence
    return table

def grab_first_field(filen):
    '''Opens file, returns the header name of the first field'''
    try:
        contents = open(filen).readlines()
    except IOError:
        sys.stderr.write("Can't open file "+ filen + "!\n")
        sys.exit(1)
    try:
        firstfield = contents[0].strip().split("\t")[0]
        return firstfield
    except IndexError:
        return ""

if __name__ == '__main__':
    global TYPE, TMPDIR, DATAPATH
    usage = "usage: %prog -i <input seqfile> -o <scrubbed seqfile>\nExamines FASTA or FASTQ for barcodes"
    parser = OptionParser(usage)
    parser.add_option("-i", "--input", dest="input", default=None, help="Input sequence file.")
    parser.add_option("-o", "--output", dest="output", default=None, help="Output scrubbed file.")
    parser.add_option("-l", "--logfile", dest="logfile", default=None, help="Output log file [default STDOUT]")
    parser.add_option("-t", "--tmpdir", dest="tmpdir", default=".", help="DIR for intermediate files [default CWD]")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Verbose [default off]")

    (opts, args) = parser.parse_args()
    if not (opts.input and os.path.isfile(opts.input):
        parser.error("Missing input file")
    if not opts.output:
        parser.error("Missing output name")
    filename = opts.input
    if filename[-6:-1] == ".fast":
        filestem = filename[:-6]
    else:
        filestem = filename

    TYPE = idfiletype(filename)
    TMPDIR = opts.tmpdir
    DATAPATH = os.path.dirname(os.path.realpath(sys.argv[0]))
    PATHPREFIX = os.path.join(TMPDIR, os.path.basename(filestem))

    idvector(filename)
    P5table = read_fasta_to_table(DATAPATH + "/data/vectors-P5.fa")
    P7table = read_fasta_to_table(DATAPATH + "/data/vectors-P7.fa")
    P5table[""] = ""
    P7table[""] = ""
    P5adaptername = grab_first_field(PATHPREFIX + ".P5.csv")
    P7adaptername = grab_first_field(PATHPREFIX + ".P7.csv")
    P5adapter = P5table[P5adaptername]
    P7adapter = P7table[P7adaptername]
    P5r = revc(P5adapter)
    P7r = revc(P7adapter)
    if opts.verbose:
        print P5adapter
        print P5r
        print P7adapter
        print P7r

    adaptorfile = PATHPREFIX + ".adapter.fa"
    skewoutname = PATHPREFIX + ".4"
    skewoptions = "-k 5 -l 0 --quiet -t 4 -r .2 -m any"
    write_adapter_fasta(adaptorfile,
                        [P5adaptername, P5adapter, P5adaptername+"_R",
                        P5r, P7adaptername, P7adapter,
                        P7adaptername+"_R", P7r])
    skewcmd = "skewer -x %s %s %s -o %s"%(adaptorfile, skewoptions, filename, skewoutname)
    if opts.verbose:
        print skewcmd
    check_call(skewcmd.split(" "))

    os.rename(skewoutname."-trimmed.fastq", opts.output)
    if (opts.logfile):
        os.rename(skewoutname."-trimmed.log", opts.logfile)
    else:
        print open(skewoutname."-trimmed.log", 'r').read()

    if not opts.verbose:
        os.remove(PATHPREFIX+".P5.csv"); os.remove(PATHPREFIX+".P7.csv");
<commit_msg>pathing fixes<commit_after>#!/usr/bin/env python

import sys, os
from optparse import OptionParser
from string import maketrans
from subprocess import check_call

def idfiletype(fname):
    with open(fname) as p:
       firstline = p.readline()
       if firstline[0] == ">":
           return "FASTA"
       if firstline[0] == "@":
           return "FASTQ"
       sys.exit("Cannot identify sequence file type")

def idvector(fname):
    '''Run bowtie2 on dataset against library of adapters; create intermediate file with sorted list of adapter names'''
    if TYPE == "FASTA":
        OPTIONS = "-f"
    else:
        OPTIONS = "-q"
    if not os.path.exists(DATAPATH+"/vectors-P5.4.bt2"):
        sys.stderr.write("Can't find bowtie2 index in data directory!\n")
        sys.exit(1)
    check_call("bowtie2 -x {}/vectors-P5 {} {} --no-head --local --upto 2000000 -p 4 > {}.P5.tmp 2> {}.P5.err".format(DATAPATH, OPTIONS, fname, fname, fname), shell=True)
    check_call("bowtie2 -x {}/vectors-P7 {} {} --no-head --local --upto 2000000 -p 4 > {}.P7.tmp 2> {}.P7.err".format(DATAPATH, OPTIONS, fname, fname, fname), shell=True)
    check_call("cut -f 3  {}.P5.tmp | grep -v '*' | head -n 100000 | sort | uniq -c | awk '{{print $2 \"\t\" $1}}' | sort -k 2 -n -r > {}.P5.csv".format(fname, fname), shell=True)
    check_call("cut -f 3  {}.P7.tmp | grep -v '*' | head -n 100000 | sort | uniq -c | awk '{{print $2 \"\t\" $1}}' | sort -k 2 -n -r > {}.P7.csv".format(fname, fname), shell=True)
    if not opts.verbose:
        os.remove(filename+".P5.tmp"); os.remove(filename+".P7.tmp"); os.remove(filename+".P5.err"); os.remove(filename+".P7.err")
    return

def revc(s):
    '''reverse complement a string'''
    t = " " * len(s)
    l = len(s)
    intab = "AaCcGgTt"
    outtab = "TtGgCcAa"
    trantab = maketrans(intab, outtab)
    t = s.translate(trantab)[::-1]
    return t

def write_adapter_fasta(filename, namesandadapters):
    '''Writes a fasta file containing a handful of sequence names, sequences'''
    fh = open(filename, 'w')
    assert len(namesandadapters) % 2 == 0
    for i in range(len(namesandadapters)/2):
        if len(namesandadapters[2*i + 1]) > 0:
            fh.write(">"+namesandadapters[2*i]+"\n" +
                     namesandadapters[2*i + 1] + "\n")
    fh.close()

def read_fasta_to_table(filen):
    '''Loads a simple fasta file into a dictionary'''
    table = {}
    try:
        FILE = open(filen)
    except IOError:
        sys.stderr.write("Can't open file "+ filen + "!\n")
        sys.exit(1)
    for line in FILE:
        if line[0] == ">":
            header = line.strip()[1:].split()[0]
        else:
            sequence = line.strip()
            table[header] = sequence
    return table

def grab_first_field(filen):
    '''Opens file, returns the header name of the first field'''
    try:
        contents = open(filen).readlines()
    except IOError:
        sys.stderr.write("Can't open file "+ filen + "!\n")
        sys.exit(1)
    try:
        firstfield = contents[0].strip().split("\t")[0]
        return firstfield
    except IndexError:
        return ""

if __name__ == '__main__':
    usage = "usage: %prog -i <input seqfile> -o <scrubbed seqfile>\nExamines FASTA or FASTQ for barcodes"
    parser = OptionParser(usage)
    parser.add_option("-i", "--input", dest="input", default=None, help="Input sequence file.")
    parser.add_option("-o", "--output", dest="output", default=None, help="Output scrubbed file.")
    parser.add_option("-l", "--logfile", dest="logfile", default=None, help="Output log file [default STDOUT]")
    parser.add_option("-t", "--tmpdir", dest="tmpdir", default=".", help="DIR for intermediate files [default CWD]")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Verbose [default off]")

    (opts, args) = parser.parse_args()
    if not (opts.input and os.path.isfile(opts.input)):
        parser.error("Missing input file")
    if not opts.output:
        parser.error("Missing output name")
    filename = opts.input
    if filename[-6:-1] == ".fast":
        filestem = filename[:-6]
    else:
        filestem = filename

    TYPE = idfiletype(filename)
    TMPDIR = opts.tmpdir
    DATAPATH = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "data")
    PATHPREFIX = os.path.join(TMPDIR, os.path.basename(filestem))

    idvector(filename)
    P5table = read_fasta_to_table(os.path.join(DATAPATH, "/vectors-P5.fa"))
    P7table = read_fasta_to_table(os.path.join(DATAPATH, "/vectors-P7.fa"))
    P5table[""] = ""
    P7table[""] = ""
    P5adaptername = grab_first_field(PATHPREFIX + ".P5.csv")
    P7adaptername = grab_first_field(PATHPREFIX + ".P7.csv")
    P5adapter = P5table[P5adaptername]
    P7adapter = P7table[P7adaptername]
    P5r = revc(P5adapter)
    P7r = revc(P7adapter)
    if opts.verbose:
        print P5adapter
        print P5r
        print P7adapter
        print P7r

    adaptorfile = PATHPREFIX + ".adapter.fa"
    skewoutname = PATHPREFIX + ".4"
    skewoptions = "-k 5 -l 0 --quiet -t 4 -r .2 -m any"
    write_adapter_fasta(adaptorfile,
                        [P5adaptername, P5adapter, P5adaptername+"_R",
                        P5r, P7adaptername, P7adapter,
                        P7adaptername+"_R", P7r])
    skewcmd = "skewer -x %s %s %s -o %s"%(adaptorfile, skewoptions, filename, skewoutname)
    if opts.verbose:
        print skewcmd
    check_call(skewcmd.split(" "))

    os.rename(skewoutname+"-trimmed.fastq", opts.output)
    if (opts.logfile):
        os.rename(skewoutname+"-trimmed.log", opts.logfile)
    else:
        print open(skewoutname+"-trimmed.log", 'r').read()

    if not opts.verbose:
        os.remove(PATHPREFIX+".P5.csv"); os.remove(PATHPREFIX+".P7.csv");
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Flask, jsonify, request, Response
from crossrefquery import *
import json

#test
# stuff = 'book' #book, dissertation, monograph, journal-article, book-chapter
# test = ['Cold War']
# z = requestarticles(stuff, test)
# print z

app = Flask(__name__)

@app.route('/v1') #http://127.0.0.1:5000/v1?words=Stalin
def hello_world():
    cat = (request.args.get('cat'))
    words = [request.args.get('words')]
    output = requestarticles(cat, words)
    length = len(output)
    resp = Response(response=json.dumps(output), status=200, mimetype="application/json")
    return resp

if __name__ == '__main__':
    app.run(host='0.0.0.0', debug = False)
    #app.run(debug = True)
<commit_msg>serialiser<commit_after># -*- coding: utf-8 -*-
from flask import Flask, jsonify, request
from crossrefquery import *
import json

#test
# stuff = 'book' #book, dissertation, monograph, journal-article, book-chapter
# test = ['Cold War']
# z = requestarticles(stuff, test)
# print z

app = Flask(__name__)

@app.route('/v1') #http://127.0.0.1:5000/v1?words=Stalin
def hello_world():
    cat = (request.args.get('cat'))
    words = [request.args.get('words')]
    output = requestarticles(cat, words)
    length = len(output)
    return jsonify({'length': length, 'results': output})

if __name__ == '__main__':
    #app.run(host='0.0.0.0', debug = False)
    app.run(debug = True)
<|endoftext|>


<commit_before>"""
Functions which are not listed in the maya documentation, such as commands created by plugins,
as well as the name parsing classes `DependNodeName`, `DagNodeName`, and `AttributeName`.
"""

import pmcmds as cmds
import re

import factories as _factories
_factories.createFunctions( __name__ )

#--------------------------
# Object Wrapper Classes
#--------------------------

class NameParser(unicode):
    def __new__(cls, strObj):
        """Casts a string to a pymel class. Use this function if you are unsure which class is the right one to use
        for your object."""

        # the if statement was failing for some types (ex: pymel.node.Vertex),
        # so forcing into unicode string:

        newcls = _getParserClass(strObj)
        self = super(NameParser, cls).__new__(newcls, strObj)
        return self

    def __repr__(self):
        return u"%s('%s')" % (self.__class__.__name__, self)

    #def __unicode__(self):
    #    return u"%s" % self

    def __getattr__(self, attr):
        """
            >>> NameParser('foo:bar').spangle
            AttributeName('foo:bar.spangle')

        """
        if attr.startswith('__') and attr.endswith('__'):
            return super(NameParser, self).__getattr__(attr)

        return AttributeName( '%s.%s' % (self, attr) )

        #raise AttributeNameError, 'AttributeName does not exist %s' % attr

    def stripNamespace(self, levels=0):
        """
        Returns a new instance of the object with its namespace removed.  The calling instance is unaffected.
        The optional levels keyword specifies how many levels of cascading namespaces to strip, starting with the topmost (leftmost).
        The default is 0 which will remove all namespaces.

            >>> NameParser('foo:bar.spangle').stripNamespace()
            AttributeName('bar.spangle')

        """

        nodes = []
        for x in self.split('|'):
            y = x.split('.')
            z = y[0].split(':')
            if levels:
                y[0] = ':'.join( z[min(len(z)-1,levels):] )

            else:
                y[0] = z[-1]
            nodes.append( '.'.join( y ) )
        return self.__class__( '|'.join( nodes) )

    def swapNamespace(self, prefix):
        """Returns a new instance of the object with its current namespace replaced with the provided one.
        The calling instance is unaffected."""
        return self.__class__.addPrefix( self.stripNamespace(), prefix+':' )

    def namespaceList(self):
        """Useful for cascading references.  Returns all of the namespaces of the calling object as a list"""
        return self.lstrip('|').rstrip('|').split('|')[-1].split(':')[:-1]

    def namespace(self):
        """Returns the namespace of the object with trailing colon included"""
        return ':'.join(self.namespaceList()) + ':'

    def addPrefix(self, prefix):
        'addPrefixToName'
        name = self
        leadingSlash = False
        if name.startswith('|'):
            name = name[1:]
            leadingSlash = True
        name = self.__class__( '|'.join( map( lambda x: prefix+x, name.split('|') ) ) )
        if leadingSlash:
            name = '|' + name
        return self.__class__( name )

    def attr(self, attr):
        """access to AttributeName of a node. returns an instance of the AttributeName class for the
        given AttributeName.

            >>> NameParser('foo:bar').attr('spangle')
            AttributeName('foo:bar.spangle')

        """
        return AttributeName( '%s.%s' % (self, attr) )

class AttributeName(NameParser):
    """

    """
    attrItemReg = re.compile( '\[(\d+:*\d*)\]$')

    #def __repr__(self):
    #    return "AttributeName('%s')" % self

    def __init__(self, attrName):
        if '.' not in attrName:
            raise TypeError, "%s: AttributeNames must include the node and the AttributeName. e.g. 'nodeName.AttributeNameName' " % self
        self.__dict__['_multiattrIndex'] = 0

    def __getitem__(self, item):
        return AttributeName('%s[%s]' % (self, item) )

    # Added the __call__ so to generate a more appropriate exception when a class method is not found
    def __call__(self, *args, **kwargs):
        raise TypeError("The object <%s> does not support the '%s' method" % (repr(self.node()), self.plugAttr()))

    def array(self):
        """
        Returns the array (multi) AttributeName of the current element
            >>> n = AttributeName('lambert1.groupNodes[0]')
            >>> n.array()
            'lambert1.groupNode'
        """
        try:
            return AttributeName(AttributeName.attrItemReg.split( self )[0])
        except:
            raise TypeError, "%s is not a multi AttributeName" % self

    def plugNode(self):
        """plugNode

        >>> NameParser('foo:bar.spangle.banner').plugNode()
        DependNodeName('foo:bar')

        """
        return NameParser( unicode(self).split('.')[0])

    node = plugNode

    def plugAttr(self):
        """plugAttr

        >>> NameParser('foo:bar.spangle.banner').plugAttr()
        u'spangle.banner'

        """
        return '.'.join(unicode(self).split('.')[1:])

    def lastPlugAttr(self):
        """
        >>> NameParser('foo:bar.spangle.banner').plugAttr()
        u'banner'

        """
        return self.split('.')[-1]

    def nodeName( self ):
        'basename'
        return self.split('|')[-1]

    def item(self, asSlice=False, asString=False):
        try:
            item = AttributeName.attrItemReg.search(self).group(1)
            if asString:
                return "[%s]" % unicode(item)
            val = item.split(":")
            val = map(int,val)
            if len(val)>1:
                return asSlice and slice(*val) or val
            return val[0]
        except: return None

    def getParent(self):
        """
        Returns the parent attribute
        """
        if self.count('.') > 1:
            return AttributeName('.'.join(self.split('.')[:-1]))

    def addAttr( self, **kwargs):
        kwargs['longName'] = self.plugAttr()
        kwargs.pop('ln', None )
        return addAttr( self.node(), **kwargs )

    def setAttr(self, *args, **kwargs):
        from pymel.core.general import setAttr
        return setAttr(self, *args, **kwargs)

    set = setAttr
    add = addAttr

class DependNodeName( NameParser ):
    #-------------------------------
    #    Name Info and Manipulation
    #-------------------------------

    def node(self):
        """for compatibility with AttributeName class"""
        return self

    def exists(self, **kwargs):
        "objExists"
        return cmds.objExists(**kwargs)

    _numPartReg = re.compile('([0-9]+)$')

    def stripNum(self):
        """Return the name of the node with trailing numbers stripped off. If no trailing numbers are found
        the name will be returned unchanged."""
        try:
            return DependNodeName._numPartReg.split(self)[0]
        except:
            return unicode(self)

    def extractNum(self):
        """Return the trailing numbers of the node name. If no trailing numbers are found
        an error will be raised."""

        try:
            return DependNodeName._numPartReg.split(self)[1]
        except:
            raise "No trailing numbers to extract on object ", self

    def nextUniqueName(self):
        """Increment the trailing number of the object until a unique name is found"""
        name = self.shortName().nextName()
        while name.exists():
            name = name.nextName()
        return name

    def nextName(self):
        """Increment the trailing number of the object by 1"""
        try:
            groups = DependNodeName._numPartReg.split(self)
            num = groups[1]
            formatStr = '%s%0' + unicode(len(num)) + 'd'
            return self.__class__(formatStr % ( groups[0], (int(num) + 1) ))
        except:
            raise "could not find trailing numbers to increment"

    def prevName(self):
        """Decrement the trailing number of the object by 1"""
        try:
            groups = DependNodeName._numPartReg.split(self)
            num = groups[1]
            formatStr = '%s%0' + unicode(len(num)) + 'd'
            return self.__class__(formatStr % ( groups[0], (int(num) - 1) ))
        except:
            raise "could not find trailing numbers to decrement"

class DagNodeName(DependNodeName):

#    def __eq__(self, other):
#        """ensures that we compare longnames when checking for dag node equality"""
#        try:
#            return unicode(self.longName()) == unicode(DagNodeName(other).longName())
#        except (TypeError,IndexError):
#            return unicode(self) == unicode(other)
#
#    def __ne__(self, other):
#        """ensures that we compare longnames when checking for dag node equality"""
#        try:
#            return unicode(self.longName()) != unicode(DagNodeName(other).longName())
#        except (TypeError,IndexError):
#            return unicode(self) != unicode(other)

    #--------------------------
    #    DagNodeName Path Info
    #--------------------------
    def root(self):
        'rootOf'
        return DagNodeName( '|' + self.longName()[1:].split('|')[0] )

    def getRoot(self):
        """unlike the root command which determines the parent via string formatting, this
        command uses the listRelatives command"""

        par = None
        cur = self
        while 1:
            par = cur.getParent()
            if not par:
                break
            cur = par
        return cur

    def firstParent(self):
        'firstParentOf'

        return DagNodeName( '|'.join( self.split('|')[:-1] ) )

    def getParent(self):
        'firstParentOf'

        return DagNodeName( '|'.join( self.split('|')[:-1] ) )

#    def shortName( self ):
#        'shortNameOf'
#        try:
#            return self.__class__( cmds.ls( self )[0] )
#        except:
#            return self

    def nodeName( self ):
        'basename'
        return self.split('|')[-1]

def _getParserClass(strObj):
    strObj = unicode(strObj)

    if '.' in strObj:
        newcls = AttributeName
            # Return Component Arrays ======================================================
            #            attr = obj.array().plugAttr()
            #            if attr in ["f","vtx","e","map"]:
            #                comps = getattr(Mesh(obj.node()), attr)
            #                return comps.__getitem__(obj.item(asSlice=1))
            #            else:
            #                return obj
            #===============================================================================

    elif '|' in strObj:
        newcls = DagNodeName
    else:
        newcls = DependNodeName
    return newcls

 <commit_msg>bugfix to DependNodeName.namespace<commit_after>"""
Functions which are not listed in the maya documentation, such as commands created by plugins,
as well as the name parsing classes `DependNodeName`, `DagNodeName`, and `AttributeName`.
"""

import pmcmds as cmds
import re

import factories as _factories
_factories.createFunctions( __name__ )

#--------------------------
# Object Wrapper Classes
#--------------------------

class NameParser(unicode):
    def __new__(cls, strObj):
        """Casts a string to a pymel class. Use this function if you are unsure which class is the right one to use
        for your object."""

        # the if statement was failing for some types (ex: pymel.node.Vertex),
        # so forcing into unicode string:

        newcls = _getParserClass(strObj)
        self = super(NameParser, cls).__new__(newcls, strObj)
        return self

    def __repr__(self):
        return u"%s('%s')" % (self.__class__.__name__, self)

    #def __unicode__(self):
    #    return u"%s" % self

    def __getattr__(self, attr):
        """
            >>> NameParser('foo:bar').spangle
            AttributeName('foo:bar.spangle')

        """
        if attr.startswith('__') and attr.endswith('__'):
            return super(NameParser, self).__getattr__(attr)

        return AttributeName( '%s.%s' % (self, attr) )

        #raise AttributeNameError, 'AttributeName does not exist %s' % attr

    def stripNamespace(self, levels=0):
        """
        Returns a new instance of the object with its namespace removed.  The calling instance is unaffected.
        The optional levels keyword specifies how many levels of cascading namespaces to strip, starting with the topmost (leftmost).
        The default is 0 which will remove all namespaces.

            >>> NameParser('foo:bar.spangle').stripNamespace()
            AttributeName('bar.spangle')

        """

        nodes = []
        for x in self.split('|'):
            y = x.split('.')
            z = y[0].split(':')
            if levels:
                y[0] = ':'.join( z[min(len(z)-1,levels):] )

            else:
                y[0] = z[-1]
            nodes.append( '.'.join( y ) )
        return self.__class__( '|'.join( nodes) )

    def swapNamespace(self, prefix):
        """Returns a new instance of the object with its current namespace replaced with the provided one.
        The calling instance is unaffected."""
        return self.__class__.addPrefix( self.stripNamespace(), prefix+':' )

    def namespaceList(self):
        """Useful for cascading references.  Returns all of the namespaces of the calling object as a list"""
        return self.lstrip('|').rstrip('|').split('|')[-1].split(':')[:-1]

    def namespace(self):
        """Returns the namespace of the object with trailing colon included"""
        nsList = self.namespaceList()
        if nsList:
            return  ':'.join(nsList) + ':'
        return ''

    def addPrefix(self, prefix):
        'addPrefixToName'
        name = self
        leadingSlash = False
        if name.startswith('|'):
            name = name[1:]
            leadingSlash = True
        name = self.__class__( '|'.join( map( lambda x: prefix+x, name.split('|') ) ) )
        if leadingSlash:
            name = '|' + name
        return self.__class__( name )

    def attr(self, attr):
        """access to AttributeName of a node. returns an instance of the AttributeName class for the
        given AttributeName.

            >>> NameParser('foo:bar').attr('spangle')
            AttributeName('foo:bar.spangle')

        """
        return AttributeName( '%s.%s' % (self, attr) )

class AttributeName(NameParser):
    """

    """
    attrItemReg = re.compile( '\[(\d+:*\d*)\]$')

    #def __repr__(self):
    #    return "AttributeName('%s')" % self

    def __init__(self, attrName):
        if '.' not in attrName:
            raise TypeError, "%s: AttributeNames must include the node and the AttributeName. e.g. 'nodeName.AttributeNameName' " % self
        self.__dict__['_multiattrIndex'] = 0

    def __getitem__(self, item):
        return AttributeName('%s[%s]' % (self, item) )

    # Added the __call__ so to generate a more appropriate exception when a class method is not found
    def __call__(self, *args, **kwargs):
        raise TypeError("The object <%s> does not support the '%s' method" % (repr(self.node()), self.plugAttr()))

    def array(self):
        """
        Returns the array (multi) AttributeName of the current element
            >>> n = AttributeName('lambert1.groupNodes[0]')
            >>> n.array()
            'lambert1.groupNode'
        """
        try:
            return AttributeName(AttributeName.attrItemReg.split( self )[0])
        except:
            raise TypeError, "%s is not a multi AttributeName" % self

    def plugNode(self):
        """plugNode

        >>> NameParser('foo:bar.spangle.banner').plugNode()
        DependNodeName('foo:bar')

        """
        return NameParser( unicode(self).split('.')[0])

    node = plugNode

    def plugAttr(self):
        """plugAttr

        >>> NameParser('foo:bar.spangle.banner').plugAttr()
        u'spangle.banner'

        """
        return '.'.join(unicode(self).split('.')[1:])

    def lastPlugAttr(self):
        """
        >>> NameParser('foo:bar.spangle.banner').plugAttr()
        u'banner'

        """
        return self.split('.')[-1]

    def nodeName( self ):
        'basename'
        return self.split('|')[-1]

    def item(self, asSlice=False, asString=False):
        try:
            item = AttributeName.attrItemReg.search(self).group(1)
            if asString:
                return "[%s]" % unicode(item)
            val = item.split(":")
            val = map(int,val)
            if len(val)>1:
                return asSlice and slice(*val) or val
            return val[0]
        except: return None

    def getParent(self):
        """
        Returns the parent attribute
        """
        if self.count('.') > 1:
            return AttributeName('.'.join(self.split('.')[:-1]))

    def addAttr( self, **kwargs):
        kwargs['longName'] = self.plugAttr()
        kwargs.pop('ln', None )
        return addAttr( self.node(), **kwargs )

    def setAttr(self, *args, **kwargs):
        from pymel.core.general import setAttr
        return setAttr(self, *args, **kwargs)

    set = setAttr
    add = addAttr

class DependNodeName( NameParser ):
    #-------------------------------
    #    Name Info and Manipulation
    #-------------------------------

    def node(self):
        """for compatibility with AttributeName class"""
        return self

    def exists(self, **kwargs):
        "objExists"
        return cmds.objExists(**kwargs)

    _numPartReg = re.compile('([0-9]+)$')

    def stripNum(self):
        """Return the name of the node with trailing numbers stripped off. If no trailing numbers are found
        the name will be returned unchanged."""
        try:
            return DependNodeName._numPartReg.split(self)[0]
        except:
            return unicode(self)

    def extractNum(self):
        """Return the trailing numbers of the node name. If no trailing numbers are found
        an error will be raised."""

        try:
            return DependNodeName._numPartReg.split(self)[1]
        except:
            raise "No trailing numbers to extract on object ", self

    def nextUniqueName(self):
        """Increment the trailing number of the object until a unique name is found"""
        name = self.shortName().nextName()
        while name.exists():
            name = name.nextName()
        return name

    def nextName(self):
        """Increment the trailing number of the object by 1"""
        try:
            groups = DependNodeName._numPartReg.split(self)
            num = groups[1]
            formatStr = '%s%0' + unicode(len(num)) + 'd'
            return self.__class__(formatStr % ( groups[0], (int(num) + 1) ))
        except:
            raise "could not find trailing numbers to increment"

    def prevName(self):
        """Decrement the trailing number of the object by 1"""
        try:
            groups = DependNodeName._numPartReg.split(self)
            num = groups[1]
            formatStr = '%s%0' + unicode(len(num)) + 'd'
            return self.__class__(formatStr % ( groups[0], (int(num) - 1) ))
        except:
            raise "could not find trailing numbers to decrement"

class DagNodeName(DependNodeName):

#    def __eq__(self, other):
#        """ensures that we compare longnames when checking for dag node equality"""
#        try:
#            return unicode(self.longName()) == unicode(DagNodeName(other).longName())
#        except (TypeError,IndexError):
#            return unicode(self) == unicode(other)
#
#    def __ne__(self, other):
#        """ensures that we compare longnames when checking for dag node equality"""
#        try:
#            return unicode(self.longName()) != unicode(DagNodeName(other).longName())
#        except (TypeError,IndexError):
#            return unicode(self) != unicode(other)

    #--------------------------
    #    DagNodeName Path Info
    #--------------------------
    def root(self):
        'rootOf'
        return DagNodeName( '|' + self.longName()[1:].split('|')[0] )

    def getRoot(self):
        """unlike the root command which determines the parent via string formatting, this
        command uses the listRelatives command"""

        par = None
        cur = self
        while 1:
            par = cur.getParent()
            if not par:
                break
            cur = par
        return cur

    def firstParent(self):
        'firstParentOf'

        return DagNodeName( '|'.join( self.split('|')[:-1] ) )

    def getParent(self):
        'firstParentOf'

        return DagNodeName( '|'.join( self.split('|')[:-1] ) )

#    def shortName( self ):
#        'shortNameOf'
#        try:
#            return self.__class__( cmds.ls( self )[0] )
#        except:
#            return self

    def nodeName( self ):
        'basename'
        return self.split('|')[-1]

def _getParserClass(strObj):
    strObj = unicode(strObj)

    if '.' in strObj:
        newcls = AttributeName
            # Return Component Arrays ======================================================
            #            attr = obj.array().plugAttr()
            #            if attr in ["f","vtx","e","map"]:
            #                comps = getattr(Mesh(obj.node()), attr)
            #                return comps.__getitem__(obj.item(asSlice=1))
            #            else:
            #                return obj
            #===============================================================================

    elif '|' in strObj:
        newcls = DagNodeName
    else:
        newcls = DependNodeName
    return newcls

 <|endoftext|>


<commit_before>#! /usr/bin/env python

# Hugo Chargois - 17 jan. 2010 - v.0.1
# Parses the output of iwlist scan into a table

# You can add or change the functions to parse the properties
# of each AP (cell) below. They take one argument, the bunch of text
# describing one cell in iwlist scan and return a property of that cell.

import re
import subprocess

def get_name(cell):
    """ Gets the name / essid of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The name / essid of the network.
    """

    return matching_line(cell, "ESSID:")[1:-1]

def get_quality(cell):
    """ Gets the quality of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The quality of the network.
    """

    quality = matching_line(cell, "Quality=").split()[0].split("/")
    return str(int(round(float(quality[0]) / float(quality[1]) * 100)))

def get_signal_level(cell):
    """ Gets the signal level of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The signal level of the network.
    """

    signal = matching_line(cell, "Signal level=").split("=")[1].split("/")
    if len(signal) == 2:
        return str(int(round(float(signal[0]) / float(signal[1]) * 100)))
    elif len(signal) == 1:
        return signal[0]
    else:
        return ""

def get_channel(cell):
    """ Gets the channel of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The channel of the network.
    """

    channel = matching_line(cell, "Channel:")
    if channel:
        return channel
    frequency = matching_line(cell, "Frequency:")
    channel = re.sub(r".*\(Channel\s(\d{1,2})\).*", r"\1", frequency)
    return channel

def get_encryption(cell):
    """ Gets the encryption type of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The encryption type of the network.
    """

    enc = ""
    if matching_line(cell, "Encryption key:") == "off":
        enc = "Open"
    else:
        for line in cell:
            matching = match(line,"IE:")
            if matching != None:
                wpa = match(matching,"WPA")
                if wpa != None:
                    enc = wpa
        if enc == "":
            enc = "WEP"
    return enc

def get_address(cell):
    """ Gets the address of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The address of the network.
    """

    return matching_line(cell, "Address: ")

def get_bit_rates(cell):
    """ Gets the bit rate of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The bit rate of the network.
    """

    return matching_line(cell, "Bit Rates:")

# Here you can choose the way of sorting the table. sortby should be a key of
# the dictionary rules.

def sort_cells(cells):
    sortby = "Quality"
    reverse = True
    cells.sort(None, lambda el:el[sortby], reverse)

# Below here goes the boring stuff. You shouldn't have to edit anything below
# this point

def matching_line(lines, keyword):
    """ Returns the first matching line in a list of lines.
    @see match()
    """
    for line in lines:
        matching = match(line,keyword)
        if matching != None:
            return matching
    return None

def match(line, keyword):
    """ If the first part of line (modulo blanks) matches keyword,
    returns the end of that line. Otherwise checks if keyword is
    anywhere in the line and returns that section, else returns None"""

    line = line.lstrip()
    length = len(keyword)
    if line[:length] == keyword:
        return line[length:]
    else:
        if keyword in line:
            return line[line.index(keyword):]
        else:
            return None

def parse_cell(cell, rules):
    """ Applies the rules to the bunch of text describing a cell.
    @param string cell
        A network / cell from iwlist scan.
    @param dictionary rules
        A dictionary of parse rules.

    @return dictionary
        parsed networks. """

    parsed_cell = {}
    for key in rules:
        rule = rules[key]
        parsed_cell.update({key: rule(cell)})
    return parsed_cell

def print_table(table):
    # Functional black magic.
    widths = map(max, map(lambda l: map(len, l), zip(*table)))

    justified_table = []
    for line in table:
        justified_line = []
        for i, el in enumerate(line):
            justified_line.append(el.ljust(widths[i] + 2))
        justified_table.append(justified_line)

    for line in justified_table:
        for el in line:
            print el,
        print

def print_cells(cells, columns):
    table = [columns]
    for cell in cells:
        cell_properties = []
        for column in columns:
            if column == 'Quality':
                # make print nicer
                cell[column] = cell[column].rjust(3) + " %"
            cell_properties.append(cell[column])
        table.append(cell_properties)
    print_table(table)

def get_parsed_cells(iw_data, rules=None):
    """ Parses iwlist output into a list of networks.
        @param list iw_data
            Output from iwlist scan.
            A list of strings.

        @return list
            properties: Name, Address, Quality, Channel, Encryption.
    """

    # Here's a dictionary of rules that will be applied to the description
    # of each cell. The key will be the name of the column in the table.
    # The value is a function defined above.
    rules = rules or {
        "Name": get_name,
        "Quality": get_quality,
        "Channel": get_channel,
        "Encryption": get_encryption,
        "Address": get_address,
        "Signal Level": get_signal_level,
        "Bit Rates": get_bit_rates,
    }

    cells = [[]]
    parsed_cells = []

    for line in iw_data:
        cell_line = match(line, "Cell ")
        if cell_line != None:
            cells.append([])
            line = cell_line[-27:]
        cells[-1].append(line.rstrip())

    cells = cells[1:]

    for cell in cells:
        parsed_cells.append(parse_cell(cell, rules))

    sort_cells(parsed_cells)
    return parsed_cells

def call_iwlist(interface='wlan0'):
    """ Get iwlist output via subprocess
        @param string interface
            interface to scan
            default is wlan0

        @return string
            properties: iwlist output
    """
    p = subprocess.Popen(['iwlist', interface, 'scanning'],
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return p.stdout.read()

def get_interfaces(interface="wlan0"):
    """ Get parsed iwlist output
        @param string interface
            interface to scan
            default is wlan0

        @param list columns
            default data attributes to return

        @return dict
            properties: dictionary of iwlist attributes
    """
    return get_parsed_cells(call_iwlist(interface).split('\n'))

<commit_msg>Retained versioning format.<commit_after>#! /usr/bin/env python

# Hugo Chargois - 17 jan. 2010 - v.0.1
# Parses the output of iwlist scan into a table

# You can add or change the functions to parse the properties
# of each AP (cell) below. They take one argument, the bunch of text
# describing one cell in iwlist scan and return a property of that cell.

import re
import subprocess

VERSION_RGX = re.compile("version\s+\d+", re.IGNORECASE)

def get_name(cell):
    """ Gets the name / essid of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The name / essid of the network.
    """

    return matching_line(cell, "ESSID:")[1:-1]

def get_quality(cell):
    """ Gets the quality of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The quality of the network.
    """

    quality = matching_line(cell, "Quality=").split()[0].split("/")
    return str(int(round(float(quality[0]) / float(quality[1]) * 100)))

def get_signal_level(cell):
    """ Gets the signal level of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The signal level of the network.
    """

    signal = matching_line(cell, "Signal level=").split("=")[1].split("/")
    if len(signal) == 2:
        return str(int(round(float(signal[0]) / float(signal[1]) * 100)))
    elif len(signal) == 1:
        return signal[0]
    else:
        return ""

def get_channel(cell):
    """ Gets the channel of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The channel of the network.
    """

    channel = matching_line(cell, "Channel:")
    if channel:
        return channel
    frequency = matching_line(cell, "Frequency:")
    channel = re.sub(r".*\(Channel\s(\d{1,2})\).*", r"\1", frequency)
    return channel

def get_encryption(cell):
    """ Gets the encryption type of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The encryption type of the network.
    """

    enc = ""
    if matching_line(cell, "Encryption key:") == "off":
        enc = "Open"
    else:
        for line in cell:
            matching = match(line,"IE:")
            if matching == None:
                continue

            wpa = match(matching,"WPA")
            if wpa == None:
                continue

            version_matches = VERSION_RGX.search(wpa)
            if len(version_matches.regs) == 1:
                version = version_matches \
                    .group(0) \
                    .lower() \
                    .replace("version", "") \
                    .strip()
                wpa = wpa.replace(version_matches.group(0), "").strip()
                enc = "{0} v.{1}".format(wpa, version)
            else:
                enc = wpa
        if enc == "":
            enc = "WEP"
    return enc

def get_address(cell):
    """ Gets the address of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The address of the network.
    """

    return matching_line(cell, "Address: ")

def get_bit_rates(cell):
    """ Gets the bit rate of a network / cell.
    @param string cell
        A network / cell from iwlist scan.

    @return string
        The bit rate of the network.
    """

    return matching_line(cell, "Bit Rates:")

# Here you can choose the way of sorting the table. sortby should be a key of
# the dictionary rules.

def sort_cells(cells):
    sortby = "Quality"
    reverse = True
    cells.sort(None, lambda el:el[sortby], reverse)

# Below here goes the boring stuff. You shouldn't have to edit anything below
# this point

def matching_line(lines, keyword):
    """ Returns the first matching line in a list of lines.
    @see match()
    """
    for line in lines:
        matching = match(line,keyword)
        if matching != None:
            return matching
    return None

def match(line, keyword):
    """ If the first part of line (modulo blanks) matches keyword,
    returns the end of that line. Otherwise checks if keyword is
    anywhere in the line and returns that section, else returns None"""

    line = line.lstrip()
    length = len(keyword)
    if line[:length] == keyword:
        return line[length:]
    else:
        if keyword in line:
            return line[line.index(keyword):]
        else:
            return None

def parse_cell(cell, rules):
    """ Applies the rules to the bunch of text describing a cell.
    @param string cell
        A network / cell from iwlist scan.
    @param dictionary rules
        A dictionary of parse rules.

    @return dictionary
        parsed networks. """

    parsed_cell = {}
    for key in rules:
        rule = rules[key]
        parsed_cell.update({key: rule(cell)})
    return parsed_cell

def print_table(table):
    # Functional black magic.
    widths = map(max, map(lambda l: map(len, l), zip(*table)))

    justified_table = []
    for line in table:
        justified_line = []
        for i, el in enumerate(line):
            justified_line.append(el.ljust(widths[i] + 2))
        justified_table.append(justified_line)

    for line in justified_table:
        for el in line:
            print el,
        print

def print_cells(cells, columns):
    table = [columns]
    for cell in cells:
        cell_properties = []
        for column in columns:
            if column == 'Quality':
                # make print nicer
                cell[column] = cell[column].rjust(3) + " %"
            cell_properties.append(cell[column])
        table.append(cell_properties)
    print_table(table)

def get_parsed_cells(iw_data, rules=None):
    """ Parses iwlist output into a list of networks.
        @param list iw_data
            Output from iwlist scan.
            A list of strings.

        @return list
            properties: Name, Address, Quality, Channel, Encryption.
    """

    # Here's a dictionary of rules that will be applied to the description
    # of each cell. The key will be the name of the column in the table.
    # The value is a function defined above.
    rules = rules or {
        "Name": get_name,
        "Quality": get_quality,
        "Channel": get_channel,
        "Encryption": get_encryption,
        "Address": get_address,
        "Signal Level": get_signal_level,
        "Bit Rates": get_bit_rates,
    }

    cells = [[]]
    parsed_cells = []

    for line in iw_data:
        cell_line = match(line, "Cell ")
        if cell_line != None:
            cells.append([])
            line = cell_line[-27:]
        cells[-1].append(line.rstrip())

    cells = cells[1:]

    for cell in cells:
        parsed_cells.append(parse_cell(cell, rules))

    sort_cells(parsed_cells)
    return parsed_cells

def call_iwlist(interface='wlan0'):
    """ Get iwlist output via subprocess
        @param string interface
            interface to scan
            default is wlan0

        @return string
            properties: iwlist output
    """
    p = subprocess.Popen(['iwlist', interface, 'scanning'],
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return p.stdout.read()

def get_interfaces(interface="wlan0"):
    """ Get parsed iwlist output
        @param string interface
            interface to scan
            default is wlan0

        @param list columns
            default data attributes to return

        @return dict
            properties: dictionary of iwlist attributes
    """
    return get_parsed_cells(call_iwlist(interface).split('\n'))

<|endoftext|>