<commit_before>from django.contrib import admin
from django.contrib.admin.templatetags.admin_list import _boolean_icon
from django.templatetags.static import static
from django.urls import reverse
from django.utils.html import format_html
from django.utils.translation import ugettext as _

from ophasebase.models import OphaseCategory
from .admin_actions import (
    helper_job_overview,
    mail_export,
    orga_job_overview,
    send_fillform_mail,
    staff_nametag_export,
    staff_overview_export,
    tutorgroup_export,
    update_attendees, mark_attendance_x, mark_attendance_a, mark_attendance_e, mark_phoned_x, mark_phoned_e,
    mark_phoned_n, mark_attendance_v, generate_orga_cert)
from .models import (
    HelperJob,
    OrgaJob,
    Person,
    Settings,
    TutorGroup,
    StaffFilterGroup,
    Attendance,
    AttendanceEvent
)

@admin.register(OrgaJob)
@admin.register(HelperJob)
class LabelSortAdmin(admin.ModelAdmin):
    """Simple ModelAdmin which just shows the field Value in the list view"""
    list_display = ['label']

class TutorFilter(admin.SimpleListFilter):
    title = _('Tutorenstatus')
    parameter_name = "tutorstatus"

    def lookups(self, request, model_admin):
        choices = [('onlytutors', _('Alle Tutoren'))]
        for gc in OphaseCategory.objects.all():
            choices.append((gc.id, gc.name))
        choices.append(('notutors', _('Keine Tutoren')))
        return choices

    def queryset(self, request, queryset):
        if self.value() is None:
            return queryset
        if self.value() == 'onlytutors':
            return queryset.filter(is_tutor=True)
        if self.value() == 'notutors':
            return queryset.filter(is_tutor=False)
        return queryset.filter(tutor_for__id=self.value())

@admin.register(Person)
class PersonAdmin(admin.ModelAdmin):
    class Media:
        css = {
             'all': ("ophasebase/components/font-awesome/css/font-awesome.min.css",)
        }

    list_display = ['prename', 'name', 'is_tutor_with_title', 'is_orga', 'is_helper', 'created_at', 'orga_annotation_status']
    list_filter = [ ("ophase", admin.RelatedOnlyFieldListFilter), TutorFilter, 'is_orga', 'is_helper']
    list_display_links = ['prename', 'name']
    search_fields = ['prename', 'name', 'phone']
    readonly_fields = ('created_at', 'updated_at')
    actions = [mail_export, staff_overview_export, staff_nametag_export, helper_job_overview, orga_job_overview, send_fillform_mail, generate_orga_cert]

    fieldsets = [
        (_('Personendaten'), {'fields':
            ['ophase', 'prename', 'name', 'email', 'phone', 'orga_annotation']}),
        (_('Bewerbung'), {'fields':
            ['matriculated_since', 'degree_course', 'experience_ophase', 'why_participate', 'remarks']}),
        (_('In der Ophase'), {'fields':
            ['is_tutor', 'tutor_for', 'tutor_experience', 'is_orga', 'orga_jobs', 'is_helper', 'helper_jobs']}),
        (_('Sonstiges'), {'fields':
            ['created_at', 'updated_at']}),
    ]

    filter_horizontal = ('orga_jobs', 'helper_jobs')

    def orga_annotation_status(self, obj):
        if obj.orga_annotation:
            return format_html('', obj.orga_annotation)
        else:
            return ""
    orga_annotation_status.short_description = _('Orga-Notiz')

    def is_tutor_with_title(self, obj):
        """If the person is a tutor the tutor_for tag is set as image title"""
        if obj.is_tutor == True and obj.tutor_for is not None:
            icon_url = static('admin/img/icon-yes.svg')
            return format_html('',
                               icon_url, obj.is_tutor, obj.tutor_for.name)
        else:
            return _boolean_icon(obj.is_tutor)

    is_tutor_with_title.short_description = _('Tutor')

@admin.register(StaffFilterGroup)
class StaffFilterGroupAdmin(admin.ModelAdmin):
    list_display = ['name', 'is_tutor', 'is_orga', 'is_helper']

@admin.register(TutorGroup)
class TutorGroupAdmin(admin.ModelAdmin):
    list_display = ['name', 'get_tutors', 'group_category']
    list_filter = ['group_category']
    actions = [tutorgroup_export]

    def get_tutors(self, obj):
        return ", ".join([str(t) for t in obj.tutors.all()])
    get_tutors.short_description = _('Tutoren')

@admin.register(Attendance)
class AttendanceAdmin(admin.ModelAdmin):
    list_display = ['link_person', 'status', 'phone_status', 'comment', 'event']
    list_filter = ['event', 'status']
    list_display_links = ['status', 'phone_status']
    actions = [mark_attendance_a, mark_attendance_v, mark_attendance_e, mark_attendance_x, mark_phoned_x, mark_phoned_e, mark_phoned_n]

    @staticmethod
    def link_person(event):
        return format_html('{name}',
                           url=reverse('admin:staff_person_change',  args=(event.person.id, )),
                           name=event.person)

    @staticmethod
    def person_phone(event):
        return event.person.phone

@admin.register(AttendanceEvent)
class AttendanceAdmin(admin.ModelAdmin):
    list_display = ['name', 'begin', 'end', 'required_for', 'link_attendance_list']
    actions = [update_attendees]

    @staticmethod
    def link_attendance_list(event):
        return format_html('{name}',
                           url=reverse('admin:staff_attendance_changelist'),
                           id=event.pk,
                           name=_('Teilnehmerliste'))

@admin.register(Settings)
class SettingsAdmin(admin.ModelAdmin):
    list_display = ['tutor_registration_enabled', 'orga_registration_enabled', 'helper_registration_enabled']
    list_display_links = ['tutor_registration_enabled', 'orga_registration_enabled', 'helper_registration_enabled']
<commit_msg>added ophase to tutorgroup filters<commit_after>from django.contrib import admin
from django.contrib.admin.templatetags.admin_list import _boolean_icon
from django.templatetags.static import static
from django.urls import reverse
from django.utils.html import format_html
from django.utils.translation import ugettext as _

from ophasebase.models import OphaseCategory
from .admin_actions import (
    helper_job_overview,
    mail_export,
    orga_job_overview,
    send_fillform_mail,
    staff_nametag_export,
    staff_overview_export,
    tutorgroup_export,
    update_attendees, mark_attendance_x, mark_attendance_a, mark_attendance_e, mark_phoned_x, mark_phoned_e,
    mark_phoned_n, mark_attendance_v, generate_orga_cert)
from .models import (
    HelperJob,
    OrgaJob,
    Person,
    Settings,
    TutorGroup,
    StaffFilterGroup,
    Attendance,
    AttendanceEvent
)

@admin.register(OrgaJob)
@admin.register(HelperJob)
class LabelSortAdmin(admin.ModelAdmin):
    """Simple ModelAdmin which just shows the field Value in the list view"""
    list_display = ['label']

class TutorFilter(admin.SimpleListFilter):
    title = _('Tutorenstatus')
    parameter_name = "tutorstatus"

    def lookups(self, request, model_admin):
        choices = [('onlytutors', _('Alle Tutoren'))]
        for gc in OphaseCategory.objects.all():
            choices.append((gc.id, gc.name))
        choices.append(('notutors', _('Keine Tutoren')))
        return choices

    def queryset(self, request, queryset):
        if self.value() is None:
            return queryset
        if self.value() == 'onlytutors':
            return queryset.filter(is_tutor=True)
        if self.value() == 'notutors':
            return queryset.filter(is_tutor=False)
        return queryset.filter(tutor_for__id=self.value())

@admin.register(Person)
class PersonAdmin(admin.ModelAdmin):
    class Media:
        css = {
             'all': ("ophasebase/components/font-awesome/css/font-awesome.min.css",)
        }

    list_display = ['prename', 'name', 'is_tutor_with_title', 'is_orga', 'is_helper', 'created_at', 'orga_annotation_status']
    list_filter = [ ("ophase", admin.RelatedOnlyFieldListFilter), TutorFilter, 'is_orga', 'is_helper']
    list_display_links = ['prename', 'name']
    search_fields = ['prename', 'name', 'phone']
    readonly_fields = ('created_at', 'updated_at')
    actions = [mail_export, staff_overview_export, staff_nametag_export, helper_job_overview, orga_job_overview, send_fillform_mail, generate_orga_cert]

    fieldsets = [
        (_('Personendaten'), {'fields':
            ['ophase', 'prename', 'name', 'email', 'phone', 'orga_annotation']}),
        (_('Bewerbung'), {'fields':
            ['matriculated_since', 'degree_course', 'experience_ophase', 'why_participate', 'remarks']}),
        (_('In der Ophase'), {'fields':
            ['is_tutor', 'tutor_for', 'tutor_experience', 'is_orga', 'orga_jobs', 'is_helper', 'helper_jobs']}),
        (_('Sonstiges'), {'fields':
            ['created_at', 'updated_at']}),
    ]

    filter_horizontal = ('orga_jobs', 'helper_jobs')

    def orga_annotation_status(self, obj):
        if obj.orga_annotation:
            return format_html('', obj.orga_annotation)
        else:
            return ""
    orga_annotation_status.short_description = _('Orga-Notiz')

    def is_tutor_with_title(self, obj):
        """If the person is a tutor the tutor_for tag is set as image title"""
        if obj.is_tutor == True and obj.tutor_for is not None:
            icon_url = static('admin/img/icon-yes.svg')
            return format_html('',
                               icon_url, obj.is_tutor, obj.tutor_for.name)
        else:
            return _boolean_icon(obj.is_tutor)

    is_tutor_with_title.short_description = _('Tutor')

@admin.register(StaffFilterGroup)
class StaffFilterGroupAdmin(admin.ModelAdmin):
    list_display = ['name', 'is_tutor', 'is_orga', 'is_helper']

@admin.register(TutorGroup)
class TutorGroupAdmin(admin.ModelAdmin):
    list_display = ['name', 'get_tutors', 'ophase', 'group_category']
    list_filter = ['ophase', 'group_category']
    actions = [tutorgroup_export]

    def get_tutors(self, obj):
        return ", ".join([str(t) for t in obj.tutors.all()])
    get_tutors.short_description = _('Tutoren')

@admin.register(Attendance)
class AttendanceAdmin(admin.ModelAdmin):
    list_display = ['link_person', 'status', 'phone_status', 'comment', 'event']
    list_filter = ['event', 'status']
    list_display_links = ['status', 'phone_status']
    actions = [mark_attendance_a, mark_attendance_v, mark_attendance_e, mark_attendance_x, mark_phoned_x, mark_phoned_e, mark_phoned_n]

    @staticmethod
    def link_person(event):
        return format_html('{name}',
                           url=reverse('admin:staff_person_change',  args=(event.person.id, )),
                           name=event.person)

    @staticmethod
    def person_phone(event):
        return event.person.phone

@admin.register(AttendanceEvent)
class AttendanceAdmin(admin.ModelAdmin):
    list_display = ['name', 'begin', 'end', 'required_for', 'link_attendance_list']
    actions = [update_attendees]

    @staticmethod
    def link_attendance_list(event):
        return format_html('{name}',
                           url=reverse('admin:staff_attendance_changelist'),
                           id=event.pk,
                           name=_('Teilnehmerliste'))

@admin.register(Settings)
class SettingsAdmin(admin.ModelAdmin):
    list_display = ['tutor_registration_enabled', 'orga_registration_enabled', 'helper_registration_enabled']
    list_display_links = ['tutor_registration_enabled', 'orga_registration_enabled', 'helper_registration_enabled']
<|endoftext|>


<commit_before>from django.core.mail import EmailMessage
from django.contrib.auth.models import Group, User, Permission
from django.urls import reverse_lazy

from django.views.generic.edit import CreateView, FormView
from django.views.generic.base import TemplateView
from django.db.models import Q

from staff.models import OrgaCandidate
from workshops.models import Workshop
from ofahrtbase.models import Ofahrt
from staff.forms import ContactForm, PasswordForm
from pyofahrt import settings

class SignUpView(TemplateView):
    template_name = "staff/signup.html"

    def get_context_data(self, **kwargs):
        ofahrt = Ofahrt.current()

        context = super(SignUpView, self).get_context_data(**kwargs)
        context["orga_reg_open"] = ofahrt.orga_reg_open
        return context

class ContactView(FormView):
    template_name = "staff/contact.html"
    form_class = ContactForm
    success_url = reverse_lazy('staff:successcontact')

    def form_valid(self, form):
        email = form.cleaned_data.get('email')
        text = form.cleaned_data.get('message')

        mail = EmailMessage()
        mail.subject = settings.MAIL_CONTACTFORM_SUBJECT
        mail.body = settings.MAIL_CONTACTFORM_TEXT % {
            'sender': email,
            'text': text
        }
        mail.to = [settings.SERVER_EMAIL]
        mail.send()

        return super(ContactView, self).form_valid(form)

class PasswordView(FormView):
    template_name = "staff/changepassword/changepassword.html"
    form_class = PasswordForm
    success_url = reverse_lazy("staff:changepasswordsuccess")

    def get_form_kwargs(self):
        kwargs = super(PasswordView, self).get_form_kwargs()
        kwargs.update({'user': self.request.user})
        return kwargs

    def form_valid(self, form):
        self.request.user.set_password(form.cleaned_data.get('newpassworda'))
        self.request.user.save()
        return super(PasswordView, self).form_valid(form)

class SuccessView(TemplateView):
    template_name = "staff/success.html"

    def get_context_data(self, **kwargs):
        ofahrt = Ofahrt.current()

        context = super(SuccessView, self).get_context_data(**kwargs)
        context["orga_reg_open"] = ofahrt.orga_reg_open
        return context

class PasswordSuccessView(TemplateView):
    template_name = "staff/changepassword/success.html"

class SignUpOrgaView(CreateView):
    model = OrgaCandidate

    template_name = "staff/signup_orga.html"
    success_url = reverse_lazy('staff:success')

    fields = [
        'first_name', 'last_name', 'email', 'phone', 'roommate_preference',
        'orga_for'
    ]

    def form_valid(self, form):
        member = form.save(commit=False)
        member.base = Ofahrt.current()

        mail = EmailMessage()
        mail.subject = settings.MAIL_NEW_ORGA_SUBJECT
        mail.body = settings.MAIL_NEW_ORGA_TEXT % {
            'firstname': member.first_name,
            'lastname': member.last_name
        }
        mail.to = [settings.SERVER_EMAIL]
        mail.send()

        return super(SignUpOrgaView, self).form_valid(form)

    def get_context_data(self, **kwargs):
        ofahrt = Ofahrt.current()
        context = super(SignUpOrgaView, self).get_context_data(**kwargs)
        context["orga_reg_open"] = ofahrt.orga_reg_open
        context['open_orga_jobs'] = Group.objects.exclude(permissions__codename='group_full').count()
        return context
<commit_msg>Added reply to, fixes #166<commit_after>from django.core.mail import EmailMessage
from django.contrib.auth.models import Group, User, Permission
from django.urls import reverse_lazy

from django.views.generic.edit import CreateView, FormView
from django.views.generic.base import TemplateView
from django.db.models import Q

from staff.models import OrgaCandidate
from workshops.models import Workshop
from ofahrtbase.models import Ofahrt
from staff.forms import ContactForm, PasswordForm
from pyofahrt import settings

class SignUpView(TemplateView):
    template_name = "staff/signup.html"

    def get_context_data(self, **kwargs):
        ofahrt = Ofahrt.current()

        context = super(SignUpView, self).get_context_data(**kwargs)
        context["orga_reg_open"] = ofahrt.orga_reg_open
        return context

class ContactView(FormView):
    template_name = "staff/contact.html"
    form_class = ContactForm
    success_url = reverse_lazy('staff:successcontact')

    def form_valid(self, form):
        email = form.cleaned_data.get('email')
        text = form.cleaned_data.get('message')

        mail = EmailMessage()
        mail.subject = settings.MAIL_CONTACTFORM_SUBJECT
        mail.body = settings.MAIL_CONTACTFORM_TEXT % {
            'sender': email,
            'text': text
        }
        mail.to = [settings.SERVER_EMAIL]
        mail.reply_to = email
        mail.send()

        return super(ContactView, self).form_valid(form)

class PasswordView(FormView):
    template_name = "staff/changepassword/changepassword.html"
    form_class = PasswordForm
    success_url = reverse_lazy("staff:changepasswordsuccess")

    def get_form_kwargs(self):
        kwargs = super(PasswordView, self).get_form_kwargs()
        kwargs.update({'user': self.request.user})
        return kwargs

    def form_valid(self, form):
        self.request.user.set_password(form.cleaned_data.get('newpassworda'))
        self.request.user.save()
        return super(PasswordView, self).form_valid(form)

class SuccessView(TemplateView):
    template_name = "staff/success.html"

    def get_context_data(self, **kwargs):
        ofahrt = Ofahrt.current()

        context = super(SuccessView, self).get_context_data(**kwargs)
        context["orga_reg_open"] = ofahrt.orga_reg_open
        return context

class PasswordSuccessView(TemplateView):
    template_name = "staff/changepassword/success.html"

class SignUpOrgaView(CreateView):
    model = OrgaCandidate

    template_name = "staff/signup_orga.html"
    success_url = reverse_lazy('staff:success')

    fields = [
        'first_name', 'last_name', 'email', 'phone', 'roommate_preference',
        'orga_for'
    ]

    def form_valid(self, form):
        member = form.save(commit=False)
        member.base = Ofahrt.current()

        mail = EmailMessage()
        mail.subject = settings.MAIL_NEW_ORGA_SUBJECT
        mail.body = settings.MAIL_NEW_ORGA_TEXT % {
            'firstname': member.first_name,
            'lastname': member.last_name
        }
        mail.to = [settings.SERVER_EMAIL]
        mail.send()

        return super(SignUpOrgaView, self).form_valid(form)

    def get_context_data(self, **kwargs):
        ofahrt = Ofahrt.current()
        context = super(SignUpOrgaView, self).get_context_data(**kwargs)
        context["orga_reg_open"] = ofahrt.orga_reg_open
        context['open_orga_jobs'] = Group.objects.exclude(permissions__codename='group_full').count()
        return context
<|endoftext|>


<commit_before>"""Virksomhed."""

from collections import OrderedDict

from re import findall

from numpy import nan

class Virksomhed(object):

    """Encapsulate a single 'virksomhed' from CVR."""

    def __init__(self, data):
        """Initial raw data storage.

        Parameters
        ----------
        data : dist
            Nested structure corresponding to JSON file from CVR dump.

        """
        self.data = data

    def __str__(self):
        """Return human readable string for object.

        Returns
        -------
        readable : str
            String showing object

        """
        return "<Virksomhed(CVR={})>".format(self.cvr_nummer)

    __repr__ = __str__

    @property
    def antal_penheder(self):
        """Return number of P-enheder."""
        value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
            'antalPenheder']
        return value

    @property
    def branche_ansvarskode(self):
        """Return 'brancheAnsvarskode'.

        This fields is often None, sometimes 65 and seldom other values (99,
        97 and 0). The returned value is of type str.

        Returns
        -------
        kode : str

        """
        value = self.data['_source']['Vrvirksomhed']['brancheAnsvarskode']
        return str(value)

    @property
    def cvr_nummer(self):
        """Return CVR number."""
        value = self.data['_source']['Vrvirksomhed']['cvrNummer']
        return value

    @property
    def nyeste_antal_ansatte(self):
        """Return 'antal ansatte'.

        Taken as the first number from the 'intervalKodeAntalAnsatte' field
        from the nyesteAarsbeskaeftigelse.

        """
        try:
            value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
                'nyesteAarsbeskaeftigelse']['intervalKodeAntalAnsatte']
            numbers = findall('\d+', value)
            return int(numbers[0])
        except:
            return nan

    @property
    def nyeste_virksomhedsform(self):
        """Return 'virksomhedsform'.

        Possible values are 'Interessentskab', 'Enkeltmandsvirksomhed',

        """
        try:
            value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
                'nyesteVirksomhedsform']['langBeskrivelse']
            return value
        except:
            return nan

    @property
    def nyeste_virksomhedsstatus(self):
        """Return last virksomhedsstatus."""
        virksomhedsstatus = self.data['_source']['Vrvirksomhed'][
            'virksomhedMetadata']['nyesteVirksomhedsstatus']['langbeskrivelse']
        return virksomhedsstatus

    @property
    def reklamebeskyttet(self):
        """Return 'reklamebeskyttet'."""
        value = self.data['_source']['Vrvirksomhed']['reklamebeskyttet']
        return value

    @property
    def sammensat_status(self):
        """Return 'sammensatStatus'."""
        value = self.data['_source']['Vrvirksomhed'][
            'virksomhedMetadata']['sammensatStatus']
        return value

    @property
    def sidste_virksomhedsstatus(self):
        """Return last virksomhedsstatus."""
        virksomhedsstatus = self.data['_source']['Vrvirksomhed'][
            'virksomhedsstatus']
        if len(virksomhedsstatus) == 0:
            return 'NORMAL'
        else:
            return virksomhedsstatus[-1]['status']

    @property
    def stiftelsesdato(self):
        value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
            'stiftelsesDato']
        return value

    @property
    def stiftelsesaar(self):
        date = self.stiftelsesdato
        try:
            year = int(date[:4])
        except:
            year = nan
        return year

    def features(self):
        """Return set of features for a virksomhed.

        Returns
        -------
        features : OrderedDict
            Ordered dictionary with feature names and feature values

        """
        return OrderedDict([
            ('cvr_nummer', self.cvr_nummer),
            ('antal_penheder', self.antal_penheder),
            ('branche_ansvarskode', self.branche_ansvarskode),
            ('nyeste_antal_ansatte', self.nyeste_antal_ansatte),
            ('nyeste_virksomhedsform', self.nyeste_virksomhedsform),
            ('reklamebeskyttet', self.reklamebeskyttet),
            ('sammensat_status', self.sammensat_status),
            ('sidste_virksomhedsstatus', self.sidste_virksomhedsstatus),
            ('stiftelsesaar', self.stiftelsesaar),
        ])
<commit_msg>Change features, fix style, add documentation<commit_after>"""Virksomhed."""

from collections import OrderedDict

from re import findall

from numpy import nan

class Virksomhed(object):
    """Encapsulate a single 'virksomhed' from CVR."""

    def __init__(self, data):
        """Initial raw data storage.

        Parameters
        ----------
        data : dist
            Nested structure corresponding to JSON file from CVR dump.

        """
        self.data = data

    def __str__(self):
        """Return human readable string for object.

        Returns
        -------
        readable : str
            String showing object

        """
        return "<Virksomhed(CVR={})>".format(self.cvr_nummer)

    __repr__ = __str__

    @property
    def antal_penheder(self):
        """Return number of P-enheder.

        Returns
        -------
        antal : int
            Number of p-enheder from the 'antalPenhder field.

        """
        value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
            'antalPenheder']
        return value

    @property
    def branche_ansvarskode(self):
        """Return 'brancheAnsvarskode'.

        This fields is often None, sometimes 65 and seldom other values (99,
        97 and 0). The returned value is of type str.

        Returns
        -------
        kode : str
            The Branch-ansvarskode as a string. If the field is None then the
            string 'None' is returned.

        """
        value = self.data['_source']['Vrvirksomhed']['brancheAnsvarskode']
        return str(value)

    @property
    def cvr_nummer(self):
        """Return CVR number."""
        value = self.data['_source']['Vrvirksomhed']['cvrNummer']
        return value

    @property
    def nyeste_antal_ansatte(self):
        """Return 'antal ansatte'.

        Taken as the first number from the 'intervalKodeAntalAnsatte' field
        from the nyesteAarsbeskaeftigelse.

        """
        try:
            value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
                'nyesteAarsbeskaeftigelse']['intervalKodeAntalAnsatte']
            numbers = findall('\d+', value)
            return int(numbers[0])
        except:
            return nan

    @property
    def nyeste_virksomhedsform(self):
        """Return 'virksomhedsform'.

        Possible values are 'Interessentskab', 'Enkeltmandsvirksomhed',

        """
        try:
            value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
                'nyesteVirksomhedsform']['langBeskrivelse']
            return value
        except:
            return nan

    @property
    def nyeste_statuskode(self):
        """Return newest 'statuskode' from 'nyestestatus.

        Return
        ------
        statuskode : int
            Integer for 'statuskode'

        """
        try:
            statuskode = self.data['_source']['Vrvirksomhed'][
                'virksomhedMetadata']['nyesteStatus']['statuskode']
            return str(statuskode)
        except:
            return 'None'

    @property
    def reklamebeskyttet(self):
        """Return 'reklamebeskyttet'."""
        value = self.data['_source']['Vrvirksomhed']['reklamebeskyttet']
        return value

    @property
    def sammensat_status(self):
        """Return 'sammensatStatus'."""
        value = self.data['_source']['Vrvirksomhed'][
            'virksomhedMetadata']['sammensatStatus']
        return value

    @property
    def sidste_virksomhedsstatus(self):
        """Return last virksomhedsstatus.

        Returns
        -------
        virksomhedsstatus : string
            Last 'virksomhedsstatus'. If the length of list of
            'virksomhedstatus' is zero an empty string is returned.

        """
        virksomhedsstatus = self.data['_source']['Vrvirksomhed'][
            'virksomhedsstatus']
        if len(virksomhedsstatus) == 0:
            return ''
        else:
            return virksomhedsstatus[-1]['status']

    @property
    def stiftelsesdato(self):
        """Return stiftelsesdato.

        Returns
        -------
        date : str
            Stiftelsesdato from 'virksomhedMetadata' as a string.

        """
        value = self.data['_source']['Vrvirksomhed']['virksomhedMetadata'][
            'stiftelsesDato']
        return value

    @property
    def stiftelsesaar(self):
        """Return year part of stiftelsesdato.

        Returns
        -------
        year : int
            Integer for year of stiftelsesdato. NaN is returned if the year
            cannot be converted from the date.

        """
        date = self.stiftelsesdato
        try:
            year = int(date[:4])
        except:
            year = nan
        return year

    def features(self):
        """Return set of features for a virksomhed.

        Returns
        -------
        features : OrderedDict
            Ordered dictionary with feature names and feature values

        """
        return OrderedDict([
            ('cvr_nummer', self.cvr_nummer),
            ('antal_penheder', self.antal_penheder),
            ('branche_ansvarskode', self.branche_ansvarskode),
            ('nyeste_antal_ansatte', self.nyeste_antal_ansatte),
            ('nyeste_virksomhedsform', self.nyeste_virksomhedsform),
            ('reklamebeskyttet', self.reklamebeskyttet),
            ('sammensat_status', self.sammensat_status),
            ('nyeste_statuskode', self.nyeste_statuskode),
            ('stiftelsesaar', self.stiftelsesaar),
        ])
<|endoftext|>


<commit_before>"""Support for executing Docker containers using the Singularity 2.x engine."""
from __future__ import absolute_import

import os
import os.path
import re
import shutil
import tempfile
import sys
from distutils import spawn
from io import open  # pylint: disable=redefined-builtin
from typing import Dict, List, MutableMapping, Optional, Tuple

from schema_salad.sourceline import SourceLine
from typing_extensions import Text  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from .context import RuntimeContext  # pylint: disable=unused-import
from .errors import WorkflowException
from .job import ContainerCommandLineJob
from .loghandler import _logger
from .pathmapper import PathMapper, MapperEnt  # pylint: disable=unused-import
from .pathmapper import ensure_writable, ensure_non_writable
from .process import UnsupportedRequirement
from .utils import docker_windows_path_adjust

if os.name == 'posix':
    if sys.version_info < (3, 5):
        from subprocess32 import (  # pylint: disable=import-error,no-name-in-module
            check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
            TimeoutExpired)
    else:
        from subprocess import (  # pylint: disable=import-error,no-name-in-module
            check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
            TimeoutExpired)

else:  # we're not on Unix, so none of this matters
    pass

_USERNS = None

def _singularity_supports_userns():  # type: ()->bool
    global _USERNS  # pylint: disable=global-statement
    if _USERNS is None:
        try:
            hello_image = os.path.join(os.path.dirname(__file__), 'hello.simg')
            result = Popen(
                [u"singularity", u"exec", u"--userns", hello_image, u"true"],
                stderr=PIPE, stdout=DEVNULL,
                universal_newlines=True).communicate(timeout=60)[1]
            _USERNS = "No valid /bin/sh" in result
        except TimeoutExpired:
            _USERNS = False
    return _USERNS

def _normalize_image_id(string):  # type: (Text)->Text
    candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)
    return re.sub(pattern=r'[:/]', repl=r'-', string=candidate) + ".img"

class SingularityCommandLineJob(ContainerCommandLineJob):

    @staticmethod
    def get_image(dockerRequirement,  # type: Dict[Text, Text]
                  pull_image,         # type: bool
                  force_pull=False    # type: bool
                 ):
        # type: (...) -> bool
        """
        Acquire the software container image in the specified dockerRequirement
        using Singularity and returns the success as a bool. Updates the
        provided dockerRequirement with the specific dockerImageId to the full
        path of the local image, if found. Likewise the
        dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.
        """
        found = False

        candidates = []

        if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
            match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
            candidate = _normalize_image_id(dockerRequirement['dockerPull'])
            candidates.append(candidate)
            dockerRequirement['dockerImageId'] = candidate
            if not match:
                dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
        elif "dockerImageId" in dockerRequirement:
            candidates.append(dockerRequirement['dockerImageId'])
            candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))

        # check if Singularity image is available in $SINGULARITY_CACHEDIR
        targets = [os.getcwd()]
        for env in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER"):
            if env in os.environ:
                targets.append(os.environ[env])
        for target in targets:
            for candidate in candidates:
                path = os.path.join(target, candidate)
                if os.path.isfile(path):
                    _logger.info(
                        "Using local copy of Singularity image found in %s",
                        target)
                    dockerRequirement["dockerImageId"] = path
                    found = True

        if (force_pull or not found) and pull_image:
            cmd = []  # type: List[Text]
            if "dockerPull" in dockerRequirement:
                cmd = ["singularity", "pull", "--force", "--name",
                       str(dockerRequirement["dockerImageId"]),
                       str(dockerRequirement["dockerPull"])]
                _logger.info(Text(cmd))
                check_call(cmd, stdout=sys.stderr)
                found = True
            elif "dockerFile" in dockerRequirement:
                raise WorkflowException(SourceLine(
                    dockerRequirement, 'dockerFile').makeError(
                        "dockerFile is not currently supported when using the "
                        "Singularity runtime for Docker containers."))
            elif "dockerLoad" in dockerRequirement:
                raise WorkflowException(SourceLine(
                    dockerRequirement, 'dockerLoad').makeError(
                        "dockerLoad is not currently supported when using the "
                        "Singularity runtime for Docker containers."))
            elif "dockerImport" in dockerRequirement:
                raise WorkflowException(SourceLine(
                    dockerRequirement, 'dockerImport').makeError(
                        "dockerImport is not currently supported when using the "
                        "Singularity runtime for Docker containers."))

        return found

    def get_from_requirements(self,
                              r,                      # type: Dict[Text, Text]
                              pull_image,             # type: bool
                              force_pull=False,       # type: bool
                              tmp_outdir_prefix=None  # type: Text
                             ):
        # type: (...) -> Optional[Text]
        """
        Returns the filename of the Singularity image (e.g.
        hello-world-latest.img).
        """

        if not bool(spawn.find_executable('singularity')):
            raise WorkflowException('singularity executable is not available')

        if not self.get_image(r, pull_image, force_pull):
            raise WorkflowException(u"Container image {} not "
                                    "found".format(r["dockerImageId"]))

        return os.path.abspath(r["dockerImageId"])

    @staticmethod
    def append_volume(runtime, source, target, writable=False):
        # type: (List[Text], Text, Text, bool) -> None
        runtime.append(u"--bind")
        runtime.append("{}:{}:{}".format(
            docker_windows_path_adjust(source),
            docker_windows_path_adjust(target), "rw" if writable else "ro"))

    def add_file_or_directory_volume(self,
                                     runtime,         # type: List[Text]
                                     volume,          # type: MapperEnt
                                     host_outdir_tgt  # type: Optional[Text]
                                    ):  # type: (...) -> None
        if host_outdir_tgt is not None:
            # workaround for lack of overlapping mounts in Singularity
            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
            # once https://github.com/sylabs/singularity/issues/1607
            # is fixed
            if volume.type == "File":
                shutil.copy(volume.resolved, host_outdir_tgt)
            else:
                shutil.copytree(volume.resolved, host_outdir_tgt)
            ensure_non_writable(host_outdir_tgt)
        elif not volume.resolved.startswith("_:"):
            self.append_volume(runtime, volume.resolved, volume.target)

    def add_writable_file_volume(self,
                                 runtime,          # type: List[Text]
                                 volume,           # type: MapperEnt
                                 host_outdir_tgt,  # type: Optional[Text]
                                 tmpdir_prefix     # type: Text
                                ):  # type: (...) -> None
        if host_outdir_tgt is not None:
            # workaround for lack of overlapping mounts in Singularity
            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
            # once https://github.com/sylabs/singularity/issues/1607
            # is fixed
            if self.inplace_update:
                try:
                    os.link(os.path.realpath(volume.resolved),
                            host_outdir_tgt)
                except os.error:
                    shutil.copy(volume.resolved, host_outdir_tgt)
            else:
                shutil.copy(volume.resolved, host_outdir_tgt)
            ensure_writable(host_outdir_tgt)
        elif self.inplace_update:
            self.append_volume(
                runtime, volume.resolved, volume.target, writable=True)
            ensure_writable(volume.resolved)
        else:
            tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
            file_copy = os.path.join(
                tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                os.path.basename(volume.resolved))
            shutil.copy(volume.resolved, file_copy)
            #volume.resolved = file_copy
            self.append_volume(
                runtime, file_copy, volume.target, writable=True)
            ensure_writable(file_copy)

    def add_writable_directory_volume(self,
                                      runtime,          # type: List[Text]
                                      volume,           # type: MapperEnt
                                      host_outdir_tgt,  # type: Optional[Text]
                                      tmpdir_prefix     # type: Text
                                     ):  # type: (...) -> None
        if volume.resolved.startswith("_:"):
            if host_outdir_tgt is not None:
                new_dir = host_outdir_tgt
            else:
                tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                new_dir = os.path.join(
                    tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                    os.path.basename(volume.resolved))
            os.makedirs(new_dir, 0o0755)
        else:
            if host_outdir_tgt is not None:
                # workaround for lack of overlapping mounts in Singularity
                # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
                # once https://github.com/sylabs/singularity/issues/1607
                # is fixed
                shutil.copytree(volume.resolved, host_outdir_tgt)
                ensure_writable(host_outdir_tgt)
            else:
                if not self.inplace_update:
                    tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                    dir_copy = os.path.join(
                        tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                        os.path.basename(volume.resolved))
                    shutil.copytree(volume.resolved, dir_copy)
                    source = dir_copy
                    #volume.resolved = dir_copy
                else:
                    source = volume.resolved
                self.append_volume(
                    runtime, source, volume.target, writable=True)
                ensure_writable(source)

    def create_runtime(self,
                       env,            # type: MutableMapping[Text, Text]
                       runtime_context  # type: RuntimeContext
                      ):  # type: (...) -> Tuple[List, Optional[Text]]
        """ Returns the Singularity runtime list of commands and options."""
        any_path_okay = self.builder.get_requirement("DockerRequirement")[1] \
            or False
        runtime = [u"singularity", u"--quiet", u"exec", u"--contain", u"--pid",
                   u"--ipc"]
        if _singularity_supports_userns():
            runtime.append(u"--userns")
        runtime.append(u"--bind")
        runtime.append(u"{}:{}:rw".format(
            docker_windows_path_adjust(os.path.realpath(self.outdir)),
            self.builder.outdir))
        runtime.append(u"--bind")
        runtime.append(u"{}:{}:rw".format(
            docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp"))

        self.add_volumes(self.pathmapper, runtime, any_path_okay=True,
                         secret_store=runtime_context.secret_store,
                         tmpdir_prefix=runtime_context.tmpdir_prefix)
        if self.generatemapper is not None:
            self.add_volumes(
                self.generatemapper, runtime, any_path_okay=any_path_okay,
                secret_store=runtime_context.secret_store,
                tmpdir_prefix=runtime_context.tmpdir_prefix)

        runtime.append(u"--pwd")
        runtime.append(u"%s" % (docker_windows_path_adjust(self.builder.outdir)))

        if runtime_context.custom_net:
            raise UnsupportedRequirement(
                "Singularity implementation does not support custom networking")
        elif runtime_context.disable_net:
            runtime.append(u"--net")

        env["SINGULARITYENV_TMPDIR"] = "/tmp"
        env["SINGULARITYENV_HOME"] = self.builder.outdir

        for name, value in self.environment.items():
            env["SINGULARITYENV_{}".format(name)] = value
        return (runtime, None)
<commit_msg>changed singularity.py to find env images and .sif files<commit_after>"""Support for executing Docker containers using the Singularity 2.x engine."""
from __future__ import absolute_import

import os
import os.path
import re
import shutil
import tempfile
import sys
from distutils import spawn
from io import open  # pylint: disable=redefined-builtin
from typing import Dict, List, MutableMapping, Optional, Tuple

from schema_salad.sourceline import SourceLine
from typing_extensions import Text  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from .context import RuntimeContext  # pylint: disable=unused-import
from .errors import WorkflowException
from .job import ContainerCommandLineJob
from .loghandler import _logger
from .pathmapper import PathMapper, MapperEnt  # pylint: disable=unused-import
from .pathmapper import ensure_writable, ensure_non_writable
from .process import UnsupportedRequirement
from .utils import docker_windows_path_adjust

if os.name == 'posix':
    if sys.version_info < (3, 5):
        from subprocess32 import (  # nosec # pylint: disable=import-error,no-name-in-module
            check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
            TimeoutExpired)
    else:
        from subprocess import (  # nosec # pylint: disable=import-error,no-name-in-module
            check_call, check_output, CalledProcessError, DEVNULL, PIPE, Popen,
            TimeoutExpired)

else:  # we're not on Unix, so none of this matters
    pass

_USERNS = None

def _singularity_supports_userns():  # type: ()->bool
    global _USERNS  # pylint: disable=global-statement
    if _USERNS is None:
        try:
            hello_image = os.path.join(os.path.dirname(__file__), 'hello.simg')
            result = Popen(  # nosec
                [u"singularity", u"exec", u"--userns", hello_image, u"true"],
                stderr=PIPE, stdout=DEVNULL,
                universal_newlines=True).communicate(timeout=60)[1]
            _USERNS = "No valid /bin/sh" in result
        except TimeoutExpired:
            _USERNS = False
    return _USERNS

def _normalize_image_id(string):  # type: (Text)->Text
    candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)
    return re.sub(pattern=r'[:/]', repl=r'-', string=candidate) + ".img"

def _normalize_sif_id(string): # type: (Text)->Text
    candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)
    candidate_new = re.sub(pattern=r'[::]', repl=r'_', string=candidate) + ".sif"
    if '/' in candidate_new:
        return candidate_new.split('/')[1]
    else:
        return candidate_new

class SingularityCommandLineJob(ContainerCommandLineJob):

    @staticmethod
    def get_image(dockerRequirement,  # type: Dict[Text, Text]
                  pull_image,         # type: bool
                  force_pull=False    # type: bool
                 ):
        # type: (...) -> bool
        """
        Acquire the software container image in the specified dockerRequirement
        using Singularity and returns the success as a bool. Updates the
        provided dockerRequirement with the specific dockerImageId to the full
        path of the local image, if found. Likewise the
        dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.
        """
        found = False

        candidates = []

        if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
            match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
            candidate_image = _normalize_image_id(dockerRequirement['dockerPull'])
            candidates.append(candidate_image)
            candidate_sif = _normalize_sif_id(dockerRequirement['dockerPull'])
            candidates.append(candidate_sif)
            dockerRequirement['dockerImageId'] = candidate_image
            if not match:
                dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
        elif "dockerImageId" in dockerRequirement:
            candidates.append(dockerRequirement['dockerImageId'])
            candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))

        # check if Singularity image is available in $SINGULARITY_CACHEDIR
        # or any subdirs created in cachedir
        targets = [os.getcwd()]
        for env in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER"):
            if env in os.environ:
                targets.append(os.environ[env])
        for target in targets:
            for dirpath, subdirs, files in os.walk(target):
                for sif in files:
                    if sif in candidates:
                        path = os.path.join(dirpath, sif)
                        if os.path.isfile(path):
                            _logger.info(
                                "Using local copy of Singularity image found in %s",
                                dirpath)
                            dockerRequirement["dockerImageId"] = path
                            found = True
        if (force_pull or not found) and pull_image:
            cmd = []  # type: List[Text]
            if "dockerPull" in dockerRequirement:
                if "SINGULARITY_PULLFOLDER" in os.environ:
                    pull_folder = str(os.environ["SINGULARITY_PULLFOLDER"])
                    cmd = ["singularity", "pull", "--force", "--name",
                           str('%s/' %pull_folder) + str(dockerRequirement["dockerImageId"]),
                           str(dockerRequirement["dockerPull"])]
                    _logger.info(Text(cmd))
                    check_call(cmd, stdout=sys.stderr)  # nosec
                    dockerRequirement["dockerImageId"] = pull_folder + '/' + str(dockerRequirement["dockerImageId"])
                    found = True
                else:
                    cmd = ["singularity", "pull", "--force", "--name",
                           str(dockerRequirement["dockerImageId"]),
                           str(dockerRequirement["dockerPull"])]
                    _logger.info(Text(cmd))
                    check_call(cmd, stdout=sys.stderr)  # nosec
                    found = True

            elif "dockerFile" in dockerRequirement:
                raise WorkflowException(SourceLine(
                    dockerRequirement, 'dockerFile').makeError(
                        "dockerFile is not currently supported when using the "
                        "Singularity runtime for Docker containers."))
            elif "dockerLoad" in dockerRequirement:
                raise WorkflowException(SourceLine(
                    dockerRequirement, 'dockerLoad').makeError(
                        "dockerLoad is not currently supported when using the "
                        "Singularity runtime for Docker containers."))
            elif "dockerImport" in dockerRequirement:
                raise WorkflowException(SourceLine(
                    dockerRequirement, 'dockerImport').makeError(
                        "dockerImport is not currently supported when using the "
                        "Singularity runtime for Docker containers."))

        return found

    def get_from_requirements(self,
                              r,                      # type: Dict[Text, Text]
                              pull_image,             # type: bool
                              force_pull=False,       # type: bool
                              tmp_outdir_prefix=None  # type: Text
                             ):
        # type: (...) -> Optional[Text]
        """
        Returns the filename of the Singularity image (e.g.
        hello-world-latest.img).
        """

        if not bool(spawn.find_executable('singularity')):
            raise WorkflowException('singularity executable is not available')

        if not self.get_image(r, pull_image, force_pull):
            raise WorkflowException(u"Container image {} not "
                                    "found".format(r["dockerImageId"]))

        if "SINGULARITY_PULLFOLDER" in os.environ:
            return str(r["dockerImageId"])
        else:
            return os.path.abspath(r["dockerImageId"])

    @staticmethod
    def append_volume(runtime, source, target, writable=False):
        # type: (List[Text], Text, Text, bool) -> None
        runtime.append(u"--bind")
        runtime.append("{}:{}:{}".format(
            docker_windows_path_adjust(source),
            docker_windows_path_adjust(target), "rw" if writable else "ro"))

    def add_file_or_directory_volume(self,
                                     runtime,         # type: List[Text]
                                     volume,          # type: MapperEnt
                                     host_outdir_tgt  # type: Optional[Text]
                                    ):  # type: (...) -> None
        if host_outdir_tgt is not None:
            # workaround for lack of overlapping mounts in Singularity
            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
            # once https://github.com/sylabs/singularity/issues/1607
            # is fixed
            if volume.type == "File":
                shutil.copy(volume.resolved, host_outdir_tgt)
            else:
                shutil.copytree(volume.resolved, host_outdir_tgt)
            ensure_non_writable(host_outdir_tgt)
        elif not volume.resolved.startswith("_:"):
            self.append_volume(runtime, volume.resolved, volume.target)

    def add_writable_file_volume(self,
                                 runtime,          # type: List[Text]
                                 volume,           # type: MapperEnt
                                 host_outdir_tgt,  # type: Optional[Text]
                                 tmpdir_prefix     # type: Text
                                ):  # type: (...) -> None
        if host_outdir_tgt is not None:
            # workaround for lack of overlapping mounts in Singularity
            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
            # once https://github.com/sylabs/singularity/issues/1607
            # is fixed
            if self.inplace_update:
                try:
                    os.link(os.path.realpath(volume.resolved),
                            host_outdir_tgt)
                except os.error:
                    shutil.copy(volume.resolved, host_outdir_tgt)
            else:
                shutil.copy(volume.resolved, host_outdir_tgt)
            ensure_writable(host_outdir_tgt)
        elif self.inplace_update:
            self.append_volume(
                runtime, volume.resolved, volume.target, writable=True)
            ensure_writable(volume.resolved)
        else:
            tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
            file_copy = os.path.join(
                tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                os.path.basename(volume.resolved))
            shutil.copy(volume.resolved, file_copy)
            #volume.resolved = file_copy
            self.append_volume(
                runtime, file_copy, volume.target, writable=True)
            ensure_writable(file_copy)

    def add_writable_directory_volume(self,
                                      runtime,          # type: List[Text]
                                      volume,           # type: MapperEnt
                                      host_outdir_tgt,  # type: Optional[Text]
                                      tmpdir_prefix     # type: Text
                                     ):  # type: (...) -> None
        if volume.resolved.startswith("_:"):
            if host_outdir_tgt is not None:
                new_dir = host_outdir_tgt
            else:
                tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                new_dir = os.path.join(
                    tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                    os.path.basename(volume.resolved))
            os.makedirs(new_dir)
        else:
            if host_outdir_tgt is not None:
                # workaround for lack of overlapping mounts in Singularity
                # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
                # once https://github.com/sylabs/singularity/issues/1607
                # is fixed
                shutil.copytree(volume.resolved, host_outdir_tgt)
                ensure_writable(host_outdir_tgt)
            else:
                if not self.inplace_update:
                    tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
                    dir_copy = os.path.join(
                        tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                        os.path.basename(volume.resolved))
                    shutil.copytree(volume.resolved, dir_copy)
                    source = dir_copy
                    #volume.resolved = dir_copy
                else:
                    source = volume.resolved
                self.append_volume(
                    runtime, source, volume.target, writable=True)
                ensure_writable(source)

    def create_runtime(self,
                       env,              # type: MutableMapping[Text, Text]
                       runtime_context   # type: RuntimeContext
                      ):  # type: (...) -> Tuple[List, Optional[Text]]
        """ Returns the Singularity runtime list of commands and options."""
        any_path_okay = self.builder.get_requirement("DockerRequirement")[1] \
            or False
        runtime = [u"singularity", u"--quiet", u"exec", u"--contain", u"--pid",
                   u"--ipc"]
        if _singularity_supports_userns():
            runtime.append(u"--userns")
        runtime.append(u"--bind")
        runtime.append(u"{}:{}:rw".format(
            docker_windows_path_adjust(os.path.realpath(self.outdir)),
            self.builder.outdir))
        runtime.append(u"--bind")
        tmpdir = "/tmp"  # nosec
        runtime.append(u"{}:{}:rw".format(
            docker_windows_path_adjust(os.path.realpath(self.tmpdir)), tmpdir))

        self.add_volumes(self.pathmapper, runtime, any_path_okay=True,
                         secret_store=runtime_context.secret_store,
                         tmpdir_prefix=runtime_context.tmpdir_prefix)
        if self.generatemapper is not None:
            self.add_volumes(
                self.generatemapper, runtime, any_path_okay=any_path_okay,
                secret_store=runtime_context.secret_store,
                tmpdir_prefix=runtime_context.tmpdir_prefix)

        runtime.append(u"--pwd")
        runtime.append(u"%s" % (docker_windows_path_adjust(self.builder.outdir)))

        if runtime_context.custom_net:
            raise UnsupportedRequirement(
                "Singularity implementation does not support custom networking")
        elif runtime_context.disable_net:
            runtime.append(u"--net")

        env["SINGULARITYENV_TMPDIR"] = tmpdir
        env["SINGULARITYENV_HOME"] = self.builder.outdir

        for name, value in self.environment.items():
            env["SINGULARITYENV_{}".format(name)] = str(value)
        return (runtime, None)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import glob
import math
import os
import re
import sys

from Pegasus.DAX3 import *

top_dir = os.getcwd()

# The name of the DAX file is the first argument
if len(sys.argv) != 2:
        sys.stderr.write("Usage: %s DAXFILE\n" % (sys.argv[0]))
        sys.exit(1)
daxfile = sys.argv[1]
USER = pwd.getpwuid(os.getuid())[0]

dax = ADAG("population")

# Add some workflow-level metadata
dax.metadata("creator", "%s@%s" % (USER, os.uname()[1]))
dax.metadata("created", time.ctime())

# common library
geospatial = File('geospatial.py')
geospatial.addPFN(PFN('file://' + top_dir + '/geospatial.py', 'local'))
dax.addFile(geospatial)

# config file
config = File('county_cohort_pop_config.ini')
config.addPFN(PFN('file://' + top_dir + '/config/county_cohort_pop_config.ini', 'local'))
dax.addFile(config)

# shapefile (set of files)
shapefiles = []
basename = "SouthSudan_CountyPopulation"
for fname in glob.glob('input/' + basename + '.*'):
    # do not include input/ in the lfn
    fname = re.sub('input/', '', fname)
    f = File(fname)
    #f.addPFN(PFN('file://' + top_dir + '/input/' + fname, 'local'))
    dax.addFile(f)
    shapefiles.append(f)

# distribution tif
dist_tif = File('ss_pop_spatial_dist.tif')
dist_tif.addPFN(PFN('file://' + top_dir + '/input/ss_pop_spatial_dist.tif', 'local'))
dax.addFile(dist_tif)

# legend
legend = File('legend.png')
legend.addPFN(PFN('file://' + top_dir + '/graphics/legend.png', 'local'))
dax.addFile(legend)

# animate - this is the final job, but will be built up in the loops below
animate = Job("animate")
out_anim = File('animation.gif')
animate.uses(out_anim, link=Link.OUTPUT, transfer=True)
dax.addJob(animate)

# add one job per year
# we have data till 2030
for year in range(2017, 2020):

    # create a job to process the shape file
    j1 = Job("county_population_raster")

    # we need the geospatial lib
    j1.uses(geospatial, link=Link.INPUT)

    # config file
    j1.uses(config, link=Link.INPUT)
    j1.addArguments('--config', config)

    # shape file
    for f in shapefiles:
        j1.uses(f, link=Link.INPUT)
    j1.addArguments('--shapefile', basename + '.shp')

    # year
    j1.addArguments('--year', str(year))

    # outputs
    tif = File('county_level_pop_' + str(year) + '.tif')
    j1.uses(tif, link=Link.OUTPUT, transfer=True)
    j1.addArguments('--outfile', tif)

    dax.addJob(j1)

    # create a full res distribution of the population
    j2 = Job('full_res_pop_raster')
    j2.uses(tif, link=Link.INPUT)
    j2.uses(dist_tif, link=Link.INPUT)
    pop_dist_tif = File('pop_dist_' + str(year) + '.tif')
    j2.uses(pop_dist_tif, link=Link.OUTPUT, transfer=True)
    j2.addArguments('--population-file', tif,
                    '--dist-file', dist_tif,
                    '--out-file', pop_dist_tif)
    dax.addJob(j2)
    dax.depends(parent=j1, child=j2)

    # second job to convert the raster tif to an annotated png
    j3 = Job("raster_to_png")
    j3.uses(pop_dist_tif, link=Link.INPUT)
    j3.uses(legend, link=Link.INPUT)
    png = File('pop_dist_' + str(year) + '.png')
    j3.uses(png, link=Link.OUTPUT, transfer=True)
    j3.addArguments(pop_dist_tif, png, str(year))
    dax.addJob(j3)
    dax.depends(parent=j2, child=j3)

    # add to animate job
    animate.uses(png, link=Link.INPUT)
    dax.depends(parent=j3, child=animate)

# write out the dax
f = open(daxfile, "w")
dax.writeXML(f)
f.close()
print "Generated dax %s" %daxfile

<commit_msg>PM-1280 fixed missing imports<commit_after>#!/usr/bin/env python

import glob
import math
import os
import re
import sys
import pwd
import time

from Pegasus.DAX3 import *

top_dir = os.getcwd()

# The name of the DAX file is the first argument
if len(sys.argv) != 2:
        sys.stderr.write("Usage: %s DAXFILE\n" % (sys.argv[0]))
        sys.exit(1)
daxfile = sys.argv[1]
USER = pwd.getpwuid(os.getuid())[0]

dax = ADAG("population")

# Add some workflow-level metadata
dax.metadata("creator", "%s@%s" % (USER, os.uname()[1]))
dax.metadata("created", time.ctime())

# common library
geospatial = File('geospatial.py')
geospatial.addPFN(PFN('file://' + top_dir + '/geospatial.py', 'local'))
dax.addFile(geospatial)

# config file
config = File('county_cohort_pop_config.ini')
config.addPFN(PFN('file://' + top_dir + '/config/county_cohort_pop_config.ini', 'local'))
dax.addFile(config)

# shapefile (set of files)
shapefiles = []
basename = "SouthSudan_CountyPopulation"
for fname in glob.glob('input/' + basename + '.*'):
    # do not include input/ in the lfn
    fname = re.sub('input/', '', fname)
    f = File(fname)
    #f.addPFN(PFN('file://' + top_dir + '/input/' + fname, 'local'))
    dax.addFile(f)
    shapefiles.append(f)

# distribution tif
dist_tif = File('ss_pop_spatial_dist.tif')
dist_tif.addPFN(PFN('file://' + top_dir + '/input/ss_pop_spatial_dist.tif', 'local'))
dax.addFile(dist_tif)

# legend
legend = File('legend.png')
legend.addPFN(PFN('file://' + top_dir + '/graphics/legend.png', 'local'))
dax.addFile(legend)

# animate - this is the final job, but will be built up in the loops below
animate = Job("animate")
out_anim = File('animation.gif')
animate.uses(out_anim, link=Link.OUTPUT, transfer=True)
dax.addJob(animate)

# add one job per year
# we have data till 2030
for year in range(2017, 2020):

    # create a job to process the shape file
    j1 = Job("county_population_raster")

    # we need the geospatial lib
    j1.uses(geospatial, link=Link.INPUT)

    # config file
    j1.uses(config, link=Link.INPUT)
    j1.addArguments('--config', config)

    # shape file
    for f in shapefiles:
        j1.uses(f, link=Link.INPUT)
    j1.addArguments('--shapefile', basename + '.shp')

    # year
    j1.addArguments('--year', str(year))

    # outputs
    tif = File('county_level_pop_' + str(year) + '.tif')
    j1.uses(tif, link=Link.OUTPUT, transfer=True)
    j1.addArguments('--outfile', tif)

    dax.addJob(j1)

    # create a full res distribution of the population
    j2 = Job('full_res_pop_raster')
    j2.uses(tif, link=Link.INPUT)
    j2.uses(dist_tif, link=Link.INPUT)
    pop_dist_tif = File('pop_dist_' + str(year) + '.tif')
    j2.uses(pop_dist_tif, link=Link.OUTPUT, transfer=True)
    j2.addArguments('--population-file', tif,
                    '--dist-file', dist_tif,
                    '--out-file', pop_dist_tif)
    dax.addJob(j2)
    dax.depends(parent=j1, child=j2)

    # second job to convert the raster tif to an annotated png
    j3 = Job("raster_to_png")
    j3.uses(pop_dist_tif, link=Link.INPUT)
    j3.uses(legend, link=Link.INPUT)
    png = File('pop_dist_' + str(year) + '.png')
    j3.uses(png, link=Link.OUTPUT, transfer=True)
    j3.addArguments(pop_dist_tif, png, str(year))
    dax.addJob(j3)
    dax.depends(parent=j2, child=j3)

    # add to animate job
    animate.uses(png, link=Link.INPUT)
    dax.depends(parent=j3, child=animate)

# write out the dax
f = open(daxfile, "w")
dax.writeXML(f)
f.close()
print "Generated dax %s" %daxfile

<|endoftext|>


<commit_before>"""
    heatExchanger.py -
        Performs basic heat exchanger calculations for a single pass
        counter-flow heat exchanger

Logarithmic Mean Temperature Difference (LMTD) Method
    Design a heat exchanger to meet prescribed heat transfer requirements

    LMTD Limitations
    1) Restricted to single pass heat exchanger
    - Multipass exchangers can be designed using empirical correction factor ‘F’
    2) Both starting and final temperature parameters must be known

NTU (effectiveness) Method
    Determine the heat transfer rate and outlet temperatures when the type and size of the heat exchanger is specified.

    NTU Limitations
    1) Effectiveness of the chosen heat exchanger must be known (empirical)

    Compatible with OpenMDAO v0.8.1
"""

from openmdao.main.api import Component
from openmdao.lib.datatypes.api import Float

from math import log

class heatExchanger(Component):
    """ Main Component """

    #--Inputs--
    #Boundary Temperatures
    T_win = Float(0.0, iotype='in', desc='Temp of water into heat exchanger')
    T_wout = Float(0.0, iotype='in', desc='Temp of water out of heat exchanger')
    T_ain = Float(0.0, iotype='in', desc='Temp of air into heat exchanger')
    T_aout = Float(0.0, iotype='in', desc='Temp of air out of heat exchanger')

    #Design Variables
    Mdot_w = = Float(0.0, iotype='in', desc='Mass flow rate of water pumped through system')
    Ao_pipe = Float(0.0, iotype='out', desc='Shell pipe (Outer) Area')
    Ai_pipe = Float(0.0, iotype='out', desc='Tube pipe (Inner) Area')

    #Assumed Constant Properties
    rho_w = Float(0.0, iotype='in', desc='density of water')
    rho_a = Float(0.0, iotype='in', desc='density of air ')
    cp_w = Float(0.0, iotype='in', desc='specific heat of water')
    cp_a = Float(0.0, iotype='in', desc='specific heat of air')
    dvisc_w = Float(0.0, iotype='in', desc='dynamic viscosity for water')
    dvis_a = Float(0.0, iotype='in', desc='dynamic viscosity for air')
    kvisc_w = Float(0.0, iotype='in', desc='kinematic viscosity for water')
    kvisc_a = Float(0.0, iotype='in', desc='kinematic viscosity for air')
    k_w = Float(0.0, iotype='in', desc='thermal conductivity for water')
    k_a = Float(0.0, iotype='in', desc='thermal conductivity for air')
    k_p = Float(0.0, iotype='in', desc='thermal conductivity of the pipe')

    #--Outputs--
    #Intermediate Variables
    Asurf_pipe = Float(0.0, iotype='out', desc='Surface Area of the Pipe')

    #Calculated Variables
    V_w = Float(0.0, iotype='out', desc='flow velocity of water')
    V_a = Float(0.0, iotype='out', desc='flow velocity of air')
    W_w = Float(0.0, iotype='out', desc='mass flow of water')
    W_a = Float(0.0, iotype='out', desc='mass flow of air')
    h_w = Float(0.0, iotype='out', desc='heat transfer of water')
    h_a = Float(0.0, iotype='out', desc='heat transfer of air')
    q_w = Float(0.0, iotype='out', desc='heat flow of water')
    q_a = Float(0.0, iotype='out', desc='heat flow of air')

    #Size/Volume Considerations
    V_water = Float(0.0, iotype='out', desc='Volume of input water tank')
    V_steam = Float(0.0, iotype='out', desc='Volume of output steam tank')
    M_water = Float(0.0, iotype='out', desc='Mass of input water tank')
    M_steam = Float(0.0, iotype='out', desc='Mass of output steam tank')

    def execute(self):
        """Calculate Various Paramters
        """

        #Determine amount of heat that needs to be transferred
        #Q_water must equal Q_air
        #Q = Mdot * Cp * delta T

        #Determine the flow rate of the water
        # Mdot = rho * Area * Volume

        #Determine Hyrdraulic Diameters

        #Determine Overall Heat Transfer Coefficient
        #1) Calculate Reynolds Number
        #2) Calculate Prandtl Number
        #3) Calculate Nusselt Number

        #Determine LMTD
        LMTD = self.LMTD #Log Mean Temp Diff
        Th_in = self.T_ain
        Tc_in = self.T_win
        Th_out = self.T_aout
        Tc_out = self.T_wout

        dT1 = (Th_out - Th_in) #Change in T1 (delta T1)
        dT2 = (Tc_out-Tc_in) #Change in T2 (delta T2)

        self.LMTD = (dT1 - dT2)/(log(dT1/(dT2), e)) #take natural log (base e)

        #Determine the required length of the heat exchanger

        #Assume fouling losses

        #Assume pipe minor losses<commit_msg>Added units, initial engineering calcs<commit_after>"""
    heatExchanger.py -
        Performs basic heat exchanger calculations for a single pass
        counter-flow shell and tube heat exchanger

Logarithmic Mean Temperature Difference (LMTD) Method
    Design a heat exchanger to meet prescribed heat transfer requirements

    LMTD Limitations
    1) Restricted to single pass heat exchanger
    - Multipass exchangers can be designed using empirical correction factor ‘F’
    2) Both starting and final temperature parameters must be known

NTU (effectiveness) Method
    Determine the heat transfer rate and outlet temperatures when the type and size of the heat exchanger is specified.

    NTU Limitations
    1) Effectiveness of the chosen heat exchanger must be known (empirical)

    Compatible with OpenMDAO v0.8.1
"""

from openmdao.main.api import Component
from openmdao.lib.datatypes.api import Float

from math import log, pi

class heatExchanger(Component):
    """ Main Component """

    #--Inputs--
    #Boundary Temperatures
    T_win = Float(0.0, units = 'K', iotype='in', desc='Temp of water into heat exchanger')
    T_wout = Float(0.0, units = 'K', iotype='in', desc='Temp of water out of heat exchanger')
    T_ain = Float(0.0, units = 'K', iotype='in', desc='Temp of air into heat exchanger')
    T_aout = Float(0.0, units = 'K', iotype='in', desc='Temp of air out of heat exchanger')

    #Design Variables
    Mdot_w = Float(0.0, units = 'kg/s', iotype='in', desc='Mass flow rate of water pumped through system')
    Mdot_a = Float(0.0, units = 'kg/s', iotype='out', desc='Mass flow rate of air')
    Di_shell = Float(0.0, units = 'm', iotype='out', desc='Shell pipe (inner) Diameter')
    Do_tube = Float(0.0, units = 'm', iotype='out', desc='Tube pipe (outer) Diameter')
    Di_tube = Float(0.0, units = 'm', iotype='out', desc='Tube pipe (inner) Diameter')

    #Assumed Constant Properties
    rho_w = Float(0.0, units = 'kg/m**3', iotype='in', desc='density of water')
    rho_a = Float(0.0, units = 'kg/m**3', iotype='in', desc='density of air ')
    cp_w = Float(0.0, units = 'J/(kg*K)', iotype='in', desc='specific heat of water')
    cp_a = Float(0.0, units = 'J/(kg*K)', iotype='in', desc='specific heat of air')
    dvisc_w = Float(0.0, units = 'kg/(m*s)', iotype='in', desc='dynamic viscosity for water')
    dvis_a = Float(0.0, units = 'kg/(m*s)', iotype='in', desc='dynamic viscosity for air')
    kvisc_w = Float(0.0, units = 'm**2/s', iotype='in', desc='kinematic viscosity for water')
    kvisc_a = Float(0.0, units = 'm**2/s', iotype='in', desc='kinematic viscosity for air')
    k_w = Float(0.0, units = 'W/(m*K)', iotype='in', desc='thermal conductivity for water')
    k_a = Float(0.0, units = 'W/(m*K)', iotype='in', desc='thermal conductivity for air')
    k_p = Float(0.0, units = 'W/(m*K)', iotype='in', desc='thermal conductivity of the pipe')

    #--Outputs--
    #Intermediate Variables
    Asurf_pipe = Float(0.0, units = 'm**2', iotype='out', desc='Surface Area of the Pipe')
    Dh = Float(0.0, units= 'm', iotype='out', desc='Hyrdraulic Diameter of the shell (annulus) for fluid flow')
    De = Float(0.0, units= 'm', iotype='out', desc='Hyrdraulic Diameter of the shell (annulus) for heat flow')

    #Calculated Variables
    V_w = Float(0.0, units= 'm/s', iotype='out', desc='flow velocity of water')
    V_a = Float(0.0, units= 'm/s', iotype='out', desc='flow velocity of air')
    h_w = Float(0.0, iotype ='out', desc='heat transfer of water')
    h_a = Float(0.0, units = 'W/(m**2)*K', iotype='out', desc='heat transfer of air')
    q_w = Float(0.0, units = 'W', iotype='out', desc='heat flow of water')
    q_a = Float(0.0, units = 'W', iotype='out', desc='heat flow of air')

    #Size/Volume Considerations
    Vol_water = Float(0.0, units= 'm**3', iotype='out', desc='Volume of input water tank')
    Vol_steam = Float(0.0, units= 'm**3', iotype='out', desc='Volume of output steam tank')
    Mass_water = Float(0.0, units= 'kg', iotype='out', desc='Mass of input water tank')
    Mass_steam = Float(0.0, units= 'kg', iotype='out', desc='Mass of output steam tank')

    def execute(self):
        """Calculate Various Paramters"""

        #Determine the area of the air tube
        A_a = pi*(self.Di_tube/2)^2
        #Determine the fluid velocity of the air
        #Rearrange Mdot = rho * Area * Velocity --> Velocity = Mdot/(rho*Area)
        Veloc_a = Mdot_a / (rho_a * A_a)

        #Determine q
        #q = mdot * cp * deltaT
        q_a = Mdot_a* cp_a * (T_aout - T_ain)

        #Energy Balance: Q_water must equal Q_air
        q_w = q_a

        #Determine water Mdot
        #q = mdot * cp * deltaT
        Mdot_w = q_w / cp_w * (T_win - T_wout)
        #Determine the flow velocity of the water
        #Rearrange Mdot = rho * Area * Velocity --> Velocity = Mdot/(rho*Area)
        Veloc_w = Mdot_w / (rho_w * A_w)

        #Hydraulic Diameter
        #D_h = (4*Af)/(Pflow) = 4*pi*(Di_shell^2 - Do_tube^2)/ 4*pi*(Di_shell - Do_tube) = Di_shell - Do_tube
        #D_e = (4*Af)/(PheatTransfer) = 4*pi*(Di_shell^2 - Do_tube^2)/ 4*pi* Do_tube = (Di_shell^2 - Do_tube^2)/Do_tube

        Dw_h = self.Di_shell - Do_tube
        Dw_e = (self.Di_shell^2 - self.Do_tube^2)/self.Do_tube

        Da_h = self.Di_shell - Do_tube
        Da_e = (self.Di_shell^2 - self.Do_tube^2)/self.Do_tube

        #Determine the Reynolds Number
        #Re = velocity * hydraulic dimater / kinematic viscostiy   (general form for pipes)
        #Re = inertial forces/ viscous forces
        Re_a = Veloc_a*Da_h/self.kvisc_a
        Re_w = Veloc_w*Dw_h/self.kvisc_w

        #Determine the Prandtl Number
        #Pr = viscous diffusion rate/ thermal diffusion rate = Cp * dyanamic viscosity / thermal conductivity
        #Pr << 1 means thermal diffusivity dominates
        #Pr >> 1 means momentum diffusivity dominates
        Pr_a =
        Pr_w =

        #Determine Overall Heat Transfer Coefficient
        #1) Calculate Reynolds Number
        #2) Calculate Prandtl Number
        #3) Calculate Nusselt Number

        #Determine LMTD
        LMTD = self.LMTD #Log Mean Temp Diff
        Th_in = self.T_ain
        Tc_in = self.T_win
        Th_out = self.T_aout
        Tc_out = self.T_wout

        dT1 = (Th_out - Th_in) #Change in T1 (delta T1)
        dT2 = (Tc_out-Tc_in) #Change in T2 (delta T2)

        self.LMTD = (dT1 - dT2)/(log(dT1/(dT2), e)) #take natural log (base e)

        #Determine the required length of the heat exchanger

        #Assume fouling losses

        #Assume pipe minor losses<|endoftext|>


<commit_before>import argparse

from .client import ArchivesSpaceClient
from .reporter import CSVReporter
from .validator import Validator

class DACSspace:
    """Base DACSspace class. Fetches data from AS, validates and reports results."""

    def run(self, published_only, invalid_only):
        client = ArchivesSpaceClient()
        validator = Validator()
        reporter = CSVReporter()
        data = client.get_resources(published_only)
        results = []
        for obj in data:
            result = validator.validate(obj)
            results.append(result)
        reporter.write_report(results, invalid_only)

parser = argparse.ArgumentParser(
    description="Fetches data from AS, validates and reports results")
parser.add_argument(
    '--published_only',
    help='Fetches only published records from AS',
    action='store_true')
parser.add_argument(
    '--invalid_only',
    help='Reports only invalid data',
    action='store_false')
args = parser.parse_args()

if __name__ == '__main__':
    DACSspace().run(args.published_only, args.invalid_only)
<commit_msg>move if statement<commit_after>import argparse

from .client import ArchivesSpaceClient
from .reporter import CSVReporter
from .validator import Validator

class DACSspace:
    """Base DACSspace class. Fetches data from AS, validates and reports results."""

    def run(self, published_only, invalid_only):
        client = ArchivesSpaceClient()
        validator = Validator()
        reporter = CSVReporter()
        data = client.get_resources(published_only)
        results = []
        for obj in data:
            result = validator.validate(obj)
            results.append(result)
        reporter.write_report(results, invalid_only)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Fetches data from AS, validates and reports results")
    parser.add_argument(
        '--published_only',
        help='Fetches only published records from AS',
        action='store_true')
    parser.add_argument(
        '--invalid_only',
        help='Reports only invalid data',
        action='store_false')
    args = parser.parse_args()

    DACSspace().run(args.published_only, args.invalid_only)
<|endoftext|>


<commit_before>import numpy as np
import os
import sys
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from scipy.spatial import cKDTree
from sklearn.neighbors import NearestNeighbors as NN
from sklearn.neighbors import RadiusNeighborsClassifier as RN
from astropy.utils.console import ProgressBar

plt.style.use('ggplot')
vecinos = 16	#1 + vecinos
radio   = False
output  = 'local_15all14155.pdf'
refer   = 'all14155.dat'
zinfo   = '../zinfo_img'

se,el,yr = np.genfromtxt(zinfo,unpack=True,usecols=(4,5,6),skiprows=6)
yrs = (yr-yr[0])/365.25

def linear(coords,a,b,c):
	x,y = coords
	return a*x + b*y + c

def quadratic(coords,a,b,c,d,e,f):
	x,y = coords
	return a + b*x + c*y + d*np.square(x) + e*np.multiply(x,y) + f*np.square(y)

def linear2(coords,a,b,c,d):
	M = np.matrix([[a,b],[c,d]])
	return np.dot(M,coords)

bid,bx,by = np.genfromtxt(refer,unpack=True,usecols=(0,3,4))

fig, ax = plt.subplots(nrows=23,ncols=3,figsize=[3.5*3,3.5*23])
fig.tight_layout()

for i,a in enumerate(np.ravel(ax)):
	#r1,d1,m,r2,d2 = np.genfromtxt('%s.dat'%(70*j/8+(i+1)),unpack=True,usecols=(1,2,5,7,8))
	id,x1,y1,m,x2,y2 = np.genfromtxt('%s.dat'%(i+1),unpack=True,usecols=(0,3,4,5,9,10))
	mask			 = (m<15)*(m>11)
	id,x1,y1,m,x2,y2   = np.transpose([id,x1,y1,m,x2,y2])[mask].T

	epinbu = np.in1d(id,bid)
	buinep = np.in1d(bid,id)

	print (id[epinbu]==bid[buinep]).sum(),id.size

	epxy = np.transpose([x2,y2])[epinbu]
	epm  = m[epinbu]

	nbrs = NN(n_neighbors=vecinos, algorithm='auto').fit(epxy)

	if radio:
		dist, idx = nbrs.radius_neighbors(np.transpose([x2,y2]),radius=400)
		nbors 	  = np.array([len(d) for d in dist])

		mednbors, minnbors = np.median(nbors),nbors.min()

		for j in range(len(dist)):
			#Elimina la misma estrella
			msk	    = dist[j]>300
			dist[j] = dist[j][msk]
			idx[j]  = idx[j][msk]

			#Toma las 50 mas brillantes
			'''
			if len(dist[j])>15:
				midx = np.argsort(epm[idx[j]])[:15]
				dist[j] = dist[j][midx]
				idx[j]  = idx[j][midx]
			'''

			if len(dist[j])>15:
				midx = np.argsort(dist[j])[:15]
				dist[j] = dist[j][midx]
				idx[j]  = idx[j][midx]

	else:
		dist,idx = nbrs.kneighbors(np.transpose([x2,y2]))
		idx		 = idx[:,1:]
		dist 	 = dist[:,1:]

	means = np.array([np.mean(d) for d in dist])
	print means.size, means.shape
	nbors 	  = np.array([len(d) for d in dist])
	mednbors, minnbors = np.median(nbors),nbors.min()

	ctx = np.zeros(x1.size)
	cty = np.zeros(y1.size)

	with ProgressBar(x1.size) as bar:
		for k in range(x1.size):
			coords = np.transpose([x2,y2])[idx[k]].T
			poptx, pcovx = curve_fit(linear,coords,x1[idx[k]])
			popty, pcovy = curve_fit(linear,coords,y1[idx[k]])

			the_x = x2[k]
			the_y = y2[k]

			ctx[k] += linear([the_x,the_y],*poptx)
			cty[k] += linear([the_x,the_y],*popty)

			bar.update()

	x0,y0 = 1352.,554.
	#x0,y0 = 1788.,-6359.
	clust = (np.sqrt((x1-x0)**2 + (y1-y0)**2) < 280.)
	#clust = np.sqrt((x1-x0)**2 + (y1-y0)**2) < 350.
	#clust = ~epinbu
	#a.plot((r1-r2)[mask],(d1-d2)[mask],'o',ms=.5,alpha=.5)
	a.scatter((x1-ctx)[~clust],(y1-cty)[~clust],s=1,rasterized=True,edgecolor='',color='#0055FF',lw=.5)
	a.scatter((x1-ctx)[clust*(m<14)],(y1-cty)[clust*(m<14)],s=1.25,rasterized=True,edgecolor='',color='#FF5500',lw=.5)
	a.text(.1,.1,u'$S = %f$\n$E = %s$\n$N = %d/%d$\n$Med_d, Max_d = %.3f, %.3f$\n$Med_n,Min_n = %d,%d$'%(se[i+1],el[i+1],(clust*(m<14)).sum(),clust.size,np.median(means),np.max(means),mednbors, minnbors),transform = a.transAxes,alpha=.66,fontsize=10)
	a.text(.1,.9,u'$%d$'%(i+2),transform = a.transAxes,alpha=.66,fontsize=14)
	#a.set_xlim(-6e-5,6e-5)
	#a.set_ylim(-6e-5,6e-5)
	a.set_xlim(-10,10)
	a.set_ylim(-10,10)
	a.set_aspect('equal')

	dx = x1-ctx
	dy = x1-cty

	#np.savetxt()

	#a.set_xticklabels([])
	#a.set_yticklabels([])

		#plt.show()
	if (i%5)==0:
		print i
		#fig.suptitle('Neigh = %d'%vecinos)
		fig.savefig(output,dpi=200)
fig.savefig(output,dpi=200)<commit_msg>Parametros y nombre de los archivos<commit_after>import numpy as np
import os
import sys
import glob
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from scipy.spatial import cKDTree
from sklearn.neighbors import NearestNeighbors as NN
from sklearn.neighbors import RadiusNeighborsClassifier as RN
from astropy.utils.console import ProgressBar

plt.style.use('ggplot')
vecinos = 16						#1 + vecinos
radio   = False						#Usar criterio de radio (True) o vecinos mas cercanos (False)
endepo  = 001						#Numero con que terminan los archivos (formato de 3 numeros ej 001)
output  = 'local_15all14155.pdf'	#PDF de Output (verificar siempre o se reemplazara sin avisar!)
refer   = 'all14155.dat'			#Catalogo con las estrellas (IDs) de referencia
zinfo   = '../zinfo_img'			#Ubicacion del archivo zinfo con los seeings y elipticidades

archivos = sorted(glob.glob(folder+'*_%d.match'%endepo))

se,el,yr = np.genfromtxt(zinfo,unpack=True,usecols=(4,5,6),skiprows=6)
yrs = (yr-yr[0])/365.25

def linear(coords,a,b,c):
	x,y = coords
	return a*x + b*y + c

def quadratic(coords,a,b,c,d,e,f):
	x,y = coords
	return a + b*x + c*y + d*np.square(x) + e*np.multiply(x,y) + f*np.square(y)

def linear2(coords,a,b,c,d):
	M = np.matrix([[a,b],[c,d]])
	return np.dot(M,coords)

bid,bx,by = np.genfromtxt(refer,unpack=True,usecols=(0,3,4))

fig, ax = plt.subplots(nrows=23,ncols=3,figsize=[3.5*3,3.5*23])
fig.tight_layout()

for i,a in enumerate(np.ravel(ax)):
	#r1,d1,m,r2,d2 = np.genfromtxt('%s.dat'%(70*j/8+(i+1)),unpack=True,usecols=(1,2,5,7,8))
	id,x1,y1,m,x2,y2 = np.genfromtxt(archivos[i],unpack=True,usecols=(0,3,4,5,9,10))
	mask			 = (m<15)*(m>11)
	id,x1,y1,m,x2,y2   = np.transpose([id,x1,y1,m,x2,y2])[mask].T

	epinbu = np.in1d(id,bid)
	buinep = np.in1d(bid,id)

	print (id[epinbu]==bid[buinep]).sum(),id.size

	epxy = np.transpose([x2,y2])[epinbu]
	epm  = m[epinbu]

	nbrs = NN(n_neighbors=vecinos, algorithm='auto').fit(epxy)

	if radio:
		dist, idx = nbrs.radius_neighbors(np.transpose([x2,y2]),radius=400)
		nbors 	  = np.array([len(d) for d in dist])

		mednbors, minnbors = np.median(nbors),nbors.min()

		for j in range(len(dist)):
			#Elimina la misma estrella
			msk	    = dist[j]>300
			dist[j] = dist[j][msk]
			idx[j]  = idx[j][msk]

			#Toma las 50 mas brillantes
			'''
			if len(dist[j])>15:
				midx = np.argsort(epm[idx[j]])[:15]
				dist[j] = dist[j][midx]
				idx[j]  = idx[j][midx]
			'''

			if len(dist[j])>15:
				midx = np.argsort(dist[j])[:15]
				dist[j] = dist[j][midx]
				idx[j]  = idx[j][midx]

	else:
		dist,idx = nbrs.kneighbors(np.transpose([x2,y2]))
		idx		 = idx[:,1:]
		dist 	 = dist[:,1:]

	means = np.array([np.mean(d) for d in dist])
	print means.size, means.shape
	nbors 	  = np.array([len(d) for d in dist])
	mednbors, minnbors = np.median(nbors),nbors.min()

	ctx = np.zeros(x1.size)
	cty = np.zeros(y1.size)

	with ProgressBar(x1.size) as bar:
		for k in range(x1.size):
			coords = np.transpose([x2,y2])[idx[k]].T
			poptx, pcovx = curve_fit(linear,coords,x1[idx[k]])
			popty, pcovy = curve_fit(linear,coords,y1[idx[k]])

			the_x = x2[k]
			the_y = y2[k]

			ctx[k] += linear([the_x,the_y],*poptx)
			cty[k] += linear([the_x,the_y],*popty)

			bar.update()

	x0,y0 = 1352.,554.
	#x0,y0 = 1788.,-6359.
	clust = (np.sqrt((x1-x0)**2 + (y1-y0)**2) < 280.)
	#clust = np.sqrt((x1-x0)**2 + (y1-y0)**2) < 350.
	#clust = ~epinbu
	#a.plot((r1-r2)[mask],(d1-d2)[mask],'o',ms=.5,alpha=.5)
	a.scatter((x1-ctx)[~clust],(y1-cty)[~clust],s=1,rasterized=True,edgecolor='',color='#0055FF',lw=.5)
	a.scatter((x1-ctx)[clust*(m<14)],(y1-cty)[clust*(m<14)],s=1.25,rasterized=True,edgecolor='',color='#FF5500',lw=.5)
	a.text(.1,.1,u'$S = %f$\n$E = %s$\n$N = %d/%d$\n$Med_d, Max_d = %.3f, %.3f$\n$Med_n,Min_n = %d,%d$'%(se[i+1],el[i+1],(clust*(m<14)).sum(),clust.size,np.median(means),np.max(means),mednbors, minnbors),transform = a.transAxes,alpha=.66,fontsize=10)
	a.text(.1,.9,u'$%d$'%(i+2),transform = a.transAxes,alpha=.66,fontsize=14)
	#a.set_xlim(-6e-5,6e-5)
	#a.set_ylim(-6e-5,6e-5)
	a.set_xlim(-10,10)
	a.set_ylim(-10,10)
	a.set_aspect('equal')

	dx = x1-ctx
	dy = x1-cty

	#np.savetxt()

	#a.set_xticklabels([])
	#a.set_yticklabels([])

		#plt.show()
	if (i%5)==0:
		print i
		#fig.suptitle('Neigh = %d'%vecinos)
		fig.savefig(output,dpi=200)
fig.savefig(output,dpi=200)<|endoftext|>


<commit_before>#!/usr/bin/env python

#-----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
<commit_msg>STY: PEP8 fixes<commit_after>#!/usr/bin/env python

# -----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# -----------------------------------------------------------------------------
<|endoftext|>