<commit_before>from django import forms
from django.contrib import admin
from django.db.models.functions import Coalesce
from django.urls import reverse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from django.contrib.gis.admin import OSMGeoAdmin
from django.db.models import F, Sum, Count, Q
from django.urls import path
from django.utils import timezone
from django.utils.encoding import force_text
from django.utils.html import format_html, escape

from agir.events.models import RSVP, Event
from agir.people.admin import PersonFormAdminMixin
from agir.people.models import PersonFormSubmission
from ...api.admin import admin_site
from ...groups.models import SupportGroup
from ...lib.admin import CenterOnFranceMixin
from ...lib.utils import front_url

from .. import models

from . import actions
from . import views
from .forms import EventAdminForm

class EventStatusFilter(admin.SimpleListFilter):
    title = _('Statut')

    parameter_name = 'status'

    def lookups(self, request, model_admin):
        return (
            ('all', _('All')),
            ('finished', _('Terminé')),
            ('current', _('En cours')),
            ('upcoming', _('À venir')),
        )

    def choices(self, changelist):
        for lookup, title in self.lookup_choices:
            yield {
                'selected': (lookup == 'upcoming' and self.value() is None) or self.value() == force_text(lookup),
                'query_string': changelist.get_query_string({self.parameter_name: lookup}, []),
                'display': title,
            }

    def queryset(self, request, queryset):
        now = timezone.now()
        if self.value() == 'finished':
            return queryset.filter(end_time__lt=now)
        elif self.value() == 'current':
            return queryset.filter(start_time__lte=now, end_time__gte=now)
        elif self.value() == 'all':
            return queryset
        else:
            return queryset.filter(start_time__gt=now)

class EventHasReportFilter(admin.SimpleListFilter):
    title = _('Compte-rendu présent')

    parameter_name = 'has_report'

    def lookups(self, request, model_admin):
        return (
            ('yes', _('Présent')),
            ('no', _('Absent')),
        )

    def queryset(self, request, queryset):
        if self.value() == 'yes':
            return queryset.exclude(report_content='')
        if self.value() == 'no':
            return queryset.filter(report_content='')

class OrganizerConfigInlineAdminForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        if self.instance._state.adding:
            self.fields['as_group'].queryset = SupportGroup.objects.none()
        else:
            self.fields['as_group'].queryset = SupportGroup.objects.filter(memberships__person=self.instance.person,
                                                                           memberships__is_manager=True)

class OrganizerConfigInline(admin.TabularInline):
    model = models.OrganizerConfig
    fields = ('person_link', 'as_group')
    readonly_fields = ('person_link', )
    extra = 0
    form = OrganizerConfigInlineAdminForm

    def person_link(self, obj):
        return mark_safe(format_html(
            '{}',
            reverse('admin:people_person_change', args=(obj.person.id,)),
            escape(obj.person.email)
        ))
    person_link.short_description = _("Personne")

    def has_add_permission(self, request):
        return False

class EventImageInline(admin.TabularInline):
    model = models.EventImage
    fields = ('image_link', 'author_link', 'legend')
    readonly_fields = ('image_link', 'author_link',)
    extra = 0

    def has_add_permission(self, request):
        return False

    def image_link(self, obj):
        return mark_safe(format_html(
            '',
            obj.image.url,
            obj.image.admin_thumbnail.url
        ))
    image_link.short_description = _('Image')

    def author_link(self, obj):
        return mark_safe(format_html(
            '{}',
            reverse('admin:people_person_change', args=(obj.author.id,)),
            escape(obj.author.email)
        ))
    author_link.short_description = _("Auteur")

@admin.register(models.Event, site=admin_site)
class EventAdmin(PersonFormAdminMixin, CenterOnFranceMixin, OSMGeoAdmin):
    form = EventAdminForm

    fieldsets = (
        (None, {
            'fields': ('id', 'name', 'link', 'created', 'modified', 'add_organizer_button')
        }),
        (_('Informations'), {
            'fields': ('subtype', 'description', 'allow_html', 'image', 'start_time', 'end_time', 'calendars', 'tags', 'published'),
        }),
        (_('Inscription'), {
            'fields': ('max_participants', 'allow_guests', 'subscription_form', 'rsvps_buttons', 'payment_parameters'),
        }),
        (_('Lieu'), {
            'fields': ('location_name', 'location_address1', 'location_address2', 'location_city', 'location_zip',
                       'location_state', 'location_country', 'coordinates', 'coordinates_type', 'redo_geocoding')
        }),
        (_('Contact'), {
            'fields': ('contact_name', 'contact_email', 'contact_phone', 'contact_hide_phone')
        }),
        (_('Compte-rendu'), {
            'fields': ('report_content', 'report_image')
        }),
        (_('NationBuilder'), {
            'fields': ('nb_id', 'nb_path', 'location_address')
        })
    )

    inlines = (OrganizerConfigInline, EventImageInline)

    readonly_fields = (
    'id', 'link', 'add_organizer_button', 'organizers', 'created', 'modified', 'coordinates_type', 'rsvps_buttons')
    date_hierarchy = 'start_time'

    list_display = ('name', 'published', 'calendar_names', 'location_short', 'attendee_count',
                    'start_time', 'created')
    list_filter = (
        'published',
        'subtype__type',
        'subtype',
        'calendars',
        EventHasReportFilter,
        EventStatusFilter,
        'tags',
    )

    search_fields = ('name', 'description', 'location_city', 'location_country')

    actions = (actions.export_events, actions.make_published, actions.unpublish)

    def location_short(self, object):
        return _('{zip} {city}, {country}').format(
            zip=object.location_zip,
            city=object.location_city,
            country=object.location_country.name
        )
    location_short.short_description = _("Lieu")
    location_short.admin_order_field = 'location_city'

    def calendar_names(self, object):
        return ', '.join(c.name for c in object.calendars.all())
    calendar_names.short_description = _('Agendas')

    def add_organizer_button(self, object):
        if object.pk is None:
            return mark_safe('-')
        return format_html(
            '+ Organisateur',
            reverse('admin:events_event_add_organizer', args=[object.pk])
        ) if object.pk else '-'
    add_organizer_button.short_description = _('Ajouter un organisateur')

    def attendee_count(self, object):
        all_attendee = object.attendee_count + object.identified_guests_count

        if object.is_free:
            return str(all_attendee)

        confirmed_attendee = object.confirmed_attendee_count + object.confirmed_identified_guests_count
        return _(f'{all_attendee} (dont {confirmed_attendee} confirmés)')

    attendee_count.short_description = _("Nombre de personnes inscrites")
    attendee_count.admin_order_field = 'attendee_count'

    def link(self, object):
        if object.pk:
            return format_html('{0}', front_url('view_event', kwargs={'pk': object.pk}))
        else:
            return '-'
    link.short_description = _("Page sur le site")

    def rsvps_buttons(self, object):
        if object.subscription_form is None :
            return mark_safe('-')
        else:
            return format_html(
                'Voir les inscriptions
'
                'Télécharger les inscriptions
',
                view_results_link=reverse('admin:events_event_rsvps_view_results', args=(object.pk,)),
                download_results_link=reverse('admin:events_event_rsvps_download_results', args=(object.pk,)),
            )
    rsvps_buttons.short_description = _("Inscriptions")

    def get_form_submission_qs(self, form):
        return PersonFormSubmission.objects.filter(Q(rsvp__event=self.instance) | Q(guest_rsvp__event=self.instance))

    def view_results(self, request, pk):
        self.instance = Event.objects.get(pk=pk)
        return super().view_results(request, self.instance.subscription_form.id)

    def download_results(self, request, pk):
        self.instance = Event.objects.get(pk=pk)
        return super().download_results(request, self.instance.subscription_form.id)

    def get_queryset(self, request):
        qs = super().get_queryset(request)

        confirmed_guests = Q(rsvps__identified_guests__status=RSVP.STATUS_CONFIRMED)
        confirmed_rsvps = Q(rsvps__status=RSVP.STATUS_CONFIRMED)

        return qs \
            .annotate(confirmed_identified_guests_count=Coalesce(Count('rsvps__identified_guests', filter=confirmed_guests), 0)) \
            .annotate(confirmed_attendee_count=Coalesce(Sum(1 + F('rsvps__guests'), filter=confirmed_rsvps), 0)) \
            .annotate(identified_guests_count=Coalesce(Count('rsvps__identified_guests'), 0)) \
            .annotate(attendee_count=Coalesce(Sum(1 + F('rsvps__guests')), 0))

    def get_urls(self):
        return [
            path('<uuid:pk>/add_organizer/', admin_site.admin_view(self.add_organizer), name="events_event_add_organizer"),
            path('<uuid:pk>/view_rsvps/', self.admin_site.admin_view(self.view_results), name="events_event_rsvps_view_results"),
            path('<uuid:pk>/download_rsvps/', self.admin_site.admin_view(self.download_results), name="events_event_rsvps_download_results"),
        ] + super().get_urls()

    def add_organizer(self, request, pk):
        return views.add_member(self, request, pk)

@admin.register(models.Calendar, site=admin_site)
class CalendarAdmin(admin.ModelAdmin):
    fields = ('name', 'slug', 'link', 'user_contributed', 'description', 'image')
    list_display = ('name', 'slug', 'user_contributed')
    readonly_fields = ('link',)

    def link(self, object):
        if object.slug:
            return format_html('{0}', front_url('view_calendar', kwargs={'slug': object.slug}))
        else:
            return '-'
    link.short_description = _("Lien vers l'agenda")

@admin.register(models.EventTag, site=admin_site)
class EventTagAdmin(admin.ModelAdmin):
    pass

@admin.register(models.EventSubtype, site=admin_site)
class EventSubtypeAdmin(admin.ModelAdmin):
    list_display = ('label', 'description', 'type')
    list_filter = ('type',)
<commit_msg>fix participants count in admin<commit_after>from django import forms
from django.contrib import admin
from django.db.models.functions import Coalesce
from django.urls import reverse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from django.contrib.gis.admin import OSMGeoAdmin
from django.db.models import F, Sum, Count, Q
from django.urls import path
from django.utils import timezone
from django.utils.encoding import force_text
from django.utils.html import format_html, escape

from agir.events.models import RSVP, Event
from agir.people.admin import PersonFormAdminMixin
from agir.people.models import PersonFormSubmission
from ...api.admin import admin_site
from ...groups.models import SupportGroup
from ...lib.admin import CenterOnFranceMixin
from ...lib.utils import front_url

from .. import models

from . import actions
from . import views
from .forms import EventAdminForm

class EventStatusFilter(admin.SimpleListFilter):
    title = _('Statut')

    parameter_name = 'status'

    def lookups(self, request, model_admin):
        return (
            ('all', _('All')),
            ('finished', _('Terminé')),
            ('current', _('En cours')),
            ('upcoming', _('À venir')),
        )

    def choices(self, changelist):
        for lookup, title in self.lookup_choices:
            yield {
                'selected': (lookup == 'upcoming' and self.value() is None) or self.value() == force_text(lookup),
                'query_string': changelist.get_query_string({self.parameter_name: lookup}, []),
                'display': title,
            }

    def queryset(self, request, queryset):
        now = timezone.now()
        if self.value() == 'finished':
            return queryset.filter(end_time__lt=now)
        elif self.value() == 'current':
            return queryset.filter(start_time__lte=now, end_time__gte=now)
        elif self.value() == 'all':
            return queryset
        else:
            return queryset.filter(start_time__gt=now)

class EventHasReportFilter(admin.SimpleListFilter):
    title = _('Compte-rendu présent')

    parameter_name = 'has_report'

    def lookups(self, request, model_admin):
        return (
            ('yes', _('Présent')),
            ('no', _('Absent')),
        )

    def queryset(self, request, queryset):
        if self.value() == 'yes':
            return queryset.exclude(report_content='')
        if self.value() == 'no':
            return queryset.filter(report_content='')

class OrganizerConfigInlineAdminForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        if self.instance._state.adding:
            self.fields['as_group'].queryset = SupportGroup.objects.none()
        else:
            self.fields['as_group'].queryset = SupportGroup.objects.filter(memberships__person=self.instance.person,
                                                                           memberships__is_manager=True)

class OrganizerConfigInline(admin.TabularInline):
    model = models.OrganizerConfig
    fields = ('person_link', 'as_group')
    readonly_fields = ('person_link', )
    extra = 0
    form = OrganizerConfigInlineAdminForm

    def person_link(self, obj):
        return mark_safe(format_html(
            '{}',
            reverse('admin:people_person_change', args=(obj.person.id,)),
            escape(obj.person.email)
        ))
    person_link.short_description = _("Personne")

    def has_add_permission(self, request):
        return False

class EventImageInline(admin.TabularInline):
    model = models.EventImage
    fields = ('image_link', 'author_link', 'legend')
    readonly_fields = ('image_link', 'author_link',)
    extra = 0

    def has_add_permission(self, request):
        return False

    def image_link(self, obj):
        return mark_safe(format_html(
            '',
            obj.image.url,
            obj.image.admin_thumbnail.url
        ))
    image_link.short_description = _('Image')

    def author_link(self, obj):
        return mark_safe(format_html(
            '{}',
            reverse('admin:people_person_change', args=(obj.author.id,)),
            escape(obj.author.email)
        ))
    author_link.short_description = _("Auteur")

@admin.register(models.Event, site=admin_site)
class EventAdmin(PersonFormAdminMixin, CenterOnFranceMixin, OSMGeoAdmin):
    form = EventAdminForm

    fieldsets = (
        (None, {
            'fields': ('id', 'name', 'link', 'created', 'modified', 'add_organizer_button')
        }),
        (_('Informations'), {
            'fields': ('subtype', 'description', 'allow_html', 'image', 'start_time', 'end_time', 'calendars', 'tags', 'published'),
        }),
        (_('Inscription'), {
            'fields': ('max_participants', 'allow_guests', 'subscription_form', 'rsvps_buttons', 'payment_parameters'),
        }),
        (_('Lieu'), {
            'fields': ('location_name', 'location_address1', 'location_address2', 'location_city', 'location_zip',
                       'location_state', 'location_country', 'coordinates', 'coordinates_type', 'redo_geocoding')
        }),
        (_('Contact'), {
            'fields': ('contact_name', 'contact_email', 'contact_phone', 'contact_hide_phone')
        }),
        (_('Compte-rendu'), {
            'fields': ('report_content', 'report_image')
        }),
        (_('NationBuilder'), {
            'fields': ('nb_id', 'nb_path', 'location_address')
        })
    )

    inlines = (OrganizerConfigInline, EventImageInline)

    readonly_fields = (
    'id', 'link', 'add_organizer_button', 'organizers', 'created', 'modified', 'coordinates_type', 'rsvps_buttons')
    date_hierarchy = 'start_time'

    list_display = ('name', 'published', 'calendar_names', 'location_short', 'attendee_count',
                    'start_time', 'created')
    list_filter = (
        'published',
        'subtype__type',
        'subtype',
        'calendars',
        EventHasReportFilter,
        EventStatusFilter,
        'tags',
    )

    search_fields = ('name', 'description', 'location_city', 'location_country')

    actions = (actions.export_events, actions.make_published, actions.unpublish)

    def location_short(self, object):
        return _('{zip} {city}, {country}').format(
            zip=object.location_zip,
            city=object.location_city,
            country=object.location_country.name
        )
    location_short.short_description = _("Lieu")
    location_short.admin_order_field = 'location_city'

    def calendar_names(self, object):
        return ', '.join(c.name for c in object.calendars.all())
    calendar_names.short_description = _('Agendas')

    def add_organizer_button(self, object):
        if object.pk is None:
            return mark_safe('-')
        return format_html(
            '+ Organisateur',
            reverse('admin:events_event_add_organizer', args=[object.pk])
        ) if object.pk else '-'
    add_organizer_button.short_description = _('Ajouter un organisateur')

    def attendee_count(self, object):
        all_attendee = object.identified_guests_count +\
                       object.rsvps.aggregate(participants=Sum(F('guests') + 1))['participants']

        if object.is_free:
            return str(all_attendee)

        confirmed_rsvps = Q(status=RSVP.STATUS_CONFIRMED)
        confirmed_attendee = object.confirmed_identified_guests_count +\
                    object.rsvps.aggregate(participants=Sum(F('guests') + 1, filter=confirmed_rsvps))['participants']
        return _(f'{all_attendee} (dont {confirmed_attendee} confirmés)')

    attendee_count.short_description = _("Nombre de personnes inscrites")
    attendee_count.admin_order_field = 'attendee_count'

    def link(self, object):
        if object.pk:
            return format_html('{0}', front_url('view_event', kwargs={'pk': object.pk}))
        else:
            return '-'
    link.short_description = _("Page sur le site")

    def rsvps_buttons(self, object):
        if object.subscription_form is None :
            return mark_safe('-')
        else:
            return format_html(
                'Voir les inscriptions
'
                'Télécharger les inscriptions
',
                view_results_link=reverse('admin:events_event_rsvps_view_results', args=(object.pk,)),
                download_results_link=reverse('admin:events_event_rsvps_download_results', args=(object.pk,)),
            )
    rsvps_buttons.short_description = _("Inscriptions")

    def get_form_submission_qs(self, form):
        return PersonFormSubmission.objects.filter(Q(rsvp__event=self.instance) | Q(guest_rsvp__event=self.instance))

    def view_results(self, request, pk):
        self.instance = Event.objects.get(pk=pk)
        return super().view_results(request, self.instance.subscription_form.id)

    def download_results(self, request, pk):
        self.instance = Event.objects.get(pk=pk)
        return super().download_results(request, self.instance.subscription_form.id)

    def get_queryset(self, request):
        qs = super().get_queryset(request)

        confirmed_guests = Q(rsvps__identified_guests__status=RSVP.STATUS_CONFIRMED)

        return qs \
            .annotate(confirmed_identified_guests_count=Coalesce(Count('rsvps__identified_guests', filter=confirmed_guests), 0)) \
            .annotate(identified_guests_count=Coalesce(Count('rsvps__identified_guests'), 0))

    def get_urls(self):
        return [
            path('<uuid:pk>/add_organizer/', admin_site.admin_view(self.add_organizer), name="events_event_add_organizer"),
            path('<uuid:pk>/view_rsvps/', self.admin_site.admin_view(self.view_results), name="events_event_rsvps_view_results"),
            path('<uuid:pk>/download_rsvps/', self.admin_site.admin_view(self.download_results), name="events_event_rsvps_download_results"),
        ] + super().get_urls()

    def add_organizer(self, request, pk):
        return views.add_member(self, request, pk)

@admin.register(models.Calendar, site=admin_site)
class CalendarAdmin(admin.ModelAdmin):
    fields = ('name', 'slug', 'link', 'user_contributed', 'description', 'image')
    list_display = ('name', 'slug', 'user_contributed')
    readonly_fields = ('link',)

    def link(self, object):
        if object.slug:
            return format_html('{0}', front_url('view_calendar', kwargs={'slug': object.slug}))
        else:
            return '-'
    link.short_description = _("Lien vers l'agenda")

@admin.register(models.EventTag, site=admin_site)
class EventTagAdmin(admin.ModelAdmin):
    pass

@admin.register(models.EventSubtype, site=admin_site)
class EventSubtypeAdmin(admin.ModelAdmin):
    list_display = ('label', 'description', 'type')
    list_filter = ('type',)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pymisp import MISPEvent
try:
    from bs4 import BeautifulSoup
    has_bs4 = True
except ImportError:
    has_bs4 = False

iocMispMapping = {
    'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'},

    'DnsEntryItem/Host': {'type': 'domain'},

    'Email/To': {'type': 'target-email'},
    'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'},
    # 'Email/Body': {'type': 'email-subject'},
    'Email/From': {'type': 'email-dst'},
    'Email/Subject': {'type': 'email-subject'},
    'Email/Attachment/Name': {'type': 'email-attachment'},

    'FileItem/Md5sum': {'type': 'md5'},
    'FileItem/Sha1sum': {'type': 'sha1'},
    'FileItem/Sha256sum': {'type': 'sha256'},

    'ServiceItem/serviceDLLmd5sum': {'type': 'md5', 'category': 'Payload installation'},
    'ServiceItem/serviceDLLsha1sum': {'type': 'sha1', 'category': 'Payload installation'},
    'ServiceItem/serviceDLLsha256sum': {'type': 'sha256', 'category': 'Payload installation'},

    'TaskItem/md5sum': {'type': 'md5'},
    'TaskItem/sha1sum': {'type': 'sha1'},
    'TaskItem/Sha256sum': {'type': 'sha256'},

    'FileItem/FileName': {'type': 'filename'},
    'FileItem/FullPath': {'type': 'filename'},
    'FileItem/FilePath': {'type': 'filename'},
    'DriverItem/DriverName': {'type': 'filename'},

    'Network/URI': {'type': 'uri'},
    'Network/DNS': {'type': 'domain'},
    'Network/String': {'type': 'ip-dst'},
    'RouteEntryItem/Destination': {'type': 'ip-dst'},
    'Network/UserAgent': {'type': 'user-agent'},

    'PortItem/localIP': {'type': 'ip-src'},
    'PortItem/remoteIP': {'type': 'ip-dst'},

    'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'},
    'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'},
    'ProcessItem/Mutex': {'type': 'mutex'},
    'ProcessItem/Pipe/Name': {'type': 'named pipe'},
    'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'},

    'CookieHistoryItem/HostName': {'type': 'hostname'},
    'FormHistoryItem/HostName': {'type': 'hostname'},
    'SystemInfoItem/HostName': {'type': 'hostname'},
    'UrlHistoryItem/HostName': {'type': 'hostname'},
    'DnsEntryItem/RecordName': {'type': 'hostname'},
    'DnsEntryItem/Host': {'type': 'hostname'},

    # Is it the regkey value?
    # 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '},
    'RegistryItem/KeyPath': {'type': 'regkey'},
    'RegistryItem/Path': {'type': 'regkey'},

    'ServiceItem/name': {'type': 'windows-service-name'},
    'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '},

    'Snort/Snort': {'type': 'snort'},
}

def extract_field(report, field_name):
    data = report.find(field_name.lower())
    if data and hasattr(data, 'text'):
        return data.text
    return None

def load_openioc(openioc):
    if not has_bs4:
        raise Exception('You need to install BeautifulSoup: pip install bs4')
    misp_event = MISPEvent()
    with open(openioc, "r") as ioc_file:
        iocreport = BeautifulSoup(ioc_file, "lxml")
        # Set event fields
        info = extract_field(iocreport, 'short_description')
        if info:
            misp_event.info = info
        date = extract_field(iocreport, 'authored_date')
        if date:
            misp_event.set_date(date)
        # Set special attributes
        description = extract_field(iocreport, 'description')
        if description:
            misp_event.add_attribute('comment', description)
        author = extract_field(iocreport, 'authored_by')
        if author:
            misp_event.add_attribute('comment', author)
        misp_event = set_all_attributes(iocreport, misp_event)
    return misp_event

def get_mapping(openioc_type):
    t = openioc_type.lower()
    for k, v in iocMispMapping.items():
        if k.lower() == t:
            return v
    return False

def set_all_attributes(openioc, misp_event):
    for item in openioc.find_all("indicatoritem"):
        attribute_values = {'comment': ''}
        if item.find('context'):
            mapping = get_mapping(item.find('context')['search'])
            if mapping:
                attribute_values.update(mapping)
            else:
                # Unknown mapping, ignoring
                # print(item.find('context'))
                continue
        else:
            continue
        value = extract_field(item, 'Content')
        if value:
            attribute_values['value'] = value
        else:
            # No value, ignoring
            continue
        comment = extract_field(item, 'Comment')
        if comment:
            attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment)
        misp_event.add_attribute(**attribute_values)
    return misp_event
<commit_msg>Allow to pass a pseudo file to OpenIOC loader.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

from pymisp import MISPEvent
try:
    from bs4 import BeautifulSoup
    has_bs4 = True
except ImportError:
    has_bs4 = False

iocMispMapping = {
    'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'},

    'DnsEntryItem/Host': {'type': 'domain'},

    'Email/To': {'type': 'target-email'},
    'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'},
    # 'Email/Body': {'type': 'email-subject'},
    'Email/From': {'type': 'email-dst'},
    'Email/Subject': {'type': 'email-subject'},
    'Email/Attachment/Name': {'type': 'email-attachment'},

    'FileItem/Md5sum': {'type': 'md5'},
    'FileItem/Sha1sum': {'type': 'sha1'},
    'FileItem/Sha256sum': {'type': 'sha256'},

    'ServiceItem/serviceDLLmd5sum': {'type': 'md5', 'category': 'Payload installation'},
    'ServiceItem/serviceDLLsha1sum': {'type': 'sha1', 'category': 'Payload installation'},
    'ServiceItem/serviceDLLsha256sum': {'type': 'sha256', 'category': 'Payload installation'},

    'TaskItem/md5sum': {'type': 'md5'},
    'TaskItem/sha1sum': {'type': 'sha1'},
    'TaskItem/Sha256sum': {'type': 'sha256'},

    'FileItem/FileName': {'type': 'filename'},
    'FileItem/FullPath': {'type': 'filename'},
    'FileItem/FilePath': {'type': 'filename'},
    'DriverItem/DriverName': {'type': 'filename'},

    'Network/URI': {'type': 'uri'},
    'Network/DNS': {'type': 'domain'},
    'Network/String': {'type': 'ip-dst'},
    'RouteEntryItem/Destination': {'type': 'ip-dst'},
    'Network/UserAgent': {'type': 'user-agent'},

    'PortItem/localIP': {'type': 'ip-src'},
    'PortItem/remoteIP': {'type': 'ip-dst'},

    'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'},
    'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'},
    'ProcessItem/Mutex': {'type': 'mutex'},
    'ProcessItem/Pipe/Name': {'type': 'named pipe'},
    'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'},

    'CookieHistoryItem/HostName': {'type': 'hostname'},
    'FormHistoryItem/HostName': {'type': 'hostname'},
    'SystemInfoItem/HostName': {'type': 'hostname'},
    'UrlHistoryItem/HostName': {'type': 'hostname'},
    'DnsEntryItem/RecordName': {'type': 'hostname'},
    'DnsEntryItem/Host': {'type': 'hostname'},

    # Is it the regkey value?
    # 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '},
    'RegistryItem/KeyPath': {'type': 'regkey'},
    'RegistryItem/Path': {'type': 'regkey'},

    'ServiceItem/name': {'type': 'windows-service-name'},
    'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '},

    'Snort/Snort': {'type': 'snort'},
}

def extract_field(report, field_name):
    data = report.find(field_name.lower())
    if data and hasattr(data, 'text'):
        return data.text
    return None

def load_openioc_file(openioc_path):
    if not os.path.exists(openioc_path):
        raise Exception("Path doesn't exists.")
    with open(openioc_path, 'r') as f:
        return load_openioc(f)

def load_openioc(openioc):
    # Takes a opened file, or a string
    if not has_bs4:
        raise Exception('You need to install BeautifulSoup: pip install bs4')
    misp_event = MISPEvent()
    iocreport = BeautifulSoup(openioc, "lxml")
    # Set event fields
    info = extract_field(iocreport, 'short_description')
    if info:
        misp_event.info = info
    date = extract_field(iocreport, 'authored_date')
    if date:
        misp_event.set_date(date)
    # Set special attributes
    description = extract_field(iocreport, 'description')
    if description:
        misp_event.add_attribute('comment', description)
    author = extract_field(iocreport, 'authored_by')
    if author:
        misp_event.add_attribute('comment', author)
    misp_event = set_all_attributes(iocreport, misp_event)
    return misp_event

def get_mapping(openioc_type):
    t = openioc_type.lower()
    for k, v in iocMispMapping.items():
        if k.lower() == t:
            return v
    return False

def set_all_attributes(openioc, misp_event):
    for item in openioc.find_all("indicatoritem"):
        attribute_values = {'comment': ''}
        if item.find('context'):
            mapping = get_mapping(item.find('context')['search'])
            if mapping:
                attribute_values.update(mapping)
            else:
                # Unknown mapping, ignoring
                # print(item.find('context'))
                continue
        else:
            continue
        value = extract_field(item, 'Content')
        if value:
            attribute_values['value'] = value
        else:
            # No value, ignoring
            continue
        comment = extract_field(item, 'Comment')
        if comment:
            attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment)
        misp_event.add_attribute(**attribute_values)
    return misp_event
<|endoftext|>


<commit_before># coding: utf-8
"""Utilities for installing Javascript extensions for the notebook"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

import os
import shutil
import sys
import tarfile
import zipfile
import uuid
from os.path import basename, join as pjoin

# Deferred imports
try:
    from urllib.parse import urlparse  # Py3
    from urllib.request import urlretrieve
except ImportError:
    from urlparse import urlparse
    from urllib import urlretrieve

from IPython.utils.path import get_ipython_dir, ensure_dir_exists
from IPython.utils.py3compat import string_types, cast_unicode_py2
from IPython.utils.tempdir import TemporaryDirectory

class ArgumentConflict(ValueError):
    pass

# Packagers: modify the next block if you store system-installed nbextensions elsewhere (unlikely)
SYSTEM_NBEXTENSIONS_DIRS = []

if os.name == 'nt':
    programdata = os.environ.get('PROGRAMDATA', None)
    if programdata: # PROGRAMDATA is not defined by default on XP.
        SYSTEM_NBEXTENSIONS_DIRS = [pjoin(programdata, 'jupyter', 'nbextensions')]
    prefixes = []
else:
    prefixes = [os.path.sep + pjoin('usr', 'local'), os.path.sep + 'usr']

# add sys.prefix at the front
if sys.prefix not in prefixes:
    prefixes.insert(0, sys.prefix)

for prefix in prefixes:
    nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
    if nbext not in SYSTEM_NBEXTENSIONS_DIRS:
        SYSTEM_NBEXTENSIONS_DIRS.append(nbext)

if os.name == 'nt':
    # PROGRAMDATA
    SYSTEM_NBEXTENSIONS_INSTALL_DIR = SYSTEM_NBEXTENSIONS_DIRS[-1]
else:
    # /usr/local
    SYSTEM_NBEXTENSIONS_INSTALL_DIR = SYSTEM_NBEXTENSIONS_DIRS[-2]

def _should_copy(src, dest, verbose=1):
    """should a file be copied?"""
    if not os.path.exists(dest):
        return True
    if os.stat(dest).st_mtime < os.stat(src).st_mtime:
        if verbose >= 2:
            print("%s is out of date" % dest)
        return True
    if verbose >= 2:
        print("%s is up to date" % dest)
    return False

def _maybe_copy(src, dest, verbose=1):
    """copy a file if it needs updating"""
    if _should_copy(src, dest, verbose):
        if verbose >= 1:
            print("copying %s -> %s" % (src, dest))
        try:
            shutil.copy2(src, dest)
        except IOError as e:
            print(str(e), file=sys.stderr)

def _safe_is_tarfile(path):
    """safe version of is_tarfile, return False on IOError"""
    try:
        return tarfile.is_tarfile(path)
    except IOError:
        return False

def check_nbextension(files, nbextensions_dir=None):
    """Check whether nbextension files have been installed

    files should be a list of relative paths within nbextensions.

    Returns True if all files are found, False if any are missing.
    """
    if nbextensions_dir:
        nbext = nbextensions_dir
    else:
        nbext = pjoin(get_ipython_dir(), u'nbextensions')
    # make sure nbextensions dir exists
    if not os.path.exists(nbext):
        return False

    if isinstance(files, string_types):
        # one file given, turn it into a list
        files = [files]

    return all(os.path.exists(pjoin(nbext, f)) for f in files)

def install_nbextension(files, overwrite=False, symlink=False, user=False, prefix=None, nbextensions_dir=None, verbose=1):
    """Install a Javascript extension for the notebook

    Stages files and/or directories into the nbextensions directory.
    By default, this compares modification time, and only stages files that need updating.
    If `overwrite` is specified, matching files are purged before proceeding.

    Parameters
    ----------

    files : list(paths or URLs) or dict(install_name: path or URL)
        One or more paths or URLs to existing files directories to install.
        If given as a list, these will be installed with their base name, so '/path/to/foo'
        will install to 'nbextensions/foo'.  If given as a dict, such as {'bar': '/path/to/foo'},
        then '/path/to/foo' will install to 'nbextensions/bar'.
        Archives (zip or tarballs) will be extracted into the nbextensions directory.
    overwrite : bool [default: False]
        If True, always install the files, regardless of what may already be installed.
    symlink : bool [default: False]
        If True, create a symlink in nbextensions, rather than copying files.
        Not allowed with URLs or archives. Windows support for symlinks requires
        Vista or above, Python 3, and a permission bit which only admin users
        have by default, so don't rely on it.
    user : bool [default: False]
        Whether to install to the user's .ipython/nbextensions directory.
        Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
    prefix : str [optional]
        Specify install prefix, if it should differ from default (e.g. /usr/local).
        Will install to prefix/share/jupyter/nbextensions
    nbextensions_dir : str [optional]
        Specify absolute path of nbextensions directory explicitly.
    verbose : int [default: 1]
        Set verbosity level. The default is 1, where file actions are printed.
        set verbose=2 for more output, or verbose=0 for silence.
    """
    if sum(map(bool, [user, prefix, nbextensions_dir])) > 1:
        raise ArgumentConflict("Cannot specify more than one of user, prefix, or nbextensions_dir.")
    if user:
        nbext = pjoin(get_ipython_dir(), u'nbextensions')
    else:
        if prefix:
            nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
        elif nbextensions_dir:
            nbext = nbextensions_dir
        else:
            nbext = SYSTEM_NBEXTENSIONS_INSTALL_DIR
    # make sure nbextensions dir exists
    ensure_dir_exists(nbext)

    if isinstance(files, string_types):
        # one file given, turn it into a list
        files = [files]
    if isinstance(files, (list,tuple)):
        # list given, turn into dict
        _files = {}
        for path in map(cast_unicode_py2, files):
            if path.startswith(('https://', 'http://')):
                destination = urlparse(path).path.split('/')[-1]
            elif path.endswith('.zip') or _safe_is_tarfile(path):
                destination = str(uuid.uuid4()) # ignored for archives
            else:
                destination = basename(path)
            _files[destination] = path
        files = _files

    for dest_basename,path in (map(cast_unicode_py2, item) for item in files.items()):

        if path.startswith(('https://', 'http://')):
            if symlink:
                raise ValueError("Cannot symlink from URLs")
            # Given a URL, download it
            with TemporaryDirectory() as td:
                filename = urlparse(path).path.split('/')[-1]
                local_path = os.path.join(td, filename)
                if verbose >= 1:
                    print("downloading %s to %s" % (path, local_path))
                urlretrieve(path, local_path)
                # now install from the local copy
                install_nbextension({dest_basename: local_path}, overwrite=overwrite, symlink=symlink, nbextensions_dir=nbext, verbose=verbose)
            continue

        # handle archives
        archive = None
        if path.endswith('.zip'):
            archive = zipfile.ZipFile(path)
        elif _safe_is_tarfile(path):
            archive = tarfile.open(path)

        if archive:
            if symlink:
                raise ValueError("Cannot symlink from archives")
            if verbose >= 1:
                print("extracting %s to %s" % (path, nbext))
            archive.extractall(nbext)
            archive.close()
            continue

        dest = pjoin(nbext, dest_basename)
        if overwrite and os.path.exists(dest):
            if verbose >= 1:
                print("removing %s" % dest)
            if os.path.isdir(dest) and not os.path.islink(dest):
                shutil.rmtree(dest)
            else:
                os.remove(dest)

        if symlink:
            path = os.path.abspath(path)
            if not os.path.exists(dest):
                if verbose >= 1:
                    print("symlink %s -> %s" % (dest, path))
                os.symlink(path, dest)
            continue

        if os.path.isdir(path):
            path = pjoin(os.path.abspath(path), '') # end in path separator
            for parent, dirs, files in os.walk(path):
                dest_dir = pjoin(dest, parent[len(path):])
                if not os.path.exists(dest_dir):
                    if verbose >= 2:
                        print("making directory %s" % dest_dir)
                    os.makedirs(dest_dir)
                for file in files:
                    src = pjoin(parent, file)
                    # print("%r, %r" % (dest_dir, file))
                    dest_file = pjoin(dest_dir, file)
                    _maybe_copy(src, dest_file, verbose)
        else:
            src = path
            _maybe_copy(src, dest, verbose)

#----------------------------------------------------------------------
# install nbextension app
#----------------------------------------------------------------------

from IPython.utils.traitlets import Bool, Enum, Unicode, TraitError
from IPython.core.application import BaseIPythonApplication

flags = {
    "overwrite" : ({
        "NBExtensionApp" : {
            "overwrite" : True,
        }}, "Force overwrite of existing files"
    ),
    "debug" : ({
        "NBExtensionApp" : {
            "verbose" : 2,
        }}, "Extra output"
    ),
    "quiet" : ({
        "NBExtensionApp" : {
            "verbose" : 0,
        }}, "Minimal output"
    ),
    "symlink" : ({
        "NBExtensionApp" : {
            "symlink" : True,
        }}, "Create symlinks instead of copying files"
    ),
    "user" : ({
        "NBExtensionApp" : {
            "user" : True,
        }}, "Install to the user's IPython directory"
    ),
}
flags['s'] = flags['symlink']

aliases = {
    "ipython-dir" : "NBExtensionApp.ipython_dir",
    "prefix" : "NBExtensionApp.prefix",
    "nbextensions" : "NBExtensionApp.nbextensions_dir",
}

class NBExtensionApp(BaseIPythonApplication):
    """Entry point for installing notebook extensions"""

    description = """Install IPython notebook extensions

    Usage

        ipython install-nbextension file [more files, folders, archives or urls]

    This copies files and/or folders into the IPython nbextensions directory.
    If a URL is given, it will be downloaded.
    If an archive is given, it will be extracted into nbextensions.
    If the requested files are already up to date, no action is taken
    unless --overwrite is specified.
    """

    examples = """
    ipython install-nbextension /path/to/d3.js /path/to/myextension
    """
    aliases = aliases
    flags = flags

    overwrite = Bool(False, config=True, help="Force overwrite of existing files")
    symlink = Bool(False, config=True, help="Create symlinks instead of copying files")
    user = Bool(False, config=True, help="Whether to do a user install")
    prefix = Unicode('', config=True, help="Installation prefix")
    nbextensions_dir = Unicode('', config=True, help="Full path to nbextensions dir (probably use prefix or user)")
    verbose = Enum((0,1,2), default_value=1, config=True,
        help="Verbosity level"
    )

    def install_extensions(self):
        install_nbextension(self.extra_args,
            overwrite=self.overwrite,
            symlink=self.symlink,
            verbose=self.verbose,
            user=self.user,
            prefix=self.prefix,
            nbextensions_dir=self.nbextensions_dir,
        )

    def start(self):
        if not self.extra_args:
            for nbext in [pjoin(self.ipython_dir, u'nbextensions')] + SYSTEM_NBEXTENSIONS_DIRS:
                if os.path.exists(nbext):
                    print("Notebook extensions in %s:" % nbext)
                    for ext in os.listdir(nbext):
                        print(u"    %s" % ext)
        else:
            try:
                self.install_extensions()
            except ArgumentConflict as e:
                print(str(e), file=sys.stderr)
                self.exit(1)

if __name__ == '__main__':
    NBExtensionApp.launch_instance()
    <commit_msg>Work around a bug in setting and getting the mtime in python 2<commit_after># coding: utf-8
"""Utilities for installing Javascript extensions for the notebook"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

import os
import shutil
import sys
import tarfile
import zipfile
import uuid
from os.path import basename, join as pjoin

# Deferred imports
try:
    from urllib.parse import urlparse  # Py3
    from urllib.request import urlretrieve
except ImportError:
    from urlparse import urlparse
    from urllib import urlretrieve

from IPython.utils.path import get_ipython_dir, ensure_dir_exists
from IPython.utils.py3compat import string_types, cast_unicode_py2
from IPython.utils.tempdir import TemporaryDirectory

class ArgumentConflict(ValueError):
    pass

# Packagers: modify the next block if you store system-installed nbextensions elsewhere (unlikely)
SYSTEM_NBEXTENSIONS_DIRS = []

if os.name == 'nt':
    programdata = os.environ.get('PROGRAMDATA', None)
    if programdata: # PROGRAMDATA is not defined by default on XP.
        SYSTEM_NBEXTENSIONS_DIRS = [pjoin(programdata, 'jupyter', 'nbextensions')]
    prefixes = []
else:
    prefixes = [os.path.sep + pjoin('usr', 'local'), os.path.sep + 'usr']

# add sys.prefix at the front
if sys.prefix not in prefixes:
    prefixes.insert(0, sys.prefix)

for prefix in prefixes:
    nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
    if nbext not in SYSTEM_NBEXTENSIONS_DIRS:
        SYSTEM_NBEXTENSIONS_DIRS.append(nbext)

if os.name == 'nt':
    # PROGRAMDATA
    SYSTEM_NBEXTENSIONS_INSTALL_DIR = SYSTEM_NBEXTENSIONS_DIRS[-1]
else:
    # /usr/local
    SYSTEM_NBEXTENSIONS_INSTALL_DIR = SYSTEM_NBEXTENSIONS_DIRS[-2]

def _should_copy(src, dest, verbose=1):
    """should a file be copied?"""
    if not os.path.exists(dest):
        return True
    if os.stat(src).st_mtime - os.stat(dest).st_mtime > 1e-6:
        # we add a fudge factor to work around a bug in python 2.x
        # that was fixed in python 3.x: http://bugs.python.org/issue12904
        if verbose >= 2:
            print("%s is out of date" % dest)
        return True
    if verbose >= 2:
        print("%s is up to date" % dest)
    return False

def _maybe_copy(src, dest, verbose=1):
    """copy a file if it needs updating"""
    if _should_copy(src, dest, verbose):
        if verbose >= 1:
            print("copying %s -> %s" % (src, dest))
        try:
            shutil.copy2(src, dest)
        except IOError as e:
            print(str(e), file=sys.stderr)

def _safe_is_tarfile(path):
    """safe version of is_tarfile, return False on IOError"""
    try:
        return tarfile.is_tarfile(path)
    except IOError:
        return False

def check_nbextension(files, nbextensions_dir=None):
    """Check whether nbextension files have been installed

    files should be a list of relative paths within nbextensions.

    Returns True if all files are found, False if any are missing.
    """
    if nbextensions_dir:
        nbext = nbextensions_dir
    else:
        nbext = pjoin(get_ipython_dir(), u'nbextensions')
    # make sure nbextensions dir exists
    if not os.path.exists(nbext):
        return False

    if isinstance(files, string_types):
        # one file given, turn it into a list
        files = [files]

    return all(os.path.exists(pjoin(nbext, f)) for f in files)

def install_nbextension(files, overwrite=False, symlink=False, user=False, prefix=None, nbextensions_dir=None, verbose=1):
    """Install a Javascript extension for the notebook

    Stages files and/or directories into the nbextensions directory.
    By default, this compares modification time, and only stages files that need updating.
    If `overwrite` is specified, matching files are purged before proceeding.

    Parameters
    ----------

    files : list(paths or URLs) or dict(install_name: path or URL)
        One or more paths or URLs to existing files directories to install.
        If given as a list, these will be installed with their base name, so '/path/to/foo'
        will install to 'nbextensions/foo'.  If given as a dict, such as {'bar': '/path/to/foo'},
        then '/path/to/foo' will install to 'nbextensions/bar'.
        Archives (zip or tarballs) will be extracted into the nbextensions directory.
    overwrite : bool [default: False]
        If True, always install the files, regardless of what may already be installed.
    symlink : bool [default: False]
        If True, create a symlink in nbextensions, rather than copying files.
        Not allowed with URLs or archives. Windows support for symlinks requires
        Vista or above, Python 3, and a permission bit which only admin users
        have by default, so don't rely on it.
    user : bool [default: False]
        Whether to install to the user's .ipython/nbextensions directory.
        Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
    prefix : str [optional]
        Specify install prefix, if it should differ from default (e.g. /usr/local).
        Will install to prefix/share/jupyter/nbextensions
    nbextensions_dir : str [optional]
        Specify absolute path of nbextensions directory explicitly.
    verbose : int [default: 1]
        Set verbosity level. The default is 1, where file actions are printed.
        set verbose=2 for more output, or verbose=0 for silence.
    """
    if sum(map(bool, [user, prefix, nbextensions_dir])) > 1:
        raise ArgumentConflict("Cannot specify more than one of user, prefix, or nbextensions_dir.")
    if user:
        nbext = pjoin(get_ipython_dir(), u'nbextensions')
    else:
        if prefix:
            nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
        elif nbextensions_dir:
            nbext = nbextensions_dir
        else:
            nbext = SYSTEM_NBEXTENSIONS_INSTALL_DIR
    # make sure nbextensions dir exists
    ensure_dir_exists(nbext)

    if isinstance(files, string_types):
        # one file given, turn it into a list
        files = [files]
    if isinstance(files, (list,tuple)):
        # list given, turn into dict
        _files = {}
        for path in map(cast_unicode_py2, files):
            if path.startswith(('https://', 'http://')):
                destination = urlparse(path).path.split('/')[-1]
            elif path.endswith('.zip') or _safe_is_tarfile(path):
                destination = str(uuid.uuid4()) # ignored for archives
            else:
                destination = basename(path)
            _files[destination] = path
        files = _files

    for dest_basename,path in (map(cast_unicode_py2, item) for item in files.items()):

        if path.startswith(('https://', 'http://')):
            if symlink:
                raise ValueError("Cannot symlink from URLs")
            # Given a URL, download it
            with TemporaryDirectory() as td:
                filename = urlparse(path).path.split('/')[-1]
                local_path = os.path.join(td, filename)
                if verbose >= 1:
                    print("downloading %s to %s" % (path, local_path))
                urlretrieve(path, local_path)
                # now install from the local copy
                install_nbextension({dest_basename: local_path}, overwrite=overwrite, symlink=symlink, nbextensions_dir=nbext, verbose=verbose)
            continue

        # handle archives
        archive = None
        if path.endswith('.zip'):
            archive = zipfile.ZipFile(path)
        elif _safe_is_tarfile(path):
            archive = tarfile.open(path)

        if archive:
            if symlink:
                raise ValueError("Cannot symlink from archives")
            if verbose >= 1:
                print("extracting %s to %s" % (path, nbext))
            archive.extractall(nbext)
            archive.close()
            continue

        dest = pjoin(nbext, dest_basename)
        if overwrite and os.path.exists(dest):
            if verbose >= 1:
                print("removing %s" % dest)
            if os.path.isdir(dest) and not os.path.islink(dest):
                shutil.rmtree(dest)
            else:
                os.remove(dest)

        if symlink:
            path = os.path.abspath(path)
            if not os.path.exists(dest):
                if verbose >= 1:
                    print("symlink %s -> %s" % (dest, path))
                os.symlink(path, dest)
            continue

        if os.path.isdir(path):
            path = pjoin(os.path.abspath(path), '') # end in path separator
            for parent, dirs, files in os.walk(path):
                dest_dir = pjoin(dest, parent[len(path):])
                if not os.path.exists(dest_dir):
                    if verbose >= 2:
                        print("making directory %s" % dest_dir)
                    os.makedirs(dest_dir)
                for file in files:
                    src = pjoin(parent, file)
                    # print("%r, %r" % (dest_dir, file))
                    dest_file = pjoin(dest_dir, file)
                    _maybe_copy(src, dest_file, verbose)
        else:
            src = path
            _maybe_copy(src, dest, verbose)

#----------------------------------------------------------------------
# install nbextension app
#----------------------------------------------------------------------

from IPython.utils.traitlets import Bool, Enum, Unicode, TraitError
from IPython.core.application import BaseIPythonApplication

flags = {
    "overwrite" : ({
        "NBExtensionApp" : {
            "overwrite" : True,
        }}, "Force overwrite of existing files"
    ),
    "debug" : ({
        "NBExtensionApp" : {
            "verbose" : 2,
        }}, "Extra output"
    ),
    "quiet" : ({
        "NBExtensionApp" : {
            "verbose" : 0,
        }}, "Minimal output"
    ),
    "symlink" : ({
        "NBExtensionApp" : {
            "symlink" : True,
        }}, "Create symlinks instead of copying files"
    ),
    "user" : ({
        "NBExtensionApp" : {
            "user" : True,
        }}, "Install to the user's IPython directory"
    ),
}
flags['s'] = flags['symlink']

aliases = {
    "ipython-dir" : "NBExtensionApp.ipython_dir",
    "prefix" : "NBExtensionApp.prefix",
    "nbextensions" : "NBExtensionApp.nbextensions_dir",
}

class NBExtensionApp(BaseIPythonApplication):
    """Entry point for installing notebook extensions"""

    description = """Install IPython notebook extensions

    Usage

        ipython install-nbextension file [more files, folders, archives or urls]

    This copies files and/or folders into the IPython nbextensions directory.
    If a URL is given, it will be downloaded.
    If an archive is given, it will be extracted into nbextensions.
    If the requested files are already up to date, no action is taken
    unless --overwrite is specified.
    """

    examples = """
    ipython install-nbextension /path/to/d3.js /path/to/myextension
    """
    aliases = aliases
    flags = flags

    overwrite = Bool(False, config=True, help="Force overwrite of existing files")
    symlink = Bool(False, config=True, help="Create symlinks instead of copying files")
    user = Bool(False, config=True, help="Whether to do a user install")
    prefix = Unicode('', config=True, help="Installation prefix")
    nbextensions_dir = Unicode('', config=True, help="Full path to nbextensions dir (probably use prefix or user)")
    verbose = Enum((0,1,2), default_value=1, config=True,
        help="Verbosity level"
    )

    def install_extensions(self):
        install_nbextension(self.extra_args,
            overwrite=self.overwrite,
            symlink=self.symlink,
            verbose=self.verbose,
            user=self.user,
            prefix=self.prefix,
            nbextensions_dir=self.nbextensions_dir,
        )

    def start(self):
        if not self.extra_args:
            for nbext in [pjoin(self.ipython_dir, u'nbextensions')] + SYSTEM_NBEXTENSIONS_DIRS:
                if os.path.exists(nbext):
                    print("Notebook extensions in %s:" % nbext)
                    for ext in os.listdir(nbext):
                        print(u"    %s" % ext)
        else:
            try:
                self.install_extensions()
            except ArgumentConflict as e:
                print(str(e), file=sys.stderr)
                self.exit(1)

if __name__ == '__main__':
    NBExtensionApp.launch_instance()
    <|endoftext|>


<commit_before>
import psycopg2
import sys

con = None

try:

    con = psycopg2.connect(database='testdb1', user='jafem')
    cur = con.cursor()
    cur.execute("CREATE TABLE metadata (Id INTEGER PRIMARY KEY, Filesize INT, Duration FLOAT, Samplerate INT, Tags VARCHAR(20) ARRAY, Type VARCHAR(7), Saliance INT, StartTime FLOAT, EndTime FLOAT, Class VARCHAR(20))")
    cur.execute('SELECT version()')
    ver = cur.fetchone()
    print(ver)

except psycopg2.DatabaseError as err:
    print( 'Error %s' % err)
    sys.exit(1)

finally:

    if con:
        con.close()
<commit_msg>Changed database table fields to more appropriate types<commit_after>
import psycopg2
import sys

con = None

try:

    con = psycopg2.connect(database='testdb1', user='jafem')
    cur = con.cursor()
    cur.execute("CREATE TABLE metadata (Id INTEGER PRIMARY KEY, Filesize INT, Duration FLOAT, Samplerate INT, Tags VARCHAR(20) ARRAY, Type VARCHAR(7), Saliance VARCHAR(5), StartTime FLOAT, EndTime FLOAT, Class VARCHAR(20))")
    cur.execute('SELECT version()')
    ver = cur.fetchone()
    print(ver)

except psycopg2.DatabaseError as err:
    print( 'Error %s' % err)
    sys.exit(1)

finally:

    if con:
        con.close()
<|endoftext|>


<commit_before>import exceptions, sys
from twisted.internet import reactor, defer
from twisted.names import dns as twisted_dns
from twisted.names import server
from datetime import datetime
from random import randrange
from threading import Thread
from time import sleep
from datetime import datetime
import dns.query, dns.rdatatype, dns.exception
import socket
import rpyc

myHostName = socket.gethostname().replace('.', '-')
myIP = socket.gethostbyname(socket.gethostname()).replace('.', '-')
myAddr = '%s-%s.nbapuns.com' % (myIP, myHostName)

# target1 = '8.8.8.8'
# target2 = 'ns1.google.com'
# target2_ip = '216.239.32.10'

###############
# RPC Service #
###############
class TurboKingService(rpyc.Service):
    def on_connect(self):
        pass

    def on_disconnect(self):
        pass

    def exposed_test(self):
        return 1

    def exposed_get_latency(self, t1, ip1, t2, ip2):
        query_id = randrange(0, sys.maxint)

        # Setup DNS Server
        factory = DNSServerFactory(query_id, t2, ip2)
        protocol = twisted_dns.DNSDatagramProtocol(factory)
        reactor.listenUDP(53, protocol)
        reactor.listenTCP(53, factory)

        # Start DNS Client
        t=DNSClient(query_id, t1, ip1)
        t.start()

        # Start DNS Server
        reactor.run(installSignalHandlers=0)

        if factory.start_time:
            return t.end_time - factory.start_time
        else:
            return None

##########
# Client #
##########
class DNSClient(Thread):
    def __init__(self, query_id, target1, target1_ip):
        Thread.__init__(self)
        self.query_id = query_id
        self.target1 = target1
        self.target1_ip = target1_ip
        self.end_time = None

    def run(self):
        sleep(1) # Wait for DNS Server to Start
        addr = "final.%i.%s" % (self.query_id, myAddr)
        query = dns.message.make_query(addr, dns.rdatatype.A)
        try:
            response = dns.query.udp(query, self.target1_ip, timeout=5)
            self.end_time = datetime.now()
            print "Recieved Response:"
            print response
        except dns.exception.Timeout, e:
            print e
        reactor.callFromThread(reactor.stop)

##############
# DNS Server #
##############
class DNSServerFactory(server.DNSServerFactory):
    def __init__(self, query_id, t2, ip2, authorities=None, caches=None, clients=None, verbose=0):
        server.DNSServerFactory.__init__(self,authorities,caches,clients,verbose)
        self.query_id = query_id
        self.target2 = t2
        self.target2_ip = ip2
        self.start_time = None

    #TODO: Stop Handling of Queries after seeing a valid one
    def handleQuery(self, message, protocol, address):
        print "Recieved Query"
        print address
        print message

        global start_time, query_id
        try:
            query = message.queries[0]
            target = query.name.name
            print target
            dummy, id, origin = target.split('.')[0:3]

            if int(id) != self.query_id:
                print "Query ID Doesn't Match"
                raise Exception
            else:
                self.start_time = datetime.now()

            NS = twisted_dns.RRHeader(name=target, type=twisted_dns.NS, cls=twisted_dns.IN, ttl=0, auth=True,
                             payload=twisted_dns.Record_NS(name=self.target2, ttl=0))
            A = twisted_dns.RRHeader(name=self.target2, type=twisted_dns.A, cls=twisted_dns.IN, ttl=0,
                            payload=twisted_dns.Record_A(address=self.target2_ip, ttl=None))

            ans = []
            auth = [NS]
            add = [A]
            args = (self, (ans, auth, add), protocol, message, address)

            return server.DNSServerFactory.gotResolverResponse(*args)
        except Exception, e:
            print "Bad Request", e

if __name__ == "__main__":
    from rpyc.utils.server import ThreadedServer
    t = ThreadedServer(TurboKingService, port = 18861)
    t.start()
<commit_msg>stop reactor<commit_after>import exceptions, sys
from twisted.internet import reactor, defer
from twisted.names import dns as twisted_dns
from twisted.names import server
from datetime import datetime
from random import randrange
from threading import Thread
from time import sleep
from datetime import datetime
import dns.query, dns.rdatatype, dns.exception
import socket
import rpyc

myHostName = socket.gethostname().replace('.', '-')
myIP = socket.gethostbyname(socket.gethostname()).replace('.', '-')
myAddr = '%s-%s.nbapuns.com' % (myIP, myHostName)

# target1 = '8.8.8.8'
# target2 = 'ns1.google.com'
# target2_ip = '216.239.32.10'

###############
# RPC Service #
###############
class TurboKingService(rpyc.Service):
    def on_connect(self):
        pass

    def on_disconnect(self):
        pass

    def exposed_test(self):
        return 1

    def exposed_get_latency(self, t1, ip1, t2, ip2):
        query_id = randrange(0, sys.maxint)

        # Setup DNS Server
        factory = DNSServerFactory(query_id, t2, ip2)
        protocol = twisted_dns.DNSDatagramProtocol(factory)
        reactor.listenUDP(53, protocol)
        reactor.listenTCP(53, factory)

        # Start DNS Client
        t=DNSClient(query_id, t1, ip1)
        t.start()

        # Start DNS Server
        reactor.run(installSignalHandlers=0)

        if factory.start_time:
            return t.end_time - factory.start_time
        else:
            return None

##########
# Client #
##########
class DNSClient(Thread):
    def __init__(self, query_id, target1, target1_ip):
        Thread.__init__(self)
        self.query_id = query_id
        self.target1 = target1
        self.target1_ip = target1_ip
        self.end_time = None

    def run(self):
        sleep(1) # Wait for DNS Server to Start
        addr = "final.%i.%s" % (self.query_id, myAddr)
        query = dns.message.make_query(addr, dns.rdatatype.A)
        try:
            response = dns.query.udp(query, self.target1_ip, timeout=5)
            self.end_time = datetime.now()
            print "Recieved Response:"
            print response
        except dns.exception.Timeout, e:
            print e
        reactor.callFromThread(reactor.stop)
        print "Stopped Reactor"

##############
# DNS Server #
##############
class DNSServerFactory(server.DNSServerFactory):
    def __init__(self, query_id, t2, ip2, authorities=None, caches=None, clients=None, verbose=0):
        server.DNSServerFactory.__init__(self,authorities,caches,clients,verbose)
        self.query_id = query_id
        self.target2 = t2
        self.target2_ip = ip2
        self.start_time = None

    #TODO: Stop Handling of Queries after seeing a valid one
    def handleQuery(self, message, protocol, address):
        print "Recieved Query"
        print address
        print message

        global start_time, query_id
        try:
            query = message.queries[0]
            target = query.name.name
            print target
            dummy, id, origin = target.split('.')[0:3]

            if int(id) != self.query_id:
                print "Query ID Doesn't Match"
                raise Exception
            else:
                self.start_time = datetime.now()

            NS = twisted_dns.RRHeader(name=target, type=twisted_dns.NS, cls=twisted_dns.IN, ttl=0, auth=True,
                             payload=twisted_dns.Record_NS(name=self.target2, ttl=0))
            A = twisted_dns.RRHeader(name=self.target2, type=twisted_dns.A, cls=twisted_dns.IN, ttl=0,
                            payload=twisted_dns.Record_A(address=self.target2_ip, ttl=None))

            ans = []
            auth = [NS]
            add = [A]
            args = (self, (ans, auth, add), protocol, message, address)

            return server.DNSServerFactory.gotResolverResponse(*args)
        except Exception, e:
            print "Bad Request", e

if __name__ == "__main__":
    from rpyc.utils.server import ThreadedServer
    t = ThreadedServer(TurboKingService, port = 18861)
    t.start()
<|endoftext|>


<commit_before>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2021, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#

import os
from copy import deepcopy

import numpy as np
import torch

from nupic.research.frameworks.vernon.distributed import experiments, mixins
from torchvision.datasets import STL10, FakeData
from torchvision import transforms
from ray import tune
import sys
from nupic.research.frameworks.greedy_infomax.models import FullVisionModel
from nupic.research.frameworks.greedy_infomax.utils.loss_utils import \
    multiple_cross_entropy

class GreedyInfoMaxExperiment(mixins.LogEveryLoss,
                              experiments.SelfSupervisedExperiment,):
    pass

# get transforms for the dataset
def get_transforms(eval=False, aug=None):
    trans = []

    if aug["randcrop"]:
        if eval:
            trans.append(transforms.RandomCrop(aug["randcrop"]))
        else:
            trans.append(transforms.CenterCrop(aug["randcrop"]))

    if aug["flip"] and not eval:
        trans.append(transforms.RandomHorizontalFlip())

    if aug["grayscale"]:
        trans.append(transforms.Grayscale())
        trans.append(transforms.ToTensor())
        trans.append(transforms.Normalize(mean=aug["bw_mean"], std=aug["bw_std"]))
    elif aug["mean"]:
        trans.append(transforms.ToTensor())
        trans.append(transforms.Normalize(mean=aug["mean"], std=aug["std"]))
    else:
        trans.append(transforms.ToTensor())

    trans = transforms.Compose(trans)
    return trans

# labeled train set: mean [0.4469, 0.4400, 0.4069], std [0.2603, 0.2566, 0.2713]
aug = {
    "randcrop": 64,
    "flip": True,
    "grayscale": True,
    "mean": [0.4313, 0.4156, 0.3663],
    "std": [0.2683, 0.2610, 0.2687],
    "bw_mean": [0.4120],
    "bw_std": [0.2570],
}
transform_unsupervised = transform_supervised = get_transforms(eval=False, aug=aug)
transform_valid = trans = get_transforms(eval=True, aug=aug)

base_dataset_args=dict(
        root="~/nta/data/STL10",
        download=False,
)

# #fake data class for debugging purposes
# def fake_data(size=256, image_size=(3, 96, 96), num_classes = 10, train=True,
#               transform=transform_valid):
#     return FakeData(size=size, image_size=image_size, num_classes=num_classes,
#     transform=transform)

unsupervised_dataset_args = deepcopy(base_dataset_args)
unsupervised_dataset_args.update(dict(transform=transform_unsupervised,
                                      split="unlabeled"))
supervised_dataset_args = deepcopy(base_dataset_args)
supervised_dataset_args.update(dict(transform=transform_supervised,
                                      split="train"))
validation_dataset_args = deepcopy(base_dataset_args)
validation_dataset_args.update(dict(transform=transform_valid,
                                      split="test"))

BATCH_SIZE = 32
NUM_CLASSES = 10
DEFAULT_BASE = dict(
    experiment_class=GreedyInfoMaxExperiment,

    # Dataset
    dataset_class=STL10,
    dataset_args=dict(unsupervised=unsupervised_dataset_args,
                      supervised=supervised_dataset_args,
                      validation=validation_dataset_args),

    num_unsupervised_samples = 1000,
    workers=1,
    reuse_actors = True,
    # Seed
    seed=tune.sample_from(lambda spec: np.random.randint(1, 10000)),

    # Number of times to sample from the hyperparameter space. If `grid_search` is
    # provided the grid will be repeated `num_samples` of times.

    # Training batch size
    batch_size=BATCH_SIZE,
    # Validation batch size
    val_batch_size=10000,
    # Number of batches per epoch. Useful for debugging
    batches_in_epoch=sys.maxsize,

    # Update this to stop training when accuracy reaches the metric value
    # For example, stop=dict(mean_accuracy=0.75),
    stop=dict(),

    # Number of epochs
    epochs=10,
    epochs_to_validate=[3,],

    # Which epochs to run and report inference over the validation dataset.
    # epochs_to_validate=range(-1, 30),  # defaults to the last 3 epochs

    # Model class. Must inherit from "torch.nn.Module"
    model_class=FullVisionModel,
    #default model arguments
    model_args = dict(negative_samples=10,
                      k_predictions=5,
                      resnet_50=False,
                      grayscale=True,
                      patch_size=16,
                      overlap=2),

    classifier_model_class = torch.nn.Linear,
    classifier_model_args = dict(in_features=100,
                                 out_features=NUM_CLASSES),

    loss_function = multiple_cross_entropy, #each GIM layer has a cross-entropy
    loss_function_supervised = torch.nn.functional.cross_entropy,

    # Optimizer class. Must inherit from "torch.optim.Optimizer"
    optimizer_class=torch.optim.Adam,

    # Optimizer class class arguments passed to the constructor
    optimizer_args=dict(
        lr=1.5e-4,
    ),

    # Classifier Optimizer class. Must inherit from "torch.optim.Optimizer"
    classifier_optimizer_class=torch.optim.Adam,
    # Optimizer class class arguments passed to the constructor
    classifier_optimizer_args=dict(
        lr=1.5e-4,
    ),

    # # Learning rate scheduler class. Must inherit from "_LRScheduler"
    # lr_scheduler_class=torch.optim.lr_scheduler.StepLR,

    # How often to checkpoint (epochs)
    checkpoint_freq=0,
    keep_checkpoints_num=1,
    checkpoint_at_end=True,
    checkpoint_score_attr="training_iteration",

    # How many times to try to recover before stopping the trial
    max_failures=3,

    # How many times to retry the epoch before stopping. This is useful when
    # using distributed training with spot instances.
    max_retries=3,

    # Python Logging level : "critical", "error", "warning", "info", "debug"
    log_level="debug",

    # Python Logging Format
    log_format="%(asctime)s:%(levelname)s:%(name)s:%(message)s",

    # Ray tune verbosity. When set to the default value of 2 it will log
    # iteration result dicts. This dict can flood the console if it contains
    # large data structures, so default to verbose=1. The SupervisedTrainable logs
    # a succinct version of the result dict.
    verbose=1,
)

CONFIGS = dict(
    default_base = DEFAULT_BASE
)
<commit_msg>changes to experiment config<commit_after>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2021, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#

import sys
from copy import deepcopy

import numpy as np
import torch
from ray import tune
from torchvision import transforms
from torchvision.datasets import STL10

from nupic.research.frameworks.greedy_infomax.models import FullVisionModel
from nupic.research.frameworks.greedy_infomax.utils.loss_utils import (
    multiple_cross_entropy,
)
from nupic.research.frameworks.vernon import experiments, mixins

class GreedyInfoMaxExperiment(
    mixins.LogEveryLoss,
    experiments.SelfSupervisedExperiment,
):
    pass

# get transforms for the dataset
def get_transforms(val=False, aug=None):
    trans = []

    if aug["randcrop"]:
        if val:
            trans.append(transforms.RandomCrop(aug["randcrop"]))
        else:
            trans.append(transforms.CenterCrop(aug["randcrop"]))

    if aug["flip"] and not val:
        trans.append(transforms.RandomHorizontalFlip())

    if aug["grayscale"]:
        trans.append(transforms.Grayscale())
        trans.append(transforms.ToTensor())
        trans.append(transforms.Normalize(mean=aug["bw_mean"], std=aug["bw_std"]))
    elif aug["mean"]:
        trans.append(transforms.ToTensor())
        trans.append(transforms.Normalize(mean=aug["mean"], std=aug["std"]))
    else:
        trans.append(transforms.ToTensor())

    trans = transforms.Compose(trans)
    return trans

# labeled train set: mean [0.4469, 0.4400, 0.4069], std [0.2603, 0.2566, 0.2713]
aug = {
    "randcrop": 64,
    "flip": True,
    "grayscale": True,
    "mean": [0.4313, 0.4156, 0.3663],
    "std": [0.2683, 0.2610, 0.2687],
    "bw_mean": [0.4120],
    "bw_std": [0.2570],
}
transform_unsupervised = transform_supervised = get_transforms(val=False, aug=aug)
transform_validation = trans = get_transforms(val=True, aug=aug)

base_dataset_args = dict(root="~/nta/data/STL10", download=False)

# #fake data class for debugging purposes
# def fake_data(size=256, image_size=(3, 96, 96), num_classes = 10, train=True,
#               transform=transform_validation):
#     return FakeData(size=size, image_size=image_size, num_classes=num_classes,
#     transform=transform)

unsupervised_dataset_args = deepcopy(base_dataset_args)
unsupervised_dataset_args.update(
    dict(transform=transform_unsupervised, split="unlabeled")
)
supervised_dataset_args = deepcopy(base_dataset_args)
supervised_dataset_args.update(dict(transform=transform_supervised, split="train"))
validation_dataset_args = deepcopy(base_dataset_args)
validation_dataset_args.update(dict(transform=transform_validation, split="test"))

BATCH_SIZE = 32
NUM_CLASSES = 10
DEFAULT_BASE = dict(
    experiment_class=GreedyInfoMaxExperiment,
    # Dataset
    dataset_class=STL10,
    dataset_args=dict(
        unsupervised=unsupervised_dataset_args,
        supervised=supervised_dataset_args,
        validation=validation_dataset_args,
    ),
    num_unsupervised_samples=32,
    num_supervised_samples=32,
    num_validation_samples=32,
    workers=1,
    reuse_actors=True,
    # Seed
    seed=tune.sample_from(lambda spec: np.random.randint(1, 10000)),
    # Number of times to sample from the hyperparameter space. If `grid_search` is
    # provided the grid will be repeated `num_samples` of times.
    # Training batch size
    batch_size=BATCH_SIZE,
    # Validation batch size
    val_batch_size=10000,
    # Number of batches per epoch. Useful for debugging
    batches_in_epoch=sys.maxsize,
    # Update this to stop training when accuracy reaches the metric value
    # For example, stop=dict(mean_accuracy=0.75),
    stop=dict(),
    # Number of epochs
    epochs=10,
    epochs_to_validate=[3],
    # Which epochs to run and report inference over the validation dataset.
    # epochs_to_validate=range(-1, 30),  # defaults to the last 3 epochs
    # Model class. Must inherit from "torch.nn.Module"
    model_class=FullVisionModel,
    # default model arguments
    model_args=dict(
        negative_samples=10,
        k_predictions=5,
        resnet_50=False,
        grayscale=True,
        patch_size=16,
        overlap=2,
    ),
    classifier_config=dict(
        model_class=torch.nn.Linear,
        model_args=dict(in_features=100, out_features=NUM_CLASSES),
        loss_function=torch.nn.functional.cross_entropy,
        # Classifier Optimizer class. Must inherit from "torch.optim.Optimizer"
        optimizer_class=torch.optim.Adam,
        # Optimizer class class arguments passed to the constructor
        optimizer_args=dict(lr=1.5e-4),
    ),
    loss_function=multiple_cross_entropy,  # each GIM layer has a cross-entropy
    # Optimizer class. Must inherit from "torch.optim.Optimizer"
    optimizer_class=torch.optim.Adam,
    # Optimizer class class arguments passed to the constructor
    optimizer_args=dict(lr=1.5e-4),
    # # Learning rate scheduler class. Must inherit from "_LRScheduler"
    # lr_scheduler_class=torch.optim.lr_scheduler.StepLR,
    # How often to checkpoint (epochs)
    checkpoint_freq=0,
    keep_checkpoints_num=1,
    checkpoint_at_end=True,
    checkpoint_score_attr="training_iteration",
    # How many times to try to recover before stopping the trial
    max_failures=3,
    # How many times to retry the epoch before stopping. This is useful when
    # using distributed training with spot instances.
    max_retries=3,
    # Python Logging level : "critical", "error", "warning", "info", "debug"
    log_level="debug",
    # Python Logging Format
    log_format="%(asctime)s:%(levelname)s:%(name)s:%(message)s",
    # Ray tune verbosity. When set to the default value of 2 it will log
    # iteration result dicts. This dict can flood the console if it contains
    # large data structures, so default to verbose=1. The SupervisedTrainable logs
    # a succinct version of the result dict.
    verbose=1,
)

CONFIGS = dict(default_base=DEFAULT_BASE)
<|endoftext|>


<commit_before>"""Stack documentation build system.
"""

__all__ = ('build_stack_docs',)

import logging
import os

from .pkgdiscovery import (
    discover_setup_packages, find_table_file, list_packages_in_eups_table,
    find_package_docs, NoPackageDocs)
from ..sphinxrunner import run_sphinx

def build_stack_docs(root_project_dir, skippedNames=None):
    """Build stack Sphinx documentation (main entrypoint).

    Parameters
    ----------
    root_project_dir : `str`
        Path to the root directory of the main documentation project. This
        is the directory containing the ``conf.py`` file.
    skippedNames : `list`, optional
        Optional list of packages to skip while creating symlinks.
    """
    logger = logging.getLogger(__name__)

    # Create the directory where module content is symlinked
    # NOTE: this path is hard-wired in for pipelines.lsst.io, but could be
    # refactored as a configuration.
    root_modules_dir = os.path.join(root_project_dir, 'modules')
    if os.path.isdir(root_modules_dir):
        logger.info('Deleting any existing modules/ symlinks')
        remove_existing_links(root_modules_dir)
    else:
        logger.info('Creating modules/ dir at {0}'.format(root_modules_dir))
        os.makedirs(root_modules_dir)

    # Create directory for package content
    root_packages_dir = os.path.join(root_project_dir, 'packages')
    if os.path.isdir(root_packages_dir):
        # Clear out existing module links
        logger.info('Deleting any existing packages/ symlinks')
        remove_existing_links(root_packages_dir)
    else:
        logger.info('Creating packages/ dir at {0}'.format(root_packages_dir))
        os.makedirs(root_packages_dir)

    # Ensure _static directory exists (but do not delete any existing
    # directory contents)
    root_static_dir = os.path.join(root_project_dir, '_static')
    if os.path.isdir(root_static_dir):
        # Clear out existing directory links
        logger.info('Deleting any existing _static/ symlinks')
        remove_existing_links(root_static_dir)
    else:
        logger.info('Creating _static/ at {0}'.format(root_static_dir))
        os.makedirs(root_static_dir)

    # Get packages explicitly required in the table file to filter out
    # implicit dependencies later.
    table_path = find_table_file(root_project_dir)
    listed_packages = list_packages_in_eups_table(table_path.read_text())
    # Find package setup by EUPS
    packages = discover_setup_packages(scope=listed_packages)

    # Link to documentation directories of packages from the root project
    for package_name, package_info in packages.items():
        try:
            package_docs = find_package_docs(
                package_info['dir'],
                skipped_names=skippedNames)
        except NoPackageDocs as e:
            logger.debug(
                'Skipping %s doc linking. %s', package_name, e)
            continue

        link_directories(root_modules_dir, package_docs.module_dirs)
        link_directories(root_packages_dir, package_docs.package_dirs)
        link_directories(root_static_dir, package_docs.static_doc_dirs)

    # Trigger the Sphinx build
    return_code = run_sphinx(root_project_dir)
    return return_code

def link_directories(root_dir, package_doc_dirs):
    """Create symlinks to package/module documentation directories from the
    root documentation project.

    Parameters
    ----------
    root_dir : `str`
        Directory in the main documentation project where links will be
        created. For example, this could be a ``'modules'`` directory
        in the ``pipelines_lsst_io`` project directory.
    package_doc_dirs : `dict`
        Dictionary that maps symlinks to be made in ``root_dir`` with
        source directories in the packages.

    Notes
    -----
    If the link already exists in the ``root_dir`` it will be automatically
    replaced.
    """
    logger = logging.getLogger(__name__)

    for dirname, source_dirname in package_doc_dirs.items():
        link_name = os.path.join(root_dir, dirname)
        if os.path.islink(link_name):
            os.remove(link_name)
        os.symlink(source_dirname, link_name)

        message = 'Linking {0} -> {1}'.format(link_name, source_dirname)
        logger.info(message)

def remove_existing_links(root_dir):
    """Delete any symlinks present at the root of a directory.

    Parameters
    ----------
    root_dir : `str`
        Directory that might contain symlinks.

    Notes
    -----
    This function is used to remove any symlinks created by `link_directories`.
    Running ``remove_existing_links`` at the beginning of a build ensures that
    builds are isolated. For example, if a package is un-setup it won't
    re-appear in the documentation because its symlink still exists.
    """
    logger = logging.getLogger(__name__)

    for name in os.listdir(root_dir):
        full_name = os.path.join(root_dir, name)
        if os.path.islink(full_name):
            logger.debug('Deleting existing symlink {0}'.format(full_name))
            os.remove(full_name)
<commit_msg>Refactor build_stack_docs to first get Packages<commit_after>"""Stack documentation build system.
"""

__all__ = ('build_stack_docs',)

import logging
import os
from typing import Dict

from .pkgdiscovery import (
    discover_setup_packages, find_table_file, list_packages_in_eups_table,
    find_package_docs, NoPackageDocs, Package)
from ..sphinxrunner import run_sphinx

def build_stack_docs(root_project_dir, skippedNames=None):
    """Build stack Sphinx documentation (main entrypoint).

    Parameters
    ----------
    root_project_dir : `str`
        Path to the root directory of the main documentation project. This
        is the directory containing the ``conf.py`` file.
    skippedNames : `list`, optional
        Optional list of packages to skip while creating symlinks.
    """
    logger = logging.getLogger(__name__)

    # Create the directory where module content is symlinked
    # NOTE: this path is hard-wired in for pipelines.lsst.io, but could be
    # refactored as a configuration.
    root_modules_dir = os.path.join(root_project_dir, 'modules')
    if os.path.isdir(root_modules_dir):
        logger.info('Deleting any existing modules/ symlinks')
        remove_existing_links(root_modules_dir)
    else:
        logger.info('Creating modules/ dir at {0}'.format(root_modules_dir))
        os.makedirs(root_modules_dir)

    # Create directory for package content
    root_packages_dir = os.path.join(root_project_dir, 'packages')
    if os.path.isdir(root_packages_dir):
        # Clear out existing module links
        logger.info('Deleting any existing packages/ symlinks')
        remove_existing_links(root_packages_dir)
    else:
        logger.info('Creating packages/ dir at {0}'.format(root_packages_dir))
        os.makedirs(root_packages_dir)

    # Ensure _static directory exists (but do not delete any existing
    # directory contents)
    root_static_dir = os.path.join(root_project_dir, '_static')
    if os.path.isdir(root_static_dir):
        # Clear out existing directory links
        logger.info('Deleting any existing _static/ symlinks')
        remove_existing_links(root_static_dir)
    else:
        logger.info('Creating _static/ at {0}'.format(root_static_dir))
        os.makedirs(root_static_dir)

    # Get packages explicitly required in the table file to filter out
    # implicit dependencies later.
    table_path = find_table_file(root_project_dir)
    listed_packages = list_packages_in_eups_table(table_path.read_text())
    # Find package setup by EUPS
    set_up_packages = discover_setup_packages(scope=listed_packages)

    # Determine what packages have documentation content, and get Package
    # metadata objects about those
    packages: Dict[str, Package] = {}
    for package_name, package_info in set_up_packages.items():
        try:
            package_docs = find_package_docs(
                package_info['dir'],
                skipped_names=skippedNames)
            packages[package_name] = package_docs
        except NoPackageDocs as e:
            logger.debug(
                'Skipping %s doc linking. %s', package_name, e)
            continue

    # Link to documentation directories of packages from the root project
    for package_name, package in packages.items():
        link_directories(root_modules_dir, package.module_dirs)
        link_directories(root_packages_dir, package.package_dirs)
        link_directories(root_static_dir, package.static_doc_dirs)

    # Trigger the Sphinx build
    return_code = run_sphinx(root_project_dir)
    return return_code

def link_directories(root_dir, package_doc_dirs):
    """Create symlinks to package/module documentation directories from the
    root documentation project.

    Parameters
    ----------
    root_dir : `str`
        Directory in the main documentation project where links will be
        created. For example, this could be a ``'modules'`` directory
        in the ``pipelines_lsst_io`` project directory.
    package_doc_dirs : `dict`
        Dictionary that maps symlinks to be made in ``root_dir`` with
        source directories in the packages.

    Notes
    -----
    If the link already exists in the ``root_dir`` it will be automatically
    replaced.
    """
    logger = logging.getLogger(__name__)

    for dirname, source_dirname in package_doc_dirs.items():
        link_name = os.path.join(root_dir, dirname)
        if os.path.islink(link_name):
            os.remove(link_name)
        os.symlink(source_dirname, link_name)

        message = 'Linking {0} -> {1}'.format(link_name, source_dirname)
        logger.info(message)

def remove_existing_links(root_dir):
    """Delete any symlinks present at the root of a directory.

    Parameters
    ----------
    root_dir : `str`
        Directory that might contain symlinks.

    Notes
    -----
    This function is used to remove any symlinks created by `link_directories`.
    Running ``remove_existing_links`` at the beginning of a build ensures that
    builds are isolated. For example, if a package is un-setup it won't
    re-appear in the documentation because its symlink still exists.
    """
    logger = logging.getLogger(__name__)

    for name in os.listdir(root_dir):
        full_name = os.path.join(root_dir, name)
        if os.path.islink(full_name):
            logger.debug('Deleting existing symlink {0}'.format(full_name))
            os.remove(full_name)
<|endoftext|>