<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from six.moves import input

from django.core.management.base import BaseCommand

from corehq.apps.export.models import ExportInstance
from dimagi.utils.couch.database import iter_bulk_delete

class Command(BaseCommand):
    help = "Delete exports in a domain"

    def add_arguments(self, parser):

        parser.add_argument(
            'domain',
            help="The domain to delete exports in"
        )
        parser.add_argument(
            '--days_inactive',
            default=0,
            help="Only delete exports that have been inactive for this many days"
        )

    def handle(self, domain, **options):
        db = ExportInstance.get_db()
        exports = db.view(
            'export_instances_by_domain/view',
            startkey=[domain],
            endkey=[domain, {}],
            include_docs=False,
            reduce=False,
        ).all()
        if not exports:
            print("No exports to delete here, exiting.")
            return

        if options['days_inactive'] > 0:
            import datetime
            inactive_since = datetime.datetime.today() - datetime.timedelta(days=int(options['days_inactive']))
            inactive_exports = []
            for export in exports:
                e = ExportInstance.get(export['id'])
                if e.last_accessed and e.last_accessed <= inactive_since:
                    inactive_exports.append(e)
            if not inactive_exports:
                print("No exports have been inactive for more than {days_inactive} days, exiting.".format(
                    days_inactive=options['days_inactive'])
                )
                return
            confirm = input(
                "There are {total_exports} exports in {domain}. Are you sure you want to delete "
                "{total_inactive_exports} that are older than {days_inactive} days [y/N]?".format(
                    total_exports=len(exports),
                    total_inactive_exports=len(inactive_exports),
                    domain=domain,
                    days_inactive=int(options['days_inactive'])
                )
            )
            if confirm.lower() == 'y':
                exports = inactive_exports
            else:
                return

        filter_exports = lambda _type: [row for row in exports if _type in row['key']]
        form_exports = filter_exports('FormExportInstance')
        case_exports = filter_exports('CaseExportInstance')

        confirm = input(
            "There are {f_count} form exports, and {c_count} case exports. "
            "Are you sure you want to delete all these exports [y/N]?\n".format(
                f_count=len(form_exports),
                c_count=len(case_exports)
            )
        )
        to_delete = []
        if confirm.lower() == 'y':
            _type = input(
                "Enter 'case' to delete all case exports, "
                "'form to delete all form exports, "
                "'all' to delete both form and case exports. "
                "Enter anything else to exit.\n"
            )
            if _type == 'form':
                to_delete = form_exports
                print("Deleting form exports")
            elif _type == 'case':
                to_delete = case_exports
                print("Deleting case exports")
            elif _type == 'all':
                to_delete = form_exports + case_exports
                print("Deleting all exports")
            else:
                print("Not deleting anything, exiting!")
                return
            total_count = iter_bulk_delete(db, [doc['id'] for doc in to_delete])
            print("Deleted total of {} exports succesfully!".format(total_count))
<commit_msg>Compile correct inactive_exports list<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from six.moves import input

from django.core.management.base import BaseCommand

from corehq.apps.export.models import ExportInstance
from dimagi.utils.couch.database import iter_bulk_delete

class Command(BaseCommand):
    help = "Delete exports in a domain"

    def add_arguments(self, parser):

        parser.add_argument(
            'domain',
            help="The domain to delete exports in"
        )
        parser.add_argument(
            '--days_inactive',
            default=0,
            help="Only delete exports that have been inactive for this many days"
        )

    def handle(self, domain, **options):
        db = ExportInstance.get_db()
        exports = db.view(
            'export_instances_by_domain/view',
            startkey=[domain],
            endkey=[domain, {}],
            include_docs=False,
            reduce=False,
        ).all()
        if not exports:
            print("No exports to delete here, exiting.")
            return

        if options['days_inactive'] > 0:
            import datetime
            inactive_since = datetime.datetime.today() - datetime.timedelta(days=int(options['days_inactive']))
            inactive_exports = []
            for export in exports:
                e = ExportInstance.get(export['id'])
                if e.last_accessed and e.last_accessed <= inactive_since:
                    inactive_exports.append(export)
            if not inactive_exports:
                print("No exports have been inactive for more than {days_inactive} days, exiting.".format(
                    days_inactive=options['days_inactive'])
                )
                return
            confirm = input(
                "There are {total_exports} exports in {domain}. Are you sure you want to delete "
                "{total_inactive_exports} that are older than {days_inactive} days [y/N]?".format(
                    total_exports=len(exports),
                    total_inactive_exports=len(inactive_exports),
                    domain=domain,
                    days_inactive=int(options['days_inactive'])
                )
            )
            if confirm.lower() == 'y':
                exports = inactive_exports
            else:
                return

        filter_exports = lambda _type: [row for row in exports if _type in row['key']]
        form_exports = filter_exports('FormExportInstance')
        case_exports = filter_exports('CaseExportInstance')

        confirm = input(
            "There are {f_count} form exports, and {c_count} case exports. "
            "Are you sure you want to delete all these exports [y/N]?\n".format(
                f_count=len(form_exports),
                c_count=len(case_exports)
            )
        )
        to_delete = []
        if confirm.lower() == 'y':
            _type = input(
                "Enter 'case' to delete all case exports, "
                "'form to delete all form exports, "
                "'all' to delete both form and case exports. "
                "Enter anything else to exit.\n"
            )
            if _type == 'form':
                to_delete = form_exports
                print("Deleting form exports")
            elif _type == 'case':
                to_delete = case_exports
                print("Deleting case exports")
            elif _type == 'all':
                to_delete = form_exports + case_exports
                print("Deleting all exports")
            else:
                print("Not deleting anything, exiting!")
                return
            total_count = iter_bulk_delete(db, [doc['id'] for doc in to_delete])
            print("Deleted total of {} exports succesfully!".format(total_count))
<|endoftext|>


<commit_before>from django import template
from django.db.models import F, Sum

from packages.models import Package, Release, ReleaseFile, ChangeLog

register = template.Library()

@register.assignment_tag
def package_download_count(package_name=None):
    # @@@ Cache this to cut down on queries
    count = 0
    if package_name is None:
        # Total Download Count
        count = ReleaseFile.objects.all().aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
    else:
        count = ReleaseFile.objects.filter(
                    release__package__name=package_name
                ).aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
    return count

@register.assignment_tag
def package_count():
    # @@@ Cache this to cut down on queries
    return Package.objects.all().count()

@register.assignment_tag
def get_oldest_package():
    # @@@ Cache this to cut down on queries
    pkgs = Package.objects.all().order_by("created")[:1]
    if pkgs:
        return pkgs[0]
    else:
        return None

@register.assignment_tag
def new_packages(num):
    return [
        x for
        x in ChangeLog.objects.filter(type=ChangeLog.TYPES.new).select_related("package", "release").prefetch_related("package__releases").order_by("-created")[:num * 3]
        if len(x.package.releases.all())
    ][:num]

@register.assignment_tag
def updated_packages(num):
    return ChangeLog.objects.filter(type=ChangeLog.TYPES.updated).select_related("package", "release").order_by("-created")[:num]

@register.assignment_tag
def featured_packages(num):
    return Package.objects.filter(featured=True).order_by("?")[:num]

@register.assignment_tag
def random_packages(num):
    return Package.objects.exclude(releases=None).order_by("?")[:num]

@register.assignment_tag
def package_versions(package_name, num=None):
    qs = Release.objects.filter(package__name=package_name).select_related("package").order_by("-order")
    if num is not None:
        qs = qs[:num]
    return qs

@register.assignment_tag
def package_version_count(package_name):
    return Release.objects.filter(package__name=package_name).count()
<commit_msg>remove unused import<commit_after>from django import template
from django.db.models import Sum

from packages.models import Package, Release, ReleaseFile, ChangeLog

register = template.Library()

@register.assignment_tag
def package_download_count(package_name=None):
    # @@@ Cache this to cut down on queries
    count = 0
    if package_name is None:
        # Total Download Count
        count = ReleaseFile.objects.all().aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
    else:
        count = ReleaseFile.objects.filter(
                    release__package__name=package_name
                ).aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
    return count

@register.assignment_tag
def package_count():
    # @@@ Cache this to cut down on queries
    return Package.objects.all().count()

@register.assignment_tag
def get_oldest_package():
    # @@@ Cache this to cut down on queries
    pkgs = Package.objects.all().order_by("created")[:1]
    if pkgs:
        return pkgs[0]
    else:
        return None

@register.assignment_tag
def new_packages(num):
    return [
        x for
        x in ChangeLog.objects.filter(type=ChangeLog.TYPES.new).select_related("package", "release").prefetch_related("package__releases").order_by("-created")[:num * 3]
        if len(x.package.releases.all())
    ][:num]

@register.assignment_tag
def updated_packages(num):
    return ChangeLog.objects.filter(type=ChangeLog.TYPES.updated).select_related("package", "release").order_by("-created")[:num]

@register.assignment_tag
def featured_packages(num):
    return Package.objects.filter(featured=True).order_by("?")[:num]

@register.assignment_tag
def random_packages(num):
    return Package.objects.exclude(releases=None).order_by("?")[:num]

@register.assignment_tag
def package_versions(package_name, num=None):
    qs = Release.objects.filter(package__name=package_name).select_related("package").order_by("-order")
    if num is not None:
        qs = qs[:num]
    return qs

@register.assignment_tag
def package_version_count(package_name):
    return Release.objects.filter(package__name=package_name).count()
<|endoftext|>


<commit_before># rain_notifier.py
#
# This is a project that uses the Grove LED attached to an umbrella to remind
# people to take their umbrella if it's raining or going to rain that day.
#
# You will need to modify the first few lines to make this project work for you.
# Fee free to modify the rest of the program to make it behave the way you want it to.
#
# Enter your zip code (or Postal Code) on the following line
# (leave the name 'zip' as is, even if you have a postal code. )
zipcode = 'YOUR_ZIP_CODE'

# once you have gotten an API Key from Wunderground (see tutorial), pleave provide it here
# Wunderground API Key
api_key = 'YOUR_WUNDERGROUND_API_KEY'

# if you wish to use two LEDs, provide the port number of the second LED
# The second LED is the one that will show there's no rain in the weather prediction
# With two LEDs, one will always be on.
# if you do not want a second LED, set this to -1
clearled = 8

# in case of rain, do you want the LED to blink in order to catch your attention more?
# set to True for blinking, set to False for a steady warning
BLINK = True

############################################################################
# no modifications are required after this
# however the next few lines can be tweaked easily
############################################################################

# this is the delay between each polling of Wunderground. It's set to poll once every 5 minutes
# you may prefer a faster polling.
# the Wunderground free account does set a limit

DELAY = 5*60 # in seconds

# Pin for the rain LED on the umbrella
rainled = 7

############################################################################
# nothing needs to be modified after this
# if you are familiar with Python, feel free to change anything to improve this
# we welcome additions and suggestions
############################################################################

import urllib2
import json
from grovepi import *
import time
import sys

def clear_led(status):
    """ change clear sky LED status """
    if clearled > -1:
        digitalWrite(clearled,status)

def assign_rain(status, blink):
    """ controls the LEDs status

    Arguments:
    status: True if it will be raining
            False if there's no rain in the forecast
    blink:  True if the rain led should blink when on
            False if the rain led does not need blinking
    no return value
    """
    if status == True: # there will be rain
        clear_led(0) # Turn off the CLEAR LED on the umbrella

        # Light the RAIN LED on the umbrella
        # check if it needs to be blinking
        if blink == True:
            led_blink(DELAY) # no sleep required here
        else:
            digitalWrite(rainled,1)
            time.sleep(DELAY)

    else: # no rain in forecast
        # turn off the RAIN LED on the umbrella
        digitalWrite(rainled,0)
        # Turn on the CLEAR LED on the umbrella
        clear_led(1)
        time.sleep(DELAY)

def led_blink(blinkdelay):
    """ forces the rain LED to blink

    rain LED will blink at a rate of 0.2 secs on, and 0.2 secs off
    it will blink for a duration set by blinkdelay
    no additional sleep() is required as it's incorporated in the blinking
    """
    count = 0.0
    blinkrate = 0.4
    while count < float(blinkdelay):
        digitalWrite(rainled,1)
        time.sleep(blinkrate)
        digitalWrite(rainled,0)
        time.sleep(blinkrate)
        count += (2*blinkrate)

#########################################################################

# Zip code of location
url='http://api.wunderground.com/api/'+api_key+'/geolookup/conditions/q/'+zip+'.json'

pinMode(rainled,"OUTPUT")

if len(sys.argv) > 1 and sys.argv[1] == 'test':
    try:
        clear_led(1)
        led_blink(5)
        clear_led(0)
    except KeyboardInterrupt:
        digitalWrite(clearled,0)
        digitalWrite(rainled,0)
    quit()

if clearled > -1:
    pinMode(clearled,"OUTPUT")

try:
    while True:
        f = urllib2.urlopen(url)
        json_string = f.read()
        f.close()
        parsed_json = json.loads(json_string)
        location = parsed_json['location']['city']
        precip_today_in = parsed_json['current_observation']['precip_today_in']

        print "Current precipitation in %s is: %s" % (location, precip_today_in)

        if float(precip_today_in) > 0 :
            print "Rain today, take the umbrella"
            assign_rain(True, BLINK)

        else:
            print "No Rain today"
            assign_rain(False, BLINK)

except KeyboardInterrupt:
    clear_led(0)
    digitalWrite(rainled,0)
<commit_msg>added test mode and change blink rate<commit_after># rain_notifier.py
#
# This is a project that uses the Grove LED attached to an umbrella to remind
# people to take their umbrella if it's raining or going to rain that day.
#
# You will need to modify the first few lines to make this project work for you.
# Fee free to modify the rest of the program to make it behave the way you want it to.
#
# Enter your zip code (or Postal Code) on the following line
# (leave the name 'zip' as is, even if you have a postal code. )
zipcode = 'YOUR_ZIP_CODE'

# once you have gotten an API Key from Wunderground (see tutorial), pleave provide it here
# Wunderground API Key
api_key = 'YOUR_WUNDERGROUND_API_KEY'

# if you wish to use two LEDs, provide the port number of the second LED
# The second LED is the one that will show there's no rain in the weather prediction
# With two LEDs, one will always be on.
# if you do not want a second LED, set this to -1
clearled = 8

# in case of rain, do you want the LED to blink in order to catch your attention more?
# set to True for blinking, set to False for a steady warning
BLINK = True

############################################################################
# no modifications are required after this
# however the next few lines can be tweaked easily
############################################################################

# this is the delay between each polling of Wunderground. It's set to poll once every 5 minutes
# you may prefer a faster polling.
# the Wunderground free account does set a limit

DELAY = 5*60 # in seconds

# Pin for the rain LED on the umbrella
rainled = 7

############################################################################
# nothing needs to be modified after this
# if you are familiar with Python, feel free to change anything to improve this
# we welcome additions and suggestions
############################################################################

import urllib2
import json
from grovepi import *
import time
import sys

def clear_led(status):
    """ change clear sky LED status """
    if clearled > -1:
        digitalWrite(clearled,status)

def assign_rain(status, blink):
    """ controls the LEDs status

    Arguments:
    status: True if it will be raining
            False if there's no rain in the forecast
    blink:  True if the rain led should blink when on
            False if the rain led does not need blinking
    no return value
    """
    if status == True: # there will be rain
        clear_led(0) # Turn off the CLEAR LED on the umbrella

        # Light the RAIN LED on the umbrella
        # check if it needs to be blinking
        if blink == True:
            led_blink(DELAY) # no sleep required here
        else:
            digitalWrite(rainled,1)
            time.sleep(DELAY)

    else: # no rain in forecast
        # turn off the RAIN LED on the umbrella
        digitalWrite(rainled,0)
        # Turn on the CLEAR LED on the umbrella
        clear_led(1)
        time.sleep(DELAY)

def led_blink(blinkdelay):
    """ forces the rain LED to blink

    rain LED will blink at a rate of 0.2 secs on, and 0.2 secs off
    it will blink for a duration set by blinkdelay
    no additional sleep() is required as it's incorporated in the blinking
    """
    count = 0.0
    blinkrate = 0.4
    while count < float(blinkdelay):
        digitalWrite(rainled,1)
        time.sleep(blinkrate)
        digitalWrite(rainled,0)
        time.sleep(blinkrate)
        count += (2*blinkrate)

#########################################################################

# Zip code of location
url='http://api.wunderground.com/api/'+api_key+'/geolookup/conditions/q/'+zip+'.json'

pinMode(rainled,"OUTPUT")

if len(sys.argv) > 1 and sys.argv[1] == 'test':
    try:
        clear_led(1)
        led_blink(5)
        clear_led(0)
    except KeyboardInterrupt:
        digitalWrite(clearled,0)
        digitalWrite(rainled,0)
    quit()

if clearled > -1:
    pinMode(clearled,"OUTPUT")

try:
    while True:
        f = urllib2.urlopen(url)
        json_string = f.read()
        f.close()
        parsed_json = json.loads(json_string)
        location = parsed_json['location']['city']
        precip_today_in = parsed_json['current_observation']['precip_today_in']

        print "Current precipitation in %s is: %s" % (location, precip_today_in)

        if float(precip_today_in) > 0 :
            print "Rain today, take the umbrella"
            assign_rain(True, BLINK)

        else:
            print "No Rain today"
            assign_rain(False, BLINK)

except KeyboardInterrupt:
    clear_led(0)
    digitalWrite(rainled,0)
<|endoftext|>


<commit_before>import logging
import os
import re
import subprocess

import utils.dockerutils

from icrawl_plugin import IContainerCrawler
from utils.namespace import run_as_another_namespace, ALL_NAMESPACES

logger = logging.getLogger('crawlutils')

class NodePackageCrawler(IContainerCrawler):

    def get_feature(self):
        return 'node-package'

    def _get_packages_by_cmd(self):
        proc = subprocess.Popen(
            ['sh', '-c', 'npm list -g'],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        output, err = proc.communicate()

        if output:
            pkg_list = output.strip('\n')
            if pkg_list:
                for pkg in pkg_list.split('\n'):
                    if pkg.split()[-1] == 'deduped':
                        continue
                    pkg_info = re.findall(r"\S*@\S*", pkg)
                    for _pkg_info in pkg_info:
                        pkg_name, pkg_version = _pkg_info.split('@')
                        yield (
                            pkg_name,
                            {"pkgname": pkg_name, "pkgversion": pkg_version},
                            'node-package')

    def _crawl_in_system(self):
        return self._get_packages_by_cmd()

    def crawl(self, container_id, avoid_setns=False, **kwargs):
        inspect = utils.dockerutils.exec_dockerinspect(container_id)
        state = inspect['State']
        pid = str(state['Pid'])
        logger.debug('Crawling OS for container %s' % container_id)

        if avoid_setns:
            raise NotImplementedError()
        else:
            return run_as_another_namespace(pid,
                                            ALL_NAMESPACES,
                                            self._crawl_in_system)
<commit_msg>flake8<commit_after>import logging
import re
import subprocess

import utils.dockerutils

from icrawl_plugin import IContainerCrawler
from utils.namespace import run_as_another_namespace, ALL_NAMESPACES

logger = logging.getLogger('crawlutils')

class NodePackageCrawler(IContainerCrawler):

    def get_feature(self):
        return 'node-package'

    def _get_packages_by_cmd(self):
        proc = subprocess.Popen(
            ['sh', '-c', 'npm list -g'],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        output, err = proc.communicate()

        if output:
            pkg_list = output.strip('\n')
            if pkg_list:
                for pkg in pkg_list.split('\n'):
                    if pkg.split()[-1] == 'deduped':
                        continue
                    pkg_info = re.findall(r"\S*@\S*", pkg)
                    for _pkg_info in pkg_info:
                        pkg_name, pkg_version = _pkg_info.split('@')
                        yield (
                            pkg_name,
                            {"pkgname": pkg_name, "pkgversion": pkg_version},
                            'node-package')

    def _crawl_in_system(self):
        return self._get_packages_by_cmd()

    def crawl(self, container_id, avoid_setns=False, **kwargs):
        inspect = utils.dockerutils.exec_dockerinspect(container_id)
        state = inspect['State']
        pid = str(state['Pid'])
        logger.debug('Crawling OS for container %s' % container_id)

        if avoid_setns:
            raise NotImplementedError()
        else:
            return run_as_another_namespace(pid,
                                            ALL_NAMESPACES,
                                            self._crawl_in_system)
<|endoftext|>


<commit_before>from kivy.properties import BooleanProperty, ObjectProperty
from kivy.core.window import Window

class HoverBehavior(object):
    """Hover behavior.
    :Events:
        `on_enter`
            Fired when mouse enter the box of the widget.
        `on_leave`
            Fired when the mouse exit the widget
    """

    hovered = BooleanProperty(False)
    border_point= ObjectProperty(None)
    '''Contains the last relevant point received by the Hoverable. This can
    be used in `on_enter` or `on_leave` in order to know where was dispatched the event.
    '''

    def __init__(self, **kwargs):
        self.register_event_type('on_enter')
        self.register_event_type('on_leave')
        Window.bind(mouse_pos=self.on_mouse_pos)
        super(HoverBehavior, self).__init__(**kwargs)

    def on_mouse_pos(self, *args):
        if not self.get_root_window():
            return # do proceed if I'm not displayed <=> If have no parent
        pos = args[1]
        # next line to_widget allow to compensate for relative layout
        inside = self.collide_point(*self.to_widget(*pos))
        if self.hovered == inside:
            # we have already done what was needed
            return
        self.border_point = pos
        self.hovered = inside
        if inside:
            self.dispatch('on_enter')
        else:
            self.dispatch('on_leave')

    def on_enter(self):
        pass

    def on_leave(self):
        pass
<commit_msg>Fix hover behaviour on mouse leave<commit_after>from kivy.properties import BooleanProperty, ObjectProperty
from kivy.core.window import Window

class HoverBehavior(object):
    """Hover behavior.
    :Events:
        `on_enter`
            Fired when mouse enter the box of the widget.
        `on_leave`
            Fired when the mouse exit the widget
    """

    hovered = BooleanProperty(False)
    border_point= ObjectProperty(None)
    '''Contains the last relevant point received by the Hoverable. This can
    be used in `on_enter` or `on_leave` in order to know where was dispatched the event.
    '''

    def __init__(self, **kwargs):
        self.register_event_type('on_enter')
        self.register_event_type('on_leave')
        Window.bind(mouse_pos=self._on_mouse_pos)
        Window.bind(on_cursor_leave=self._on_cursor_leave)
        super(HoverBehavior, self).__init__(**kwargs)

    def _on_mouse_pos(self, *args):
        if not self.get_root_window():
            return # do proceed if I'm not displayed <=> If have no parent
        pos = args[1]
        # next line to_widget allow to compensate for relative layout
        inside = self.collide_point(*self.to_widget(*pos))
        if self.hovered == inside:
            # we have already done what was needed
            return
        self.border_point = pos
        self.hovered = inside
        if inside:
            self.dispatch('on_enter')
        else:
            self.dispatch('on_leave')

    def _on_cursor_leave(self, *args):
        if (self.hovered):
            self.hovered = False
            self.dispatch('on_leave')

    def on_enter(self):
        pass

    def on_leave(self):
        pass
<|endoftext|>


<commit_before># Copyright (c) 2014. Mount Sinai School of Medicine
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division, absolute_import
import logging
from subprocess import check_output
import tempfile

from typechecks import require_string, require_integer

from .common import check_sequence_dictionary
from .alleles import normalize_allele_name, AlleleParseError
from .base_predictor import BasePredictor, UnsupportedAllele
from .process_helpers import run_command
from .cleanup_context import CleanupFiles
from .epitope_collection import EpitopeCollection
from .file_formats import create_input_fasta_files
from .process_helpers import run_multiple_commands_redirect_stdout

class BaseCommandlinePredictor(BasePredictor):
    """
    Base class for MHC binding predictors that run a local external
    program and write their output to a local file.
    """
    def __init__(
            self,
            program_name,
            alleles,
            epitope_lengths,
            parse_output_fn,
            supported_alleles_flag="-listMHC",
            input_fasta_flag="-f",
            length_flag="-length",
            allele_flag="-a",
            tempdir_flag="-tdir",
            extra_flags=[],
            max_file_records=None,
            process_limit=0):
        """
        Parameters
        ----------
        program_name : str
            Name of prediction program to run
            (e.g. "netMHCcons" or "NetMHCIIpan")

        alleles : list of str
            MHC alleles

        epitope_lengths : list of int

        supported_alleles_flag : str
            Flag to pass to the predictor to get a list of supported alleles
            (e.g. "-A", "-list", "-listMHC")

        parse_output_fn : fn
            Takes the stdout string from the predictor and returns a collection
            of BindingPrediction objects

        input_fasta_flag : str
            How to specify the input FASTA file of source sequences (e.g. "-f")

        length_flag : str
            How to specify the desired predicted epitope length (e.g. "-length")

        allele_flag : str
            How to specify the allele we want predictions for (e.g. "-a")

        tempdir_flag : str, optional
            How to specify the predictor's temporary directory (e.g. "-tdir")

        extra_flags : list of str
            Extra flags to pass to the predictor

        max_file_records : int, optional
            Maximum number of sequences per input FASTA file

        process_limit : int, optional
            Maximum number of parallel processes to start
        """

        require_string(program_name, "Predictor program name")
        self.program_name = program_name

        if supported_alleles_flag is not None:
            require_string(supported_alleles_flag, "Supported alleles flag")
        self.supported_alleles_flag = supported_alleles_flag

        require_string(input_fasta_flag, "Input FASTA file flag")
        self.input_fasta_flag = input_fasta_flag

        require_string(allele_flag, "Allele flag")
        self.allele_flag = allele_flag

        require_string(length_flag, "Peptide length flag")
        self.length_flag = length_flag

        if tempdir_flag is not None:
            require_string(tempdir_flag, "Temporary directory flag")
        self.tempdir_flag = tempdir_flag

        self.extra_flags = extra_flags

        if max_file_records is not None:
            require_integer(
                    max_file_records,
                    "Maximum number of sequences per input files")
        self.max_file_records = max_file_records

        require_integer(process_limit, "Maximum number of processes")
        self.process_limit = process_limit

        self.parse_output_fn = parse_output_fn

        if self.supported_alleles_flag:
            valid_alleles = self._determine_supported_alleles(
                self.program_name,
                self.supported_alleles_flag)
        else:
            # if we're not running the tool to determine supported alleles
            # then at least try running it by itself to determine if it's
            # it's present
            try:
                run_command([self.program_name])
            except:
                raise SystemError("Failed to run %s" % self.program_name)
            valid_alleles = None

        try:
            BasePredictor.__init__(
                self,
                alleles,
                epitope_lengths,
                valid_alleles=valid_alleles)
        except UnsupportedAllele as e:
            if self.supported_alleles_flag:
                additional_message = (
                    "\nRun command %s %s to see a list of valid alleles" % (
                        self.program_name,
                        self.supported_alleles_flag))
            else:
                additional_message = ""
            raise UnsupportedAllele(str(e) + additional_message)

    @staticmethod
    def _determine_supported_alleles(command, supported_allele_flag):
        """
        Try asking the commandline predictor (e.g. netMHCpan)
        which alleles it supports.
        """
        try:
            # convert to str since Python3 returns a `bytes` object
            supported_alleles_output = check_output([
                command, supported_allele_flag
            ])
            supported_alleles_str = supported_alleles_output.decode("ascii", "ignore")
            assert len(supported_alleles_str) > 0, \
                '%s returned empty allele list' % command
            supported_alleles = set([])
            for line in supported_alleles_str.split("\n"):
                line = line.strip()
                if not line.startswith('#') and len(line) > 0:
                    try:
                        # We need to normalize these alleles (the output of the predictor
                        # when it lists its supported alleles) so that they are comparable with
                        # our own alleles.
                        supported_alleles.add(normalize_allele_name(line))
                    except AlleleParseError as error:
                        logging.info("Skipping allele %s: %s" % (
                            line, error))
                        continue
            return supported_alleles
        except:
            logging.warning(
                "Failed to run %s %s",
                command,
                supported_allele_flag)
            raise

    def prepare_allele_name(self, allele_name):
        """
        How does the predictor expect to see allele names?
        """
        return allele_name.replace("*", "")

    def _build_command(self, input_filename, allele, length, temp_dirname=None):
        args = [self.program_name]
        if self.input_fasta_flag:
            args.extend([self.input_fasta_flag, input_filename])
        else:
            args.append(input_filename)
        args.extend([
            self.allele_flag, allele,
            self.length_flag, str(length),
        ])
        if self.tempdir_flag and temp_dirname:
            args.extend([self.tempdir_flag, temp_dirname])
        args.extend(self.extra_flags)
        return args

    def predict(self, fasta_dictionary):
        """
        Run multiple predictors simultaneously, split across:
            1) multiple input files, each containing self.max_file_records
            2) every allele + peptide length combination
        """
        fasta_dictionary = check_sequence_dictionary(fasta_dictionary)
        input_filenames, sequence_key_mapping = create_input_fasta_files(
            fasta_dictionary,
            max_file_records=self.max_file_records)
        output_files = {}
        commands = {}
        dirs = []
        alleles = [
            self.prepare_allele_name(allele)
            for allele in self.alleles
        ]
        for i, input_filename in enumerate(input_filenames):
            for j, allele in enumerate(alleles):
                for length in self.epitope_lengths:
                    if self.tempdir_flag:
                        temp_dirname = tempfile.mkdtemp(
                            prefix="tmp_%s_length_%d" % (
                                self.program_name, length))
                        logging.info(
                            "Created temporary directory %s for allele %s, length %d",
                            temp_dirname,
                            allele,
                            length)
                        dirs.append(temp_dirname)
                    else:
                        temp_dirname = None
                    output_file = tempfile.NamedTemporaryFile(
                        "w",
                        prefix="%s_output_%d_%d" % (self.program_name, i, j),
                        delete=False)
                    commands[output_file] = self._build_command(
                        input_filename=input_filename,
                        allele=allele,
                        length=length,
                        temp_dirname=temp_dirname)

        epitope_collections = []

        # Cleanup either when finished or if an exception gets raised by
        # deleting the input and output files
        filenames_to_delete = input_filenames
        for f in output_files.keys():
            filenames_to_delete.append(f.name)

        with CleanupFiles(
                filenames=filenames_to_delete,
                directories=dirs):
            run_multiple_commands_redirect_stdout(
                commands,
                print_commands=True,
                process_limit=self.process_limit)
            for output_file, command in commands.items():
                # closing/opening looks insane
                # but I was getting empty files otherwise
                output_file.close()
                with open(output_file.name, 'r') as f:
                    epitope_collection = self.parse_output_fn(
                        stdout=f.read(),
                        fasta_dictionary=fasta_dictionary,
                        sequence_key_mapping=sequence_key_mapping,
                        prediction_method_name=self.program_name)
                    epitope_collections.append(epitope_collection)

        if len(epitope_collections) != len(commands):
            raise ValueError(
                ("Expected an epitope collection for each "
                 "command (%d), but instead there are %d") % (
                    len(commands),
                    len(epitope_collections)))

        if len(epitope_collections) == 0:
            raise ValueError("No epitopes from %s" % self.program_name)

        # flatten all epitope collections into a single object
        return EpitopeCollection([
            binding_prediction
            for sublist in epitope_collections
            for binding_prediction in sublist])
<commit_msg>got rid of default values for commandline flags<commit_after># Copyright (c) 2014. Mount Sinai School of Medicine
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division, absolute_import
import logging
from subprocess import check_output
import tempfile

from typechecks import require_string, require_integer

from .common import check_sequence_dictionary
from .alleles import normalize_allele_name, AlleleParseError
from .base_predictor import BasePredictor, UnsupportedAllele
from .process_helpers import run_command
from .cleanup_context import CleanupFiles
from .epitope_collection import EpitopeCollection
from .file_formats import create_input_fasta_files
from .process_helpers import run_multiple_commands_redirect_stdout

class BaseCommandlinePredictor(BasePredictor):
    """
    Base class for MHC binding predictors that run a local external
    program and write their output to a local file.
    """
    def __init__(
            self,
            program_name,
            alleles,
            epitope_lengths,
            parse_output_fn,
            supported_alleles_flag,
            input_fasta_flag,
            length_flag,
            allele_flag,
            tempdir_flag=None,
            extra_flags=[],
            max_file_records=None,
            process_limit=0):
        """
        Parameters
        ----------
        program_name : str
            Name of prediction program to run
            (e.g. "netMHCcons" or "NetMHCIIpan")

        alleles : list of str
            MHC alleles

        epitope_lengths : list of int

        supported_alleles_flag : str
            Flag to pass to the predictor to get a list of supported alleles
            (e.g. "-A", "-list", "-listMHC")

        parse_output_fn : fn
            Takes the stdout string from the predictor and returns a collection
            of BindingPrediction objects

        input_fasta_flag : str
            How to specify the input FASTA file of source sequences (e.g. "-f")

        length_flag : str
            How to specify the desired predicted epitope length (e.g. "-length")

        allele_flag : str
            How to specify the allele we want predictions for (e.g. "-a")

        tempdir_flag : str, optional
            How to specify the predictor's temporary directory (e.g. "-tdir")

        extra_flags : list of str
            Extra flags to pass to the predictor

        max_file_records : int, optional
            Maximum number of sequences per input FASTA file

        process_limit : int, optional
            Maximum number of parallel processes to start
        """

        require_string(program_name, "Predictor program name")
        self.program_name = program_name

        if supported_alleles_flag is not None:
            require_string(supported_alleles_flag, "Supported alleles flag")
        self.supported_alleles_flag = supported_alleles_flag

        require_string(input_fasta_flag, "Input FASTA file flag")
        self.input_fasta_flag = input_fasta_flag

        require_string(allele_flag, "Allele flag")
        self.allele_flag = allele_flag

        require_string(length_flag, "Peptide length flag")
        self.length_flag = length_flag

        if tempdir_flag is not None:
            require_string(tempdir_flag, "Temporary directory flag")
        self.tempdir_flag = tempdir_flag

        self.extra_flags = extra_flags

        if max_file_records is not None:
            require_integer(
                    max_file_records,
                    "Maximum number of sequences per input files")
        self.max_file_records = max_file_records

        require_integer(process_limit, "Maximum number of processes")
        self.process_limit = process_limit

        self.parse_output_fn = parse_output_fn

        if self.supported_alleles_flag:
            valid_alleles = self._determine_supported_alleles(
                self.program_name,
                self.supported_alleles_flag)
        else:
            # if we're not running the tool to determine supported alleles
            # then at least try running it by itself to determine if it's
            # it's present
            try:
                run_command([self.program_name])
            except:
                raise SystemError("Failed to run %s" % self.program_name)
            valid_alleles = None

        try:
            BasePredictor.__init__(
                self,
                alleles,
                epitope_lengths,
                valid_alleles=valid_alleles)
        except UnsupportedAllele as e:
            if self.supported_alleles_flag:
                additional_message = (
                    "\nRun command %s %s to see a list of valid alleles" % (
                        self.program_name,
                        self.supported_alleles_flag))
            else:
                additional_message = ""
            raise UnsupportedAllele(str(e) + additional_message)

    @staticmethod
    def _determine_supported_alleles(command, supported_allele_flag):
        """
        Try asking the commandline predictor (e.g. netMHCpan)
        which alleles it supports.
        """
        try:
            # convert to str since Python3 returns a `bytes` object
            supported_alleles_output = check_output([
                command, supported_allele_flag
            ])
            supported_alleles_str = supported_alleles_output.decode("ascii", "ignore")
            assert len(supported_alleles_str) > 0, \
                '%s returned empty allele list' % command
            supported_alleles = set([])
            for line in supported_alleles_str.split("\n"):
                line = line.strip()
                if not line.startswith('#') and len(line) > 0:
                    try:
                        # We need to normalize these alleles (the output of the predictor
                        # when it lists its supported alleles) so that they are comparable with
                        # our own alleles.
                        supported_alleles.add(normalize_allele_name(line))
                    except AlleleParseError as error:
                        logging.info("Skipping allele %s: %s" % (
                            line, error))
                        continue
            return supported_alleles
        except:
            logging.warning(
                "Failed to run %s %s",
                command,
                supported_allele_flag)
            raise

    def prepare_allele_name(self, allele_name):
        """
        How does the predictor expect to see allele names?
        """
        return allele_name.replace("*", "")

    def _build_command(self, input_filename, allele, length, temp_dirname=None):
        args = [self.program_name]
        if self.input_fasta_flag:
            args.extend([self.input_fasta_flag, input_filename])
        else:
            args.append(input_filename)
        args.extend([
            self.allele_flag, allele,
            self.length_flag, str(length),
        ])
        if self.tempdir_flag and temp_dirname:
            args.extend([self.tempdir_flag, temp_dirname])
        args.extend(self.extra_flags)
        return args

    def predict(self, fasta_dictionary):
        """
        Run multiple predictors simultaneously, split across:
            1) multiple input files, each containing self.max_file_records
            2) every allele + peptide length combination
        """
        fasta_dictionary = check_sequence_dictionary(fasta_dictionary)
        input_filenames, sequence_key_mapping = create_input_fasta_files(
            fasta_dictionary,
            max_file_records=self.max_file_records)
        output_files = {}
        commands = {}
        dirs = []
        alleles = [
            self.prepare_allele_name(allele)
            for allele in self.alleles
        ]
        for i, input_filename in enumerate(input_filenames):
            for j, allele in enumerate(alleles):
                for length in self.epitope_lengths:
                    if self.tempdir_flag:
                        temp_dirname = tempfile.mkdtemp(
                            prefix="tmp_%s_length_%d" % (
                                self.program_name, length))
                        logging.info(
                            "Created temporary directory %s for allele %s, length %d",
                            temp_dirname,
                            allele,
                            length)
                        dirs.append(temp_dirname)
                    else:
                        temp_dirname = None
                    output_file = tempfile.NamedTemporaryFile(
                        "w",
                        prefix="%s_output_%d_%d" % (self.program_name, i, j),
                        delete=False)
                    commands[output_file] = self._build_command(
                        input_filename=input_filename,
                        allele=allele,
                        length=length,
                        temp_dirname=temp_dirname)

        epitope_collections = []

        # Cleanup either when finished or if an exception gets raised by
        # deleting the input and output files
        filenames_to_delete = input_filenames
        for f in output_files.keys():
            filenames_to_delete.append(f.name)

        with CleanupFiles(
                filenames=filenames_to_delete,
                directories=dirs):
            run_multiple_commands_redirect_stdout(
                commands,
                print_commands=True,
                process_limit=self.process_limit)
            for output_file, command in commands.items():
                # closing/opening looks insane
                # but I was getting empty files otherwise
                output_file.close()
                with open(output_file.name, 'r') as f:
                    epitope_collection = self.parse_output_fn(
                        stdout=f.read(),
                        fasta_dictionary=fasta_dictionary,
                        sequence_key_mapping=sequence_key_mapping,
                        prediction_method_name=self.program_name)
                    epitope_collections.append(epitope_collection)

        if len(epitope_collections) != len(commands):
            raise ValueError(
                ("Expected an epitope collection for each "
                 "command (%d), but instead there are %d") % (
                    len(commands),
                    len(epitope_collections)))

        if len(epitope_collections) == 0:
            raise ValueError("No epitopes from %s" % self.program_name)

        # flatten all epitope collections into a single object
        return EpitopeCollection([
            binding_prediction
            for sublist in epitope_collections
            for binding_prediction in sublist])
<|endoftext|>


<commit_before>
def register(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def sensor(nodenet, node=None, datasource=None, **params):
    node.gates["gen"].gate_function(nodenet.world.get_datasource(nodenet.uid, datasource))

def actor(nodenet, node=None, datatarget=None, **params):
    node.nodenet.world.set_datatarget(nodenet.uid, datatarget, node.get_slot("gen").activation)
    node.activation = node.get_slot("gen").activation

def concept(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def pipe(nodenet, node=None, **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0

    gen += node.get_slot("sur").activation
    if gen < 0: gen = 0
    if gen > 1: gen = 1

    sub += node.get_slot("gen").activation * 3
    sub += node.get_slot("sub").activation
    sub += node.get_slot("por").activation
    if sub == 0: sub = -1
    if sub > 1: sub = 1

    sur += node.get_slot("sur").activation * \
           (1+node.get_slot("sub").activation) * \
           (1+node.get_slot("ret").activation) * \
           (1+node.get_slot("por").activation)
    if sur < 0: sur = 0

    por += node.get_slot("sur").activation
    por += node.get_slot("por").activation
    if por == 0: por = -1
    if por > 1: por = 1

    ret += node.get_slot("ret").activation
    if ret == 0: ret = -1

    node.get_gate("gen").gate_function(gen)
    node.get_gate("por").gate_function(por)
    node.get_gate("ret").gate_function(ret)
    node.get_gate("sub").gate_function(sub)
    node.get_gate("sur").gate_function(sur)

def label(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def event(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def activator(nodenet, node, **params):
    node.activation = node.get_slot("gen").activation
    nodenet.nodespaces[node.parent_nodespace].activators[node.parameters["type"]] = node.activation
<commit_msg>enter fuzzynes<commit_after>
def register(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def sensor(nodenet, node=None, datasource=None, **params):
    node.gates["gen"].gate_function(nodenet.world.get_datasource(nodenet.uid, datasource))

def actor(nodenet, node=None, datatarget=None, **params):
    node.nodenet.world.set_datatarget(nodenet.uid, datatarget, node.get_slot("gen").activation)
    node.activation = node.get_slot("gen").activation

def concept(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def pipe(nodenet, node=None, **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0

    gen += node.get_slot("sur").voted_activation
    if gen < 0: gen = 0
    if gen > 1: gen = 1

    sub += node.get_slot("gen").activation * 3
    sub += node.get_slot("sub").activation
    sub += node.get_slot("por").activation
    if sub == 0: sub = -1
    if sub > 1: sub = 1

    sur += node.get_slot("sur").voted_activation
    if sur < 0: sur = 0

    por += node.get_slot("sur").activation * \
           (1+node.get_slot("sub").activation) * \
           (1+node.get_slot("por").activation)
    por += node.get_slot("por").activation * \
           (1+node.get_slot("ret").activation)
    if por == 0: por = -1
    if por > 1: por = 1

    ret += node.get_slot("ret").activation
    if ret == 0: ret = -1

    node.get_gate("gen").gate_function(gen)
    node.get_gate("por").gate_function(por)
    node.get_gate("ret").gate_function(ret)
    node.get_gate("sub").gate_function(sub)
    node.get_gate("sur").gate_function(sur)

def label(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def event(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def activator(nodenet, node, **params):
    node.activation = node.get_slot("gen").activation
    nodenet.nodespaces[node.parent_nodespace].activators[node.parameters["type"]] = node.activation
<|endoftext|>


<commit_before>import json
import logging

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject, MISPAttribute

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url',
                             'AutonomousSystem': 'AS'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            for n in neighboors['objs']:
                yield n

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        for obs_to_add in self.get_neighboors(obs['id']):
            object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
            if object_misp_domain_ip:
                self.misp_event.add_object(object_misp_domain_ip)
            object_misp_url = self.__get_object_url(obs_to_add)
            if object_misp_url:
                self.misp_event.add_object(object_misp_url)
            if not object_misp_url and not object_misp_url:
                attr = self.__get_attribute(obs_to_add)
                if attr:
                    self.misp_event.add_attribute(attr.type, attr.value, tags=['test','toto'])

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def __get_attribute(self, obs_to_add):
        attr = MISPAttribute()
        attr.value = obs_to_add['value']
        try:
            attr.type = self.misp_mapping[obs_to_add['type']]
        except KeyError:
            logging.error('type not found %s' % obs_to_add['type'])
            return

        for t in obs_to_add['tags']:
            attr.tags.append(t['name'])
        return attr

    def __get_object_domain_ip(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
                                           obj_to_add['value'])
            domain_ip_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False),
                                           self.attribute['value'])
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')

            return domain_ip_object

    def __get_object_url(self, obj_to_add):
        if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or (
            obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url'
        ):
            url_object = MISPObject('url')
            obj_relation = self.__get_relation(obj_to_add)
            if obj_relation:
                url_object.add_attribute(obj_relation, obj_to_add['value'])
            obj_relation = self.__get_relation(self.attribute, is_yeti_object=False)
            if obj_relation:
                url_object.add_attribute(obj_relation,
                                         self.attribute['value'])
            url_object.add_reference(self.attribute['uuid'], 'related_to')

            return url_object

    def __get_relation(self, obj, is_yeti_object=True):
        if is_yeti_object:
            type_attribute = self.misp_mapping[obj['type']]
        else:
            type_attribute = obj['type']
        if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
            return 'ip'
        elif 'domain' == type_attribute:
            return 'domain'
        elif 'hostname' == type_attribute:
            return 'domain'
        elif type_attribute == 'url':
            return type_attribute

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return {'results': yeti_client.get_result()}
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<commit_msg>Update yeti.py<commit_after>import json
import logging

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject, MISPAttribute

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url',
                             'AutonomousSystem': 'AS'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            for n in neighboors['objs']:
                yield n

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        for obs_to_add in self.get_neighboors(obs['id']):
            object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
            if object_misp_domain_ip:
                self.misp_event.add_object(object_misp_domain_ip)
            object_misp_url = self.__get_object_url(obs_to_add)
            if object_misp_url:
                self.misp_event.add_object(object_misp_url)
            if not object_misp_url and not object_misp_url:
                attr = self.__get_attribute(obs_to_add)
                if attr:
                    self.misp_event.add_attribute(attr.type, attr.value)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def __get_attribute(self, obs_to_add):
        attr = MISPAttribute()
        attr.value = obs_to_add['value']
        try:
            attr.type = self.misp_mapping[obs_to_add['type']]
        except KeyError:
            logging.error('type not found %s' % obs_to_add['type'])
            return

        for t in obs_to_add['tags']:
            attr.tags.append(t['name'])
        return attr

    def __get_object_domain_ip(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
                                           obj_to_add['value'])
            domain_ip_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False),
                                           self.attribute['value'])
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')

            return domain_ip_object

    def __get_object_url(self, obj_to_add):
        if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or (
            obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url'
        ):
            url_object = MISPObject('url')
            obj_relation = self.__get_relation(obj_to_add)
            if obj_relation:
                url_object.add_attribute(obj_relation, obj_to_add['value'])
            obj_relation = self.__get_relation(self.attribute, is_yeti_object=False)
            if obj_relation:
                url_object.add_attribute(obj_relation,
                                         self.attribute['value'])
            url_object.add_reference(self.attribute['uuid'], 'related_to')

            return url_object

    def __get_relation(self, obj, is_yeti_object=True):
        if is_yeti_object:
            type_attribute = self.misp_mapping[obj['type']]
        else:
            type_attribute = obj['type']
        if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
            return 'ip'
        elif 'domain' == type_attribute:
            return 'domain'
        elif 'hostname' == type_attribute:
            return 'domain'
        elif type_attribute == 'url':
            return type_attribute

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return {'results': yeti_client.get_result()}
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<|endoftext|>


<commit_before>import json

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            for n in neighboors['objs']:
                yield n

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        self.misp_event.add_attribute(**self.attribute)
        for obs_to_add in self.get_neighboors(obs['id']):
            object_misp = self.get_object(obs_to_add)
            self.misp_event.add_object(object_misp)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def get_object(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add))
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')
            domain_ip_object.add_attribute(**self.attribute)
            return domain_ip_object

    def __get_attribute(self, obj_yeti):
        typ_attribute = self.misp_mapping[obj_yeti['type']]
        attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']}
        if typ_attribute == 'ip-src' or typ_attribute =='ip-dst':
            attr_misp['object_relation'] = 'ip'
        elif 'domain' == typ_attribute:
            attr_misp['object_relation'] = 'domain'
        else:
            attr_misp['object_relation'] = None
        return attr_misp

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return yeti_client.get_result()
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<commit_msg>Update yeti.py<commit_after>import json

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            for n in neighboors['objs']:
                yield n

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        self.misp_event.add_attribute(**self.attribute)
        for obs_to_add in self.get_neighboors(obs['id']):
            object_misp = self.get_object(obs_to_add)
            self.misp_event.add_object(object_misp)
            print(self.misp_event)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def get_object(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add))
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')
            domain_ip_object.add_attribute(**self.attribute)
            return domain_ip_object

    def __get_attribute(self, obj_yeti):
        typ_attribute = self.misp_mapping[obj_yeti['type']]
        attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']}
        if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst':
            attr_misp['object_relation'] = 'ip'
        elif 'domain' == typ_attribute:
            attr_misp['object_relation'] = 'domain'
        else:
            attr_misp['object_relation'] = None
        return attr_misp

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return yeti_client.get_result()
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<|endoftext|>


<commit_before>import torch

from mmdet.ops.nms import batched_nms

def multiclass_nms(multi_bboxes,
                   multi_scores,
                   score_thr,
                   nms_cfg,
                   max_num=-1,
                   score_factors=None):
    """NMS for multi-class bboxes.

    Args:
        multi_bboxes (Tensor): shape (n, #class*4) or (n, 4)
        multi_scores (Tensor): shape (n, #class), where the 0th column
            contains scores of the background class, but this will be ignored.
        score_thr (float): bbox threshold, bboxes with scores lower than it
            will not be considered.
        nms_thr (float): NMS IoU threshold
        max_num (int): if there are more than max_num bboxes after NMS,
            only top max_num will be kept.
        score_factors (Tensor): The factors multiplied to scores before
            applying NMS

    Returns:
        tuple: (bboxes, labels), tensors of shape (k, 5) and (k, 1). Labels
            are 0-based.
    """
    num_classes = multi_scores.size(1) - 1
    # exclude background category
    if multi_bboxes.shape[1] > 4:
        bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4)
    else:
        bboxes = multi_bboxes[:, None].expand(-1, num_classes, 4)
    scores = multi_scores[:, :-1]

    # filter out boxes with low scores
    valid_mask = scores > score_thr
    bboxes = bboxes[valid_mask]
    if score_factors is not None:
        scores = scores * score_factors[:, None]
    scores = scores[valid_mask]
    labels = valid_mask.nonzero()[:, 1]

    if bboxes.numel() == 0:
        bboxes = multi_bboxes.new_zeros((0, 5))
        labels = multi_bboxes.new_zeros((0, ), dtype=torch.long)
        return bboxes, labels

    dets, keep = batched_nms(bboxes, scores, labels, nms_cfg)
    return dets[:max_num], labels[keep[:max_num]]
<commit_msg>Check if max_num > 0 before slicing in nms (#2486)<commit_after>import torch

from mmdet.ops.nms import batched_nms

def multiclass_nms(multi_bboxes,
                   multi_scores,
                   score_thr,
                   nms_cfg,
                   max_num=-1,
                   score_factors=None):
    """NMS for multi-class bboxes.

    Args:
        multi_bboxes (Tensor): shape (n, #class*4) or (n, 4)
        multi_scores (Tensor): shape (n, #class), where the 0th column
            contains scores of the background class, but this will be ignored.
        score_thr (float): bbox threshold, bboxes with scores lower than it
            will not be considered.
        nms_thr (float): NMS IoU threshold
        max_num (int): if there are more than max_num bboxes after NMS,
            only top max_num will be kept.
        score_factors (Tensor): The factors multiplied to scores before
            applying NMS

    Returns:
        tuple: (bboxes, labels), tensors of shape (k, 5) and (k, 1). Labels
            are 0-based.
    """
    num_classes = multi_scores.size(1) - 1
    # exclude background category
    if multi_bboxes.shape[1] > 4:
        bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4)
    else:
        bboxes = multi_bboxes[:, None].expand(-1, num_classes, 4)
    scores = multi_scores[:, :-1]

    # filter out boxes with low scores
    valid_mask = scores > score_thr
    bboxes = bboxes[valid_mask]
    if score_factors is not None:
        scores = scores * score_factors[:, None]
    scores = scores[valid_mask]
    labels = valid_mask.nonzero()[:, 1]

    if bboxes.numel() == 0:
        bboxes = multi_bboxes.new_zeros((0, 5))
        labels = multi_bboxes.new_zeros((0, ), dtype=torch.long)
        return bboxes, labels

    dets, keep = batched_nms(bboxes, scores, labels, nms_cfg)

    if max_num > 0:
        dets = dets[:max_num]
        keep = keep[:max_num]

    return dets, labels[keep]
<|endoftext|>


<commit_before>from molly.utils.views import BaseView
from molly.utils.breadcrumbs import *

from .models import Idea

class IndexView(BaseView):
    @BreadcrumbFactory
    def breadcrumb(cls, request, context):
        return Breacrumb(
            cls.conf.local_name,
            None,
            'Feature suggestions',
            lazy_reverse('feature_voting:index'),
        )

    def initial_context(cls, request):
        return {
            'ideas': Idea.objects.all(),
        }

    def handle_GET(cls, request, context):
        return cls.render(request, context, 'feature_voting/index')

<commit_msg>Fixed typo 'Breacrumb'<commit_after>from molly.utils.views import BaseView
from molly.utils.breadcrumbs import *

from .models import Idea

class IndexView(BaseView):
    @BreadcrumbFactory
    def breadcrumb(cls, request, context):
        return Breadcrumb(
            cls.conf.local_name,
            None,
            'Feature suggestions',
            lazy_reverse('feature_voting:index'),
        )

    def initial_context(cls, request):
        return {
            'ideas': Idea.objects.all(),
        }

    def handle_GET(cls, request, context):
        return cls.render(request, context, 'feature_voting/index')

<|endoftext|>


<commit_before># Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import monascastatsd
import os

from oslo_config import cfg
from oslo_log import log

LOG = log.getLogger(__name__)
CONF = cfg.CONF

STATSD_PORT = int(os.getenv('STATSD_PORT', '8125'))
STATSD_HOST = os.getenv('STATSD_HOST', '127.0.0.1')

_DEFAULT_HOST = STATSD_HOST
_DEFAULT_PORT = STATSD_PORT
_DEFAULT_BUFFER_SIZE = 50
_DEFAULT_DIMENSIONS = {
    'service': 'monitoring',
    'component': 'monasca-persister'
}
_CLIENT_NAME = 'monasca'

monitoring_opts = [
    cfg.IPOpt('statsd_host',
              default=_DEFAULT_HOST,
              help=('IP address of statsd server, default to %s'
                    % _DEFAULT_HOST)),
    cfg.PortOpt('statsd_port',
                default=_DEFAULT_PORT,
                help='Port of statsd server, default to %d' % _DEFAULT_PORT),
    cfg.IntOpt('statsd_buffer',
               default=_DEFAULT_BUFFER_SIZE,
               required=True,
               help=('Maximum number of metric to buffer before sending, '
                     'default to %d' % _DEFAULT_BUFFER_SIZE)),
    cfg.DictOpt('dimensions')
]

monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring')

cfg.CONF.register_group(monitoring_group)
cfg.CONF.register_opts(monitoring_opts, monitoring_group)

def get_client(dimensions=None):
    """Creates statsd client

    Creates monasca-statsd client using configuration from
    config file and supplied dimensions.

    Configuration is composed out of ::

        [monitoring]
        statsd_host = 192.168.10.4
        statsd_port = 8125
        statsd_buffer = 50

    Dimensions are appended to following dictionary ::

        {
            'service': 'monitoring',
            'component': 'monasca-log-api'
        }

    Note:
        Passed dimensions do not override those specified in
        dictionary above

    :param dict dimensions: Optional dimensions
    :return: statsd client
    :rtype: monascastatsd.Client
    """
    dims = _DEFAULT_DIMENSIONS.copy()
    if dimensions:
        for key, val in dimensions.items():
            if key not in _DEFAULT_DIMENSIONS:
                dims[key] = val
            else:
                LOG.warn('Cannot override fixed dimension %s=%s', key,
                         _DEFAULT_DIMENSIONS[key])

    connection = monascastatsd.Connection(
        host=CONF.monitoring.statsd_host,
        port=CONF.monitoring.statsd_port,
        max_buffer_size=CONF.monitoring.statsd_buffer
    )
    client = monascastatsd.Client(name=_CLIENT_NAME,
                                  connection=connection,
                                  dimensions=dims)

    LOG.debug('Created statsd client %s[%s] = %s:%d', _CLIENT_NAME, dims,
              CONF.monitoring.statsd_host, CONF.monitoring.statsd_port)

    return client
<commit_msg>update to monasca-statsd 2<commit_after># Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os

import monascastatsd
from oslo_config import cfg
from oslo_log import log

LOG = log.getLogger(__name__)
CONF = cfg.CONF

STATSD_PORT = int(os.getenv('STATSD_PORT', '8125'))
STATSD_HOST = os.getenv('STATSD_HOST', '127.0.0.1')

_DEFAULT_HOST = STATSD_HOST
_DEFAULT_PORT = STATSD_PORT
_DEFAULT_BUFFER_SIZE = 50
_DEFAULT_DIMENSIONS = {
    'service': 'monitoring',
    'component': 'monasca-persister'
}
_CLIENT_NAME = 'monasca'

monitoring_opts = [
    cfg.IPOpt('statsd_host',
              default=_DEFAULT_HOST,
              help=('IP address of statsd server, default to %s'
                    % _DEFAULT_HOST)),
    cfg.PortOpt('statsd_port',
                default=_DEFAULT_PORT,
                help='Port of statsd server, default to %d' % _DEFAULT_PORT),
    cfg.IntOpt('statsd_buffer',
               default=_DEFAULT_BUFFER_SIZE,
               required=True,
               help=('Maximum number of metric to buffer before sending, '
                     'default to %d' % _DEFAULT_BUFFER_SIZE)),
    cfg.DictOpt('dimensions')
]

monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring')

cfg.CONF.register_group(monitoring_group)
cfg.CONF.register_opts(monitoring_opts, monitoring_group)

def get_client(dimensions=None):
    """Creates statsd client

    Creates monasca-statsd client using configuration from
    config file and supplied dimensions.

    Configuration is composed out of ::

        [monitoring]
        statsd_host = 192.168.10.4
        statsd_port = 8125
        statsd_buffer = 50

    Dimensions are appended to following dictionary ::

        {
            'service': 'monitoring',
            'component': 'monasca-log-api'
        }

    Note:
        Passed dimensions do not override those specified in
        dictionary above

    :param dict dimensions: Optional dimensions
    :return: statsd client
    :rtype: monascastatsd.Client
    """
    dims = _DEFAULT_DIMENSIONS.copy()
    if dimensions:
        for key, val in dimensions.items():
            if key not in _DEFAULT_DIMENSIONS:
                dims[key] = val
            else:
                LOG.warn('Cannot override fixed dimension %s=%s', key,
                         _DEFAULT_DIMENSIONS[key])

    client = monascastatsd.Client(name=_CLIENT_NAME, host=CONF.monitoring.statsd_host,
                                  port=CONF.monitoring.statsd_port, auto_buffer=True,
                                  max_buffer_size=CONF.monitoring.statsd_buffer, dimensions=dims)

    LOG.debug('Created statsd client %s[%s] = %s:%d', _CLIENT_NAME, dims,
              CONF.monitoring.statsd_host, CONF.monitoring.statsd_port)

    return client
<|endoftext|>


<commit_before>####################################
## batch code for WF simulation
####################################
import sys, array, os, getpass
from subprocess import call
import subprocess as subp
import time
import math as math
from subprocess import Popen, PIPE

if len(sys.argv)<2:
    print("========= Syntax ========")
    print("python BatchSimulation.py ....")
    print("<Output path (abs.)>")
    print("<number of jobs>")
    print("<number of events in each job>")
    print("<enable PMT after pulses ?(0 for disable)>")
    print("<enable S2 after pulses ?(0 for disable)>")
    print("<photon number lower>")
    print("<photon number upper>")
    print("<electron number lower>")
    print("<electron number upper>")
    print("<If enable S1-S2 correlation (0 for no, 1 for yes)>")
    print("<If use Public node (0 for no(xenon1t nodes); 1 for yes; 2 for kicp nodes)>")
    exit()

OutputGeneralPath = sys.argv[1]
NumJobs = int(sys.argv[2])
NumEvents = int(sys.argv[3])
PMTAfterpulseFlag = int(sys.argv[4])
S2AfterpulseFlag = int(sys.argv[5])
PhotonNumLower = int(sys.argv[6])
PhotonNumUpper = int(sys.argv[7])
ElectronNumLower = int(sys.argv[8])
ElectronNumUpper = int(sys.argv[9])
IfEnableS1S2Correlation = int(sys.argv[10])
IfUsePublicNodes = int(sys.argv[11])

MaxNumJob = 64
if not IfUsePublicNodes:
    MaxNumJob=200

##### Start batching #########
CurrentPath = os.getcwd()
print (CurrentPath)
CurrentUser = getpass.getuser()
for i in range(NumJobs):

    RunString = "%06d" % i

    # create folder
    OutputPath = OutputGeneralPath + "/" + RunString
    if os.path.exists(OutputPath):
        subp.call("rm -r "+OutputPath, shell=True)
    subp.call("mkdir -p "+OutputPath, shell=True)

    # define filenames
    SubmitFile = OutputPath+"/submit_"+ RunString + ".sh"
    SubmitOutputFilename = OutputPath+"/submit_"+ RunString + ".log"
    SubmitErrorFilename = OutputPath+"/submit_"+ RunString + ".log"

    # create the basic submit
    subp.call("echo '#!/bin/bash\n' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --output="+SubmitOutputFilename+"' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --error="+SubmitErrorFilename+"' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --time=03:59:00' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --account=pi-lgrandi' >> "+SubmitFile, shell=True)
    if IfUsePublicNodes==0:
        subp.call("echo '#SBATCH --qos=xenon1t' >> "+SubmitFile, shell=True)
        subp.call("echo '#SBATCH --partition=xenon1t\n' >> "+SubmitFile, shell=True)
    elif IfUsePublicNodes==2:
        subp.call("echo '#SBATCH --qos=xenon1t-kicp' >> "+SubmitFile, shell=True)
        subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True)

    Command = CurrentPath+"/./run_fax.sh "+str(PhotonNumLower)+" "+str(PhotonNumUpper)+" "+str(ElectronNumLower)+" "+str(ElectronNumUpper)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString+" "+str(IfEnableS1S2Correlation)
    subp.call("echo '"+Command+"\n' >> "+SubmitFile, shell=True)

    SubmitPath = OutputPath

    #submit
    IfSubmitted=0
    while IfSubmitted==0:
        Partition = "sandyb" # public
        if not IfUsePublicNodes:
            Partition = "xenon1t"
        elif IfUsePublicNodes==2:
            Partition = "kicp"
        p1 = Popen(["squeue","--partition="+Partition, "--user="+CurrentUser], stdout=PIPE)
        p2 = Popen(["wc", "-l"], stdin=p1.stdout, stdout=PIPE)
        p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
        output = p2.communicate()[0]
        Status=subp.call("squeue --partition="+Partition+" --user=mcfate | wc -l", shell=True)
        Output=int(output)
        #print(Status)

        print("Current job running number "+str(Output))

        if Status==0 and Output<MaxNumJob:
            #sbatch it
            subp.call("cd "+SubmitPath+";sbatch "+SubmitFile+";cd -", shell=True)
            IfSubmitted=1
            time.sleep(2.0)
        else:
            time.sleep(30)
<commit_msg>Bug fixed<commit_after>####################################
## batch code for WF simulation
####################################
import sys, array, os, getpass
from subprocess import call
import subprocess as subp
import time
import math as math
from subprocess import Popen, PIPE

if len(sys.argv)<2:
    print("========= Syntax ========")
    print("python BatchSimulation.py ....")
    print("<Output path (abs.)>")
    print("<number of jobs>")
    print("<number of events in each job>")
    print("<enable PMT after pulses ?(0 for disable)>")
    print("<enable S2 after pulses ?(0 for disable)>")
    print("<photon number lower>")
    print("<photon number upper>")
    print("<electron number lower>")
    print("<electron number upper>")
    print("<If enable S1-S2 correlation (0 for no, 1 for yes)>")
    print("<If use Public node (0 for no(xenon1t nodes); 1 for yes; 2 for kicp nodes)>")
    exit()

OutputGeneralPath = sys.argv[1]
NumJobs = int(sys.argv[2])
NumEvents = int(sys.argv[3])
PMTAfterpulseFlag = int(sys.argv[4])
S2AfterpulseFlag = int(sys.argv[5])
PhotonNumLower = int(sys.argv[6])
PhotonNumUpper = int(sys.argv[7])
ElectronNumLower = int(sys.argv[8])
ElectronNumUpper = int(sys.argv[9])
IfEnableS1S2Correlation = int(sys.argv[10])
IfUsePublicNodes = int(sys.argv[11])

MaxNumJob = 64
if not IfUsePublicNodes:
    MaxNumJob=200

##### Start batching #########
CurrentPath = os.getcwd()
print (CurrentPath)
CurrentUser = getpass.getuser()
for i in range(NumJobs):

    RunString = "%06d" % i

    # create folder
    OutputPath = OutputGeneralPath + "/" + RunString
    if os.path.exists(OutputPath):
        subp.call("rm -r "+OutputPath, shell=True)
    subp.call("mkdir -p "+OutputPath, shell=True)

    # define filenames
    SubmitFile = OutputPath+"/submit_"+ RunString + ".sh"
    SubmitOutputFilename = OutputPath+"/submit_"+ RunString + ".log"
    SubmitErrorFilename = OutputPath+"/submit_"+ RunString + ".log"

    # create the basic submit
    subp.call("echo '#!/bin/bash\n' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --output="+SubmitOutputFilename+"' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --error="+SubmitErrorFilename+"' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --time=03:59:00' >> "+SubmitFile, shell=True)
    subp.call("echo '#SBATCH --account=pi-lgrandi' >> "+SubmitFile, shell=True)
    if IfUsePublicNodes==0:
        subp.call("echo '#SBATCH --qos=xenon1t' >> "+SubmitFile, shell=True)
        subp.call("echo '#SBATCH --partition=xenon1t\n' >> "+SubmitFile, shell=True)
    elif IfUsePublicNodes==2:
        subp.call("echo '#SBATCH --qos=xenon1t-kicp' >> "+SubmitFile, shell=True)
        subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True)

    Command = CurrentPath+"/./run_fax.sh "+str(PhotonNumLower)+" "+str(PhotonNumUpper)+" "+str(ElectronNumLower)+" "+str(ElectronNumUpper)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString+" "+str(IfEnableS1S2Correlation)
    subp.call("echo '"+Command+"\n' >> "+SubmitFile, shell=True)

    SubmitPath = OutputPath

    #submit
    IfSubmitted=0
    while IfSubmitted==0:
        Partition = "sandyb" # public
        if not IfUsePublicNodes:
            Partition = "xenon1t"
        elif IfUsePublicNodes==2:
            Partition = "kicp"
        p1 = Popen(["squeue","--partition="+Partition, "--user="+CurrentUser], stdout=PIPE)
        p2 = Popen(["wc", "-l"], stdin=p1.stdout, stdout=PIPE)
        p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
        output = p2.communicate()[0]
        Status=subp.call("squeue --partition="+Partition+" --user="+CurrentUser | wc -l", shell=True)
        Output=int(output)
        #print(Status)

        print("Current job running number "+str(Output))

        if Status==0 and Output<MaxNumJob:
            #sbatch it
            subp.call("cd "+SubmitPath+";sbatch "+SubmitFile+";cd -", shell=True)
            IfSubmitted=1
            time.sleep(2.0)
        else:
            time.sleep(30)
<|endoftext|>