<commit_before># Copyright (c) Metakernel Development Team.
# Distributed under the terms of the Modified BSD License.

from IPython.html import widgets
from metakernel import Magic, option
import os
import fcntl
import getpass
import datetime

def touch(fname, times=None):
    with open(fname, 'a'):
        os.utime(fname, times)

class Activity(object):
    def __init__(self):
        self.questions = []
        self.filename = None
        self.results_filename = None
        self.instructor = None

    def load(self, filename):
        if filename.startswith("~"):
            filename = os.path.expanduser(filename)
        filename = os.path.abspath(filename)
        self.filename = filename
        with open(self.filename) as fp:
            json_text = "".join(fp.readlines())
        self.load_json(json_text)
        if self.results_filename is None:
            self.results_filename = filename + ".results"
        touch(self.results_filename)

    def load_json(self, json_text):
        from IPython.html import widgets
        # Allow use of widgets:
        json = eval(json_text, {key: getattr(widgets, key) for key in dir(widgets)})
        if json.get("results_filename", None):
            self.results_filename = json["results_filename"]
        if json.get("instructor", None):
            self.instructor = json["instructor"]
        if json["activity"] == "poll":
            self.index = 0
            for item in json["items"]:
                if item["type"] == "multiple choice":
                    # FIXME: allow widgets; need to rewrite create/show:
                    question = item["question"]
                    #if isinstance(question, str):
                    #    question = widgets.HTML(question)
                    options = item["options"]
                    #for pos in range(len(options)):
                    #    option = options[pos]
                    #    if isinstance(option, str):
                    #        options[pos] = widgets.HTML(option)
                    q = Question(item["id"], question, options)
                    self.questions.append(q)
                else:
                    raise Exception("not a valid question 'type': use ['multiple choice']")
            self.create_widget()
            self.use_question(self.index)
        else:
            raise Exception("not a valid 'activity': use ['poll']")

    def use_question(self, index):
        self.set_question(self.questions[index].question)
        self.set_id(self.questions[index].id)
        self.results_html.visible = False
        self.results_button.visible = getpass.getuser() == self.instructor
        for i in range(5):
            self.choice_row_list[i].visible = False
        for i in range(len(self.questions[index].options)):
            self.choice_widgets[i].value = self.questions[index].options[i]
            self.choice_row_list[i].visible = True

    def create_widget(self):
        self.id_widget = widgets.HTML("")
        self.question_widget = widgets.HTML("")
        self.choice_widgets = []
        self.choice_row_list = []
        for count in range(1, 5 + 1):
            self.choice_widgets.append(widgets.HTML(""))
            self.choice_row_list.append(widgets.HBox([widgets.HTML("%s)  " % count),
                                                      self.choice_widgets[-1]]))
        self.buttons = []
        # FIXME: get from options:
        for i in range(1, 5 + 1):
            button = widgets.Button(description = str(i))
            button.on_click(self.handle_submit)
            button.margin = 20
            self.buttons.append(button)
        self.respond_row_widgets = widgets.HBox([widgets.HTML("""

Respond: """)] + self.buttons)
        self.next_button = widgets.Button(description="Next")
        self.next_button.on_click(self.handle_next)
        self.results_button = widgets.Button(description="Results")
        self.results_button.on_click(self.handle_results)
        self.prev_button = widgets.Button(description="Previous")
        self.prev_button.on_click(self.handle_prev)
        self.results_html = widgets.HTML("")
        self.top_level = widgets.HBox([widgets.VBox([self.id_widget, self.question_widget] + self.choice_row_list +
                                                    [self.respond_row_widgets,
                                                     widgets.HBox([self.prev_button, self.results_button, self.next_button])]),
                                       self.results_html])

    def set_question(self, question):
        self.question_widget.value = "
%s
" % question

    def set_id(self, id):
        self.id_widget.value = "
Question ID: %s
" % id
        self.id = id

    def handle_results(self, sender):
        index = list(range(5))
        data = {}
        with open(self.results_filename) as fp:
            line = fp.readline()
            while line:
                if "::" in line:
                    id, user, time, choice = line.split("::")
                    data[user.strip()] = choice.strip()
                line = fp.readline()
        # FIXME: get from options:
        choices = {str(i): 0 for i in range(1, 5 + 1)}
        for datum in data.values():
            if datum not in choices:
                choices[datum] = 0
            choices[datum] += 1
        barvalues = [int(value) for key,value in sorted(choices.items())]
        try:
            from calysto.graphics import BarChart
            barchart = BarChart(size=(300, 400), data=barvalues, labels=sorted(choices.keys()))
            self.results_html.value = str(barchart)
            self.results_html.visible = True
        except:
            print(sorted(choices.keys()))
            print(barvalues)

    def handle_submit(self, sender):
        from metakernel.display import clear_output
        with open(self.results_filename, "a+") as g:
            fcntl.flock(g, fcntl.LOCK_EX)
            g.write("%s::%s::%s::%s\n" % (self.id, getpass.getuser(), datetime.datetime.today(), sender.description))
            fcntl.flock(g, fcntl.LOCK_UN)
        clear_output()
        print("Received: " + sender.description)

    def handle_next(self, sender):
        from metakernel.display import clear_output
        if self.index < len(self.questions) - 1:
            self.index += 1
            self.use_question(self.index)
            clear_output()

    def handle_prev(self, sender):
        from metakernel.display import clear_output
        if self.index > 0:
            self.index -= 1
            self.use_question(self.index)
            clear_output()

    def render(self):
        from metakernel.display import display
        display(self.top_level)

class Question(object):
    def __init__(self, id, question, options):
        self.id = id
        self.question = question
        self.options = options

class ActivityMagic(Magic):

    def line_activity(self, filename):
        """
        %activity FILENAME - run a widget-based activity
          (poll, classroom response, clicker-like activity)

        This magic will load the JSON in the filename.

        Example:
            %activity /home/teacher/activity1
        """
        activity = Activity()
        activity.load(filename)
        activity.render()

    def cell_activity(self, filename):
        """
        %%activity FILENAME - make an activity from
          a JSON structure

        This magic will construct a Python file from the cell's
        content, a JSON structure.

        Example:
            %%activity /home/teacher/activity1
            {"activity": "poll",
             "instructor": "teacher01",
             "results_file": "/home/teacher/activity1.results",
             "items": [{"id": "...",
                        "type": "multiple choice",
                        "question": "...",
                        "options": ["...", ...]
                       }, ...]
            }

        In this example, users will load
        /home/teacher/activity1
        """
        if filename.startswith("~"):
            filename = os.path.expanduser(filename)
        filename = os.path.abspath(filename)
        with open(filename, "w") as fp:
            fp.write(self.code)
        # Make sure results file is writable:
        activity = Activity()
        activity.load(filename)
        os.chmod(activity.results_filename, 0o777)
        # Ok, let's test it:
        self.code = "%activity " + filename

def register_magics(kernel):
    kernel.register_magics(ActivityMagic)

<commit_msg>Activity: allow use in IPython<commit_after># Copyright (c) Metakernel Development Team.
# Distributed under the terms of the Modified BSD License.

from IPython.html import widgets
from metakernel import Magic, option
import os
import fcntl
import getpass
import datetime

def touch(fname, times=None):
    with open(fname, 'a'):
        os.utime(fname, times)

class Activity(object):
    def __init__(self):
        self.questions = []
        self.filename = None
        self.results_filename = None
        self.instructor = None

    def load(self, filename):
        if filename.startswith("~"):
            filename = os.path.expanduser(filename)
        filename = os.path.abspath(filename)
        self.filename = filename
        with open(self.filename) as fp:
            json_text = "".join(fp.readlines())
        self.load_json(json_text)
        if self.results_filename is None:
            self.results_filename = filename + ".results"
        touch(self.results_filename)

    def load_json(self, json_text):
        from IPython.html import widgets
        # Allow use of widgets:
        json = eval(json_text.strip(), {key: getattr(widgets, key) for key in dir(widgets)})
        if json.get("results_filename", None):
            self.results_filename = json["results_filename"]
        if json.get("instructor", None):
            self.instructor = json["instructor"]
        if json["activity"] == "poll":
            self.index = 0
            for item in json["items"]:
                if item["type"] == "multiple choice":
                    # FIXME: allow widgets; need to rewrite create/show:
                    question = item["question"]
                    #if isinstance(question, str):
                    #    question = widgets.HTML(question)
                    options = item["options"]
                    #for pos in range(len(options)):
                    #    option = options[pos]
                    #    if isinstance(option, str):
                    #        options[pos] = widgets.HTML(option)
                    q = Question(item["id"], question, options)
                    self.questions.append(q)
                else:
                    raise Exception("not a valid question 'type': use ['multiple choice']")
            self.create_widget()
            self.use_question(self.index)
        else:
            raise Exception("not a valid 'activity': use ['poll']")

    def use_question(self, index):
        self.set_question(self.questions[index].question)
        self.set_id(self.questions[index].id)
        self.results_html.visible = False
        self.results_button.visible = getpass.getuser() == self.instructor
        for i in range(5):
            self.choice_row_list[i].visible = False
        for i in range(len(self.questions[index].options)):
            self.choice_widgets[i].value = self.questions[index].options[i]
            self.choice_row_list[i].visible = True

    def create_widget(self):
        self.id_widget = widgets.HTML("")
        self.question_widget = widgets.HTML("")
        self.choice_widgets = []
        self.choice_row_list = []
        for count in range(1, 5 + 1):
            self.choice_widgets.append(widgets.HTML(""))
            self.choice_row_list.append(widgets.HBox([widgets.HTML("%s)  " % count),
                                                      self.choice_widgets[-1]]))
        self.buttons = []
        # FIXME: get from options:
        for i in range(1, 5 + 1):
            button = widgets.Button(description = str(i))
            button.on_click(self.handle_submit)
            button.margin = 20
            self.buttons.append(button)
        self.respond_row_widgets = widgets.HBox([widgets.HTML("""

Respond: """)] + self.buttons)
        self.next_button = widgets.Button(description="Next")
        self.next_button.on_click(self.handle_next)
        self.results_button = widgets.Button(description="Results")
        self.results_button.on_click(self.handle_results)
        self.prev_button = widgets.Button(description="Previous")
        self.prev_button.on_click(self.handle_prev)
        self.results_html = widgets.HTML("")
        self.top_level = widgets.HBox([widgets.VBox([self.id_widget, self.question_widget] + self.choice_row_list +
                                                    [self.respond_row_widgets,
                                                     widgets.HBox([self.prev_button, self.results_button, self.next_button])]),
                                       self.results_html])

    def set_question(self, question):
        self.question_widget.value = "
%s
" % question

    def set_id(self, id):
        self.id_widget.value = "
Question ID: %s
" % id
        self.id = id

    def handle_results(self, sender):
        index = list(range(5))
        data = {}
        with open(self.results_filename) as fp:
            line = fp.readline()
            while line:
                if "::" in line:
                    id, user, time, choice = line.split("::")
                    data[user.strip()] = choice.strip()
                line = fp.readline()
        # FIXME: get from options:
        choices = {str(i): 0 for i in range(1, 5 + 1)}
        for datum in data.values():
            if datum not in choices:
                choices[datum] = 0
            choices[datum] += 1
        barvalues = [int(value) for key,value in sorted(choices.items())]
        try:
            from calysto.graphics import BarChart
            barchart = BarChart(size=(300, 400), data=barvalues, labels=sorted(choices.keys()))
            self.results_html.value = str(barchart)
            self.results_html.visible = True
        except:
            print(sorted(choices.keys()))
            print(barvalues)

    def handle_submit(self, sender):
        from metakernel.display import clear_output
        with open(self.results_filename, "a+") as g:
            fcntl.flock(g, fcntl.LOCK_EX)
            g.write("%s::%s::%s::%s\n" % (self.id, getpass.getuser(), datetime.datetime.today(), sender.description))
            fcntl.flock(g, fcntl.LOCK_UN)
        clear_output()
        print("Received: " + sender.description)

    def handle_next(self, sender):
        from metakernel.display import clear_output
        if self.index < len(self.questions) - 1:
            self.index += 1
            self.use_question(self.index)
            clear_output()

    def handle_prev(self, sender):
        from metakernel.display import clear_output
        if self.index > 0:
            self.index -= 1
            self.use_question(self.index)
            clear_output()

    def render(self):
        from metakernel.display import display
        display(self.top_level)

class Question(object):
    def __init__(self, id, question, options):
        self.id = id
        self.question = question
        self.options = options

class ActivityMagic(Magic):

    def line_activity(self, filename):
        """
        %activity FILENAME - run a widget-based activity
          (poll, classroom response, clicker-like activity)

        This magic will load the JSON in the filename.

        Example:
            %activity /home/teacher/activity1
        """
        activity = Activity()
        activity.load(filename)
        activity.render()

    def cell_activity(self, filename):
        """
        %%activity FILENAME - make an activity from
          a JSON structure

        This magic will construct a Python file from the cell's
        content, a JSON structure.

        Example:
            %%activity /home/teacher/activity1
            {"activity": "poll",
             "instructor": "teacher01",
             "results_file": "/home/teacher/activity1.results",
             "items": [{"id": "...",
                        "type": "multiple choice",
                        "question": "...",
                        "options": ["...", ...]
                       }, ...]
            }

        In this example, users will load
        /home/teacher/activity1
        """
        if filename.startswith("~"):
            filename = os.path.expanduser(filename)
        filename = os.path.abspath(filename)
        with open(filename, "w") as fp:
            fp.write(self.code)
        # Make sure results file is writable:
        activity = Activity()
        activity.load(filename)
        os.chmod(activity.results_filename, 0o777)
        # Ok, let's test it (MetaKernel):
        self.code = "%activity " + filename

def register_magics(kernel):
    kernel.register_magics(ActivityMagic)

def register_ipython_magics():
    from metakernel import IPythonKernel
    from IPython.core.magic import register_line_magic, register_cell_magic
    kernel = IPythonKernel()
    magic = ActivityMagic(kernel)
    # Make magics callable:
    kernel.line_magics["activity"] = magic
    kernel.cell_magics["activity"] = magic

    @register_line_magic
    def activity(line):
        kernel.call_magic("%activity " + line)

    @register_cell_magic
    def activity(line, cell):
        magic.code = cell
        magic.cell_activity(line)
<|endoftext|>


<commit_before>import getpass
import json
import logging

import requests

requests.packages.urllib3.disable_warnings()

try:
    import configparser
except ImportError:
    import ConfigParser as configparser
import re
import errno
import os
import datetime
import time

try:
    input = raw_input
except NameError:
    pass

from pnc_cli.swagger_client.rest import ApiException

__author__ = 'thauser'

envconfig = os.environ.get('PNC_CONFIG_LOCATION')
if envconfig is not None:
    if not envconfig.endswith("/"):
        envconfig += "/"
    CONFIG_LOCATION = envconfig
else:
    CONFIG_LOCATION = os.path.expanduser("~") + "/.config/pnc-cli/"
CONFIG_FILENAME = "pnc-cli.conf"
CONFIG = CONFIG_LOCATION + CONFIG_FILENAME

def is_valid_version(version, regex):
    if version is not None:
        pattern = re.compile(regex)
        return pattern.match(version)

def checked_api_call(api, func, **kwargs):
    try:
        response = getattr(api, func)(**kwargs)
    except ApiException as e:
        print(e)
    else:
        return response

def format_json_list(page_model):
    if page_model is not None:
        json_list = []
        for element in page_model:
            json_list.append(format_json(element))
        return "[" + ",".join(json_list) + "]"
    else:
        return None

def format_json(rest_model):
    return json.dumps(rest_model.to_dict(), sort_keys=True, indent=4, separators=(',', ': '))

def format_dict(rest_model):
    return json.dumps(rest_model, sort_keys=True, indent=4, separators=(',', ': '))

epoch = datetime.datetime.utcfromtimestamp(0)

def unix_time_millis(dt):
    millis = int((dt - epoch).total_seconds() * 1000.0)
    return millis

def current_time_millis():
    millis = int(round(time.time() * 1000))
    return millis

def contains_only_none_values(dictionary):
    for key in dictionary.keys():
        if dictionary[key] is not None:
            return False
    return True

def get_config():
    config = configparser.ConfigParser(allow_no_value=False)
    configparser.ConfigParser(allow_no_value=False)
    configfilename = CONFIG
    configdirname = CONFIG_LOCATION

    try:
        os.makedirs(configdirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    found = config.read(os.path.join(configfilename))

    if not found:
        config.add_section('PNC')
        config.set('PNC', 'pncUrl', 'http://localhost:8080/')
        # prompt user for his keycloak username / passwords
        config.set('PNC', 'keycloakUrl', '')
        config.set('PNC', 'keycloakRealm', 'pncredhat')
        config.set('PNC', 'keycloakClientId', 'pncdirect')
        username = input('Username: ')
        config.set('PNC', 'username', username)
        with open(os.path.join(configfilename), 'w') as configfile:
            config.write(configfile)
        logging.warning("New config file written to %s. Configure pncUrl and keycloakUrl values." % configfilename)
        exit(1)
    return config

def get_internal_repo_start(environment):
    ### Devel urls ###
    # http://pnc-orch-master-autodeploy.cloud.pnc.devel.engineering.redhat.com/
    # http://pnc-orch-branch-autodeploy.cloud.pnc.devel.engineering.redhat.com/
    ### Stage URLS ###
    # http://pnc-orch-branch-nightly.cloud.pnc.stage.engineering.redhat.com/
    # http://pnc-orch-branch-autodeploy.cloud.pnc.stage.engineering.redhat.com/
    if "stage" in environment:
        return 'git+ssh://pnc-gerrit-stage@code-stage.eng.nay.redhat.com:29418/'
    elif "devel" in environment:
        return 'git+ssh://user-pnc-gerrit@pnc-gerrit.pnc.dev.eng.bos.redhat.com:29418/'
<commit_msg>Write ApiExceptions to stderr instead of stdout<commit_after>import getpass
import json
import logging

import requests

requests.packages.urllib3.disable_warnings()

try:
    import configparser
except ImportError:
    import ConfigParser as configparser
import re
import sys
import errno
import os
import datetime
import time

try:
    input = raw_input
except NameError:
    pass

from pnc_cli.swagger_client.rest import ApiException

__author__ = 'thauser'

envconfig = os.environ.get('PNC_CONFIG_LOCATION')
if envconfig is not None:
    if not envconfig.endswith("/"):
        envconfig += "/"
    CONFIG_LOCATION = envconfig
else:
    CONFIG_LOCATION = os.path.expanduser("~") + "/.config/pnc-cli/"
CONFIG_FILENAME = "pnc-cli.conf"
CONFIG = CONFIG_LOCATION + CONFIG_FILENAME

def is_valid_version(version, regex):
    if version is not None:
        pattern = re.compile(regex)
        return pattern.match(version)

def checked_api_call(api, func, **kwargs):
    try:
        response = getattr(api, func)(**kwargs)
    except ApiException as e:
        sys.stderr.write(e)
        sys.stderr.write("\n")
    else:
        return response

def format_json_list(page_model):
    if page_model is not None:
        json_list = []
        for element in page_model:
            json_list.append(format_json(element))
        return "[" + ",".join(json_list) + "]"
    else:
        return None

def format_json(rest_model):
    return json.dumps(rest_model.to_dict(), sort_keys=True, indent=4, separators=(',', ': '))

def format_dict(rest_model):
    return json.dumps(rest_model, sort_keys=True, indent=4, separators=(',', ': '))

epoch = datetime.datetime.utcfromtimestamp(0)

def unix_time_millis(dt):
    millis = int((dt - epoch).total_seconds() * 1000.0)
    return millis

def current_time_millis():
    millis = int(round(time.time() * 1000))
    return millis

def contains_only_none_values(dictionary):
    for key in dictionary.keys():
        if dictionary[key] is not None:
            return False
    return True

def get_config():
    config = configparser.ConfigParser(allow_no_value=False)
    configparser.ConfigParser(allow_no_value=False)
    configfilename = CONFIG
    configdirname = CONFIG_LOCATION

    try:
        os.makedirs(configdirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    found = config.read(os.path.join(configfilename))

    if not found:
        config.add_section('PNC')
        config.set('PNC', 'pncUrl', 'http://localhost:8080/')
        # prompt user for his keycloak username / passwords
        config.set('PNC', 'keycloakUrl', '')
        config.set('PNC', 'keycloakRealm', 'pncredhat')
        config.set('PNC', 'keycloakClientId', 'pncdirect')
        username = input('Username: ')
        config.set('PNC', 'username', username)
        with open(os.path.join(configfilename), 'w') as configfile:
            config.write(configfile)
        logging.warning("New config file written to %s. Configure pncUrl and keycloakUrl values." % configfilename)
        exit(1)
    return config

def get_internal_repo_start(environment):
    ### Devel urls ###
    # http://pnc-orch-master-autodeploy.cloud.pnc.devel.engineering.redhat.com/
    # http://pnc-orch-branch-autodeploy.cloud.pnc.devel.engineering.redhat.com/
    ### Stage URLS ###
    # http://pnc-orch-branch-nightly.cloud.pnc.stage.engineering.redhat.com/
    # http://pnc-orch-branch-autodeploy.cloud.pnc.stage.engineering.redhat.com/
    if "stage" in environment:
        return 'git+ssh://pnc-gerrit-stage@code-stage.eng.nay.redhat.com:29418/'
    elif "devel" in environment:
        return 'git+ssh://user-pnc-gerrit@pnc-gerrit.pnc.dev.eng.bos.redhat.com:29418/'
<|endoftext|>


<commit_before>"""
Methods for model data conversion between different representations.
"""

import logging

from django.db.models import OneToOneField, Sum

from metashare.repository.models import resourceInfoType_model, \
    corpusInfoType_model, lexicalConceptualResourceInfoType_model, \
    languageDescriptionInfoType_model
    # toolServiceInfoType_model
from metashare.settings import LOG_HANDLER
from metashare.stats.models import LRStats

# Setup logging support.
LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(LOG_HANDLER)

def get_related_models(instance, parent):
    """
    Returns a list containing model names of classes that reference instance.

    Related models for a given class X are all classes Y that have a relation
    set to X via a ForeignKey field.
    """
    LOGGER.info('get_related_models({0}) called.'.format(
      instance.__class__.__name__))

    # If a parent is given, compute the corresponding parent_set String.
    parent_set = None
    if parent:
        parent_set = '{0}_set'.format(parent).lower()

    # Related model class names should have this suffix.
    related_suffix = '_{0}'.format(instance.__class__.__name__)

    result = []
    for name in dir(instance):
        if name.endswith('_set'):
            if name != parent_set:
                model_instance = getattr(instance, name)
                model_name = model_instance.model.__name__

                LOGGER.info('{0} ({1})'.format(name, model_name))

                if model_name.endswith(related_suffix):
                    result.append(model_name)

    return result

def get_root_resources(*instances):
    """
    Returns the set of `resourceInfoType_model` instances which somewhere
    contain the given model instances.

    If any of the given instances is a `resourceInfoType_model` itself, then
    this instance will be in the returned set, too. The returned set can be
    empty.
    """
    return _get_root_resources(set(), *instances)

def _get_root_resources(ignore, *instances):
    """
    Returns the set of `resourceInfoType_model` instances which somewhere
    contain the given model instances.

    If any of the given instances is a `resourceInfoType_model` itself, then
    this instance will be in the returned set, too. The returned set can be
    empty. All instances which are found in the given ignore set will not be
    looked at. The ignore set will be extended with the given instances.
    """
    result = set()
    for instance in instances:
        if instance in ignore:
            continue
        ignore.add(instance)

        # `resourceInfoType_model` instances are our actual results
        if isinstance(instance, resourceInfoType_model):
            result.add(instance)
        # an instance may be None, in which case we ignore it
        elif instance:
            # There are 3 possibilities for going backward in our model graph:

            # case (1): we have to follow a `ForeignKey` with a name starting
            #   with "back_to_":
            for rel in [r for r in instance._meta.get_all_field_names()
                        if r.startswith('back_to_')]:
                result.update(_get_root_resources(ignore,
                                                  getattr(instance, rel)))

            # case (2): we have to follow "reverse" `ForeignKey`s and
            #   `OneToOneField`s which are pointing at the current instance from
            #   a model which is closer to the searched root model:
            for rel in instance._meta.get_all_related_objects():
                accessor_name = rel.get_accessor_name()
                # the accessor name may point to a field which is None, so test
                # first, if a field instance is actually available (?)
                if hasattr(instance, accessor_name):
                    accessor = getattr(instance, accessor_name)
                    if isinstance(rel.field, OneToOneField):
                        # in the case of `OneToOneField`s the accessor is the
                        # new instance itself
                        result.update(_get_root_resources(ignore, accessor))
                    else:
                        result.update(_get_root_resources(ignore,
                                                          *accessor.all()))

            # case (2): we have to follow the "reverse" part of a `ManyToMany`
            #   field which is pointing at the current instance from a model
            #   which is closer to the searched root model:
            for rel in instance._meta.get_all_related_many_to_many_objects():
                result.update(_get_root_resources(ignore,
                        *getattr(instance, rel.get_accessor_name()).all()))

    return result

def get_resource_linguality_infos(res_obj):
    """
    Returns a list of all linguality types of the given language resource
    instance.
    """
    result = []
    corpus_media = res_obj.resourceComponentType.as_subclass()

    if isinstance(corpus_media, corpusInfoType_model):
        media_type = corpus_media.corpusMediaType
        for corpus_info in media_type.corpustextinfotype_model_set.all():
            result.append(corpus_info.lingualityInfo
                          .get_lingualityType_display())
        # if media_type.corpusAudioInfo:
        #     result.append(media_type.corpusAudioInfo.lingualityInfo \
        #                   .get_lingualityType_display())
        for corpus_info in media_type.corpusvideoinfotype_model_set.all():
            if corpus_info.lingualityInfo:
                result.append(corpus_info.lingualityInfo \
                              .get_lingualityType_display())
        # if media_type.corpusTextNgramInfo:
        #     result.append(media_type.corpusTextNgramInfo.lingualityInfo \
        #                   .get_lingualityType_display())
        # if media_type.corpusImageInfo and \
        #         media_type.corpusImageInfo.lingualityInfo:
        #     result.append(media_type.corpusImageInfo.lingualityInfo \
        #                   .get_lingualityType_display())

    elif isinstance(corpus_media, lexicalConceptualResourceInfoType_model):
        lcr_media_type = corpus_media.lexicalConceptualResourceMediaType
        if lcr_media_type.lexicalConceptualResourceTextInfo:
            result.append(lcr_media_type.lexicalConceptualResourceTextInfo \
                          .lingualityInfo.get_lingualityType_display())
        # if lcr_media_type.lexicalConceptualResourceAudioInfo and \
        #         lcr_media_type.lexicalConceptualResourceAudioInfo \
        #             .lingualityInfo:
        #     result.append(lcr_media_type \
        #             .lexicalConceptualResourceAudioInfo.lingualityInfo \
        #             .get_lingualityType_display())
        # if lcr_media_type.lexicalConceptualResourceVideoInfo and \
        #         lcr_media_type.lexicalConceptualResourceVideoInfo \
        #             .lingualityInfo:
        #     result.append(lcr_media_type \
        #             .lexicalConceptualResourceVideoInfo.lingualityInfo \
        #             .get_lingualityType_display())
        # if lcr_media_type.lexicalConceptualResourceImageInfo and \
        #         lcr_media_type.lexicalConceptualResourceImageInfo \
        #             .lingualityInfo:
        #     result.append(lcr_media_type \
        #             .lexicalConceptualResourceImageInfo.lingualityInfo \
        #             .get_lingualityType_display())

    elif isinstance(corpus_media, languageDescriptionInfoType_model):
        ld_media_type = corpus_media.languageDescriptionMediaType
        if ld_media_type.languageDescriptionTextInfo:
            result.append(ld_media_type.languageDescriptionTextInfo \
                          .lingualityInfo.get_lingualityType_display())
        # if ld_media_type.languageDescriptionVideoInfo and \
        #         ld_media_type.languageDescriptionVideoInfo.lingualityInfo:
        #     result.append(ld_media_type.languageDescriptionVideoInfo \
        #                   .lingualityInfo.get_lingualityType_display())
        # if ld_media_type.languageDescriptionImageInfo and \
        #         ld_media_type.languageDescriptionImageInfo.lingualityInfo:
        #     result.append(ld_media_type.languageDescriptionImageInfo \
        #                   .lingualityInfo.get_lingualityType_display())

    return result

def get_resource_license_types(res_obj):
    """
    Returns a list of license under which the given language resource is
    available.
    """
    return [licence for licence_info in
            res_obj.distributionInfo.licenceinfotype_model_set.all()
            for licence in licence_info.get_licence_display_list()]

def get_resource_media_types(res_obj):
    """
    Returns a list of all media types of the given language resource instance.
    """
    result = []
    corpus_media = res_obj.resourceComponentType.as_subclass()

    if isinstance(corpus_media, corpusInfoType_model):
        media_type = corpus_media.corpusMediaType
        for corpus_info in media_type.corpustextinfotype_model_set.all():
            result.append(corpus_info.mediaType)
        # if media_type.corpusAudioInfo:
        #     result.append(media_type.corpusAudioInfo.mediaType)
        for corpus_info in media_type.corpusvideoinfotype_model_set.all():
            result.append(corpus_info.mediaType)
        # if media_type.corpusTextNgramInfo:
        #     result.append(media_type.corpusTextNgramInfo.mediaType)
        # if media_type.corpusImageInfo:
        #     result.append(media_type.corpusImageInfo.mediaType)
        # if media_type.corpusTextNumericalInfo:
        #     result.append(media_type.corpusTextNumericalInfo.mediaType)

    elif isinstance(corpus_media, lexicalConceptualResourceInfoType_model):
        lcr_media_type = corpus_media.lexicalConceptualResourceMediaType
        if lcr_media_type.lexicalConceptualResourceTextInfo:
            result.append(
                lcr_media_type.lexicalConceptualResourceTextInfo.mediaType)
        # if lcr_media_type.lexicalConceptualResourceAudioInfo:
        #     result.append(
        #         lcr_media_type.lexicalConceptualResourceAudioInfo.mediaType)
        # if lcr_media_type.lexicalConceptualResourceVideoInfo:
        #     result.append(
        #         lcr_media_type.lexicalConceptualResourceVideoInfo.mediaType)
        # if lcr_media_type.lexicalConceptualResourceImageInfo:
        #     result.append(
        #         lcr_media_type.lexicalConceptualResourceImageInfo.mediaType)

    elif isinstance(corpus_media, languageDescriptionInfoType_model):
        ld_media_type = corpus_media.languageDescriptionMediaType
        if ld_media_type.languageDescriptionTextInfo:
            result.append(ld_media_type.languageDescriptionTextInfo.mediaType)
        # if ld_media_type.languageDescriptionVideoInfo:
        #     result.append(ld_media_type.languageDescriptionVideoInfo.mediaType)
        # if ld_media_type.languageDescriptionImageInfo:
        #     result.append(ld_media_type.languageDescriptionImageInfo.mediaType)

    # elif isinstance(corpus_media, toolServiceInfoType_model):
    #     if corpus_media.inputInfo:
    #         result.extend(corpus_media.inputInfo \
    #                       .get_mediaType_display_list())
    #     if corpus_media.outputInfo:
    #         result.extend(corpus_media.outputInfo \
    #                       .get_mediaType_display_list())

    result_lower = []
    for res in result:
        result_lower.append(res.lower())

    return result_lower

def get_lr_stat_action_count(obj_identifier, stats_action):
    """
    Returns the count of the given stats action for the given resource instance.

    The obj_identifier is the identifier from the storage object.
    """
    result = LRStats.objects.filter(lrid=obj_identifier, action=stats_action) \
        .aggregate(Sum('count'))['count__sum']
    # `result` may be None in case the filter doesn't match any LRStats for the
    # specified resource and action
    if result is not None:
        return result
    else:
        return 0

def get_lr_master_url(resource):
    """
    Returns the full URL of the master copy of the given resource object.

    This URL points to the single resource view of the resource on the resource'
    master node.
    """
    return "{0}/{1}".format(resource.storage_object.source_url,
                            resource.get_relative_url())
<commit_msg>remove references to unused components<commit_after>"""
Methods for model data conversion between different representations.
"""

import logging

from django.db.models import OneToOneField, Sum

from metashare.repository.models import resourceInfoType_model, \
    corpusInfoType_model, lexicalConceptualResourceInfoType_model, \
    languageDescriptionInfoType_model
    # toolServiceInfoType_model
from metashare.settings import LOG_HANDLER
from metashare.stats.models import LRStats

# Setup logging support.
LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(LOG_HANDLER)

def get_related_models(instance, parent):
    """
    Returns a list containing model names of classes that reference instance.

    Related models for a given class X are all classes Y that have a relation
    set to X via a ForeignKey field.
    """
    LOGGER.info('get_related_models({0}) called.'.format(
      instance.__class__.__name__))

    # If a parent is given, compute the corresponding parent_set String.
    parent_set = None
    if parent:
        parent_set = '{0}_set'.format(parent).lower()

    # Related model class names should have this suffix.
    related_suffix = '_{0}'.format(instance.__class__.__name__)

    result = []
    for name in dir(instance):
        if name.endswith('_set'):
            if name != parent_set:
                model_instance = getattr(instance, name)
                model_name = model_instance.model.__name__

                LOGGER.info('{0} ({1})'.format(name, model_name))

                if model_name.endswith(related_suffix):
                    result.append(model_name)

    return result

def get_root_resources(*instances):
    """
    Returns the set of `resourceInfoType_model` instances which somewhere
    contain the given model instances.

    If any of the given instances is a `resourceInfoType_model` itself, then
    this instance will be in the returned set, too. The returned set can be
    empty.
    """
    return _get_root_resources(set(), *instances)

def _get_root_resources(ignore, *instances):
    """
    Returns the set of `resourceInfoType_model` instances which somewhere
    contain the given model instances.

    If any of the given instances is a `resourceInfoType_model` itself, then
    this instance will be in the returned set, too. The returned set can be
    empty. All instances which are found in the given ignore set will not be
    looked at. The ignore set will be extended with the given instances.
    """
    result = set()
    for instance in instances:
        if instance in ignore:
            continue
        ignore.add(instance)

        # `resourceInfoType_model` instances are our actual results
        if isinstance(instance, resourceInfoType_model):
            result.add(instance)
        # an instance may be None, in which case we ignore it
        elif instance:
            # There are 3 possibilities for going backward in our model graph:

            # case (1): we have to follow a `ForeignKey` with a name starting
            #   with "back_to_":
            for rel in [r for r in instance._meta.get_all_field_names()
                        if r.startswith('back_to_')]:
                result.update(_get_root_resources(ignore,
                                                  getattr(instance, rel)))

            # case (2): we have to follow "reverse" `ForeignKey`s and
            #   `OneToOneField`s which are pointing at the current instance from
            #   a model which is closer to the searched root model:
            for rel in instance._meta.get_all_related_objects():
                accessor_name = rel.get_accessor_name()
                # the accessor name may point to a field which is None, so test
                # first, if a field instance is actually available (?)
                if hasattr(instance, accessor_name):
                    accessor = getattr(instance, accessor_name)
                    if isinstance(rel.field, OneToOneField):
                        # in the case of `OneToOneField`s the accessor is the
                        # new instance itself
                        result.update(_get_root_resources(ignore, accessor))
                    else:
                        result.update(_get_root_resources(ignore,
                                                          *accessor.all()))

            # case (2): we have to follow the "reverse" part of a `ManyToMany`
            #   field which is pointing at the current instance from a model
            #   which is closer to the searched root model:
            for rel in instance._meta.get_all_related_many_to_many_objects():
                result.update(_get_root_resources(ignore,
                        *getattr(instance, rel.get_accessor_name()).all()))

    return result

def get_resource_linguality_infos(res_obj):
    """
    Returns a list of all linguality types of the given language resource
    instance.
    """
    result = []
    corpus_media = res_obj.resourceComponentType.as_subclass()

    if isinstance(corpus_media, corpusInfoType_model):
        media_type = corpus_media.corpusMediaType
        for corpus_info in media_type.corpustextinfotype_model_set.all():
            result.append(corpus_info.lingualityInfo
                          .get_lingualityType_display())
        # if media_type.corpusAudioInfo:
        #     result.append(media_type.corpusAudioInfo.lingualityInfo \
        #                   .get_lingualityType_display())
        # for corpus_info in media_type.corpusvideoinfotype_model_set.all():
        #     if corpus_info.lingualityInfo:
        #         result.append(corpus_info.lingualityInfo \
        #                       .get_lingualityType_display())
        # if media_type.corpusTextNgramInfo:
        #     result.append(media_type.corpusTextNgramInfo.lingualityInfo \
        #                   .get_lingualityType_display())
        # if media_type.corpusImageInfo and \
        #         media_type.corpusImageInfo.lingualityInfo:
        #     result.append(media_type.corpusImageInfo.lingualityInfo \
        #                   .get_lingualityType_display())

    elif isinstance(corpus_media, lexicalConceptualResourceInfoType_model):
        lcr_media_type = corpus_media.lexicalConceptualResourceMediaType
        if lcr_media_type.lexicalConceptualResourceTextInfo:
            result.append(lcr_media_type.lexicalConceptualResourceTextInfo \
                          .lingualityInfo.get_lingualityType_display())
        # if lcr_media_type.lexicalConceptualResourceAudioInfo and \
        #         lcr_media_type.lexicalConceptualResourceAudioInfo \
        #             .lingualityInfo:
        #     result.append(lcr_media_type \
        #             .lexicalConceptualResourceAudioInfo.lingualityInfo \
        #             .get_lingualityType_display())
        # if lcr_media_type.lexicalConceptualResourceVideoInfo and \
        #         lcr_media_type.lexicalConceptualResourceVideoInfo \
        #             .lingualityInfo:
        #     result.append(lcr_media_type \
        #             .lexicalConceptualResourceVideoInfo.lingualityInfo \
        #             .get_lingualityType_display())
        # if lcr_media_type.lexicalConceptualResourceImageInfo and \
        #         lcr_media_type.lexicalConceptualResourceImageInfo \
        #             .lingualityInfo:
        #     result.append(lcr_media_type \
        #             .lexicalConceptualResourceImageInfo.lingualityInfo \
        #             .get_lingualityType_display())

    elif isinstance(corpus_media, languageDescriptionInfoType_model):
        ld_media_type = corpus_media.languageDescriptionMediaType
        if ld_media_type.languageDescriptionTextInfo:
            result.append(ld_media_type.languageDescriptionTextInfo \
                          .lingualityInfo.get_lingualityType_display())
        # if ld_media_type.languageDescriptionVideoInfo and \
        #         ld_media_type.languageDescriptionVideoInfo.lingualityInfo:
        #     result.append(ld_media_type.languageDescriptionVideoInfo \
        #                   .lingualityInfo.get_lingualityType_display())
        # if ld_media_type.languageDescriptionImageInfo and \
        #         ld_media_type.languageDescriptionImageInfo.lingualityInfo:
        #     result.append(ld_media_type.languageDescriptionImageInfo \
        #                   .lingualityInfo.get_lingualityType_display())

    return result

def get_resource_license_types(res_obj):
    """
    Returns a list of license under which the given language resource is
    available.
    """
    return [licence for licence_info in
            res_obj.distributionInfo.licenceinfotype_model_set.all()
            for licence in licence_info.get_licence_display_list()]

def get_resource_media_types(res_obj):
    """
    Returns a list of all media types of the given language resource instance.
    """
    result = []
    corpus_media = res_obj.resourceComponentType.as_subclass()

    if isinstance(corpus_media, corpusInfoType_model):
        media_type = corpus_media.corpusMediaType
        for corpus_info in media_type.corpustextinfotype_model_set.all():
            result.append(corpus_info.mediaType)
        # if media_type.corpusAudioInfo:
        #     result.append(media_type.corpusAudioInfo.mediaType)
        # for corpus_info in media_type.corpusvideoinfotype_model_set.all():
        #     result.append(corpus_info.mediaType)
        # if media_type.corpusTextNgramInfo:
        #     result.append(media_type.corpusTextNgramInfo.mediaType)
        # if media_type.corpusImageInfo:
        #     result.append(media_type.corpusImageInfo.mediaType)
        # if media_type.corpusTextNumericalInfo:
        #     result.append(media_type.corpusTextNumericalInfo.mediaType)

    elif isinstance(corpus_media, lexicalConceptualResourceInfoType_model):
        lcr_media_type = corpus_media.lexicalConceptualResourceMediaType
        if lcr_media_type.lexicalConceptualResourceTextInfo:
            result.append(
                lcr_media_type.lexicalConceptualResourceTextInfo.mediaType)
        # if lcr_media_type.lexicalConceptualResourceAudioInfo:
        #     result.append(
        #         lcr_media_type.lexicalConceptualResourceAudioInfo.mediaType)
        # if lcr_media_type.lexicalConceptualResourceVideoInfo:
        #     result.append(
        #         lcr_media_type.lexicalConceptualResourceVideoInfo.mediaType)
        # if lcr_media_type.lexicalConceptualResourceImageInfo:
        #     result.append(
        #         lcr_media_type.lexicalConceptualResourceImageInfo.mediaType)

    elif isinstance(corpus_media, languageDescriptionInfoType_model):
        ld_media_type = corpus_media.languageDescriptionMediaType
        if ld_media_type.languageDescriptionTextInfo:
            result.append(ld_media_type.languageDescriptionTextInfo.mediaType)
        # if ld_media_type.languageDescriptionVideoInfo:
        #     result.append(ld_media_type.languageDescriptionVideoInfo.mediaType)
        # if ld_media_type.languageDescriptionImageInfo:
        #     result.append(ld_media_type.languageDescriptionImageInfo.mediaType)

    # elif isinstance(corpus_media, toolServiceInfoType_model):
    #     if corpus_media.inputInfo:
    #         result.extend(corpus_media.inputInfo \
    #                       .get_mediaType_display_list())
    #     if corpus_media.outputInfo:
    #         result.extend(corpus_media.outputInfo \
    #                       .get_mediaType_display_list())

    result_lower = []
    for res in result:
        result_lower.append(res.lower())

    return result_lower

def get_lr_stat_action_count(obj_identifier, stats_action):
    """
    Returns the count of the given stats action for the given resource instance.

    The obj_identifier is the identifier from the storage object.
    """
    result = LRStats.objects.filter(lrid=obj_identifier, action=stats_action) \
        .aggregate(Sum('count'))['count__sum']
    # `result` may be None in case the filter doesn't match any LRStats for the
    # specified resource and action
    if result is not None:
        return result
    else:
        return 0

def get_lr_master_url(resource):
    """
    Returns the full URL of the master copy of the given resource object.

    This URL points to the single resource view of the resource on the resource'
    master node.
    """
    return "{0}/{1}".format(resource.storage_object.source_url,
                            resource.get_relative_url())
<|endoftext|>


<commit_before>import numpy as np
from numpy.testing import assert_equal, assert_array_equal
from nose.tools import assert_true

from ..cluster_level import permutation_cluster_test, \
                            permutation_cluster_1samp_test

noiselevel = 20

normfactor = np.hanning(20).sum()

condition1 = np.random.randn(40, 350) * noiselevel
for c in condition1:
    c[:] = np.convolve(c, np.hanning(20), mode="same") / normfactor

condition2 = np.random.randn(33, 350) * noiselevel
for c in condition2:
    c[:] = np.convolve(c, np.hanning(20), mode="same") / normfactor

pseudoekp = 5 * np.hanning(150)[None, :]
condition1[:, 100:250] += pseudoekp
condition2[:, 100:250] -= pseudoekp

def test_cluster_permutation_test():
    """Test cluster level permutations tests."""
    T_obs, clusters, cluster_p_values, hist = permutation_cluster_test(
                                [condition1, condition2], n_permutations=500,
                                tail=1)
    assert_equal(np.sum(cluster_p_values < 0.05), 1)

    T_obs, clusters, cluster_p_values, hist = permutation_cluster_test(
                                [condition1, condition2], n_permutations=500,
                                tail=0)
    assert_equal(np.sum(cluster_p_values < 0.05), 1)

def test_cluster_permutation_t_test():
    """Test cluster level permutations T-test."""
    my_condition1 = condition1[:, :, None]  # to test 2D also
    T_obs, clusters, cluster_p_values, hist = permutation_cluster_1samp_test(
                                my_condition1, n_permutations=500, tail=0)
    assert_equal(np.sum(cluster_p_values < 0.05), 1)

    T_obs_pos, _, cluster_p_values_pos, _ = permutation_cluster_1samp_test(
                                my_condition1, n_permutations=500, tail=1,
                                threshold=1.67)

    T_obs_neg, _, cluster_p_values_neg, _ = permutation_cluster_1samp_test(
                                -my_condition1, n_permutations=500, tail=-1,
                                threshold=-1.67)
    assert_array_equal(T_obs_pos, -T_obs_neg)
    assert_array_equal(cluster_p_values_pos < 0.05,
                       cluster_p_values_neg < 0.05)

def test_cluster_permutation_t_test_with_connectivity():
    """Test cluster level permutations T-test with connectivity matrix."""
    try:
        from scikits.learn.feature_extraction.image import grid_to_graph
    except ImportError:
        pass
    else:
        out = permutation_cluster_1samp_test(condition1, n_permutations=500)
        connectivity = grid_to_graph(1, condition1.shape[1])
        out_connectivity = permutation_cluster_1samp_test(condition1,
                                 n_permutations=500, connectivity=connectivity)
        assert_array_equal(out[0], out_connectivity[0])
        for a, b in zip(out_connectivity[1], out[1]):
            assert_true(np.sum(out[0][a]) == np.sum(out[0][b]))
            assert_true(np.all(a[b]))
<commit_msg>ENH : update sklearn import path + fix heisen test by fixing the seed<commit_after>import numpy as np
from numpy.testing import assert_equal, assert_array_equal
from nose.tools import assert_true

from ..cluster_level import permutation_cluster_test, \
                            permutation_cluster_1samp_test

noiselevel = 20

normfactor = np.hanning(20).sum()

rng = np.random.RandomState(42)
condition1 = rng.randn(40, 350) * noiselevel
for c in condition1:
    c[:] = np.convolve(c, np.hanning(20), mode="same") / normfactor

condition2 = rng.randn(33, 350) * noiselevel
for c in condition2:
    c[:] = np.convolve(c, np.hanning(20), mode="same") / normfactor

pseudoekp = 5 * np.hanning(150)[None, :]
condition1[:, 100:250] += pseudoekp
condition2[:, 100:250] -= pseudoekp

def test_cluster_permutation_test():
    """Test cluster level permutations tests."""
    T_obs, clusters, cluster_p_values, hist = permutation_cluster_test(
                                [condition1, condition2], n_permutations=500,
                                tail=1)
    assert_equal(np.sum(cluster_p_values < 0.05), 1)

    T_obs, clusters, cluster_p_values, hist = permutation_cluster_test(
                                [condition1, condition2], n_permutations=500,
                                tail=0)
    assert_equal(np.sum(cluster_p_values < 0.05), 1)

def test_cluster_permutation_t_test():
    """Test cluster level permutations T-test."""
    my_condition1 = condition1[:, :, None]  # to test 2D also
    T_obs, clusters, cluster_p_values, hist = permutation_cluster_1samp_test(
                                my_condition1, n_permutations=500, tail=0)
    assert_equal(np.sum(cluster_p_values < 0.05), 1)

    T_obs_pos, _, cluster_p_values_pos, _ = permutation_cluster_1samp_test(
                                my_condition1, n_permutations=500, tail=1,
                                threshold=1.67)

    T_obs_neg, _, cluster_p_values_neg, _ = permutation_cluster_1samp_test(
                                -my_condition1, n_permutations=500, tail=-1,
                                threshold=-1.67)
    assert_array_equal(T_obs_pos, -T_obs_neg)
    assert_array_equal(cluster_p_values_pos < 0.05,
                       cluster_p_values_neg < 0.05)

def test_cluster_permutation_t_test_with_connectivity():
    """Test cluster level permutations T-test with connectivity matrix."""
    try:
        try:
            from sklearn.feature_extraction.image import grid_to_graph
        except ImportError:
            from scikits.learn.feature_extraction.image import grid_to_graph
    except ImportError:
        return

    out = permutation_cluster_1samp_test(condition1, n_permutations=500)
    connectivity = grid_to_graph(1, condition1.shape[1])
    out_connectivity = permutation_cluster_1samp_test(condition1,
                             n_permutations=500, connectivity=connectivity)
    assert_array_equal(out[0], out_connectivity[0])
    for a, b in zip(out_connectivity[1], out[1]):
        assert_true(np.sum(out[0][a]) == np.sum(out[0][b]))
        assert_true(np.all(a[b]))
<|endoftext|>


<commit_before>import pickle
import itertools
from copy import deepcopy
from os.path import join as pjoin
from pysb.export.kappa import KappaExporter
from indra.tools import assemble_corpus as ac
from indra.tools.gene_network import GeneNetwork
from indra.statements import Agent, Activation, Inhibition
from indra.assemblers import PysbAssembler, SifAssembler, CxAssembler

import process_data
from read_phosphosite import read_phosphosite

def build_prior(genes, out_file):
    gn = GeneNetwork(genes, 'korkut')
    stmts = gn.get_statements(filter=False)
    ac.dump_statements(stmts, out_file)
    return stmts

def assemble_pysb(stmts, data_genes, out_file):
    """Return an assembled PySB model."""
    stmts = ac.filter_belief(stmts, 0.95)
    stmts = ac.filter_gene_list(stmts, data_genes, 'all')
    print(len(stmts))
    stmts = ac.reduce_activities(stmts)
    pa = PysbAssembler()
    pa.add_statements(stmts)
    model = pa.make_model()
    pa.set_context('SKMEL28_SKIN')
    pa.save_model(out_file)
    return model

def assemble_sif(stmts, data, out_file):
    """Return an assembled SIF."""
    # Filter for high-belief statements
    stmts = ac.filter_belief(stmts, 0.99)
    # Filter for Activation / Inhibition
    stmts_act = ac.filter_by_type(stmts, Activation)
    stmts_inact = ac.filter_by_type(stmts, Inhibition)
    stmts = stmts_act + stmts_inact
    # Get the drugs inhibiting their targets as INDRA
    # statements
    def get_drug_statements():
        drug_targets = process_data.get_drug_targets()
        drug_stmts = []
        for dn, tns in drug_targets.items():
            da = Agent(dn + ':Drugs')
            for tn in tns:
                ta = Agent(tn)
                drug_stmt = Inhibition(da, ta)
                drug_stmts.append(drug_stmt)
        return drug_stmts
    drug_stmts = get_drug_statements()
    stmts = stmts + drug_stmts
    # Because of a bug in CNO, node names containing AND
    # need to be replaced
    def rename_and_nodes(st):
        for s in st:
            for a in s.agent_list():
                if a is not None:
                    if a.name.find('AND') != -1:
                        a.name = a.name.replace('AND', 'A_ND')
    rename_and_nodes(stmts)
    # Rewrite statements to replace genes with their corresponding
    # antibodies when possible
    stmts = rewrite_ab_stmts(stmts, data)
    def filter_ab_edges(st, policy='all'):
        st_out = []
        for s in st:
            if policy == 'all':
                all_ab = True
                for a in s.agent_list():
                    if a is not None:
                        if a.name.find('_p') == -1 and \
                            a.name.find('Drugs') == -1:
                            all_ab = False
                            break
                if all_ab:
                    st_out.append(s)
            elif policy == 'one':
                any_ab = False
                for a in s.agent_list():
                    if a is not None and a.name.find('_p') != -1:
                        any_ab = True
                        break
                if any_ab:
                    st_out.append(s)
        return st_out
    stmts = filter_ab_edges(stmts, 'one')
    print(len(stmts))
    # Make the SIF model
    sa = SifAssembler(stmts)
    sa.make_model(use_name_as_key=True)
    sif_str = sa.print_model()
    with open(out_file, 'wb') as fh:
        fh.write(sif_str.encode('utf-8'))
    # Make the MIDAS data file used for training the model
    midas_data = process_data.get_midas_data(data)
    return sif_str

def assemble_cx(stmts, out_file):
    """Return a CX assembler."""
    stmts = ac.filter_belief(stmts, 0.95)
    stmts = ac.strip_agent_context(stmts)
    ca = CxAssembler()
    ca.add_statements(stmts)
    model = ca.make_model()
    ca.save_model(out_file)
    return ca

def get_prior_genes(fname):
    """Get the list of prior genes."""
    with open(fname, 'rt') as fh:
        genes = fh.read().strip().split('\n')
        return genes

def rewrite_ab_stmts(stmts_in, data):
    """Replace corresponding Agent names to AB names.

    This is used for logical model building.
    """
    # Build AB <-> Gene maps
    ab_phos = process_data.get_phos_antibodies(data)
    ab_dict = {}
    ab_dict_rev = {}
    for ab in ab_phos:
        up_ids = process_data.get_antibody_genes(data, ab)
        for up_id in up_ids:
            agent = process_data.get_agent_from_upid(up_id)
            if ab in ab_dict:
                ab_dict[ab].append(agent)
            else:
                ab_dict[ab] = [agent]
            if agent.name in ab_dict_rev:
                ab_dict_rev[agent.name].append(ab)
            else:
                ab_dict_rev[agent.name] = [ab]
    stmts_out = []
    for stmt in stmts_in:
        new_agents = []
        for agent in stmt.agent_list():
            if agent is not None:
                ab_names = ab_dict_rev.get(agent.name)
                if not ab_names:
                    new_agents.append([agent])
                else:
                    new_agents.append([Agent(ab) for ab in ab_names])
            else:
                new_agents.append([None])
        for na in itertools.product(*new_agents):
            stmt_new = deepcopy(stmt)
            stmt_new.set_agent_list(na)
            stmts_out.append(stmt_new)
    return stmts_out

if __name__ == '__main__':
    #outf = '/home/beni/data/darpa/phase3_eval/'
    outf = 'output/'
    data = process_data.read_data(process_data.data_file)
    data_genes = process_data.get_all_gene_names(data)
    reassemble = True
    #phos_stmts, antibody_map = read_phosphosite('annotated_kinases_v3.csv')
    if not reassemble:
        stmts = ac.load_statements(pjoin(outf, 'top_level.pkl'))
    else:
        #prior_stmts = build_prior(data_genes, pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.load_statements(pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.map_grounding(prior_stmts, dump_pkl=pjoin(outf, 'gmapped_prior.pkl'))
        reading_stmts = ac.load_statements(pjoin(outf, 'phase3_stmts.pkl'))
        reading_stmts = ac.map_grounding(reading_stmts, dump_pkl=pjoin(outf, 'gmapped_reading.pkl'))
        stmts = prior_stmts + reading_stmts

        stmts = ac.filter_grounded_only(stmts)
        print(len(stmts))
        stmts = ac.filter_genes_only(stmts, specific_only=False)
        print(len(stmts))
        stmts = ac.filter_human_only(stmts)
        print(len(stmts))
        stmts = ac.expand_families(stmts)
        print(len(stmts))
        stmts = ac.filter_gene_list(stmts, data_genes, 'one')
        print(len(stmts))
        stmts = ac.map_sequence(stmts, dump_pkl=pjoin(outf, 'smapped.pkl'))
        print(len(stmts))
        stmts = ac.run_preassembly(stmts, dump_pkl=pjoin(outf, 'top_level.pkl'))
        print(len(stmts))

    assemble_models = []
    #assemble_models.append('sif')
    #assemble_models.append('pysb')
    #assemble_models.append('cx')

    ### PySB assembly
    if 'pysb' in assemble_models:
        pysb_model = assemble_pysb(stmts, data_genes,
                                   pjoin(outf, 'korkut_model_pysb.py'))
        ke = KappaExporter(pysb_model)
        with open(pjoin(outf, 'korkut_model.ka'), 'wb') as fh:
            fh.write(ke.export().encode('utf-8'))
    ### SIF assembly
    if 'sif' in assemble_models:
        sif_str = assemble_sif(stmts, data, pjoin(outf, 'PKN-korkut_one_ab.sif'))
    ### CX assembly
    if 'cx' in assemble_models:
        cxa = assemble_cx(stmts, pjoin(outf, 'korkut_full_high_belief.cx'))
<commit_msg>Index card assembly and clean up<commit_after>import pickle
import itertools
from copy import deepcopy
from os.path import join as pjoin
from pysb.export.kappa import KappaExporter
from indra.tools import assemble_corpus as ac
from indra.tools.gene_network import GeneNetwork
from indra.assemblers import IndexCardAssembler
from indra.statements import *
from indra.assemblers import PysbAssembler, SifAssembler, CxAssembler

import process_data
from read_phosphosite import read_phosphosite

def build_prior(genes, out_file):
    gn = GeneNetwork(genes, 'korkut')
    stmts = gn.get_statements(filter=False)
    ac.dump_statements(stmts, out_file)
    return stmts

def assemble_index_cards(stmts):
    counter = 1
    for st in stmts:
        if isinstance(st, Modification) and st.enz is None:
            continue
        pmids = [ev.pmid for ev in st.evidence if ev.pmid is not None]
        if pmids:
            pmids = ','.join(['PMID%s' % pm for pm in list(set(pmids))])
        else:
            pmids = 'N/A'
        ica = IndexCardAssembler([st], pmc_override=pmids)
        ica.make_model()
        if ica.cards:
            ica.save_model('output/index_cards/index_card_%d.json' % counter)
            counter += 1

def assemble_pysb(stmts, data_genes, out_file):
    """Return an assembled PySB model."""
    stmts = ac.filter_belief(stmts, 0.95)
    stmts = ac.filter_gene_list(stmts, data_genes, 'all')
    stmts = ac.reduce_activities(stmts)
    pa = PysbAssembler()
    pa.add_statements(stmts)
    model = pa.make_model()
    pa.set_context('SKMEL28_SKIN')
    pa.save_model(out_file)
    return model

def assemble_sif(stmts, data, out_file):
    """Return an assembled SIF."""
    # Filter for high-belief statements
    stmts = ac.filter_belief(stmts, 0.99)
    # Filter for Activation / Inhibition
    stmts_act = ac.filter_by_type(stmts, Activation)
    stmts_inact = ac.filter_by_type(stmts, Inhibition)
    stmts = stmts_act + stmts_inact
    # Get the drugs inhibiting their targets as INDRA
    # statements
    def get_drug_statements():
        drug_targets = process_data.get_drug_targets()
        drug_stmts = []
        for dn, tns in drug_targets.items():
            da = Agent(dn + ':Drugs')
            for tn in tns:
                ta = Agent(tn)
                drug_stmt = Inhibition(da, ta)
                drug_stmts.append(drug_stmt)
        return drug_stmts
    drug_stmts = get_drug_statements()
    stmts = stmts + drug_stmts
    # Because of a bug in CNO, node names containing AND
    # need to be replaced
    def rename_and_nodes(st):
        for s in st:
            for a in s.agent_list():
                if a is not None:
                    if a.name.find('AND') != -1:
                        a.name = a.name.replace('AND', 'A_ND')
    rename_and_nodes(stmts)
    # Rewrite statements to replace genes with their corresponding
    # antibodies when possible
    stmts = rewrite_ab_stmts(stmts, data)
    def filter_ab_edges(st, policy='all'):
        st_out = []
        for s in st:
            if policy == 'all':
                all_ab = True
                for a in s.agent_list():
                    if a is not None:
                        if a.name.find('_p') == -1 and \
                            a.name.find('Drugs') == -1:
                            all_ab = False
                            break
                if all_ab:
                    st_out.append(s)
            elif policy == 'one':
                any_ab = False
                for a in s.agent_list():
                    if a is not None and a.name.find('_p') != -1:
                        any_ab = True
                        break
                if any_ab:
                    st_out.append(s)
        return st_out
    stmts = filter_ab_edges(stmts, 'one')
    print(len(stmts))
    # Make the SIF model
    sa = SifAssembler(stmts)
    sa.make_model(use_name_as_key=True)
    sif_str = sa.print_model()
    with open(out_file, 'wb') as fh:
        fh.write(sif_str.encode('utf-8'))
    # Make the MIDAS data file used for training the model
    midas_data = process_data.get_midas_data(data)
    return sif_str

def assemble_cx(stmts, out_file):
    """Return a CX assembler."""
    stmts = ac.filter_belief(stmts, 0.95)
    stmts = ac.strip_agent_context(stmts)
    ca = CxAssembler()
    ca.add_statements(stmts)
    model = ca.make_model()
    ca.save_model(out_file)
    return ca

def get_prior_genes(fname):
    """Get the list of prior genes."""
    with open(fname, 'rt') as fh:
        genes = fh.read().strip().split('\n')
        return genes

def rewrite_ab_stmts(stmts_in, data):
    """Replace corresponding Agent names to AB names.

    This is used for logical model building.
    """
    # Build AB <-> Gene maps
    ab_phos = process_data.get_phos_antibodies(data)
    ab_dict = {}
    ab_dict_rev = {}
    for ab in ab_phos:
        up_ids = process_data.get_antibody_genes(data, ab)
        for up_id in up_ids:
            agent = process_data.get_agent_from_upid(up_id)
            if ab in ab_dict:
                ab_dict[ab].append(agent)
            else:
                ab_dict[ab] = [agent]
            if agent.name in ab_dict_rev:
                ab_dict_rev[agent.name].append(ab)
            else:
                ab_dict_rev[agent.name] = [ab]
    stmts_out = []
    for stmt in stmts_in:
        new_agents = []
        for agent in stmt.agent_list():
            if agent is not None:
                ab_names = ab_dict_rev.get(agent.name)
                if not ab_names:
                    new_agents.append([agent])
                else:
                    new_agents.append([Agent(ab) for ab in ab_names])
            else:
                new_agents.append([None])
        for na in itertools.product(*new_agents):
            stmt_new = deepcopy(stmt)
            stmt_new.set_agent_list(na)
            stmts_out.append(stmt_new)
    return stmts_out

if __name__ == '__main__':
    outf = 'output/'
    data = process_data.read_data(process_data.data_file)
    data_genes = process_data.get_all_gene_names(data)
    reassemble = False
    if not reassemble:
        stmts = ac.load_statements(pjoin(outf, 'top_level.pkl'))
    else:
        #prior_stmts = build_prior(data_genes, pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.load_statements(pjoin(outf, 'prior.pkl'))
        prior_stmts = ac.map_grounding(prior_stmts, dump_pkl=pjoin(outf, 'gmapped_prior.pkl'))
        reading_stmts = ac.load_statements(pjoin(outf, 'phase3_stmts.pkl'))
        reading_stmts = ac.map_grounding(reading_stmts, dump_pkl=pjoin(outf, 'gmapped_reading.pkl'))
        stmts = prior_stmts + reading_stmts

        stmts = ac.filter_grounded_only(stmts)
        stmts = ac.filter_genes_only(stmts, specific_only=False)
        stmts = ac.filter_human_only(stmts)
        stmts = ac.expand_families(stmts)
        stmts = ac.filter_gene_list(stmts, data_genes, 'one')
        stmts = ac.map_sequence(stmts, dump_pkl=pjoin(outf, 'smapped.pkl'))
        stmts = ac.run_preassembly(stmts, dump_pkl=pjoin(outf, 'top_level.pkl'))

    assemble_models = []
    assemble_models.append('sif')
    assemble_models.append('pysb')
    assemble_models.append('cx')

    ### PySB assembly
    if 'pysb' in assemble_models:
        pysb_model = assemble_pysb(stmts, data_genes,
                                   pjoin(outf, 'korkut_model_pysb.py'))
        ke = KappaExporter(pysb_model)
        with open(pjoin(outf, 'korkut_model.ka'), 'wb') as fh:
            fh.write(ke.export().encode('utf-8'))
    ### SIF assembly
    if 'sif' in assemble_models:
        sif_str = assemble_sif(stmts, data, pjoin(outf, 'PKN-korkut_one_ab.sif'))
    ### CX assembly
    if 'cx' in assemble_models:
        cxa = assemble_cx(stmts, pjoin(outf, 'korkut_full_high_belief.cx'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Test links:
#   http://filecrypt.cc/Container/64E039F859.html

import binascii
import re
import urlparse

import Crypto.Cipher

from module.plugins.internal.Crypter import Crypter
from module.plugins.captcha.ReCaptcha import ReCaptcha
from module.plugins.captcha.SolveMedia import SolveMedia

class FilecryptCc(Crypter):
    __name__    = "FilecryptCc"
    __type__    = "crypter"
    __version__ = "0.23"
    __status__  = "testing"

    __pattern__ = r'https?://(?:www\.)?filecrypt\.cc/Container/\w+'
    __config__  = [("activated", "bool", "Activated", True)]

    __description__ = """Filecrypt.cc decrypter plugin"""
    __license__     = "GPLv3"
    __authors__     = [("zapp-brannigan", "fuerst.reinje@web.de"),
                       ("GammaC0de"     , None                  )]

    # URL_REPLACEMENTS  = [(r'.html$', ""), (r'$', ".html")]  #@TODO: Extend SimpleCrypter

    DLC_LINK_PATTERN = r'onclick="DownloadDLC\(\'(.+)\'\);">'
    WEBLINK_PATTERN  = r"openLink.?'([\w\-]*)',"

    CAPTCHA_PATTERN          = r'class="safety">Sicherheitsabfrage<'
    INTERNAL_CAPTCHA_PATTERN = r'<img id="nc" src="(.+?)"'
    CIRCLE_CAPTCHA_PATTERN   = r'<input type="image" src="(.+?)"'
    KEY_CAPTCHA_PATTERN      = r"


<commit_before># -*- coding: utf-8 -*-

import re

from module.plugins.internal.XFSHoster import XFSHoster

class FilejokerNet(XFSHoster):
    __name__    = "FilejokerNet"
    __type__    = "hoster"
    __version__ = "0.01"
    __status__  = "testing"

    __pattern__ = r'https?://(?:www\.)?filejoker\.net/\w{12}'
    __config__  = [("activated"   , "bool", "Activated"                                        , True),
                   ("use_premium" , "bool", "Use premium account if available"                 , True),
                   ("fallback"    , "bool", "Fallback to free download if premium fails"       , True),
                   ("chk_filesize", "bool", "Check file size"                                  , True),
                   ("max_wait"    , "int" , "Reconnect if waiting time is greater than minutes", 10  )]

    __description__ = """Filejoker.net hoster plugin"""
    __license__     = "GPLv3"
    __authors__     = [("GammaC0de", "nitzo2001[AT]yahoo[DOT]com")]

    PLUGIN_DOMAIN   = "filejoker.net"

    WAIT_PATTERN      = r'Please [Ww]ait (?:)?([\w ]+?)(?: seconds
| until the next download)'
    RECAPTCHA_PATTERN = r'

'

    INFO_PATTERN      = r'
(?P<N>.+?) \((?P<S>[\d.,]+) (?P[\w^_]+)\)
'
    SIZE_REPLACEMENTS = [('Kb','KB'), ('Mb','MB'), ('Gb','GB')]

    LINK_PATTERN      = r'
\s+<a href="(.+?)"'
<commit_msg>[FilejokerNet] Update<commit_after># -*- coding: utf-8 -*-

import re

from module.plugins.internal.XFSHoster import XFSHoster

class FilejokerNet(XFSHoster):
    __name__    = "FilejokerNet"
    __type__    = "hoster"
    __version__ = "0.02"
    __status__  = "testing"

    __pattern__ = r'https?://(?:www\.)?filejoker\.net/\w{12}'
    __config__  = [("activated"   , "bool", "Activated"                                        , True),
                   ("use_premium" , "bool", "Use premium account if available"                 , True),
                   ("fallback"    , "bool", "Fallback to free download if premium fails"       , True),
                   ("chk_filesize", "bool", "Check file size"                                  , True),
                   ("max_wait"    , "int" , "Reconnect if waiting time is greater than minutes", 10  )]

    __description__ = """Filejoker.net hoster plugin"""
    __license__     = "GPLv3"
    __authors__     = [("GammaC0de", "nitzo2001[AT]yahoo[DOT]com")]

    PLUGIN_DOMAIN   = "filejoker.net"

    WAIT_PATTERN      = r'Please [Ww]ait (?:)?([\w ]+?)(?: seconds
| until the next download)'
    RECAPTCHA_PATTERN = r'

'
    ERROR_PATTERN     = r'Wrong Captcha'

    INFO_PATTERN      = r'
(?P<N>.+?) \((?P<S>[\d.,]+) (?P[\w^_]+)\)
'
    SIZE_REPLACEMENTS = [('Kb','KB'), ('Mb','MB'), ('Gb','GB')]

    LINK_PATTERN      = r'
\s+<a href="(.+?)"'
<|endoftext|>


<commit_before>from module.plugins.Hoster import Hoster as _Hoster

def create_getInfo(plugin):
    def getInfo(urls):
        for url in urls:
            yield '#N/A: ' + url, 0, 1, url
    return getInfo

class Hoster(_Hoster):
    __name__ = "DeadHoster"
    __type__ = "hoster"
    __pattern__ = r""
    __version__ = "0.1"
    __description__ = """Hoster is no longer available"""
    __author_name__ = ("zoidberg")
    __author_mail__ = ("zoidberg@mujmail.cz")

    def init(self):
        self.fail("Hoster is no longer available") <commit_msg>fix fixed hoster fix<commit_after>from module.plugins.Hoster import Hoster as _Hoster

def create_getInfo(plugin):
    def getInfo(urls):
        yield [('#N/A: ' + url, 0, 1, url) for url in urls]
    return getInfo

class DeadHoster(_Hoster):
    __name__ = "DeadHoster"
    __type__ = "hoster"
    __pattern__ = r""
    __version__ = "0.11"
    __description__ = """Hoster is no longer available"""
    __author_name__ = ("zoidberg")
    __author_mail__ = ("zoidberg@mujmail.cz")

    def setup(self):
        self.fail("Hoster is no longer available")<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
scans IPv4 subnet for SSH servers on Port 22 or other server ports.
Useful for machines that don't/can't have NMAP installed (e.g. Windows), and device does not have Avahi server.
Yes this can be done simply via Linux shell, but I wanted to make it as cross-platform as possible
where the user would have only basic Python installed (Windows)

Michael Hirsch

Note: if using Python < 3.3, you will need
pip install ipaddress

Note: timeout value bare minimum is 0.15 seconds for LAN, suggest using higher values say 0.25 or 0.35 if you can stand the wait 254*0.35 seconds

"""
from __future__ import division,unicode_literals
from six import PY2
from time import time
import socket
from ipaddress import ip_address,ip_network,IPv4Network
#
if PY2: ConnectionRefusedError = OSError
#%% (1) get LAN IP of laptop
def getLANip():
    """ get IP of own interface
     ref: http://stackoverflow.com/a/23822431
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    # don't use link local here (169.254.x.x) unless you have a specific need
    try:
        s.connect(('<broadcast>', 0))
    except OSError:
        s.connect(('1.1.1.1',0)) # for BSD/Mac
    name= s.getsockname()[0]
    s.close()
    return ip_address(name.encode('utf-8').decode('utf-8')) #encode.decode is used for python2 and python3 compatibility
#%% (2) scan subnet for SSH servers
def isportopen(host,port,service,timeout=0.3):
    h = host.exploded
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(timeout) #seconds
    try:
        s.connect((h,port))
        b=s.recv(32) #arbitrary number of bytes

#%% service decode (optional)
        return validateservice(service,h,b)

    except (ConnectionRefusedError,socket.timeout,socket.error):
        return
    finally:
        s.close()

def validateservice(service,h,b):
    if not b: #empty reply
        return
#%% non-empty reply
    try:
        u = b.splitlines()[0].decode('utf-8') #splitlines is in case the ASCII/UTF8 response is less than 32 bytes, hoping server sends a \r\n
    except UnicodeDecodeError: # must not have been utf8 encoding..., maybe latin1 or something else..
        u = b

    print(u)
#%% optional service validation
    if service:
        try:
            if service in u.lower():
                return True
        except UnicodeDecodeError:
            print('unable to decode {} response'.format(h))
            return
    else:
        return True

def netfromaddress(addr):
    addr = ip_address(addr) #in case it's string
    if addr.version == 4:
        return ip_network(ownip.exploded.rsplit('.',1)[0]+'.0/24')
    else: #addr.version ==6
        raise NotImplementedError('https://www.6net.org/publications/standards/draft-chown-v6ops-port-scanning-implications-00.txt')
#%% main loop
def scanhosts(net,port,service,timeout):
    """
    loops over xxx.xxx.xxx.1-254
    IPv4 only.
    """
    if not isinstance(net,IPv4Network):
        net = netfromaddress(net)

    if net.version != 4:
        raise NotImplementedError('https://www.6net.org/publications/standards/draft-chown-v6ops-port-scanning-implications-00.txt')

    servers = []
    print('searching {}'.format(net))
    for t,a in enumerate(net.hosts()):
        if isportopen(a,port,service):
            servers.append(a)
            print('found {} on {} port {}'.format(service,a,port))
        if not t % 20:
            print('{:.1f} % done, {} {} servers detected on port {}'.format(t/255*100.,len(servers),service,port))
    return servers

if __name__ == '__main__':
    from argparse import ArgumentParser
    p = ArgumentParser('scan for hosts with open port, without NMAP')
    p.add_argument('-p','--port',help='single port to try',default=22,type=int)
    p.add_argument('-s','--service',help='string to match to qualify detections',default='')
    p.add_argument('-t','--timeout',help='timeout to wait for server',default=0.3,type=float)
    p.add_argument('-b','--baseip',help='instead of using own IP, set a specific subnet to scan')
    p = p.parse_args()

    tic = time()
    ownip = getLANip() if not p.baseip else p.baseip
    print('own address ' + str(ownip))
    servers = scanhosts(ownip,p.port,p.service,p.timeout)
    print('\n*** RESULTS ***\n')
    print('found {} {} server IPs in {:.1f} seconds: \n'.format(
                         len(servers),p.service,time()-tic)+str(servers))
<commit_msg>mac fallback<commit_after>#!/usr/bin/env python3
"""
scans IPv4 subnet for SSH servers on Port 22 or other server ports.
Useful for machines that don't/can't have NMAP installed (e.g. Windows), and device does not have Avahi server.
Yes this can be done simply via Linux shell, but I wanted to make it as cross-platform as possible
where the user would have only basic Python installed (Windows)

Michael Hirsch

Note: if using Python < 3.3, you will need
pip install ipaddress

Note: timeout value bare minimum is 0.15 seconds for LAN, suggest using higher values say 0.25 or 0.35 if you can stand the wait 254*0.35 seconds

"""
from __future__ import division,unicode_literals
from six import PY2
from time import time
import socket
from ipaddress import ip_address,ip_network,IPv4Network
#
if PY2: ConnectionRefusedError = OSError
#%% (1) get LAN IP of laptop
def getLANip():
    """ get IP of own interface
     ref: http://stackoverflow.com/a/23822431
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    # don't use link local here (169.254.x.x) unless you have a specific need
    try:
        s.connect(('<broadcast>', 0))
    except OSError:
        s.connect(('8.8.8.8',0)) # for BSD/Mac
    name= s.getsockname()[0]
    s.close()
    return ip_address(name.encode('utf-8').decode('utf-8')) #encode.decode is used for python2 and python3 compatibility
#%% (2) scan subnet for SSH servers
def isportopen(host,port,service,timeout=0.3):
    h = host.exploded
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(timeout) #seconds
    try:
        s.connect((h,port))
        b=s.recv(32) #arbitrary number of bytes

#%% service decode (optional)
        return validateservice(service,h,b)

    except (ConnectionRefusedError,socket.timeout,socket.error):
        return
    finally:
        s.close()

def validateservice(service,h,b):
    if not b: #empty reply
        return
#%% non-empty reply
    try:
        u = b.splitlines()[0].decode('utf-8') #splitlines is in case the ASCII/UTF8 response is less than 32 bytes, hoping server sends a \r\n
    except UnicodeDecodeError: # must not have been utf8 encoding..., maybe latin1 or something else..
        u = b

    print(u)
#%% optional service validation
    if service:
        try:
            if service in u.lower():
                return True
        except UnicodeDecodeError:
            print('unable to decode {} response'.format(h))
            return
    else:
        return True

def netfromaddress(addr):
    addr = ip_address(addr) #in case it's string
    if addr.version == 4:
        return ip_network(ownip.exploded.rsplit('.',1)[0]+'.0/24')
    else: #addr.version ==6
        raise NotImplementedError('https://www.6net.org/publications/standards/draft-chown-v6ops-port-scanning-implications-00.txt')
#%% main loop
def scanhosts(net,port,service,timeout):
    """
    loops over xxx.xxx.xxx.1-254
    IPv4 only.
    """
    if not isinstance(net,IPv4Network):
        net = netfromaddress(net)

    if net.version != 4:
        raise NotImplementedError('https://www.6net.org/publications/standards/draft-chown-v6ops-port-scanning-implications-00.txt')

    servers = []
    print('searching {}'.format(net))
    for t,a in enumerate(net.hosts()):
        if isportopen(a,port,service):
            servers.append(a)
            print('found {} on {} port {}'.format(service,a,port))
        if not t % 20:
            print('{:.1f} % done, {} {} servers detected on port {}'.format(t/255*100.,len(servers),service,port))
    return servers

if __name__ == '__main__':
    from argparse import ArgumentParser
    p = ArgumentParser('scan for hosts with open port, without NMAP')
    p.add_argument('-p','--port',help='single port to try',default=22,type=int)
    p.add_argument('-s','--service',help='string to match to qualify detections',default='')
    p.add_argument('-t','--timeout',help='timeout to wait for server',default=0.3,type=float)
    p.add_argument('-b','--baseip',help='instead of using own IP, set a specific subnet to scan')
    p = p.parse_args()

    tic = time()
    ownip = getLANip() if not p.baseip else p.baseip
    print('own address ' + str(ownip))
    servers = scanhosts(ownip,p.port,p.service,p.timeout)
    print('\n*** RESULTS ***\n')
    print('found {} {} server IPs in {:.1f} seconds: \n'.format(
                         len(servers),p.service,time()-tic)+str(servers))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
scans own LAN subnet for SSH servers on Port 22.
Useful for machines that don't/can't have NMAP installed (e.g. Windows)
Yes this can be done simply via Linux shell, but I wanted to make it as cross-platform as possible
where the user would have only basic Python installed (Windows)

Michael Hirsch
"""
from __future__ import division
from time import time
import socket
SERVICE='ssh' #try to match this string in server response
#%% (1) get LAN IP of laptop
# ref: http://stackoverflow.com/a/23822431
def getLANip():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    s.connect(('<broadcast>', 0))
    return s.getsockname()[0]
#%% (2) scan own subnet for SSH servers
def isportopen(host,port,timeout=0.15):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(timeout) #seconds
    try:
        s.connect((host,port))
        r=s.recv(32).decode('utf8') #arbitrary number of bytes
        print(r)
        s.close()
    except (OSError,socket.timeout,socket.error): #,ConnectionRefusedError
        return

    if SERVICE in r.lower():
        return True
#%% main loop
def scanhosts(ownip,port,timeout):
    """
    loops over xxx.xxx.xxx.1-254
    """
    base = ownip.rsplit('.',1)[0]
    tail = range(1,255)
    servers = []
    print('searching {}.*'.format(base))
    for t in tail:
        host = '.'.join((base,str(t)))
        if isportopen(host,port):
            servers.append(host)
            print('found {} on {} port {}'.format(SERVICE,host,port))
        if not t % 10:
            print('{:.1f} % done, {} servers detected on port {}'.format(t/255*100.,len(servers),port))
    return servers

if __name__ == '__main__':
    from argparse import ArgumentParser
    p = ArgumentParser('scan for hosts with open port, without NMAP')
    p.add_argument('-p','--port',help='single port to try',default=22,type=int)
    p.add_argument('-t','--timeout',help='timeout to wait for server',default=0.1,type=float)
    p.add_argument('-b','--baseip',help='instead of using own IP, set a specific subnet to scan')
    p = p.parse_args()

    tic = time()
    ownip = getLANip() if not p.baseip else p.baseip
    print('own IPv4 ' + str(ownip))
    servers = scanhosts(ownip,p.port,p.timeout)
    print('found {} {} server IPs in {:.1f} seconds: \n'.format(len(servers),SERVICE,time()-tic)+str(servers))
<commit_msg>use Python 3.3+ ipaddress module<commit_after>#!/usr/bin/env python3
"""
scans own LAN subnet for SSH servers on Port 22.
Useful for machines that don't/can't have NMAP installed (e.g. Windows)
Yes this can be done simply via Linux shell, but I wanted to make it as cross-platform as possible
where the user would have only basic Python installed (Windows)

Michael Hirsch

Note: if using Python < 3.4, you will need
pip insall py2-ipaddress
"""
from __future__ import division
from six import PY2
from time import time
import socket
from ipaddress import ip_address,ip_network,ip_interface
#
if PY2: ConnectionRefusedError = OSError
#%% (1) get LAN IP of laptop
def getLANip():
    """ get IP of own interface
     ref: http://stackoverflow.com/a/23822431
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    s.connect(('<broadcast>', 0))
    name= s.getsockname()[0]
    s.close()
    return ip_address(name)
#%% (2) scan subnet for SSH servers
def isportopen(host,port,service,timeout=0.15):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(timeout) #seconds
    try:
        s.connect((host.exploded,port))
        r=s.recv(32).decode('utf8') #arbitrary number of bytes
        print(r)
        s.close()
    except (ConnectionRefusedError,socket.timeout,socket.error):
        return

    if service in r.lower():
        return True
#%% main loop
def scanhosts(ownip,port,service,timeout):
    """
    loops over xxx.xxx.xxx.1-254
    """
    ownip = ip_address(ownip) #in case it's string
    net = ip_network(ownip.exploded.rsplit('.',1)[0]+'.0/24')
    servers = []
    print('searching {}'.format(net))
    for t,a in enumerate(net.hosts()):
        if isportopen(a,port,service):
            servers.append(a)
            print('found {} on {} port {}'.format(service,a,port))
        if not t % 20:
            print('{:.1f} % done, {} {} servers detected on port {}'.format(t/255*100.,len(servers),service,port))
    return servers

if __name__ == '__main__':
    from argparse import ArgumentParser
    p = ArgumentParser('scan for hosts with open port, without NMAP')
    p.add_argument('-p','--port',help='single port to try',default=22,type=int)
    p.add_argument('-s','--service',help='string to match to qualify detections',default='ssh')
    p.add_argument('-t','--timeout',help='timeout to wait for server',default=0.1,type=float)
    p.add_argument('-b','--baseip',help='instead of using own IP, set a specific subnet to scan')
    p = p.parse_args()

    tic = time()
    ownip = getLANip() if not p.baseip else p.baseip
    print('own IPv4 ' + str(ownip))
    servers = scanhosts(ownip,p.port,p.service,p.timeout)
    print('found {} {} server IPs in {:.1f} seconds: \n'.format(
                         len(servers),p.service,time()-tic)+str(servers))
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseHalaroseCsvImporter

class Command(BaseHalaroseCsvImporter):
    council_id = "E07000120"
    addresses_name = "local.2018-05-03/Version 1/polling_station_export-2018-02-16.csv"
    stations_name = "local.2018-05-03/Version 1/polling_station_export-2018-02-16.csv"
    elections = ["local.2018-05-03"]
    csv_encoding = "windows-1252"
<commit_msg>Import Hyndburn (#2176)<commit_after>import re

from data_collection.ems_importers import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000120"
    addresses_name = "parl.2019-12-12/Version 1/Democracy_Club__12December2019hynd.tsv"
    stations_name = "parl.2019-12-12/Version 1/Democracy_Club__12December2019hynd.tsv"
    elections = ["parl.2019-12-12"]
    csv_encoding = "windows-1252"
    csv_delimiter = "\t"
    allow_station_point_from_postcode = False

    def station_record_to_dict(self, record):
        if record.polling_place_id == "824":
            record = record._replace(polling_place_postcode="BB5 5DH")
        return super().station_record_to_dict(record)

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if re.search(r"\bWhalley Road\b", record.addressline1) or re.search(
            r"\bWhalley Road\b", record.addressline2
        ):
            # This road passes through Great Harwood and Clayton le Moors with duplicated house numbers.
            # The council have misrecorded UPRNs for a lot of these properties
            rec["accept_suggestion"] = False

        # More misrecorded UPRNs
        if record.post_code in [
            "BB5 4PP",  # Confusion between Stanhill [Road] and Stanhill Lane
            "BB5 4PS",  # Confusion between Stanhill [Road] and Stanhill Lane
            "BB5 3RU",  # Confusion between Hoyle Bottom and Hoyle Bottom Farm Cottages
            "BB5 0HD",  # Confusion between George Streets in Accrington and Oswaldtwistle
            "BB5 4BQ",  # Confusion between Queens Road West in Accrington and Church
            "BB5 3BB",  # Confusion between High Streets in Accrington and Oswaldtwistle
        ]:
            rec["accept_suggestion"] = False

        if uprn == "10070894060":
            # Looks assigned to the wrong polling place
            return None

        if uprn in [
            "10009970126",  # BB56EW -> BB56QT : Peel Park Hotel, Turkey Street, Accrington, Lancashire
            "10009971723",  # BB56EW -> BB56QT : Peel Park Cottage, Turkey Street, Accrington, Lancashire
            "100012544978",  # BB51EH -> BB51EN : 33 Abbey Street, Accrington, Lancashire
            "10070886537",  # BB56NT -> BB56PN : Flat Over, Hyndburn Private Clinic, Avenue Parade, Accrington
            "100012546116",  # BB55UT -> BB55UR : Waterside Bungalow, Altham, Accrington, Lancashire
            "100010451238",  # BB67QL -> BB67QQ : 19-21 Queen Street, Great Harwood, Blackburn, Lancashire
            "100010446019",  # BB67DF -> BB67DE : 45 Blackburn Road, Great Harwood, Blackburn, Lancashire
            "10009971276",  # BB55QA -> BB55PZ : Royal Oak Hotel, 35 Sparth Road, Clayton-Le-Moors, Accrington
            "100010446896",  # BB67NF -> BB67QB : 17 Church Street, Great Harwood, Blackburn, Lancashire
            "100012392089",  # BB54QA -> BB54NN : All Saint's House, Aspen Lane, Oswaldtwistle, Accrington, Lancashire
            "100012392479",  # BB53RG -> BB53RJ : Usha Lounge Restaurant, Haslingden Old Road, Oswaldtwistle, Accrington
            "100012544307",  # BB50LN -> BB50LR : St. Peter's Vicarage, 151 Willows Lane, Accrington, Lancashire
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "100010431355",  # BB56LH -> BB56LF : 1 Pipers Row, Huncoat, Accrington, Lancashire
            "100012392809",  # BB56HA -> BB56UE : Woodside House, Burnley Road, Huncoat, Accrington, Lancashire
            "10009970822",  # BB67WA -> BB67UQ : The Bungalow, 27 Mill Lane, Great Harwood, Blackburn, Lancashire
            "100012533996",  # BB14JJ -> BB14LL : 1B Henry Street, Rishton, Blackburn, Lancashire
            "100012392555",  # BB52DL -> BB52DZ : Higher Hey Cottage, Kings Highway, Accrington, Lancashire
            "10009971952",  # BB53SW -> BB53SL : Nigher Friar Hill Farm, Roundhill Road, Accrington, Lancashire
            "100012544405",  # BB50QR -> BB50SH : 43 Fountain Street, Accrington, Lancashire
            "100012392714",  # BB52JX -> BB50NY : Pine Lodge, Rothwell House, Priestley Clough, Accrington, Lancashire
            "100012392419",  # BB51SW -> BB51SP : Church View Home For the Elderly, Emma Street, Accrington, Lancashire
            "10070894405",  # BB50DN -> BB50DP : 74A Market Street, Church, Accrington, Lancashire
            "100012392942",  # BB53JB -> BB53EG : 230A Union Road, Oswaldtwistle, Accrington, Lancashire
            "100012545127",  # BB56DH -> BB51AR : Broadway Hotel, Burnley Road, Accrington, Lancashire
            "10009967793",  # BB50PS -> BB53BB : 5 High Street, Accrington, Lancashire
        ]:
            rec["accept_suggestion"] = False

        return rec
<|endoftext|>