<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import subprocess
import time
from contextlib import contextmanager

from lxml.cssselect import CSSSelector
from selenium.common.exceptions import NoSuchElementException

from splinter.driver import DriverAPI, ElementAPI
from splinter.element_list import ElementList
from splinter.utils import warn_deprecated
from tempfile import TemporaryFile

class BaseWebDriver(DriverAPI):
    old_popen = subprocess.Popen

    def __init__(self, wait_time=2):
        self.wait_time = wait_time

    def _patch_subprocess(self):
        loggers_to_silence = [
            'selenium.webdriver.firefox.extension_connection',
            'selenium.webdriver.remote.remote_connection',
            'selenium.webdriver.remote.utils',
        ]

        class MutedHandler(logging.Handler):
            pass

        for name in loggers_to_silence:
            logger = logging.getLogger(name)
            logger.addHandler(MutedHandler())
            logger.setLevel(99999)

        # selenium is such a verbose guy let's make it open the
        # browser without showing all the meaningless output
        def MyPopen(*args, **kw):
            kw['stdout'] = TemporaryFile()
            kw['stderr'] = TemporaryFile()
            kw['close_fds'] = True
            return self.old_popen(*args, **kw)

        subprocess.Popen = MyPopen

    def _unpatch_subprocess(self):
        # cleaning up the house
        subprocess.Popen = self.old_popen

    @property
    def title(self):
        return self.driver.title

    @property
    def html(self):
        return self.driver.page_source

    @property
    def url(self):
        return self.driver.current_url

    def visit(self, url):
        self.connect(url)
        self.ensure_success_response()
        self.driver.get(url)

    def back(self):
        self.driver.back()

    def forward(self):
        self.driver.forward()

    def reload(self):
        self.driver.refresh()

    def execute_script(self, script):
        self.driver.execute_script(script)

    def evaluate_script(self, script):
        return self.driver.execute_script("return %s" % script)

    def is_element_present(self, finder, selector, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            if finder(selector):
                return True
        return False

    def is_element_not_present(self, finder, selector, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            if not finder(selector):
                return True
        return False

    def is_element_present_by_css(self, css_selector, wait_time=None):
        return self.is_element_present(self.find_by_css, css_selector, wait_time)

    is_element_present_by_css_selector = warn_deprecated(is_element_present_by_css, 'is_element_present_by_css_selector')

    def is_element_not_present_by_css(self, css_selector, wait_time=None):
        return self.is_element_not_present(self.find_by_css, css_selector, wait_time)

    is_element_not_present_by_css_selector = warn_deprecated(is_element_not_present_by_css, 'is_element_not_present_by_css_selector')

    def is_element_present_by_xpath(self, xpath, wait_time=None):
        return self.is_element_present(self.find_by_xpath, xpath, wait_time)

    def is_element_not_present_by_xpath(self, xpath, wait_time=None):
        return self.is_element_not_present(self.find_by_xpath, xpath, wait_time)

    def is_element_present_by_tag(self, tag, wait_time=None):
        return self.is_element_present(self.find_by_tag, tag, wait_time)

    def is_element_not_present_by_tag(self, tag, wait_time=None):
        return self.is_element_not_present(self.find_by_tag, tag, wait_time)

    def is_element_present_by_name(self, name, wait_time=None):
        return self.is_element_present(self.find_by_name, name, wait_time)

    def is_element_not_present_by_name(self, name, wait_time=None):
        return self.is_element_not_present(self.find_by_name, name, wait_time)

    def is_element_present_by_value(self, value, wait_time=None):
        return self.is_element_present(self.find_by_value, value, wait_time)

    def is_element_not_present_by_value(self, value, wait_time=None):
        return self.is_element_not_present(self.find_by_value, value, wait_time)

    def is_element_present_by_id(self, id, wait_time=None):
        return self.is_element_present(self.find_by_id, id, wait_time)

    def is_element_not_present_by_id(self, id, wait_time=None):
        return self.is_element_not_present(self.find_by_id, id, wait_time)

    def get_alert(self):
        return AlertElement(self.driver.switch_to_alert())

    def is_text_present(self, text, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            try:
                self.driver.find_element_by_tag_name('body').text.index(text)
                return True
            except ValueError:
                pass
        return False

    def is_text_not_present(self, text, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            try:
                self.driver.find_element_by_tag_name('body').text.index(text)
            except ValueError:
                return True
        return False

    @contextmanager
    def get_iframe(self, id):
        self.driver.switch_to_frame(id)
        try:
            yield self
        finally:
            self.driver.switch_to_frame(None)

    def find_option_by_value(self, value):
        return self.find_by_xpath('//option[@value="%s"]' % value, original_find="option by value", original_query=value)

    def find_option_by_text(self, text):
        return self.find_by_xpath('//option[normalize-space(text())="%s"]' % text, original_find="option by text", original_query=text)

    def find_link_by_href(self, href):
        return self.find_by_xpath('//a[@href="%s"]' % href, original_find="link by href", original_query=href)

    def find_link_by_partial_href(self, partial_href):
        return self.find_by_xpath('//a[contains(@href, "%s")]' % partial_href, original_find="link by partial href", original_query=partial_href)

    def find_link_by_partial_text(self, partial_text):
        return ElementList([self.element_class(element, self) for element in self.driver.find_elements_by_partial_link_text(partial_text)], find_by="partial text", query=partial_text)

    def find_link_by_text(self, text):
        return ElementList([self.element_class(element, self) for element in self.driver.find_elements_by_link_text(text)], find_by="link by text", query=text)

    def find_by(self, finder, selector, original_find=None, original_query=None):
        elements = None
        end_time = time.time() + self.wait_time

        func_name = finder.im_func.func_name
        find_by = original_find or func_name[func_name.rfind('_by_') + 4:]
        query = original_query or selector

        while time.time() < end_time:
            try:
                elements = finder(selector)
                if not isinstance(elements, list):
                    elements = [elements]
            except NoSuchElementException:
                pass

            if elements:
                return ElementList([self.element_class(element, self) for element in elements], find_by=find_by, query=query)
        return ElementList([], find_by=find_by, query=query)

    def find_by_css(self, css_selector):
        selector = CSSSelector(css_selector)
        return self.find_by(self.driver.find_elements_by_xpath, selector.path, original_find='css', original_query=css_selector)

    find_by_css_selector = warn_deprecated(find_by_css, 'find_by_css_selector')

    def find_by_xpath(self, xpath, original_find=None, original_query=None):
        original_find = original_find or "xpath"
        original_query = original_query or xpath
        return self.find_by(self.driver.find_elements_by_xpath, xpath, original_find=original_find, original_query=original_query)

    def find_by_name(self, name):
        return self.find_by(self.driver.find_elements_by_name, name)

    def find_by_tag(self, tag):
        return self.find_by(self.driver.find_elements_by_tag_name, tag)

    def find_by_value(self, value):
        return self.find_by_xpath('//*[@value="%s"]' % value, original_find='value', original_query=value)

    def find_by_id(self, id):
        return self.find_by(self.driver.find_element_by_id, id)

    def fill(self, name, value):
        field = self.find_by_name(name).first
        field.value = value

    fill_in = warn_deprecated(fill, 'fill_in')
    attach_file = fill

    def type(self, name, value, slowly=False):
        element = self.driver.find_element_by_css_selector('input[name="%s"]' % name)
        if slowly:
            return TypeIterator(element, value)
        element.send_keys(value)
        return value

    def choose(self, name, value):
        fields = self.find_by_name(name)
        for field in fields:
            if field.value == value:
                field.click()

    def check(self, name):
        field = self.find_by_name(name).first
        field.check()

    def uncheck(self, name):
        field = self.find_by_name(name).first
        field.uncheck()

    def select(self, name, value):
        element = self.find_by_xpath('//select[@name="%s"]/option[@value="%s"]' % (name, value)).first._element
        element.click()

    def quit(self):
        self.driver.quit()

    @property
    def cookies(self):
        return self._cookie_manager

class TypeIterator(object):

    def __init__(self, element, keys):
        self._element = element
        self._keys = keys

    def __iter__(self):
        for key in self._keys:
            self._element.send_keys(key)
            yield key

class WebDriverElement(ElementAPI):

    def __init__(self, element, parent):
        self._element = element
        self.parent = parent

    def _get_value(self):
        value = self["value"]
        if value:
            return value
        else:
            return self._element.text

    def _set_value(self, value):
        self._element.clear()
        self._element.send_keys(value)

    value = property(_get_value, _set_value)

    @property
    def text(self):
        return self._element.text

    def click(self):
        self._element.click()

    def check(self):
        if not self.checked:
            self._element.click()

    def uncheck(self):
        if self.checked:
            self._element.click()

    @property
    def checked(self):
        return self._element.is_selected()

    selected = checked

    @property
    def visible(self):
        return self._element.is_displayed()

    def find_by_css(self, selector, original_find=None, original_query=None):
        find_by = original_find or 'css'
        query = original_query or selector

        elements = self._element.find_elements_by_css_selector(selector)
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by=find_by, query=query)

    def find_by_xpath(self, selector):
        elements = ElementList(self._element.find_elements_by_xpath(selector))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='xpath', query=selector)

    def find_by_name(self, name):
        elements = ElementList(self._element.find_elements_by_name(name))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='name', query=name)

    def find_by_tag(self, tag):
        elements = ElementList(self._element.find_elements_by_tag_name(tag))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='tag', query=tag)

    def find_by_value(self, value):
        selector = '[value="%s"]' % value
        return self.find_by_css(selector, original_find='value', original_query=value)

    def find_by_id(self, id):
        elements = ElementList(self._element.find_elements_by_id(id))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='id', query=id)

    def __getitem__(self, attr):
        return self._element.get_attribute(attr)

class AlertElement(object):

    def __init__(self, alert):
        self._alert = alert
        self.text = alert.text

    def accept(self):
        self._alert.accept()

    def dismiss(self):
        self._alert.dismiss()

    def fill_with(self, text):
        self._alert.send_keys(text)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

<commit_msg>mouseover for webdriver (currently works only for Chrome)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import subprocess
import time
from contextlib import contextmanager

from lxml.cssselect import CSSSelector
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains

from splinter.driver import DriverAPI, ElementAPI
from splinter.element_list import ElementList
from splinter.utils import warn_deprecated
from tempfile import TemporaryFile

class BaseWebDriver(DriverAPI):
    old_popen = subprocess.Popen

    def __init__(self, wait_time=2):
        self.wait_time = wait_time

    def _patch_subprocess(self):
        loggers_to_silence = [
            'selenium.webdriver.firefox.extension_connection',
            'selenium.webdriver.remote.remote_connection',
            'selenium.webdriver.remote.utils',
        ]

        class MutedHandler(logging.Handler):
            pass

        for name in loggers_to_silence:
            logger = logging.getLogger(name)
            logger.addHandler(MutedHandler())
            logger.setLevel(99999)

        # selenium is such a verbose guy let's make it open the
        # browser without showing all the meaningless output
        def MyPopen(*args, **kw):
            kw['stdout'] = TemporaryFile()
            kw['stderr'] = TemporaryFile()
            kw['close_fds'] = True
            return self.old_popen(*args, **kw)

        subprocess.Popen = MyPopen

    def _unpatch_subprocess(self):
        # cleaning up the house
        subprocess.Popen = self.old_popen

    @property
    def title(self):
        return self.driver.title

    @property
    def html(self):
        return self.driver.page_source

    @property
    def url(self):
        return self.driver.current_url

    def visit(self, url):
        self.connect(url)
        self.ensure_success_response()
        self.driver.get(url)

    def back(self):
        self.driver.back()

    def forward(self):
        self.driver.forward()

    def reload(self):
        self.driver.refresh()

    def execute_script(self, script):
        self.driver.execute_script(script)

    def evaluate_script(self, script):
        return self.driver.execute_script("return %s" % script)

    def is_element_present(self, finder, selector, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            if finder(selector):
                return True
        return False

    def is_element_not_present(self, finder, selector, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            if not finder(selector):
                return True
        return False

    def is_element_present_by_css(self, css_selector, wait_time=None):
        return self.is_element_present(self.find_by_css, css_selector, wait_time)

    is_element_present_by_css_selector = warn_deprecated(is_element_present_by_css, 'is_element_present_by_css_selector')

    def is_element_not_present_by_css(self, css_selector, wait_time=None):
        return self.is_element_not_present(self.find_by_css, css_selector, wait_time)

    is_element_not_present_by_css_selector = warn_deprecated(is_element_not_present_by_css, 'is_element_not_present_by_css_selector')

    def is_element_present_by_xpath(self, xpath, wait_time=None):
        return self.is_element_present(self.find_by_xpath, xpath, wait_time)

    def is_element_not_present_by_xpath(self, xpath, wait_time=None):
        return self.is_element_not_present(self.find_by_xpath, xpath, wait_time)

    def is_element_present_by_tag(self, tag, wait_time=None):
        return self.is_element_present(self.find_by_tag, tag, wait_time)

    def is_element_not_present_by_tag(self, tag, wait_time=None):
        return self.is_element_not_present(self.find_by_tag, tag, wait_time)

    def is_element_present_by_name(self, name, wait_time=None):
        return self.is_element_present(self.find_by_name, name, wait_time)

    def is_element_not_present_by_name(self, name, wait_time=None):
        return self.is_element_not_present(self.find_by_name, name, wait_time)

    def is_element_present_by_value(self, value, wait_time=None):
        return self.is_element_present(self.find_by_value, value, wait_time)

    def is_element_not_present_by_value(self, value, wait_time=None):
        return self.is_element_not_present(self.find_by_value, value, wait_time)

    def is_element_present_by_id(self, id, wait_time=None):
        return self.is_element_present(self.find_by_id, id, wait_time)

    def is_element_not_present_by_id(self, id, wait_time=None):
        return self.is_element_not_present(self.find_by_id, id, wait_time)

    def get_alert(self):
        return AlertElement(self.driver.switch_to_alert())

    def is_text_present(self, text, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            try:
                self.driver.find_element_by_tag_name('body').text.index(text)
                return True
            except ValueError:
                pass
        return False

    def is_text_not_present(self, text, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time

        while time.time() < end_time:
            try:
                self.driver.find_element_by_tag_name('body').text.index(text)
            except ValueError:
                return True
        return False

    @contextmanager
    def get_iframe(self, id):
        self.driver.switch_to_frame(id)
        try:
            yield self
        finally:
            self.driver.switch_to_frame(None)

    def find_option_by_value(self, value):
        return self.find_by_xpath('//option[@value="%s"]' % value, original_find="option by value", original_query=value)

    def find_option_by_text(self, text):
        return self.find_by_xpath('//option[normalize-space(text())="%s"]' % text, original_find="option by text", original_query=text)

    def find_link_by_href(self, href):
        return self.find_by_xpath('//a[@href="%s"]' % href, original_find="link by href", original_query=href)

    def find_link_by_partial_href(self, partial_href):
        return self.find_by_xpath('//a[contains(@href, "%s")]' % partial_href, original_find="link by partial href", original_query=partial_href)

    def find_link_by_partial_text(self, partial_text):
        return ElementList([self.element_class(element, self) for element in self.driver.find_elements_by_partial_link_text(partial_text)], find_by="partial text", query=partial_text)

    def find_link_by_text(self, text):
        return ElementList([self.element_class(element, self) for element in self.driver.find_elements_by_link_text(text)], find_by="link by text", query=text)

    def find_by(self, finder, selector, original_find=None, original_query=None):
        elements = None
        end_time = time.time() + self.wait_time

        func_name = finder.im_func.func_name
        find_by = original_find or func_name[func_name.rfind('_by_') + 4:]
        query = original_query or selector

        while time.time() < end_time:
            try:
                elements = finder(selector)
                if not isinstance(elements, list):
                    elements = [elements]
            except NoSuchElementException:
                pass

            if elements:
                return ElementList([self.element_class(element, self) for element in elements], find_by=find_by, query=query)
        return ElementList([], find_by=find_by, query=query)

    def find_by_css(self, css_selector):
        selector = CSSSelector(css_selector)
        return self.find_by(self.driver.find_elements_by_xpath, selector.path, original_find='css', original_query=css_selector)

    find_by_css_selector = warn_deprecated(find_by_css, 'find_by_css_selector')

    def find_by_xpath(self, xpath, original_find=None, original_query=None):
        original_find = original_find or "xpath"
        original_query = original_query or xpath
        return self.find_by(self.driver.find_elements_by_xpath, xpath, original_find=original_find, original_query=original_query)

    def find_by_name(self, name):
        return self.find_by(self.driver.find_elements_by_name, name)

    def find_by_tag(self, tag):
        return self.find_by(self.driver.find_elements_by_tag_name, tag)

    def find_by_value(self, value):
        return self.find_by_xpath('//*[@value="%s"]' % value, original_find='value', original_query=value)

    def find_by_id(self, id):
        return self.find_by(self.driver.find_element_by_id, id)

    def fill(self, name, value):
        field = self.find_by_name(name).first
        field.value = value

    fill_in = warn_deprecated(fill, 'fill_in')
    attach_file = fill

    def type(self, name, value, slowly=False):
        element = self.driver.find_element_by_css_selector('input[name="%s"]' % name)
        if slowly:
            return TypeIterator(element, value)
        element.send_keys(value)
        return value

    def choose(self, name, value):
        fields = self.find_by_name(name)
        for field in fields:
            if field.value == value:
                field.click()

    def check(self, name):
        field = self.find_by_name(name).first
        field.check()

    def uncheck(self, name):
        field = self.find_by_name(name).first
        field.uncheck()

    def select(self, name, value):
        element = self.find_by_xpath('//select[@name="%s"]/option[@value="%s"]' % (name, value)).first._element
        element.click()

    def quit(self):
        self.driver.quit()

    @property
    def cookies(self):
        return self._cookie_manager

class TypeIterator(object):

    def __init__(self, element, keys):
        self._element = element
        self._keys = keys

    def __iter__(self):
        for key in self._keys:
            self._element.send_keys(key)
            yield key

class WebDriverElement(ElementAPI):

    def __init__(self, element, parent):
        self._element = element
        self.parent = parent
        self.action_chains = ActionChains(parent.driver)

    def _get_value(self):
        value = self["value"]
        if value:
            return value
        else:
            return self._element.text

    def _set_value(self, value):
        self._element.clear()
        self._element.send_keys(value)

    value = property(_get_value, _set_value)

    @property
    def text(self):
        return self._element.text

    def click(self):
        self._element.click()

    def check(self):
        if not self.checked:
            self._element.click()

    def uncheck(self):
        if self.checked:
            self._element.click()

    @property
    def checked(self):
        return self._element.is_selected()

    selected = checked

    @property
    def visible(self):
        return self._element.is_displayed()

    def find_by_css(self, selector, original_find=None, original_query=None):
        find_by = original_find or 'css'
        query = original_query or selector

        elements = self._element.find_elements_by_css_selector(selector)
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by=find_by, query=query)

    def find_by_xpath(self, selector):
        elements = ElementList(self._element.find_elements_by_xpath(selector))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='xpath', query=selector)

    def find_by_name(self, name):
        elements = ElementList(self._element.find_elements_by_name(name))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='name', query=name)

    def find_by_tag(self, tag):
        elements = ElementList(self._element.find_elements_by_tag_name(tag))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='tag', query=tag)

    def find_by_value(self, value):
        selector = '[value="%s"]' % value
        return self.find_by_css(selector, original_find='value', original_query=value)

    def find_by_id(self, id):
        elements = ElementList(self._element.find_elements_by_id(id))
        return ElementList([self.__class__(element, self.parent) for element in elements], find_by='id', query=id)

    def mouseover(self):
        self.action_chains.move_to_element(self._element)
        self.action_chains.perform()

    def __getitem__(self, attr):
        return self._element.get_attribute(attr)

class AlertElement(object):

    def __init__(self, alert):
        self._alert = alert
        self.text = alert.text

    def accept(self):
        self._alert.accept()

    def dismiss(self):
        self._alert.dismiss()

    def fill_with(self, text):
        self._alert.send_keys(text)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

<|endoftext|>


<commit_before><commit_msg>Do a component-wise multiply for scaling instead of flat addition<commit_after><|endoftext|>


<commit_before>from os.path import abspath, basename, join, exists
from os import listdir, sep
from re import findall

class JobInputs(object):
    """
    Abstractions over dynamic inputs created for a given job (namely the command to
    execute and created configfiles).

    **Parameters**

    command_line : str
        Local command to execute for this job. (To be rewritten.)
    config_files : str
        Config files created for this job. (To be rewritten.)

    >>> import tempfile
    >>> tf = tempfile.NamedTemporaryFile()
    >>> def setup_inputs(tf):
    ...     open(tf.name, "w").write("world /path/to/input the rest")
    ...     inputs = JobInputs("hello /path/to/input", [tf.name])
    ...     return inputs
    >>> inputs = setup_inputs(tf)
    >>> inputs.rewrite_paths("/path/to/input", 'C:\\input')
    >>> inputs.rewritten_command_line
    'hello C:\\\\input'
    >>> inputs.rewritten_config_files[tf.name]
    'world C:\\\\input the rest'
    >>> tf.close()
    >>> tf = tempfile.NamedTemporaryFile()
    >>> inputs = setup_inputs(tf)
    >>> inputs.find_referenced_subfiles('/path/to')
    ['/path/to/input']
    >>> inputs.path_referenced('/path/to')
    True
    >>> inputs.path_referenced('/path/to/input')
    True
    >>> inputs.path_referenced('/path/to/notinput')
    False
    >>> tf.close()
    """

    def __init__(self, command_line, config_files):
        self.rewritten_command_line = command_line
        self.rewritten_config_files = {}
        for config_file in config_files or []:
            config_contents = _read(config_file)
            self.rewritten_config_files[config_file] = config_contents

    def find_referenced_subfiles(self, directory):
        """
        Return list of files below specified `directory` in job inputs. Could
        use more sophisticated logic (match quotes to handle spaces, handle
        subdirectories, etc...).

        **Parameters**

        directory : str
            Full path to directory to search.

        """
        pattern = r"(%s%s\S+)" % (directory, sep)
        referenced_files = set()
        for input_contents in self.__items():
            referenced_files.update(findall(pattern, input_contents))
        return list(referenced_files)

    def path_referenced(self, path):
        pattern = r"%s" % path
        found = False
        for input_contents in self.__items():
            if findall(pattern, input_contents):
                found = True
                break
        return found

    def rewrite_paths(self, local_path, remote_path):
        """
        Rewrite references to `local_path` with  `remote_path` in job inputs.
        """
        self.__rewrite_command_line(local_path, remote_path)
        self.__rewrite_config_files(local_path, remote_path)

    def __rewrite_command_line(self, local_path, remote_path):
        self.rewritten_command_line = self.rewritten_command_line.replace(local_path, remote_path)

    def __rewrite_config_files(self, local_path, remote_path):
        for config_file, rewritten_contents in self.rewritten_config_files.iteritems():
            self.rewritten_config_files[config_file] = rewritten_contents.replace(local_path, remote_path)

    def __items(self):
        items = [self.rewritten_command_line]
        items.extend(self.rewritten_config_files.values())
        return items

class TransferTracker(object):

    def __init__(self, client, job_inputs):
        self.client = client
        self.job_inputs = job_inputs
        self.file_renames = {}

    def action(self, path, type):
        return ('transfer',)

    def register_rewrite(self, local_path, remote_path):
        self.file_renames[local_path] = remote_path

    def rewrite_input_paths(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        for local_path, remote_path in self.file_renames.iteritems():
            self.job_inputs.rewrite_paths(local_path, remote_path)

class FileStager(object):
    """
    Objects of the FileStager class interact with an LWR client object to
    stage the files required to run jobs on a remote LWR server.

    **Parameters**

    client : Client
        LWR client object.
    command_line : str
        The local command line to execute, this will be rewritten for the remote server.
    config_files : list
        List of Galaxy 'configfile's produced for this job. These will be rewritten and sent to remote server.
    input_files :  list
        List of input files used by job. These will be transferred and references rewritten.
    output_files : list
        List of output_files produced by job.
    tool_dir : str
        Directory containing tool to execute (if a wrapper is used, it will be transferred to remote server).
    working_directory : str
        Local path created by Galaxy for running this job.

    """

    def __init__(self, client, tool, command_line, config_files, input_files, output_files, working_directory):
        """
        """
        self.client = client
        self.command_line = command_line
        self.config_files = config_files
        self.input_files = input_files
        self.output_files = output_files
        self.tool_id = tool.id
        self.tool_version = tool.version
        self.tool_dir = abspath(tool.tool_dir)
        self.working_directory = working_directory

        # Setup job inputs, these will need to be rewritten before
        # shipping off to remote LWR server.
        self.job_inputs = JobInputs(self.command_line, self.config_files)

        self.transfer_tracker = TransferTracker(client, self.job_inputs)

        self.__handle_setup()
        self.__initialize_referenced_tool_files()
        self.__upload_tool_files()
        self.__upload_input_files()
        self.__upload_working_directory_files()
        self.__initialize_output_file_renames()
        self.__initialize_task_output_file_renames()
        self.__initialize_config_file_renames()
        self.__handle_rewrites()
        self.__upload_rewritten_config_files()

    def __handle_setup(self):
        job_config = self.client.setup(self.tool_id, self.tool_version)

        self.new_working_directory = job_config['working_directory']
        self.new_outputs_directory = job_config['outputs_directory']
        # Default configs_directory to match remote working_directory to mimic
        # behavior of older LWR servers.
        self.new_configs_drectory = job_config.get('configs_directory', self.new_working_directory)
        self.remote_path_separator = job_config['path_separator']
        # If remote LWR server assigned job id, use that otherwise
        # just use local job_id assigned.
        galaxy_job_id = self.client.job_id
        self.job_id = job_config.get('job_id', galaxy_job_id)
        if self.job_id != galaxy_job_id:
            # Remote LWR server assigned an id different than the
            # Galaxy job id, update client to reflect this.
            self.client.job_id = self.job_id

    def __initialize_referenced_tool_files(self):
        self.referenced_tool_files = self.job_inputs.find_referenced_subfiles(self.tool_dir)

    def __upload_tool_files(self):
        for referenced_tool_file in self.referenced_tool_files:
            action = self.transfer_tracker.action(referenced_tool_file, 'tool_file')
            if action[0] == 'transfer':
                tool_upload_response = self.client.upload_tool_file(referenced_tool_file)
                self.transfer_tracker.register_rewrite(referenced_tool_file, tool_upload_response['path'])

    def __upload_input_files(self):
        for input_file in self.input_files:
            self.__upload_input_file(input_file)
            self.__upload_input_extra_files(input_file)

    def __upload_input_file(self, input_file):
        if self.job_inputs.path_referenced(input_file):
            action = self.transfer_tracker.action(input_file, 'input')
            if action[0] == 'transfer':
                input_upload_response = self.client.upload_input(input_file)
                self.transfer_tracker.register_rewrite(input_file, input_upload_response['path'])

    def __upload_input_extra_files(self, input_file):
        # TODO: Determine if this is object store safe and what needs to be
        # done if it is not.
        files_path = "%s_files" % input_file[0:-len(".dat")]
        if exists(files_path) and self.job_inputs.path_referenced(files_path):
            for extra_file in listdir(files_path):
                extra_file_path = join(files_path, extra_file)
                relative_path = basename(files_path)
                extra_file_relative_path = join(relative_path, extra_file)
                action = self.transfer_tracker.action(input_file, 'input_extra')
                if action[0] == 'transfer':
                    response = self.client.upload_extra_input(extra_file_path, extra_file_relative_path)
                    self.transfer_tracker.register_rewrite(extra_file_path, response['path'])

    def __upload_working_directory_files(self):
        # Task manager stages files into working directory, these need to be
        # uploaded if present.
        for working_directory_file in listdir(self.working_directory):
            path = join(self.working_directory, working_directory_file)
            action = self.transfer_tracker.action(path, 'working')
            if action[0] == 'transfer':
                working_file_response = self.client.upload_working_directory_file(path)
                self.transfer_tracker.register_rewrite(path, working_file_response['path'])

    def __initialize_output_file_renames(self):
        for output_file in self.output_files:
            remote_path = r'%s%s%s' % (self.new_outputs_directory, self.remote_path_separator, basename(output_file))
            self.transfer_tracker.register_rewrite(output_file, remote_path)

    def __initialize_task_output_file_renames(self):
        for output_file in self.output_files:
            name = basename(output_file)
            task_file = join(self.working_directory, name)
            remote_path = r'%s%s%s' % (self.new_working_directory, self.remote_path_separator, name)
            self.transfer_tracker.register_rewrite(task_file, remote_path)

    def __initialize_config_file_renames(self):
        for config_file in self.config_files:
            remote_path = r'%s%s%s' % (self.new_configs_drectory, self.remote_path_separator, basename(config_file))
            self.transfer_tracker.register_rewrite(config_file, remote_path)

    def __handle_rewrites(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        self.transfer_tracker.rewrite_input_paths()

    def __upload_rewritten_config_files(self):
        for config_file, new_config_contents in self.job_inputs.rewritten_config_files.iteritems():
            self.client.upload_config_file(config_file, new_config_contents)

    def get_rewritten_command_line(self):
        """
        Returns the rewritten version of the command line to execute suitable
        for remote host.
        """
        return self.job_inputs.rewritten_command_line

def _read(path):
    """
    Utility method to quickly read small files (config files and tool
    wrappers) into memory as strings.
    """
    input = open(path, "r")
    try:
        return input.read()
    finally:
        input.close()

__all__ = [FileStager]
<commit_msg>Remove duplicated code in previous FileStager refactoring.<commit_after>from os.path import abspath, basename, join, exists
from os import listdir, sep
from re import findall

class JobInputs(object):
    """
    Abstractions over dynamic inputs created for a given job (namely the command to
    execute and created configfiles).

    **Parameters**

    command_line : str
        Local command to execute for this job. (To be rewritten.)
    config_files : str
        Config files created for this job. (To be rewritten.)

    >>> import tempfile
    >>> tf = tempfile.NamedTemporaryFile()
    >>> def setup_inputs(tf):
    ...     open(tf.name, "w").write("world /path/to/input the rest")
    ...     inputs = JobInputs("hello /path/to/input", [tf.name])
    ...     return inputs
    >>> inputs = setup_inputs(tf)
    >>> inputs.rewrite_paths("/path/to/input", 'C:\\input')
    >>> inputs.rewritten_command_line
    'hello C:\\\\input'
    >>> inputs.rewritten_config_files[tf.name]
    'world C:\\\\input the rest'
    >>> tf.close()
    >>> tf = tempfile.NamedTemporaryFile()
    >>> inputs = setup_inputs(tf)
    >>> inputs.find_referenced_subfiles('/path/to')
    ['/path/to/input']
    >>> inputs.path_referenced('/path/to')
    True
    >>> inputs.path_referenced('/path/to/input')
    True
    >>> inputs.path_referenced('/path/to/notinput')
    False
    >>> tf.close()
    """

    def __init__(self, command_line, config_files):
        self.rewritten_command_line = command_line
        self.rewritten_config_files = {}
        for config_file in config_files or []:
            config_contents = _read(config_file)
            self.rewritten_config_files[config_file] = config_contents

    def find_referenced_subfiles(self, directory):
        """
        Return list of files below specified `directory` in job inputs. Could
        use more sophisticated logic (match quotes to handle spaces, handle
        subdirectories, etc...).

        **Parameters**

        directory : str
            Full path to directory to search.

        """
        pattern = r"(%s%s\S+)" % (directory, sep)
        referenced_files = set()
        for input_contents in self.__items():
            referenced_files.update(findall(pattern, input_contents))
        return list(referenced_files)

    def path_referenced(self, path):
        pattern = r"%s" % path
        found = False
        for input_contents in self.__items():
            if findall(pattern, input_contents):
                found = True
                break
        return found

    def rewrite_paths(self, local_path, remote_path):
        """
        Rewrite references to `local_path` with  `remote_path` in job inputs.
        """
        self.__rewrite_command_line(local_path, remote_path)
        self.__rewrite_config_files(local_path, remote_path)

    def __rewrite_command_line(self, local_path, remote_path):
        self.rewritten_command_line = self.rewritten_command_line.replace(local_path, remote_path)

    def __rewrite_config_files(self, local_path, remote_path):
        for config_file, rewritten_contents in self.rewritten_config_files.iteritems():
            self.rewritten_config_files[config_file] = rewritten_contents.replace(local_path, remote_path)

    def __items(self):
        items = [self.rewritten_command_line]
        items.extend(self.rewritten_config_files.values())
        return items

class TransferTracker(object):

    def __init__(self, client, job_inputs):
        self.client = client
        self.job_inputs = job_inputs
        self.file_renames = {}

    def handle_transfer(self, path, type, transfer_action):
        action = self.__action(path, type)
        if action[0] == 'transfer':
            response = transfer_action(path)
            self.register_rewrite(path, response['path'])

    def register_rewrite(self, local_path, remote_path):
        self.file_renames[local_path] = remote_path

    def rewrite_input_paths(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        for local_path, remote_path in self.file_renames.iteritems():
            self.job_inputs.rewrite_paths(local_path, remote_path)

    def __action(self, path, type):
        return ('transfer',)

class FileStager(object):
    """
    Objects of the FileStager class interact with an LWR client object to
    stage the files required to run jobs on a remote LWR server.

    **Parameters**

    client : Client
        LWR client object.
    command_line : str
        The local command line to execute, this will be rewritten for the remote server.
    config_files : list
        List of Galaxy 'configfile's produced for this job. These will be rewritten and sent to remote server.
    input_files :  list
        List of input files used by job. These will be transferred and references rewritten.
    output_files : list
        List of output_files produced by job.
    tool_dir : str
        Directory containing tool to execute (if a wrapper is used, it will be transferred to remote server).
    working_directory : str
        Local path created by Galaxy for running this job.

    """

    def __init__(self, client, tool, command_line, config_files, input_files, output_files, working_directory):
        """
        """
        self.client = client
        self.command_line = command_line
        self.config_files = config_files
        self.input_files = input_files
        self.output_files = output_files
        self.tool_id = tool.id
        self.tool_version = tool.version
        self.tool_dir = abspath(tool.tool_dir)
        self.working_directory = working_directory

        # Setup job inputs, these will need to be rewritten before
        # shipping off to remote LWR server.
        self.job_inputs = JobInputs(self.command_line, self.config_files)

        self.transfer_tracker = TransferTracker(client, self.job_inputs)

        self.__handle_setup()
        self.__initialize_referenced_tool_files()
        self.__upload_tool_files()
        self.__upload_input_files()
        self.__upload_working_directory_files()
        self.__initialize_output_file_renames()
        self.__initialize_task_output_file_renames()
        self.__initialize_config_file_renames()
        self.__handle_rewrites()
        self.__upload_rewritten_config_files()

    def __handle_setup(self):
        job_config = self.client.setup(self.tool_id, self.tool_version)

        self.new_working_directory = job_config['working_directory']
        self.new_outputs_directory = job_config['outputs_directory']
        # Default configs_directory to match remote working_directory to mimic
        # behavior of older LWR servers.
        self.new_configs_drectory = job_config.get('configs_directory', self.new_working_directory)
        self.remote_path_separator = job_config['path_separator']
        # If remote LWR server assigned job id, use that otherwise
        # just use local job_id assigned.
        galaxy_job_id = self.client.job_id
        self.job_id = job_config.get('job_id', galaxy_job_id)
        if self.job_id != galaxy_job_id:
            # Remote LWR server assigned an id different than the
            # Galaxy job id, update client to reflect this.
            self.client.job_id = self.job_id

    def __initialize_referenced_tool_files(self):
        self.referenced_tool_files = self.job_inputs.find_referenced_subfiles(self.tool_dir)

    def __upload_tool_files(self):
        for referenced_tool_file in self.referenced_tool_files:
            self.transfer_tracker.handle_transfer(referenced_tool_file, 'tool_file', self.client.upload_tool_file)

    def __upload_input_files(self):
        for input_file in self.input_files:
            self.__upload_input_file(input_file)
            self.__upload_input_extra_files(input_file)

    def __upload_input_file(self, input_file):
        if self.job_inputs.path_referenced(input_file):
            self.transfer_tracker.handle_transfer(input_file, 'input', self.client.upload_input)

    def __upload_input_extra_files(self, input_file):
        # TODO: Determine if this is object store safe and what needs to be
        # done if it is not.
        files_path = "%s_files" % input_file[0:-len(".dat")]
        if exists(files_path) and self.job_inputs.path_referenced(files_path):
            for extra_file in listdir(files_path):
                extra_file_path = join(files_path, extra_file)
                relative_path = basename(files_path)
                extra_file_relative_path = join(relative_path, extra_file)
                transfer_action = lambda extra_file_path: self.client.upload_extra_input(extra_file_path, extra_file_relative_path)
                self.transfer_tracker.handle_transfer(extra_file_path, 'input_extra', transfer_action)

    def __upload_working_directory_files(self):
        # Task manager stages files into working directory, these need to be
        # uploaded if present.
        for working_directory_file in listdir(self.working_directory):
            path = join(self.working_directory, working_directory_file)
            self.transfer_tracker.handle_transfer(path, 'working', self.client.upload_working_directory_file)

    def __initialize_output_file_renames(self):
        for output_file in self.output_files:
            remote_path = r'%s%s%s' % (self.new_outputs_directory, self.remote_path_separator, basename(output_file))
            self.transfer_tracker.register_rewrite(output_file, remote_path)

    def __initialize_task_output_file_renames(self):
        for output_file in self.output_files:
            name = basename(output_file)
            task_file = join(self.working_directory, name)
            remote_path = r'%s%s%s' % (self.new_working_directory, self.remote_path_separator, name)
            self.transfer_tracker.register_rewrite(task_file, remote_path)

    def __initialize_config_file_renames(self):
        for config_file in self.config_files:
            remote_path = r'%s%s%s' % (self.new_configs_drectory, self.remote_path_separator, basename(config_file))
            self.transfer_tracker.register_rewrite(config_file, remote_path)

    def __handle_rewrites(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        self.transfer_tracker.rewrite_input_paths()

    def __upload_rewritten_config_files(self):
        for config_file, new_config_contents in self.job_inputs.rewritten_config_files.iteritems():
            self.client.upload_config_file(config_file, new_config_contents)

    def get_rewritten_command_line(self):
        """
        Returns the rewritten version of the command line to execute suitable
        for remote host.
        """
        return self.job_inputs.rewritten_command_line

def _read(path):
    """
    Utility method to quickly read small files (config files and tool
    wrappers) into memory as strings.
    """
    input = open(path, "r")
    try:
        return input.read()
    finally:
        input.close()

__all__ = [FileStager]
<|endoftext|>


<commit_before>"""Calculating \\(\\pi\\) the simple way, but this is called
dummypi to avoid conflict with simplepi in the databench_examples repo."""

import math
from time import sleep
from random import random

import databench

class Analysis(databench.Analysis):

    def __init__(self):
        self.samples = 500

    def on_run(self):
        """Run when button is pressed."""

        inside = 0
        for i in xrange(self.samples):
            sleep(0.001)
            r1, r2 = (random(), random())
            if r1*r1 + r2*r2 < 1.0:
                inside += 1

            if (i+1) % 100 == 0:
                draws = i+1
                self.emit('log', {
                    'draws': draws,
                    'inside': inside,
                    'r1': r1,
                    'r2': r2,
                })

                p = float(inside)/draws
                uncertainty = 4.0*math.sqrt(draws*p*(1.0 - p)) / draws
                self.emit('status', {
                    'pi-estimate': 4.0*inside/draws,
                    'pi-uncertainty': uncertainty
                })

        self.emit('log', {'action': 'done'})

    def on_samples(self, value):
        """Sets the number of samples to generate per run."""
        self.samples = value

    def on_test_fn(self, first_param, second_param=100):
        """Echo params."""
        print(first_param, second_param)
        self.emit('test_fn', {
            'first_param': first_param,
            'second_param': second_param,
        })

META = databench.Meta('dummypi', __name__, __doc__, Analysis)
<commit_msg>python3<commit_after>"""Calculating \\(\\pi\\) the simple way, but this is called
dummypi to avoid conflict with simplepi in the databench_examples repo."""

import math
from time import sleep
from random import random

import databench

class Analysis(databench.Analysis):

    def __init__(self):
        self.samples = 500

    def on_run(self):
        """Run when button is pressed."""

        inside = 0
        for i in range(self.samples):
            sleep(0.001)
            r1, r2 = (random(), random())
            if r1*r1 + r2*r2 < 1.0:
                inside += 1

            if (i+1) % 100 == 0:
                draws = i+1
                self.emit('log', {
                    'draws': draws,
                    'inside': inside,
                    'r1': r1,
                    'r2': r2,
                })

                p = float(inside)/draws
                uncertainty = 4.0*math.sqrt(draws*p*(1.0 - p)) / draws
                self.emit('status', {
                    'pi-estimate': 4.0*inside/draws,
                    'pi-uncertainty': uncertainty
                })

        self.emit('log', {'action': 'done'})

    def on_samples(self, value):
        """Sets the number of samples to generate per run."""
        self.samples = value

    def on_test_fn(self, first_param, second_param=100):
        """Echo params."""
        print(first_param, second_param)
        self.emit('test_fn', {
            'first_param': first_param,
            'second_param': second_param,
        })

META = databench.Meta('dummypi', __name__, __doc__, Analysis)
<|endoftext|>


<commit_before>from pprint import pprint

from django.contrib.auth.mixins import LoginRequiredMixin

from fo2.connections import db_cursor_so

from base.views import O2BaseGetPostView
from utils.classes import TermalPrint
from utils.table_defs import TableDefs

import lotes.models

from logistica.forms.nf import NfForm
from logistica.queries.etiqueta_nf import get_dados_nf

class EtiquetaNf(LoginRequiredMixin, O2BaseGetPostView):

    def __init__(self, *args, **kwargs):
        super(EtiquetaNf, self).__init__(*args, **kwargs)
        self.Form_class = NfForm
        self.template_name = 'logistica/etiqueta_nf.html'
        self.title_name = 'Etiqueta de NF'
        self.cleaned_data2self = True

        self.init_defs()

    def init_defs(self):
        self.col_defs = TableDefs(
            {
                'empr_nome': ["Empresa"],
                'nf': ["NF"],
                'vols': ["Volumes", 'c'],
                'peso_tot': ["Peso Total"],
                'ped': ["Pedido Tussor"],
                'ped_cli': ["Pedido do cliente"],
                'transp_nome': ["Transportadora"],
                'cli_nome': ["Cliente"],
                'ende': ["Endereço"],
                'end_num': ["Número"],
                'end_compl': ["Complemento"],
                'end_bairro': ["Bairro"],
                'end_cid': ["Cidade"],
                'end_uf': ["UF"],
                'cli_cep': ["CEP"],
            },
            ['header', '+style'],
            style = {'_': 'text-align'},
        )

    def mount_context(self):
        cursor = db_cursor_so(self.request)

        dados_nf = get_dados_nf(cursor, self.nf)
        if len(dados_nf) == 0:
            self.context.update({
                'msg_erro': 'NF não encontrada',
            })
            return
        row = dados_nf[0]

        vol_inicial_val = int(self.vol_inicial) if self.vol_inicial else 1
        vol_final_val = int(self.vol_final) if self.vol_final else row['vols']

        if not (1 <= vol_inicial_val <= row['vols']):
            self.context.update({
                'msg_erro': 'Caixa inicial inválida',
            })
            return

        if not (vol_inicial_val <= vol_final_val <= row['vols']):
            self.context.update({
                'msg_erro': 'Caixa final inválida',
            })
            return

        qtd_vols = vol_final_val - vol_inicial_val + 1

        self.context.update(self.col_defs.hfs_dict())
        self.context.update({
            'data': dados_nf,
            'qtd_vols': qtd_vols,
        })

        if 'print' not in self.request.POST:
            return

        nome_impresso = 'etiqueta_nf'
        try:
            impresso = lotes.models.Impresso.objects.get(
                nome=nome_impresso)
        except lotes.models.Impresso.DoesNotExist:
            self.context.update({
                'msg_erro': 'Impresso não cadastrado',
            })
            return

        try:
            usuario_impresso = lotes.models.UsuarioImpresso.objects.get(
                usuario=self.request.user, impresso=impresso)
        except lotes.models.UsuarioImpresso.DoesNotExist:
            self.context.update({
                'msg_erro': 'Impresso não cadastrado para o usuário',
            })
            return

        teg = TermalPrint(
            usuario_impresso.impressora_termica.nome,
            file_dir=f"impresso/{nome_impresso}/%Y/%m"
        )
        teg.template(usuario_impresso.modelo.gabarito, '\r\n')
        teg.printer_start()
        try:
            for i in range(row['vols']):
                row['vol'] = i + 1
                teg.context(row)
                try:
                    teg.printer_send()
                except Exception as e:
                    self.context.update({
                        'msg_erro': f'Erro ao imprimir <{e}>',
                    })
                    return
        except Exception as e:
            raise e
        finally:
            teg.printer_end()
<commit_msg>calculando campos a mais necessários na impressão e outras melhorias<commit_after>from pprint import pprint

from django.contrib.auth.mixins import LoginRequiredMixin

from fo2.connections import db_cursor_so

from base.views import O2BaseGetPostView
from utils.classes import TermalPrint
from utils.table_defs import TableDefs

import lotes.models

from logistica.forms.nf import NfForm
from logistica.queries.etiqueta_nf import get_dados_nf

class EtiquetaNf(LoginRequiredMixin, O2BaseGetPostView):

    def __init__(self, *args, **kwargs):
        super(EtiquetaNf, self).__init__(*args, **kwargs)
        self.Form_class = NfForm
        self.template_name = 'logistica/etiqueta_nf.html'
        self.title_name = "Etiqueta de NF"
        self.cleaned_data2self = True

        self.init_defs()

    def init_defs(self):
        self.col_defs = TableDefs(
            {
                'remet_nome': ["Emissor"],
                'remet_cnpj': ["CNPJ"],
                'nf': ["NF"],
                'vols': ["Volumes", 'c'],
                'peso_tot': ["Peso Total"],
                'ped': ["Pedido Tussor"],
                'ped_cli': ["Pedido do cliente"],
                'transp_nome': ["Transportadora"],
                'cli_nome': ["Cliente"],
                'ende': ["Endereço"],
                'end_num': ["Número"],
                'end_compl': ["Complemento"],
                'end_bairro': ["Bairro"],
                'end_cid': ["Cidade"],
                'end_uf': ["UF"],
                'cli_cep': ["CEP"],
            },
            ['header', '+style'],
            style = {'_': 'text-align'},
        )

    def mount_context(self):
        cursor = db_cursor_so(self.request)

        dados_nf = get_dados_nf(cursor, self.nf)
        if len(dados_nf) == 0:
            self.context.update({
                'msg_erro': "NF não encontrada",
            })
            return
        row = dados_nf[0]

        vol_inicial_val = int(self.vol_inicial) if self.vol_inicial else 1
        vol_final_val = int(self.vol_final) if self.vol_final else row['vols']

        if not (1 <= vol_inicial_val <= row['vols']):
            self.context.update({
                'msg_erro': "Caixa inicial inválida",
            })
            return

        if not (vol_inicial_val <= vol_final_val <= row['vols']):
            self.context.update({
                'msg_erro': "Caixa final inválida",
            })
            return

        qtd_vols = vol_final_val - vol_inicial_val + 1

        self.context.update(self.col_defs.hfs_dict())
        self.context.update({
            'data': dados_nf,
            'qtd_vols': qtd_vols,
        })

        if 'print' not in self.request.POST:
            return

        slug_impresso = 'etiqueta-nf'
        # impresso = lotes.models.Impresso.objects.all().values()
        # pprint(impresso)
        try:
            impresso = lotes.models.Impresso.objects.get(
                slug=slug_impresso)
        except lotes.models.Impresso.DoesNotExist:
            self.context.update({
                'msg_erro': f"Impresso '{nome_impresso}' não cadastrado",
            })
            return

        try:
            usuario_impresso = lotes.models.UsuarioImpresso.objects.get(
                usuario=self.request.user, impresso=impresso)
        except lotes.models.UsuarioImpresso.DoesNotExist:
            self.context.update({
                'msg_erro': (
                    f"Impresso '{slug_impresso}' não cadastrado para o usuário '{self.request.user}'"
                ),
            })
            return

        teg = TermalPrint(
            usuario_impresso.impressora_termica.nome,
            file_dir=f"impresso/{slug_impresso}/%Y/%m"
        )
        teg.template(usuario_impresso.modelo.gabarito, '\r\n')
        teg.printer_start()
        try:
            for i in range(row['vols']):
                vol = i + 1
                if vol < vol_inicial_val or vol > vol_inicial_val:
                    continue
                row['svol'] = f"{vol:04d}"
                row['svols'] = f"{row['vols']:04d}"
                peso = row['peso_tot']/row['vols']
                row['peso'] = f"{peso:7.2f}"
                row['nf_num9'] = f"{row['nf_num']:09d}"
                pprint(row)
                teg.context(row)
                try:
                    teg.printer_send()
                except Exception as e:
                    self.context.update({
                        'msg_erro': f"Erro ao imprimir <{e}>",
                    })
                    return
        except Exception as e:
            raise e
        finally:
            teg.printer_end()
<|endoftext|>


<commit_before>from django import template
from django.core.cache import get_cache
from django.conf import settings

from bs4 import BeautifulSoup
import urllib2

register = template.Library()

def fetchFTPs():
    response = urllib2.urlopen('http://porygon.rez', timeout=1).read()
    soup = BeautifulSoup(response)
    ftps = []
    for ftp in soup.select('#ftp_list a'):
        ftps.append({
            'name': ftp.text,
            'link': ftp.get('href')
            })
    return ftps

@register.inclusion_tag('home/ftps_list.html')
def get_ftps():
    cache = get_cache('default')
    ftps = cache.get(settings.KEY_CACHE_FTPS)
    if ftps is None:
        ftps = fetchFTPs()
        # Cache ftps info for 25 min
        cache.set(settings.KEY_CACHE_FTPS, ftps, 60*25)
    return {'ftps': ftps}
<commit_msg>Fixes #30: added guard for ftp information retrieval<commit_after>from django import template
from django.core.cache import get_cache
from django.conf import settings

from bs4 import BeautifulSoup
import urllib2

register = template.Library()

def fetchFTPs():
    ftps = []
    try:
        response = urllib2.urlopen('http://porygon.rez', timeout=1).read()
        soup = BeautifulSoup(response)
        for ftp in soup.select('#ftp_list a'):
            ftps.append({
                'name': ftp.text,
                'link': ftp.get('href')
                })
    except urllib2.URLError:
        pass
    return ftps

@register.inclusion_tag('home/ftps_list.html')
def get_ftps():
    cache = get_cache('default')
    ftps = cache.get(settings.KEY_CACHE_FTPS)
    if ftps is None:
        ftps = fetchFTPs()
        # Cache ftps info for 25 min
        cache.set(settings.KEY_CACHE_FTPS, ftps, 60*25)
    return {'ftps': ftps}
<|endoftext|>


<commit_before>version = "0.0.8"
<commit_msg>bumpoing version number<commit_after>version = "0.0.9"
<|endoftext|>


<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import json

import cidr

from mock import Mock

class BaseClass:
    def setup_method(self, test_method):
        """To setup mock values for http requests"""
        os.environ['BUILD_SPECIFIC_GCLOUD_PROJECT'] = 'random'
        mock_values = {
            cidr.goog_url: open("samplefiles/goog.json"),
            cidr.cloud_url: open("samplefiles/cloud.json"),
        }
        _mock_fns = Mock()
        _mock_fns.side_effect = lambda x: mock_values.get(x)
        if sys.version_info.major == 3:
            cidr.urllib.request.urlopen = _mock_fns
        else:
            cidr.urllib.urlopen = _mock_fns

class TestHttpRequests(BaseClass):
    def test_goog_url(self):
        output = cidr.read_url(cidr.goog_url)
        expected_output = json.loads(open("samplefiles/goog.json").read())
        assert sorted(output.items()) == sorted(expected_output.items())

    def test_cloud_url(self):
        output = cidr.read_url(cidr.cloud_url)
        expected_output = json.loads(open("samplefiles/cloud.json").read())
        assert sorted(output.items()) == sorted(expected_output.items())

class TestMain(BaseClass):
    def test_main(self):
        try:
            import StringIO as io
        except ModuleNotFoundError:
            import io
        capturedOutput = io.StringIO()
        sys.stdout = capturedOutput
        cidr.main()
        sys.stdout = sys.__stdout__
        current_output = capturedOutput.getvalue().strip()
        expected_output = open("samplefiles/output.txt").read().strip()
        assert len(set(expected_output) - set(current_output)) == 0
<commit_msg>minor formatting<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import sys

from mock import Mock

import cidr

class BaseClass:
    def setup_method(self, test_method):
        """To setup mock values for http requests"""
        os.environ["BUILD_SPECIFIC_GCLOUD_PROJECT"] = "random"
        mock_values = {
            cidr.goog_url: open("samplefiles/goog.json"),
            cidr.cloud_url: open("samplefiles/cloud.json"),
        }
        _mock_fns = Mock()
        _mock_fns.side_effect = lambda x: mock_values.get(x)
        if sys.version_info.major == 3:
            cidr.urllib.request.urlopen = _mock_fns
        else:
            cidr.urllib.urlopen = _mock_fns

class TestHttpRequests(BaseClass):
    def test_goog_url(self):
        output = cidr.read_url(cidr.goog_url)
        expected_output = json.loads(open("samplefiles/goog.json").read())
        assert sorted(output.items()) == sorted(expected_output.items())

    def test_cloud_url(self):
        output = cidr.read_url(cidr.cloud_url)
        expected_output = json.loads(open("samplefiles/cloud.json").read())
        assert sorted(output.items()) == sorted(expected_output.items())

class TestMain(BaseClass):
    def test_main(self):
        try:
            import StringIO as io
        except ModuleNotFoundError:
            import io
        capturedOutput = io.StringIO()
        sys.stdout = capturedOutput
        cidr.main()
        sys.stdout = sys.__stdout__
        current_output = capturedOutput.getvalue().strip()
        expected_output = open("samplefiles/output.txt").read().strip()
        assert len(set(expected_output) - set(current_output)) == 0
<|endoftext|>


<commit_before>import os
import shutil
import subprocess
import tempfile
import time

from celery import shared_task

from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.contrib import messages

from excerptconverter.baseexcerptconverter import BaseExcerptConverter
from excerptconverter import ConverterHelper

from osmaxx.excerptexport import models
from osmaxx.utils import private_storage

class GisExcerptConverter(BaseExcerptConverter):
    @staticmethod
    def name():
        return 'GIS'

    @staticmethod
    def export_formats():
        return {
            'spatialite': {
                'name': 'SpatiaLite (SQLite)',
                'file_extension': 'sqlite'
            },
            'gpkg': {
                'name': 'Geo package',
                'file_extension': 'gpkg'
            },
            'shp': {
                'name': 'Shape file',
                'file_extension': 'shp'
            }
        }

    @staticmethod
    def export_options():
        return {
            # has to be implemented next:
            # 'coordinate_reference_system': {
            #     'label': 'Coordinate reference system',
            #     'type': 'choice',
            #     'default': 'pseudomerkator',
            #     'groups': [
            #         {
            #             'name': 'Global coordinate reference systems',
            #             'values': [
            #                 {'name': 'pseudomerkator', 'label': 'Pseudo merkator'},
            #                 {'name': 'wgs72', 'label': 'WGS 72'},
            #                 {'name': 'wgs84', 'label': 'WGS 84'}
            #             ]
            #         },
            #         {
            #             'name': 'UTM zones for your export',
            #             'values': [
            #                 {'name': 'utm32', 'label': 'UTM zone 32'},
            #                 {'name': 'utm33', 'label': 'UTM zone 33'}
            #             ]
            #         }
            #     ]
            # },
            # 'detail_level': {
            #     'label': 'Detail level',
            #     'type': 'choice',
            #     'default': 'verbatim',
            #     'values': [
            #         {'name': 'verbatim', 'label': 'Verbatim'},
            #         {'name': 'simplified', 'label': 'Simplified'}
            #     ]
            # }
        }

    @staticmethod
    def extract_excerpts(execution_configuration, extraction_order, bbox_args, converter_helper):
        """
        Extract excerpt for chosen formats (execution_configuration) using docker-compose
        to trigger the conversion process (defined in blackbox/docker-compose-conversion-blackbox.yml)

        :param execution_configuration:
        :param extraction_order:
        :param bbox_args example: '8.775449276 47.1892350573 8.8901920319 47.2413633153'
        :return:
        """
        for export_format_key, export_format_config in GisExcerptConverter.export_formats().items():
            index = 0
            if export_format_key in execution_configuration['formats']:
                index += 1
                extraction_command = "docker-compose run excerpt python excerpt.py %(bbox_args)s -f %(format)s" % {
                    'bbox_args':  bbox_args,
                    'format': export_format_key
                }
                subprocess.check_call(extraction_command.split(' '))

                if len(os.listdir(settings.RESULT_MEDIA_ROOT)) > 0:
                    for result_file_name in os.listdir(settings.RESULT_MEDIA_ROOT):
                        # gis files are packaged in a zip file
                        if GisExcerptConverter.create_output_file(extraction_order, result_file_name):
                            converter_helper.inform_user(
                                messages.SUCCESS,
                                _('Extraction of "%(file_type)s" of extraction order "%(order_id)s" was successful. '
                                  '(of %(number_of_files)s files of %(converter_name)s converter)') % {
                                    'file_type': export_format_config['name'],
                                    'file_index': index,
                                    'number_of_files': len(execution_configuration['formats']),
                                    'converter_name': GisExcerptConverter.name(),
                                    'order_id': extraction_order.id
                                },
                                email=False
                            )
                        else:
                            converter_helper.inform_user(
                                messages.ERROR,
                                _('The extraction of "%(file)s" of extraction order "%(order_id)s" failed.') % {
                                    'file': result_file_name,
                                    'order_id': extraction_order.id
                                },
                                email=False
                            )
                else:
                    converter_helper.inform_user(
                        messages.ERROR,
                        _('The extraction of "%(file_type)s" of extraction order "%(order_id)s" failed.') % {
                            'file_type': export_format_config['name'],
                            'order_id': extraction_order.id
                        },
                        email=False
                    )

    @staticmethod
    def create_output_file(extraction_order, result_file_name):
        """
        Move file to private media storage and add OutputFile to Extractionorder

        :return: True if file created successful
        """
        output_file = models.OutputFile.objects.create(
            mime_type='application/zip',
            extraction_order=extraction_order
        )

        if not os.path.exists(private_storage.location):
            os.makedirs(private_storage.location)

        file_name = str(output_file.public_identifier) + '.zip'
        result_file_path = os.path.abspath(os.path.join(settings.RESULT_MEDIA_ROOT, result_file_name))
        target_file_path = os.path.abspath(os.path.join(private_storage.location, file_name))

        shutil.move(result_file_path, target_file_path)
        output_file.file = private_storage.open(target_file_path)
        output_file.save()

        return os.path.isfile(target_file_path)

    @staticmethod
    @shared_task
    def execute_task(extraction_order_id, supported_export_formats, execution_configuration):
        wait_time = 0
        # wait for the db to be updated!
        extraction_order = None
        while extraction_order is None:
            try:
                extraction_order = models.ExtractionOrder.objects.get(pk=extraction_order_id)
            except models.ExtractionOrder.DoesNotExist:
                time.sleep(5)
                wait_time += 5
                if wait_time > 30:
                    raise

        converter_helper = ConverterHelper(extraction_order)
        extraction_order.state = models.ExtractionOrderState.WAITING
        extraction_order.save()

        with tempfile.TemporaryDirectory() as tmp_dir:
            original_cwd = os.getcwd()

            try:
                shutil.copyfile(
                    os.path.join(os.path.dirname(__file__), 'blackbox', 'docker-compose-conversion-blackbox.yml'),
                    os.path.join(tmp_dir, 'docker-compose.yml')
                )
                os.chdir(tmp_dir)
                # use newest images
                subprocess.check_output("docker-compose pull".split(' '))
                # database needs to be ready
                subprocess.check_output("docker-compose up -d db".split(' '))
                # wait for the db to be up
                subprocess.check_output("sleep 10".split(' '))

                converter_helper.inform_user(
                    messages.INFO,
                    _('The GIS extraction of the order "%s" is has been started.') % extraction_order.id,
                    email=False
                )

                bounding_geometry = extraction_order.excerpt.bounding_geometry
                if type(bounding_geometry) == models.BBoxBoundingGeometry:
                    bbox_args = ' '.join([
                        bounding_geometry.south,
                        bounding_geometry.west,
                        bounding_geometry.north,
                        bounding_geometry.east
                    ])

                    if len(execution_configuration['formats']) > 0:
                        subprocess.check_call(("docker-compose run bootstrap sh main-bootstrap.sh %s" %
                                               bbox_args).split(' '))
                        extraction_order.state = models.ExtractionOrderState.PROCESSING
                        extraction_order.save()
                        GisExcerptConverter.extract_excerpts(
                            execution_configuration,
                            extraction_order,
                            bbox_args,
                            converter_helper
                        )

                elif type(bounding_geometry) == models.OsmosisPolygonFilterBoundingGeometry:
                    converter_helper.inform_user(
                        messages.ERROR,
                        _('GIS excerpt converter is not yet able to extract polygon excerpts.'),
                        email=False
                    )

                else:
                    converter_helper.inform_user(
                        messages.ERROR,
                        _('GIS excerpt converter is not yet able to extract excerpts of type %s.') %
                        type(bounding_geometry).__name__,
                        email=False
                    )

                subprocess.check_call("docker-compose stop --timeout 0".split(' '))
                subprocess.check_call("docker-compose rm -vf".split(' '))

                converter_helper.file_conversion_finished()
            except:
                extraction_order.state = models.ExtractionOrderState.FAILED
                extraction_order.save()

                converter_helper.inform_user(
                    messages.ERROR,
                    _('The extraction of order %(order_id)s failed. '
                      'Please contact an administrator.') % {
                        'order_id': extraction_order.id
                    },
                    email=False
                )
                raise
            finally:
                os.chdir(original_cwd)
<commit_msg>Fix bbox_args (cast to string)<commit_after>import os
import shutil
import subprocess
import tempfile
import time

from celery import shared_task

from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.contrib import messages

from excerptconverter.baseexcerptconverter import BaseExcerptConverter
from excerptconverter import ConverterHelper

from osmaxx.excerptexport import models
from osmaxx.utils import private_storage

class GisExcerptConverter(BaseExcerptConverter):
    @staticmethod
    def name():
        return 'GIS'

    @staticmethod
    def export_formats():
        return {
            'spatialite': {
                'name': 'SpatiaLite (SQLite)',
                'file_extension': 'sqlite'
            },
            'gpkg': {
                'name': 'Geo package',
                'file_extension': 'gpkg'
            },
            'shp': {
                'name': 'Shape file',
                'file_extension': 'shp'
            }
        }

    @staticmethod
    def export_options():
        return {
            # has to be implemented next:
            # 'coordinate_reference_system': {
            #     'label': 'Coordinate reference system',
            #     'type': 'choice',
            #     'default': 'pseudomerkator',
            #     'groups': [
            #         {
            #             'name': 'Global coordinate reference systems',
            #             'values': [
            #                 {'name': 'pseudomerkator', 'label': 'Pseudo merkator'},
            #                 {'name': 'wgs72', 'label': 'WGS 72'},
            #                 {'name': 'wgs84', 'label': 'WGS 84'}
            #             ]
            #         },
            #         {
            #             'name': 'UTM zones for your export',
            #             'values': [
            #                 {'name': 'utm32', 'label': 'UTM zone 32'},
            #                 {'name': 'utm33', 'label': 'UTM zone 33'}
            #             ]
            #         }
            #     ]
            # },
            # 'detail_level': {
            #     'label': 'Detail level',
            #     'type': 'choice',
            #     'default': 'verbatim',
            #     'values': [
            #         {'name': 'verbatim', 'label': 'Verbatim'},
            #         {'name': 'simplified', 'label': 'Simplified'}
            #     ]
            # }
        }

    @staticmethod
    def extract_excerpts(execution_configuration, extraction_order, bbox_args, converter_helper):
        """
        Extract excerpt for chosen formats (execution_configuration) using docker-compose
        to trigger the conversion process (defined in blackbox/docker-compose-conversion-blackbox.yml)

        :param execution_configuration:
        :param extraction_order:
        :param bbox_args example: '8.775449276 47.1892350573 8.8901920319 47.2413633153'
        :return:
        """
        for export_format_key, export_format_config in GisExcerptConverter.export_formats().items():
            index = 0
            if export_format_key in execution_configuration['formats']:
                index += 1
                extraction_command = "docker-compose run excerpt python excerpt.py %(bbox_args)s -f %(format)s" % {
                    'bbox_args':  bbox_args,
                    'format': export_format_key
                }
                subprocess.check_call(extraction_command.split(' '))

                if len(os.listdir(settings.RESULT_MEDIA_ROOT)) > 0:
                    for result_file_name in os.listdir(settings.RESULT_MEDIA_ROOT):
                        # gis files are packaged in a zip file
                        if GisExcerptConverter.create_output_file(extraction_order, result_file_name):
                            converter_helper.inform_user(
                                messages.SUCCESS,
                                _('Extraction of "%(file_type)s" of extraction order "%(order_id)s" was successful. '
                                  '(of %(number_of_files)s files of %(converter_name)s converter)') % {
                                    'file_type': export_format_config['name'],
                                    'file_index': index,
                                    'number_of_files': len(execution_configuration['formats']),
                                    'converter_name': GisExcerptConverter.name(),
                                    'order_id': extraction_order.id
                                },
                                email=False
                            )
                        else:
                            converter_helper.inform_user(
                                messages.ERROR,
                                _('The extraction of "%(file)s" of extraction order "%(order_id)s" failed.') % {
                                    'file': result_file_name,
                                    'order_id': extraction_order.id
                                },
                                email=False
                            )
                else:
                    converter_helper.inform_user(
                        messages.ERROR,
                        _('The extraction of "%(file_type)s" of extraction order "%(order_id)s" failed.') % {
                            'file_type': export_format_config['name'],
                            'order_id': extraction_order.id
                        },
                        email=False
                    )

    @staticmethod
    def create_output_file(extraction_order, result_file_name):
        """
        Move file to private media storage and add OutputFile to Extractionorder

        :return: True if file created successful
        """
        output_file = models.OutputFile.objects.create(
            mime_type='application/zip',
            extraction_order=extraction_order
        )

        if not os.path.exists(private_storage.location):
            os.makedirs(private_storage.location)

        file_name = str(output_file.public_identifier) + '.zip'
        result_file_path = os.path.abspath(os.path.join(settings.RESULT_MEDIA_ROOT, result_file_name))
        target_file_path = os.path.abspath(os.path.join(private_storage.location, file_name))

        shutil.move(result_file_path, target_file_path)
        output_file.file = private_storage.open(target_file_path)
        output_file.save()

        return os.path.isfile(target_file_path)

    @staticmethod
    @shared_task
    def execute_task(extraction_order_id, supported_export_formats, execution_configuration):
        wait_time = 0
        # wait for the db to be updated!
        extraction_order = None
        while extraction_order is None:
            try:
                extraction_order = models.ExtractionOrder.objects.get(pk=extraction_order_id)
            except models.ExtractionOrder.DoesNotExist:
                time.sleep(5)
                wait_time += 5
                if wait_time > 30:
                    raise

        converter_helper = ConverterHelper(extraction_order)
        extraction_order.state = models.ExtractionOrderState.WAITING
        extraction_order.save()

        with tempfile.TemporaryDirectory() as tmp_dir:
            original_cwd = os.getcwd()

            try:
                shutil.copyfile(
                    os.path.join(os.path.dirname(__file__), 'blackbox', 'docker-compose-conversion-blackbox.yml'),
                    os.path.join(tmp_dir, 'docker-compose.yml')
                )
                os.chdir(tmp_dir)
                # use newest images
                subprocess.check_output("docker-compose pull".split(' '))
                # database needs to be ready
                subprocess.check_output("docker-compose up -d db".split(' '))
                # wait for the db to be up
                subprocess.check_output("sleep 10".split(' '))

                converter_helper.inform_user(
                    messages.INFO,
                    _('The GIS extraction of the order "%s" is has been started.') % extraction_order.id,
                    email=False
                )

                bounding_geometry = extraction_order.excerpt.bounding_geometry
                if type(bounding_geometry) == models.BBoxBoundingGeometry:
                    bbox_args = ' '.join(str(coordinate) for coordinate in [
                        bounding_geometry.south,
                        bounding_geometry.west,
                        bounding_geometry.north,
                        bounding_geometry.east
                    ])

                    if len(execution_configuration['formats']) > 0:
                        subprocess.check_call(("docker-compose run bootstrap sh main-bootstrap.sh %s" %
                                               bbox_args).split(' '))
                        extraction_order.state = models.ExtractionOrderState.PROCESSING
                        extraction_order.save()
                        GisExcerptConverter.extract_excerpts(
                            execution_configuration,
                            extraction_order,
                            bbox_args,
                            converter_helper
                        )

                elif type(bounding_geometry) == models.OsmosisPolygonFilterBoundingGeometry:
                    converter_helper.inform_user(
                        messages.ERROR,
                        _('GIS excerpt converter is not yet able to extract polygon excerpts.'),
                        email=False
                    )

                else:
                    converter_helper.inform_user(
                        messages.ERROR,
                        _('GIS excerpt converter is not yet able to extract excerpts of type %s.') %
                        type(bounding_geometry).__name__,
                        email=False
                    )

                subprocess.check_call("docker-compose stop --timeout 0".split(' '))
                subprocess.check_call("docker-compose rm -vf".split(' '))

                converter_helper.file_conversion_finished()
            except:
                extraction_order.state = models.ExtractionOrderState.FAILED
                extraction_order.save()

                converter_helper.inform_user(
                    messages.ERROR,
                    _('The extraction of order %(order_id)s failed. '
                      'Please contact an administrator.') % {
                        'order_id': extraction_order.id
                    },
                    email=False
                )
                raise
            finally:
                os.chdir(original_cwd)
<|endoftext|>


<commit_before>#Join of MAXMIND and FORENSICSWIKI

import sys
from pyspark import SparkCountext
import socket,struct
from weblog import Weblog

def ip2long(ip):
	packedIP = socket.inet_aton(ip)
	return struct.unpack("!L",packedIP)[0]
def longtoip(long):
	return socket.inet_ntoa(struct.pack("!L",long))

if __name__ == "__main__":
	wiki_path = 's3://gu-anly502/ps03/forensicswiki.2012-01.unzipped/access.long.2012-01-01'
	#wiki_path = 's3://gu-anly502/ps03/forensicswiki.2012.txt'
	path1 = 's3://gu-anly502/maxmind/GeoLite2-Country-Blocks-IPv4.csv'
	path2 = 's3://gu-anly502/maxmind/GeoLite2-Country-Locations-en.csv'
	sc = SparkContext(appName='IP Join')

	#Join The MaxMind Date
	ref1 = sc.textFile(path1).zipWithIndex().filter(lambda x:x[1] > 0)
	ref2 = sc.textFile(path2).zipWithIndex().filter(lambda x:x[1] > 0)
	"""
	head1,head2 = ref1.first(),ref2.first()
	ref1 = ref1.filter(lambda x:x !=head1)
	ref2 = ref2.filter(lambda x:x !=head2)
	"""
	pair1 = ref1.map(lambda x:(x.split(',')[0],x.split(',')[-1]))#genoame_id,country_name
	pair2 = ref2.map(lambda x:(x.split(',')[1],x.split(',')[0].split('/')[0]))#genoame_id,ip
	pair3 = pair.join(pair2)
	country_ip = pair3.map(lambda x:(ip2long(x[1][1]),x[1][0]))#ip,country_name
	#send this structure to all nodes

	#Forenisicswiki data for 2012
	wiki = sc.textFile(wiki_path)
	wiki_data = wiki.map(lambda x:Weblog(x))
	wiki_pair = wiki_data.map(lambda x:x.ipaddr,x.)

<commit_msg>draft geo<commit_after>#Join of MAXMIND and FORENSICSWIKI

import sys
from pyspark import SparkContext
import socket,struct
from weblog import Weblog
import bisect

def ip2long(ip):
	packedIP = socket.inet_aton(ip)
	return struct.unpack("!L",packedIP)[0]
def longtoip(long):
	return socket.inet_ntoa(struct.pack("!L",long))

def check(tmp):
    for i in tmp:
        print i
def get_ip_url(line):
    return 0

def bsearch(x,lookup):
    ip = x

    return 0
def change_to_integer(x):
    #x[0] ip x[1] country
    integer,nn = x[0].split('/')
    integer = str(ip2long(integer))
    return integer+'/'+nn,x[1]

if __name__ == "__main__":
	wiki_path = 's3://gu-anly502/ps03/forensicswiki.2012-01.unzipped/access.log.2012-01-01'
	#wiki_path = 's3://gu-anly502/ps03/forensicswiki.2012.txt'
	path1 = 's3://gu-anly502/maxmind/GeoLite2-Country-Blocks-IPv4.csv'
	path2 = 's3://gu-anly502/maxmind/GeoLite2-Country-Locations-en.csv'
	sc = SparkContext(appName='IP Join')

	#Join The MaxMind Date
	ref1 = sc.textFile(path1).zipWithIndex().filter(lambda x:x[1] > 0).map(lambda x:x[0])#ip file
	ref2 = sc.textFile(path2).zipWithIndex().filter(lambda x:x[1] > 0).map(lambda x:x[0])#country
        pair1 = ref1.map(lambda x:(x.split(',')[1],x.split(',')[0]))#genoame_id,ip
	pair2 = ref2.map(lambda x:(x.split(',')[0],x.split(',')[-1]))#genoame_id,country_name
        pair3 = pair1.join(pair2)
	country_ip = pair3.map(lambda x:(x[1]))#ip,country_name
	#country_ip = country_ip.sortByKey(True)
        #country_ip = country_ip.map(lanbda x: "/".join(ip2long(x[0].split('/')[0]), x[0].split('/')[1]),x[1])
        country_ip = country_ip.map(lambda x:change_to_integer(x)).sortByKey(True)
        tmp = country_ip.collect()
        check(tmp)
        #check(tmp)
        #send this structure to all nodes
        lookup = sc.broadcast(tmp)
        # print lookup.value

	#Forenisicswiki data for 2012
	wiki = sc.textFile(wiki_path)
        wiki_data = wiki.map(lambda x:x.split()[0])
        check(wiki_data.take(10))
        match_each_ip = wiki_data(lambda x : bsearch(x,lookup),1)

<|endoftext|>


<commit_before>"""
SAGA package version
"""
__version__ = "0.51.1"
<commit_msg>increase version<commit_after>"""
SAGA package version
"""
__version__ = "0.52.0"
<|endoftext|>


<commit_before>"""
Auxilliary Plugin does the things that other plugins don't want to do.
Keep Alives, Handshaking, Respawning, and that one encryption packet all
get to live here because they don't really belong other places.
All functionality is configurable.
"""

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import padding

from spockbot.mcdata import constants
from spockbot.mcp import proto
from spockbot.plugins.base import PluginBase

backend = default_backend()

class AuxiliaryPlugin(PluginBase):
    requires = 'Auth', 'Net'
    defaults = {
        'Encryption': True,
        'Handshake': True,
        'KeepAlive': True,
        'Respawn': True,
    }
    settings_map = {
        'Encryption': ('LOGIN<Encryption Request', 'handle_encrypt_request'),
        'Handshake': ('net_connect', 'handshake_and_login_start'),
        'KeepAlive': ('PLAY<Keep Alive', 'handle_keep_alive'),
        'Respawn': ('client_death', 'handle_death'),
    }

    def __init__(self, ploader, settings):
        super(AuxiliaryPlugin, self).__init__(ploader, settings)
        for key, val in self.settings.items():
            if val:
                event, attr = self.settings_map[key]
                ploader.reg_event_handler(event, getattr(self, attr))

    def handshake_and_login_start(self, _, __):
        self.net.push_packet('HANDSHAKE>Handshake', {
            'protocol_version': proto.MC_PROTOCOL_VERSION,
            'host': self.net.host,
            'port': self.net.port,
            'next_state': proto.LOGIN_STATE
        })
        self.net.push_packet('LOGIN>Login Start', {'name': self.auth.username})

    # Encryption Key Request - Request for client to start encryption
    def handle_encrypt_request(self, name, packet):
        pubkey_raw = packet.data['public_key']
        if self.auth.online_mode:
            self.auth.send_session_auth(pubkey_raw, packet.data['server_id'])
        pubkey = serialization.load_der_public_key(pubkey_raw, backend)
        encrypt = lambda data: pubkey.encrypt(data, padding.PKCS1v15())  # flake8: noqa
        self.net.push_packet(
            'LOGIN>Encryption Response',
            {
                'shared_secret': encrypt(self.auth.shared_secret),
                'verify_token': encrypt(packet.data['verify_token']),
            }
        )
        self.net.enable_crypto(self.auth.shared_secret)

    # Keep Alive - Reflects data back to server
    def handle_keep_alive(self, name, packet):
        packet.new_ident('PLAY>Keep Alive')
        self.net.push(packet)

    # You be dead
    def handle_death(self, name, data):
        self.net.push_packet(
            'PLAY>Client Status', {'action': constants.CL_STATUS_RESPAWN}
        )
<commit_msg>Formatting<commit_after>"""
Auxilliary Plugin does the things that other plugins don't want to do.
Keep Alives, Handshaking, Respawning, and that one encryption packet all
get to live here because they don't really belong other places.
All functionality is configurable.
"""

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import padding

from spockbot.mcdata import constants
from spockbot.mcp import proto
from spockbot.plugins.base import PluginBase

backend = default_backend()

class AuxiliaryPlugin(PluginBase):
    requires = 'Auth', 'Net'
    defaults = {
        'Encryption': True,
        'Handshake': True,
        'KeepAlive': True,
        'Respawn': True,
    }
    settings_map = {
        'Encryption': ('LOGIN<Encryption Request', 'handle_encrypt_request'),
        'Handshake': ('net_connect', 'handshake_and_login_start'),
        'KeepAlive': ('PLAY<Keep Alive', 'handle_keep_alive'),
        'Respawn': ('client_death', 'handle_death'),
    }

    def __init__(self, ploader, settings):
        super(AuxiliaryPlugin, self).__init__(ploader, settings)
        for key, val in self.settings.items():
            if val:
                event, attr = self.settings_map[key]
                ploader.reg_event_handler(event, getattr(self, attr))

    def handshake_and_login_start(self, _, __):
        self.net.push_packet('HANDSHAKE>Handshake', {
            'protocol_version': proto.MC_PROTOCOL_VERSION,
            'host': self.net.host,
            'port': self.net.port,
            'next_state': proto.LOGIN_STATE,
        })
        self.net.push_packet('LOGIN>Login Start', {'name': self.auth.username})

    # Encryption Key Request - Request for client to start encryption
    def handle_encrypt_request(self, name, packet):
        pubkey_raw = packet.data['public_key']
        if self.auth.online_mode:
            self.auth.send_session_auth(pubkey_raw, packet.data['server_id'])
        pubkey = serialization.load_der_public_key(pubkey_raw, backend)
        encrypt = lambda data: pubkey.encrypt(data, padding.PKCS1v15())  # flake8: noqa
        self.net.push_packet('LOGIN>Encryption Response', {
            'shared_secret': encrypt(self.auth.shared_secret),
            'verify_token': encrypt(packet.data['verify_token']),
        })
        self.net.enable_crypto(self.auth.shared_secret)

    # Keep Alive - Reflects data back to server
    def handle_keep_alive(self, name, packet):
        packet.new_ident('PLAY>Keep Alive')
        self.net.push(packet)

    # You be dead
    def handle_death(self, name, data):
        self.net.push_packet('PLAY>Client Status',
                             {'action': constants.CL_STATUS_RESPAWN})
<|endoftext|>