<commit_before>"""
    selenium_data_attributes.element
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module implements the core object used to create selenium structures.

    :copyright: (c) 2016 FanThreeSixty
    :author: John Lane <jlane@fanthreesixty.com>
    :license: MIT, see LICENSE for more details.
"""

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select as SeleniumSelect
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import ElementNotVisibleException, WebDriverException, NoSuchElementException, \
    InvalidSelectorException

__author__ = 'jlane'
__copyright__ = 'Copyright (c) 2016 FanThreeSixty'
__license__ = "MIT"
__version__ = '0.0.1'
__contact__ = 'jlane@fanthreesixty.com'
__status__ = 'Alpha'
__docformat__ = 'reStructuredText'

__all__ = ['Element']

class Element(object):
    """Abstract web structure class
    """

    def __init__(self, web_driver, value, by=By.XPATH):
        """Basic Selenium element

        :param web_driver: Selenium webdriver
        :param str by: By selector
        :param str value: selection value
        :return:
        """

        self.driver = web_driver
        self.search_term = (by, value)

    def __contains__(self, attribute):
        """Returns True if element contains attribute

        :param str attribute: Element attribute
        :return: True, if the element contains that attribute
        :rtype: bool
        """

        if self.exists() and isinstance(attribute, str):

            try:

                self.element().find_element_by_xpath('//self::*[@{0}]'.format(attribute))
                return True

            except NoSuchElementException:
                pass

        return False

    def __getitem__(self, attribute):
        """Returns the value of an attribute

        :param str attribute: Element attribute
        :return: Returns the string value
        :rtype: str
        """

        if self.exists() and isinstance(attribute, str):
            if self.__contains__(attribute):
                return self.element().get_attribute(attribute).encode('ascii', 'ignore')

        return ''

    def __repr__(self):
        """Returns HTML representation of the element

        :return: HTML representation of the element
        :rtype: str
        """

        if self.exists():
            return self.element().get_attribute('outerHTML')

        return ''

    def angular_hidden(self):
        """Returns True if the element is hidden by angular

        :return: True, if the element is hidden by angular
        :rtype: bool
        """

        if 'ng-hide' in self.__getitem__('class'):
            return True

        return False

    def blur(self):
        """Simulate moving the cursor out of focus of this element

        :return:
        """

        if not self.angular_hidden():
            if self.exists():
                self.driver.execute_script('arguments[0].blur();', self.element())

    def element(self):
        """Return the selenium webelement object

        :return: Selenium WebElement
        :rtype: WebElement
        """

        # If the search term passed through was an element
        if self.search_term[0] == 'element' and isinstance(self.search_term[1], WebElement):
            return self.search_term[1]

        # If the search term is a valid term
        elif By.is_valid(self.search_term[0]):

            try:

                # Locate element
                element = self.driver.find_elements(*self.search_term)

            except InvalidSelectorException:
                element = []

            if len(element) > 0:
                return element[0]

        return None

    def _click(self):
        """Click element

        :return:
        """

        if self.exists():

            try:
                self.element().click()

            # If the object is not within the view try to scroll to the element
            except (ElementNotVisibleException, WebDriverException):

                # Scroll to Element
                self._scroll_to()

                try:
                    self.element().click()

                except (ElementNotVisibleException, WebDriverException):
                    pass

    def _deselect_by_index(self, option):
        """Deselect option by index [i]

        :param option: Select option index
        :return: True, if option is deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                if isinstance(option, int):

                    select = SeleniumSelect(element)

                    try:
                        select.deselect_by_index(option)
                        return True

                    except NoSuchElementException:
                        pass

                elif isinstance(option, str):

                    if option.isdigit():

                        select = SeleniumSelect(element)

                        try:
                            select.deselect_by_index(int(option))
                            return True

                        except NoSuchElementException:
                            pass

        return False

    def _deselect_by_text(self, option):
        """Deselect option by display text

        :param option: Select option
        :return: True, if option is deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.deselect_by_visible_text(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _deselect_by_value(self, option):
        """Deselect option by option value

        :param option: Select option value
        :return: True, if option is deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.deselect_by_value(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _deselect_all(self):
        """Deselect all selected options

        :return: True, if all options are deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                select = SeleniumSelect(element)

                try:

                    select.deselect_all()
                    return True

                except NotImplementedError:
                    pass

        return False

    def exists(self):
        """Returns True if element can be located by selenium

        :return: Returns True, if the element can be located
        :rtype: bool
        """

        element = self.element()

        if element:
            return True

        return False

    def focus(self):
        """Simulate element being in focus

        :return:
        """

        if self.angular_hidden():
            if self.exists():
                self.driver.execute_script('arguments[0].focus();', self.element())

    def _input(self, text, clear=True):
        """Send text to a input field

        :param str text: Text to send to the input field
        :param bool clear: True if user wants to clear the field before assigning text
        :return: True, if text is assigned
        :rtype: bool
        """

        if self.exists() and isinstance(text, str):

            element = self.element()

            if clear:
                element.clear()

            element.send_keys(text)

            return True

        return False

    def _options(self):
        """Returns all Select options

        :return: List of options
        :rtype: list
        """

        if self.exists():

            element = self.element()

            if element.tag_name == u'select':

                select = SeleniumSelect(element)

                options = []

                for option in select.options:
                    options.append(option.text.encode('ascii', 'ignore'))

                return options

        return []

    def _scroll_to(self):
        """Scroll to the location of the element

        :return:
        """

        if self.exists():

            element = self.element()

            # Scroll to Element
            self.driver.execute_script("window.scrollTo(0, %i)" % (element.location['y'] - element.size['height']))

    def _selected(self):
        """Return True if element is selected

        :return: True, if the element is selected
        :rtype: bool
        """

        if self.exists():
            return self.element().is_selected()

        return False

    def _selected_options(self):
        """Returns a list of selected options

        :return: List of options
        :rtype: list
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                select = SeleniumSelect(element)
                options = []

                for option in select.all_selected_options:
                    options.append(option.text.encode('ascii', 'ignore'))

                return options

        return []

    def _selected_first(self):
        """Select first option

        :return:
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                select = SeleniumSelect(element)
                options = select.all_selected_options

                if len(options) > 0:
                    return options[0]

        return None

    def _select_by_index(self, option):
        """Select option at index [i]

        :param int option: Select index
        :return: True, if the option is selected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                if isinstance(option, int):

                    select = SeleniumSelect(element)

                    try:
                        select.select_by_index(option)
                        return True

                    except NoSuchElementException:
                        pass

                elif isinstance(option, str):

                    if option.isdigit():

                        select = SeleniumSelect(element)

                        try:
                            select.select_by_index(int(option))
                            return True

                        except NoSuchElementException:
                            pass

        return False

    def _select_by_text(self, option):
        """Select option by display text

        :param str option: Select option
        :return: True, if the option is selected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.select_by_visible_text(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _select_by_value(self, option):
        """Select option by option value

        :param str option: Select option value
        :return: True, if the option is selected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.select_by_value(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _text(self):
        """Returns the text within an element

        :return: Element text
        :rtype: str
        """

        if self.exists():

            element = self.element()
            text = element.text.encode('ascii', 'ignore').strip()

            if text == '':
                return self.driver.execute_script('return arguments[0].textContent',
                                                  element).encode('ascii', 'ignore').strip()

            return text

    def wait_until_present(self):
        """Wait until element is present

        :return:
        """

        wait = WebDriverWait(self.driver, 30)
        wait.until(ec.presence_of_element_located((By.XPATH, self.search_term[1])))
<commit_msg>Push to PyPI<commit_after>"""
    selenium_data_attributes.element
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module implements the core object used to create selenium structures.

    :copyright: (c) 2016 FanThreeSixty
    :author: John Lane <jlane@fanthreesixty.com>
    :license: MIT, see LICENSE for more details.
"""

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select as SeleniumSelect
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import ElementNotVisibleException, WebDriverException, NoSuchElementException, \
    InvalidSelectorException

__author__ = 'jlane'
__copyright__ = 'Copyright (c) 2016 FanThreeSixty'
__license__ = "MIT"
__version__ = '0.2'
__contact__ = 'jlane@fanthreesixty.com'
__status__ = 'Alpha'
__docformat__ = 'reStructuredText'

__all__ = ['Element']

class Element(object):
    """Abstract web structure class
    """

    def __init__(self, web_driver, value, by=By.XPATH):
        """Basic Selenium element

        :param web_driver: Selenium webdriver
        :param str by: By selector
        :param str value: selection value
        :return:
        """

        self.driver = web_driver
        self.search_term = (by, value)

    def __contains__(self, attribute):
        """Returns True if element contains attribute

        :param str attribute: Element attribute
        :return: True, if the element contains that attribute
        :rtype: bool
        """

        if self.exists() and isinstance(attribute, str):

            try:

                self.element().find_element_by_xpath('//self::*[@{0}]'.format(attribute))
                return True

            except NoSuchElementException:
                pass

        return False

    def __getitem__(self, attribute):
        """Returns the value of an attribute

        :param str attribute: Element attribute
        :return: Returns the string value
        :rtype: str
        """

        if self.exists() and isinstance(attribute, str):
            if self.__contains__(attribute):
                return self.element().get_attribute(attribute).encode('ascii', 'ignore')

        return ''

    def __repr__(self):
        """Returns HTML representation of the element

        :return: HTML representation of the element
        :rtype: str
        """

        if self.exists():
            return self.element().get_attribute('outerHTML')

        return ''

    def angular_hidden(self):
        """Returns True if the element is hidden by angular

        :return: True, if the element is hidden by angular
        :rtype: bool
        """

        if 'ng-hide' in self.__getitem__('class'):
            return True

        return False

    def blur(self):
        """Simulate moving the cursor out of focus of this element

        :return:
        """

        if not self.angular_hidden():
            if self.exists():
                self.driver.execute_script('arguments[0].blur();', self.element())

    def element(self):
        """Return the selenium webelement object

        :return: Selenium WebElement
        :rtype: WebElement
        """

        # If the search term passed through was an element
        if self.search_term[0] == 'element' and isinstance(self.search_term[1], WebElement):
            return self.search_term[1]

        # If the search term is a valid term
        elif By.is_valid(self.search_term[0]):

            try:

                # Locate element
                element = self.driver.find_elements(*self.search_term)

            except InvalidSelectorException:
                element = []

            if len(element) > 0:
                return element[0]

        return None

    def _click(self):
        """Click element

        :return:
        """

        if self.exists():

            try:
                self.element().click()

            # If the object is not within the view try to scroll to the element
            except (ElementNotVisibleException, WebDriverException):

                # Scroll to Element
                self._scroll_to()

                try:
                    self.element().click()

                except (ElementNotVisibleException, WebDriverException):
                    pass

    def _deselect_by_index(self, option):
        """Deselect option by index [i]

        :param option: Select option index
        :return: True, if option is deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                if isinstance(option, int):

                    select = SeleniumSelect(element)

                    try:
                        select.deselect_by_index(option)
                        return True

                    except NoSuchElementException:
                        pass

                elif isinstance(option, str):

                    if option.isdigit():

                        select = SeleniumSelect(element)

                        try:
                            select.deselect_by_index(int(option))
                            return True

                        except NoSuchElementException:
                            pass

        return False

    def _deselect_by_text(self, option):
        """Deselect option by display text

        :param option: Select option
        :return: True, if option is deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.deselect_by_visible_text(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _deselect_by_value(self, option):
        """Deselect option by option value

        :param option: Select option value
        :return: True, if option is deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.deselect_by_value(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _deselect_all(self):
        """Deselect all selected options

        :return: True, if all options are deselected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                select = SeleniumSelect(element)

                try:

                    select.deselect_all()
                    return True

                except NotImplementedError:
                    pass

        return False

    def exists(self):
        """Returns True if element can be located by selenium

        :return: Returns True, if the element can be located
        :rtype: bool
        """

        element = self.element()

        if element:
            return True

        return False

    def focus(self):
        """Simulate element being in focus

        :return:
        """

        if self.angular_hidden():
            if self.exists():
                self.driver.execute_script('arguments[0].focus();', self.element())

    def _input(self, text, clear=True):
        """Send text to a input field

        :param str text: Text to send to the input field
        :param bool clear: True if user wants to clear the field before assigning text
        :return: True, if text is assigned
        :rtype: bool
        """

        if self.exists() and isinstance(text, str):

            element = self.element()

            if clear:
                element.clear()

            element.send_keys(text)

            return True

        return False

    def _options(self):
        """Returns all Select options

        :return: List of options
        :rtype: list
        """

        if self.exists():

            element = self.element()

            if element.tag_name == u'select':

                select = SeleniumSelect(element)

                options = []

                for option in select.options:
                    options.append(option.text.encode('ascii', 'ignore'))

                return options

        return []

    def _scroll_to(self):
        """Scroll to the location of the element

        :return:
        """

        if self.exists():

            element = self.element()

            # Scroll to Element
            self.driver.execute_script("window.scrollTo(0, %i)" % (element.location['y'] - element.size['height']))

    def _selected(self):
        """Return True if element is selected

        :return: True, if the element is selected
        :rtype: bool
        """

        if self.exists():
            return self.element().is_selected()

        return False

    def _selected_options(self):
        """Returns a list of selected options

        :return: List of options
        :rtype: list
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                select = SeleniumSelect(element)
                options = []

                for option in select.all_selected_options:
                    options.append(option.text.encode('ascii', 'ignore'))

                return options

        return []

    def _selected_first(self):
        """Select first option

        :return:
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                select = SeleniumSelect(element)
                options = select.all_selected_options

                if len(options) > 0:
                    return options[0]

        return None

    def _select_by_index(self, option):
        """Select option at index [i]

        :param int option: Select index
        :return: True, if the option is selected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select':

                if isinstance(option, int):

                    select = SeleniumSelect(element)

                    try:
                        select.select_by_index(option)
                        return True

                    except NoSuchElementException:
                        pass

                elif isinstance(option, str):

                    if option.isdigit():

                        select = SeleniumSelect(element)

                        try:
                            select.select_by_index(int(option))
                            return True

                        except NoSuchElementException:
                            pass

        return False

    def _select_by_text(self, option):
        """Select option by display text

        :param str option: Select option
        :return: True, if the option is selected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.select_by_visible_text(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _select_by_value(self, option):
        """Select option by option value

        :param str option: Select option value
        :return: True, if the option is selected
        :rtype: bool
        """

        element = self.element()

        if element:

            if element.tag_name == u'select' and isinstance(option, str):

                select = SeleniumSelect(element)

                try:

                    select.select_by_value(option)
                    return True

                except NoSuchElementException:
                    pass

        return False

    def _text(self):
        """Returns the text within an element

        :return: Element text
        :rtype: str
        """

        if self.exists():

            element = self.element()
            text = element.text.encode('ascii', 'ignore').strip()

            if text == '':
                return self.driver.execute_script('return arguments[0].textContent',
                                                  element).encode('ascii', 'ignore').strip()

            return text

    def wait_until_present(self):
        """Wait until element is present

        :return:
        """

        wait = WebDriverWait(self.driver, 30)
        wait.until(ec.presence_of_element_located((By.XPATH, self.search_term[1])))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch

import os
import re

from pilot.user.atlas.setup import get_asetup
from pilot.user.atlas.setup import get_file_system_root_path
from pilot.info import infosys
from pilot.util.config import config
# import pilot.info.infoservice as infosys

import logging
logger = logging.getLogger(__name__)

def wrapper(executable, **kwargs):
    """
    Wrapper function for any container specific usage.
    This function will be called by pilot.util.container.execute() and prepends the executable with a container command.

    :param executable: command to be executed (string).
    :param kwargs:
    :return: executable wrapped with container command (string).
    """

    workdir = kwargs.get('workdir', '.')
    pilot_home = os.environ.get('PILOT_HOME', '')
    job = kwargs.get('job')

    if workdir == '.' and pilot_home != '':
        workdir = pilot_home

    # if job.imagename (from --containerimage <image>) is set, then always use raw singularity
    if config.Container.setup_type == "ALRB" and not job.imagename:
        fctn = alrb_wrapper
    else:
        fctn = singularity_wrapper
    return fctn(executable, workdir, job)

# def use_payload_container(job):
#     pass

def use_middleware_container():
    """
    Should middleware from container be used?
    In case middleware, i.e. the copy command for stage-in/out, should be taken from a container this function should
    return True.

    :return: True if middleware should be taken from container. False otherwise.
    """

    if get_middleware_type() == 'container':
        return True
    else:
        return False

def get_middleware_container():
    pass

def extract_platform_and_os(platform):
    """
    Extract the platform and OS substring from platform

    :param platform (string): E.g. "x86_64-slc6-gcc48-opt"
    :return: extracted platform specifics (string). E.g. "x86_64-slc6". In case of failure, return the full platform
    """

    pattern = r"([A-Za-z0-9_-]+)-.+-.+"
    a = re.findall(re.compile(pattern), platform)

    if len(a) > 0:
        ret = a[0]
    else:
        logger.warning("could not extract architecture and OS substring using pattern=%s from platform=%s"
                       "(will use %s for image name)" % (pattern, platform, platform))
        ret = platform

    return ret

def get_grid_image_for_singularity(platform):
    """
    Return the full path to the singularity grid image

    :param platform (string): E.g. "x86_64-slc6"
    :return: full path to grid image (string).
    """

    if not platform or platform == "":
        platform = "x86_64-slc6"
        logger.warning("using default platform=%s (cmtconfig not set)" % (platform))

    arch_and_os = extract_platform_and_os(platform)
    image = arch_and_os + ".img"
    _path = os.path.join(get_file_system_root_path(), "atlas.cern.ch/repo/containers/images/singularity")
    path = os.path.join(_path, image)
    if not os.path.exists(path):
        image = 'x86_64-centos7.img'
        logger.warning('path does not exist: %s (trying with image %s instead)' % (path, image))
        path = os.path.join(_path, image)
        if not os.path.exists(path):
            logger.warning('path does not exist either: %s' % path)
            path = ""

    return path

def get_middleware_type():
    """
    Return the middleware type from the container type.
    E.g. container_type = 'singularity:pilot;docker:wrapper;middleware:container'
    get_middleware_type() -> 'container', meaning that middleware should be taken from the container. The default
    is otherwise 'workernode', i.e. middleware is assumed to be present on the worker node.

    :return: middleware_type (string)
    """

    middleware_type = ""
    container_type = infosys.queuedata.container_type

    mw = 'middleware'
    if container_type and container_type != "" and mw in container_type:
        try:
            container_names = container_type.split(';')
            for name in container_names:
                t = name.split(':')
                if mw == t[0]:
                    middleware_type = t[1]
        except Exception as e:
            logger.warning("failed to parse the container name: %s, %s" % (container_type, e))
    else:
        # logger.warning("container middleware type not specified in queuedata")
        # no middleware type was specified, assume that middleware is present on worker node
        middleware_type = "workernode"

    return middleware_type

def alrb_wrapper(cmd, workdir, job):
    """
    Wrap the given command with the special ALRB setup for containers
    E.g. cmd = /bin/bash hello_world.sh
    ->
    export thePlatform="x86_64-slc6-gcc48-opt"
    export ALRB_CONT_RUNPAYLOAD="cmd'
    setupATLAS -c $thePlatform

    :param cmd (string): command to be executed in a container.
    :param workdir: (not used)
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    if container_name == 'singularity':
        # first get the full setup, which should be removed from cmd (or ALRB setup won't work)
        _asetup = get_asetup()
        cmd = cmd.replace(_asetup, "asetup ")
        # get simplified ALRB setup (export)
        asetup = get_asetup(alrb=True)

        # Get the singularity options
        singularity_options = queuedata.container_options
        logger.debug(
            "resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        _cmd = asetup
        if job.platform:
            _cmd += 'export thePlatform=\"%s\";' % job.platform
        #if '--containall' not in singularity_options:
        #    singularity_options += ' --containall'
        if singularity_options != "":
            _cmd += 'export ALRB_CONT_CMDOPTS=\"%s\";' % singularity_options
        _cmd += 'export ALRB_CONT_RUNPAYLOAD=\"%s\";' % cmd

        if '--containerImage' in job.jobparams:
            job.jobparams, container_path = remove_container_string(job.jobparams)
            if container_path != "":
                _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % container_path
            else:
                logger.warning('failed to extract container path from %s' % job.jobparams)
                _cmd = ""
        else:
            _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c images'
            if job.platform:
                _cmd += '+$thePlatform'

        _cmd = _cmd.replace('  ', ' ')
        cmd = _cmd
        logger.info("Updated command: %s" % cmd)

    return cmd

def remove_container_string(job_params):
    """ Retrieve the container string from the job parameters """

    pattern = r" \'?\-\-containerImage\=?\ ?([\S]+)\ ?\'?"
    compiled_pattern = re.compile(pattern)

    # remove any present ' around the option as well
    job_params = re.sub(r'\'\ \'', ' ', job_params)

    # extract the container path
    found = re.findall(compiled_pattern, job_params)
    container_path = found[0] if len(found) > 0 else ""

    # Remove the pattern and update the job parameters
    job_params = re.sub(pattern, ' ', job_params)

    return job_params, container_path

def singularity_wrapper(cmd, workdir, job):
    """
    Prepend the given command with the singularity execution command
    E.g. cmd = /bin/bash hello_world.sh
    -> singularity_command = singularity exec -B <bindmountsfromcatchall> <img> /bin/bash hello_world.sh
    singularity exec -B <bindmountsfromcatchall>  /cvmfs/atlas.cern.ch/repo/images/singularity/x86_64-slc6.img <script>

    :param cmd (string): command to be prepended.
    :param workdir: explicit work directory where the command should be executed (needs to be set for Singularity).
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    logger.debug("resolved container_name from queuedata.contaner_type: %s" % container_name)

    if container_name == 'singularity':
        logger.info("singularity has been requested")

        # Get the singularity options
        singularity_options = queuedata.container_options
        logger.debug("resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        if not singularity_options:
            logger.warning('singularity options not set')

        # Get the image path
        image_path = get_grid_image_for_singularity(job.platform)

        # Does the image exist?
        if image_path != '':
            # Prepend it to the given command
            cmd = "export workdir=" + workdir + "; singularity exec " + singularity_options + " " + image_path + \
                  " /bin/bash -c \'cd $workdir;pwd;" + cmd.replace("\'", "\\'").replace('\"', '\\"') + "\'"
        else:
            logger.warning("singularity options found but image does not exist")

        logger.info("Updated command: %s" % cmd)

    return cmd
<commit_msg>Deprecated remove_container_string()<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch

import os
import re

from pilot.user.atlas.setup import get_asetup
from pilot.user.atlas.setup import get_file_system_root_path
from pilot.info import infosys
from pilot.util.config import config
# import pilot.info.infoservice as infosys

import logging
logger = logging.getLogger(__name__)

def wrapper(executable, **kwargs):
    """
    Wrapper function for any container specific usage.
    This function will be called by pilot.util.container.execute() and prepends the executable with a container command.

    :param executable: command to be executed (string).
    :param kwargs:
    :return: executable wrapped with container command (string).
    """

    workdir = kwargs.get('workdir', '.')
    pilot_home = os.environ.get('PILOT_HOME', '')
    job = kwargs.get('job')

    if workdir == '.' and pilot_home != '':
        workdir = pilot_home

    # if job.imagename (from --containerimage <image>) is set, then always use raw singularity
    if config.Container.setup_type == "ALRB" and not job.imagename:
        fctn = alrb_wrapper
    else:
        fctn = singularity_wrapper
    return fctn(executable, workdir, job)

# def use_payload_container(job):
#     pass

def use_middleware_container():
    """
    Should middleware from container be used?
    In case middleware, i.e. the copy command for stage-in/out, should be taken from a container this function should
    return True.

    :return: True if middleware should be taken from container. False otherwise.
    """

    if get_middleware_type() == 'container':
        return True
    else:
        return False

def get_middleware_container():
    pass

def extract_platform_and_os(platform):
    """
    Extract the platform and OS substring from platform

    :param platform (string): E.g. "x86_64-slc6-gcc48-opt"
    :return: extracted platform specifics (string). E.g. "x86_64-slc6". In case of failure, return the full platform
    """

    pattern = r"([A-Za-z0-9_-]+)-.+-.+"
    a = re.findall(re.compile(pattern), platform)

    if len(a) > 0:
        ret = a[0]
    else:
        logger.warning("could not extract architecture and OS substring using pattern=%s from platform=%s"
                       "(will use %s for image name)" % (pattern, platform, platform))
        ret = platform

    return ret

def get_grid_image_for_singularity(platform):
    """
    Return the full path to the singularity grid image

    :param platform (string): E.g. "x86_64-slc6"
    :return: full path to grid image (string).
    """

    if not platform or platform == "":
        platform = "x86_64-slc6"
        logger.warning("using default platform=%s (cmtconfig not set)" % (platform))

    arch_and_os = extract_platform_and_os(platform)
    image = arch_and_os + ".img"
    _path = os.path.join(get_file_system_root_path(), "atlas.cern.ch/repo/containers/images/singularity")
    path = os.path.join(_path, image)
    if not os.path.exists(path):
        image = 'x86_64-centos7.img'
        logger.warning('path does not exist: %s (trying with image %s instead)' % (path, image))
        path = os.path.join(_path, image)
        if not os.path.exists(path):
            logger.warning('path does not exist either: %s' % path)
            path = ""

    return path

def get_middleware_type():
    """
    Return the middleware type from the container type.
    E.g. container_type = 'singularity:pilot;docker:wrapper;middleware:container'
    get_middleware_type() -> 'container', meaning that middleware should be taken from the container. The default
    is otherwise 'workernode', i.e. middleware is assumed to be present on the worker node.

    :return: middleware_type (string)
    """

    middleware_type = ""
    container_type = infosys.queuedata.container_type

    mw = 'middleware'
    if container_type and container_type != "" and mw in container_type:
        try:
            container_names = container_type.split(';')
            for name in container_names:
                t = name.split(':')
                if mw == t[0]:
                    middleware_type = t[1]
        except Exception as e:
            logger.warning("failed to parse the container name: %s, %s" % (container_type, e))
    else:
        # logger.warning("container middleware type not specified in queuedata")
        # no middleware type was specified, assume that middleware is present on worker node
        middleware_type = "workernode"

    return middleware_type

def alrb_wrapper(cmd, workdir, job):
    """
    Wrap the given command with the special ALRB setup for containers
    E.g. cmd = /bin/bash hello_world.sh
    ->
    export thePlatform="x86_64-slc6-gcc48-opt"
    export ALRB_CONT_RUNPAYLOAD="cmd'
    setupATLAS -c $thePlatform

    :param cmd (string): command to be executed in a container.
    :param workdir: (not used)
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    if container_name == 'singularity':
        # first get the full setup, which should be removed from cmd (or ALRB setup won't work)
        _asetup = get_asetup()
        cmd = cmd.replace(_asetup, "asetup ")
        # get simplified ALRB setup (export)
        asetup = get_asetup(alrb=True)

        # Get the singularity options
        singularity_options = queuedata.container_options
        logger.debug(
            "resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        _cmd = asetup
        if job.platform:
            _cmd += 'export thePlatform=\"%s\";' % job.platform
        #if '--containall' not in singularity_options:
        #    singularity_options += ' --containall'
        if singularity_options != "":
            _cmd += 'export ALRB_CONT_CMDOPTS=\"%s\";' % singularity_options
        _cmd += 'export ALRB_CONT_RUNPAYLOAD=\"%s\";' % cmd

        # this should not be necessary after the extract_container_image() in JobData update
        # containerImage should have been removed already
        if '--containerImage' in job.jobparams:
            job.jobparams, container_path = remove_container_string(job.jobparams)
            if container_path != "":
                _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % container_path
            else:
                logger.warning('failed to extract container path from %s' % job.jobparams)
                _cmd = ""
        else:
            _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c images'
            if job.platform:
                _cmd += '+$thePlatform'

        _cmd = _cmd.replace('  ', ' ')
        cmd = _cmd
        logger.info("Updated command: %s" % cmd)

    return cmd

## DEPRECATED, remove after verification with user container job
def remove_container_string(job_params):
    """ Retrieve the container string from the job parameters """

    pattern = r" \'?\-\-containerImage\=?\ ?([\S]+)\ ?\'?"
    compiled_pattern = re.compile(pattern)

    # remove any present ' around the option as well
    job_params = re.sub(r'\'\ \'', ' ', job_params)

    # extract the container path
    found = re.findall(compiled_pattern, job_params)
    container_path = found[0] if len(found) > 0 else ""

    # Remove the pattern and update the job parameters
    job_params = re.sub(pattern, ' ', job_params)

    return job_params, container_path

def singularity_wrapper(cmd, workdir, job):
    """
    Prepend the given command with the singularity execution command
    E.g. cmd = /bin/bash hello_world.sh
    -> singularity_command = singularity exec -B <bindmountsfromcatchall> <img> /bin/bash hello_world.sh
    singularity exec -B <bindmountsfromcatchall>  /cvmfs/atlas.cern.ch/repo/images/singularity/x86_64-slc6.img <script>

    :param cmd (string): command to be prepended.
    :param workdir: explicit work directory where the command should be executed (needs to be set for Singularity).
    :param job: job object.
    :return: prepended command with singularity execution command (string).
    """

    queuedata = job.infosys.queuedata

    container_name = queuedata.container_type.get("pilot")  # resolve container name for user=pilot
    logger.debug("resolved container_name from queuedata.contaner_type: %s" % container_name)

    if container_name == 'singularity':
        logger.info("singularity has been requested")

        # Get the singularity options
        singularity_options = queuedata.container_options
        logger.debug("resolved singularity_options from queuedata.container_options: %s" % singularity_options)

        if not singularity_options:
            logger.warning('singularity options not set')

        # Get the image path
        image_path = get_grid_image_for_singularity(job.platform)

        # Does the image exist?
        if image_path != '':
            # Prepend it to the given command
            cmd = "export workdir=" + workdir + "; singularity exec " + singularity_options + " " + image_path + \
                  " /bin/bash -c \'cd $workdir;pwd;" + cmd.replace("\'", "\\'").replace('\"', '\\"') + "\'"
        else:
            logger.warning("singularity options found but image does not exist")

        logger.info("Updated command: %s" % cmd)

    return cmd
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import hashlib
import os
from contextlib import contextmanager
from shutil import rmtree

from pip.download import is_file_url, url_to_path
from pip.index import PackageFinder
from pip.req.req_set import RequirementSet
from pip.wheel import Wheel
try:
    from pip.utils.hashes import FAVORITE_HASH
except ImportError:
    FAVORITE_HASH = 'sha256'

from ..cache import CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
                     make_install_requirement, pip_version_info)
from .base import BaseRepository

try:
    from tempfile import TemporaryDirectory  # added in 3.2
except ImportError:
    from .._compat import TemporaryDirectory

class PyPIRepository(BaseRepository):
    DEFAULT_INDEX_URL = 'https://pypi.python.org/simple'

    """
    The PyPIRepository will use the provided Finder instance to lookup
    packages.  Typically, it looks up packages on PyPI (the default implicit
    config), but any other PyPI mirror can be used if index_urls is
    changed/configured on the Finder.
    """
    def __init__(self, pip_options, session):
        self.session = session

        index_urls = [pip_options.index_url] + pip_options.extra_index_urls
        if pip_options.no_index:
            index_urls = []

        self.finder = PackageFinder(
            find_links=pip_options.find_links,
            index_urls=index_urls,
            trusted_hosts=pip_options.trusted_hosts,
            allow_all_prereleases=pip_options.pre,
            process_dependency_links=pip_options.process_dependency_links,
            session=self.session,
        )

        # Caches
        # stores project_name => InstallationCandidate mappings for all
        # versions reported by PyPI, so we only have to ask once for each
        # project
        self._available_candidates_cache = {}

        # stores InstallRequirement => list(InstallRequirement) mappings
        # of all secondary dependencies for the given requirement, so we
        # only have to go to disk once for each requirement
        self._dependencies_cache = {}

        # Setup file paths
        self.freshen_build_caches()
        self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs'))
        self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels'))

    def freshen_build_caches(self):
        """
        Start with fresh build/source caches.  Will remove any old build
        caches from disk automatically.
        """
        self._build_dir = TemporaryDirectory(fs_str('build'))
        self._source_dir = TemporaryDirectory(fs_str('source'))

    @property
    def build_dir(self):
        return self._build_dir.name

    @property
    def source_dir(self):
        return self._source_dir.name

    def clear_caches(self):
        rmtree(self._download_dir, ignore_errors=True)
        rmtree(self._wheel_download_dir, ignore_errors=True)

    def find_all_candidates(self, req_name):
        if req_name not in self._available_candidates_cache:
            # pip 8 changed the internal API, making this a public method
            if pip_version_info >= (8, 0):
                candidates = self.finder.find_all_candidates(req_name)
            else:
                candidates = self.finder._find_all_versions(req_name)
            self._available_candidates_cache[req_name] = candidates
        return self._available_candidates_cache[req_name]

    def find_best_match(self, ireq, prereleases=None):
        """
        Returns a Version object that indicates the best match for the given
        InstallRequirement according to the external repository.
        """
        if ireq.editable:
            return ireq  # return itself as the best match

        all_candidates = self.find_all_candidates(ireq.name)
        candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
        matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
                                                  prereleases=prereleases)

        # Reuses pip's internal candidate sort key to sort
        matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
        if not matching_candidates:
            raise NoCandidateFound(ireq, all_candidates)
        best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)

        # Turn the candidate into a pinned InstallRequirement
        return make_install_requirement(
            best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint
        )

    def get_dependencies(self, ireq):
        """
        Given a pinned or an editable InstallRequirement, returns a set of
        dependencies (also InstallRequirements, but not necessarily pinned).
        They indicate the secondary dependencies for the given requirement.
        """
        if not (ireq.editable or is_pinned_requirement(ireq)):
            raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))

        if ireq not in self._dependencies_cache:
            if ireq.link and not ireq.link.is_artifact:
                # No download_dir for VCS sources.  This also works around pip
                # using git-checkout-index, which gets rid of the .git dir.
                download_dir = None
            else:
                download_dir = self._download_dir
                if not os.path.isdir(download_dir):
                    os.makedirs(download_dir)
            if not os.path.isdir(self._wheel_download_dir):
                os.makedirs(self._wheel_download_dir)

            reqset = RequirementSet(self.build_dir,
                                    self.source_dir,
                                    download_dir=download_dir,
                                    wheel_download_dir=self._wheel_download_dir,
                                    session=self.session)
            self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq)
        return set(self._dependencies_cache[ireq])

    def get_hashes(self, ireq):
        """
        Given a pinned InstallRequire, returns a set of hashes that represent
        all of the files for a given requirement. It is not acceptable for an
        editable or unpinned requirement to be passed to this function.
        """
        if not is_pinned_requirement(ireq):
            raise TypeError(
                "Expected pinned requirement, not unpinned or editable, got {}".format(ireq))

        # We need to get all of the candidates that match our current version
        # pin, these will represent all of the files that could possibly
        # satisfy this constraint.
        all_candidates = self.find_all_candidates(ireq.name)
        candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
        matching_versions = list(
            ireq.specifier.filter((candidate.version for candidate in all_candidates)))
        matching_candidates = candidates_by_version[matching_versions[0]]

        return {
            self._get_file_hash(candidate.location)
            for candidate in matching_candidates
        }

    def _get_file_hash(self, location):
        h = hashlib.new(FAVORITE_HASH)
        with open_local_or_remote_file(location, self.session) as fp:
            for chunk in iter(lambda: fp.read(8096), b""):
                h.update(chunk)
        return ":".join([FAVORITE_HASH, h.hexdigest()])

    @contextmanager
    def allow_all_wheels(self):
        """
        Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.

        This also saves the candidate cache and set a new one, or else the results from the
        previous non-patched calls will interfere.
        """
        def _wheel_supported(self, tags=None):
            # Ignore current platform. Support everything.
            return True

        def _wheel_support_index_min(self, tags=None):
            # All wheels are equal priority for sorting.
            return 0

        original_wheel_supported = Wheel.supported
        original_support_index_min = Wheel.support_index_min
        original_cache = self._available_candidates_cache

        Wheel.supported = _wheel_supported
        Wheel.support_index_min = _wheel_support_index_min
        self._available_candidates_cache = {}

        try:
            yield
        finally:
            Wheel.supported = original_wheel_supported
            Wheel.support_index_min = original_support_index_min
            self._available_candidates_cache = original_cache

@contextmanager
def open_local_or_remote_file(link, session):
    """
    Open local or remote file for reading.

    :type link: pip.index.Link
    :type session: requests.Session
    :raises ValueError: If link points to a local directory.
    :return: a context manager to the opened file-like object
    """
    url = link.url_without_fragment

    if is_file_url(link):
        # Local URL
        local_path = url_to_path(url)
        if os.path.isdir(local_path):
            raise ValueError("Cannot open directory for read: {}".format(url))
        else:
            with open(local_path, 'rb') as local_file:
                yield local_file
    else:
        # Remote URL
        headers = {"Accept-Encoding": "identity"}
        response = session.get(url, headers=headers, stream=True)
        try:
            yield response.raw
        finally:
            response.close()
<commit_msg>fixed a bottleneck where pip-tools archives the entire folder of an editable requirement and this can take a really long time<commit_after># coding: utf-8
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import hashlib
import os
from contextlib import contextmanager
from shutil import rmtree

from pip.download import is_file_url, url_to_path
from pip.index import PackageFinder
from pip.req.req_set import RequirementSet
from pip.wheel import Wheel
try:
    from pip.utils.hashes import FAVORITE_HASH
except ImportError:
    FAVORITE_HASH = 'sha256'

from ..cache import CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
                     make_install_requirement, pip_version_info)
from .base import BaseRepository

try:
    from tempfile import TemporaryDirectory  # added in 3.2
except ImportError:
    from .._compat import TemporaryDirectory

class PyPIRepository(BaseRepository):
    DEFAULT_INDEX_URL = 'https://pypi.python.org/simple'

    """
    The PyPIRepository will use the provided Finder instance to lookup
    packages.  Typically, it looks up packages on PyPI (the default implicit
    config), but any other PyPI mirror can be used if index_urls is
    changed/configured on the Finder.
    """
    def __init__(self, pip_options, session):
        self.session = session

        index_urls = [pip_options.index_url] + pip_options.extra_index_urls
        if pip_options.no_index:
            index_urls = []

        self.finder = PackageFinder(
            find_links=pip_options.find_links,
            index_urls=index_urls,
            trusted_hosts=pip_options.trusted_hosts,
            allow_all_prereleases=pip_options.pre,
            process_dependency_links=pip_options.process_dependency_links,
            session=self.session,
        )

        # Caches
        # stores project_name => InstallationCandidate mappings for all
        # versions reported by PyPI, so we only have to ask once for each
        # project
        self._available_candidates_cache = {}

        # stores InstallRequirement => list(InstallRequirement) mappings
        # of all secondary dependencies for the given requirement, so we
        # only have to go to disk once for each requirement
        self._dependencies_cache = {}

        # Setup file paths
        self.freshen_build_caches()
        self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs'))
        self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels'))

    def freshen_build_caches(self):
        """
        Start with fresh build/source caches.  Will remove any old build
        caches from disk automatically.
        """
        self._build_dir = TemporaryDirectory(fs_str('build'))
        self._source_dir = TemporaryDirectory(fs_str('source'))

    @property
    def build_dir(self):
        return self._build_dir.name

    @property
    def source_dir(self):
        return self._source_dir.name

    def clear_caches(self):
        rmtree(self._download_dir, ignore_errors=True)
        rmtree(self._wheel_download_dir, ignore_errors=True)

    def find_all_candidates(self, req_name):
        if req_name not in self._available_candidates_cache:
            # pip 8 changed the internal API, making this a public method
            if pip_version_info >= (8, 0):
                candidates = self.finder.find_all_candidates(req_name)
            else:
                candidates = self.finder._find_all_versions(req_name)
            self._available_candidates_cache[req_name] = candidates
        return self._available_candidates_cache[req_name]

    def find_best_match(self, ireq, prereleases=None):
        """
        Returns a Version object that indicates the best match for the given
        InstallRequirement according to the external repository.
        """
        if ireq.editable:
            return ireq  # return itself as the best match

        all_candidates = self.find_all_candidates(ireq.name)
        candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
        matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
                                                  prereleases=prereleases)

        # Reuses pip's internal candidate sort key to sort
        matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
        if not matching_candidates:
            raise NoCandidateFound(ireq, all_candidates)
        best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)

        # Turn the candidate into a pinned InstallRequirement
        return make_install_requirement(
            best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint
        )

    def get_dependencies(self, ireq):
        """
        Given a pinned or an editable InstallRequirement, returns a set of
        dependencies (also InstallRequirements, but not necessarily pinned).
        They indicate the secondary dependencies for the given requirement.
        """
        if not (ireq.editable or is_pinned_requirement(ireq)):
            raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))

        if ireq not in self._dependencies_cache:
            if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
                # No download_dir for locally available editable requirements.
                # If a download_dir is passed, pip will  unnecessarely
                # archive the entire source directory
                download_dir = None
            elif ireq.link and not ireq.link.is_artifact:
                # No download_dir for VCS sources.  This also works around pip
                # using git-checkout-index, which gets rid of the .git dir.
                download_dir = None
            else:
                download_dir = self._download_dir
                if not os.path.isdir(download_dir):
                    os.makedirs(download_dir)
            if not os.path.isdir(self._wheel_download_dir):
                os.makedirs(self._wheel_download_dir)

            reqset = RequirementSet(self.build_dir,
                                    self.source_dir,
                                    download_dir=download_dir,
                                    wheel_download_dir=self._wheel_download_dir,
                                    session=self.session)
            self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq)
        return set(self._dependencies_cache[ireq])

    def get_hashes(self, ireq):
        """
        Given a pinned InstallRequire, returns a set of hashes that represent
        all of the files for a given requirement. It is not acceptable for an
        editable or unpinned requirement to be passed to this function.
        """
        if not is_pinned_requirement(ireq):
            raise TypeError(
                "Expected pinned requirement, not unpinned or editable, got {}".format(ireq))

        # We need to get all of the candidates that match our current version
        # pin, these will represent all of the files that could possibly
        # satisfy this constraint.
        all_candidates = self.find_all_candidates(ireq.name)
        candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
        matching_versions = list(
            ireq.specifier.filter((candidate.version for candidate in all_candidates)))
        matching_candidates = candidates_by_version[matching_versions[0]]

        return {
            self._get_file_hash(candidate.location)
            for candidate in matching_candidates
        }

    def _get_file_hash(self, location):
        h = hashlib.new(FAVORITE_HASH)
        with open_local_or_remote_file(location, self.session) as fp:
            for chunk in iter(lambda: fp.read(8096), b""):
                h.update(chunk)
        return ":".join([FAVORITE_HASH, h.hexdigest()])

    @contextmanager
    def allow_all_wheels(self):
        """
        Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.

        This also saves the candidate cache and set a new one, or else the results from the
        previous non-patched calls will interfere.
        """
        def _wheel_supported(self, tags=None):
            # Ignore current platform. Support everything.
            return True

        def _wheel_support_index_min(self, tags=None):
            # All wheels are equal priority for sorting.
            return 0

        original_wheel_supported = Wheel.supported
        original_support_index_min = Wheel.support_index_min
        original_cache = self._available_candidates_cache

        Wheel.supported = _wheel_supported
        Wheel.support_index_min = _wheel_support_index_min
        self._available_candidates_cache = {}

        try:
            yield
        finally:
            Wheel.supported = original_wheel_supported
            Wheel.support_index_min = original_support_index_min
            self._available_candidates_cache = original_cache

@contextmanager
def open_local_or_remote_file(link, session):
    """
    Open local or remote file for reading.

    :type link: pip.index.Link
    :type session: requests.Session
    :raises ValueError: If link points to a local directory.
    :return: a context manager to the opened file-like object
    """
    url = link.url_without_fragment

    if is_file_url(link):
        # Local URL
        local_path = url_to_path(url)
        if os.path.isdir(local_path):
            raise ValueError("Cannot open directory for read: {}".format(url))
        else:
            with open(local_path, 'rb') as local_file:
                yield local_file
    else:
        # Remote URL
        headers = {"Accept-Encoding": "identity"}
        response = session.get(url, headers=headers, stream=True)
        try:
            yield response.raw
        finally:
            response.close()
<|endoftext|>


<commit_before>from osv import osv, fields
from tools import debug
import time
import pooler
from dateutil import parser
from datetime import date
from datetime import timedelta
import calendar
from tools.translate import _

class ccorp_addons_account_assets(osv.osv):
	_name = 'account.asset.asset'
	_inherit = 'account.asset.asset'

	def _process(self, s):
		return (s or '') % {
			'year':time.strftime('%Y'),
			'month': time.strftime('%m'),
			'day':time.strftime('%d'),
			'y': time.strftime('%y'),
			'doy': time.strftime('%j'),
			'woy': time.strftime('%W'),
			'weekday': time.strftime('%w'),
			'h24': time.strftime('%H'),
			'h12': time.strftime('%I'),
			'min': time.strftime('%M'),
			'sec': time.strftime('%S'),
		}

	def _getCompany(self, cr, uid, vals, context=None):
		user = pooler.get_pool(cr.dbname).get('res.users').browse(cr, uid, [uid], context=context)[0]
		if user.company_id:
			return user.company_id.id

	def get_location1(self, cr, uid, ids, pprodlot, context=None):
		product_lot= self.pool.get('stock.production.lot').browse(cr, uid, pprodlot)
		debug(product_lot)
		#debug(product_lot.product_id)
		res = {}
		saved_move_location = 0
		saved_move_date = 0

		for move_object in product_lot.move_ids:
			#debug(key)
			#move_object= self.pool.get('stock.move').browse(cr, uid, key)
			debug(move_object)
			debug(move_object.date)
			temp_saved_move_date = parser.parse(move_object.date).date()
			if saved_move_date == 0:
				saved_move_location = move_object.location_dest_id.name
				saved_move_date = parser.parse(move_object.date).date()

			elif saved_move_date < temp_saved_move_date:
				saved_move_location = move_object.location_dest_id.name
				saved_move_date = parser.parse(move_object.date).date()

		return saved_move_location

	def get_id_search(self, cr, uid, sequence_id, test='id', context=None):
		assert test in ('code','id')
		company_ids = self.pool.get('res.company').search(cr, uid, [], context=context)
		cr.execute('''SELECT id, number_next, prefix, suffix, padding
					FROM ir_sequence
					WHERE %s=%%s
					AND active=true
					AND (company_id in %%s or company_id is NULL)
					ORDER BY company_id, id
					FOR UPDATE NOWAIT''' % test,
					(sequence_id, tuple(company_ids)))
		res = cr.dictfetchone()
		if res:
			if res['number_next']:
				return self._process(res['prefix']) + '%%0%sd' % res['padding'] % res['number_next'] + self._process(res['suffix'])
			else:
				return self._process(res['prefix']) + self._process(res['suffix'])
		return False

	def get_search(self, cr, uid, context=None):
		code='account.asset.asset'
		return self.get_id_search(cr, uid, code, test='code')

	def get_product1(self, cr, uid, ids, pprodlot, context=None):
		product_lot= self.pool.get('stock.production.lot').browse(cr, uid, pprodlot)
		debug(product_lot)
		#debug(product_lot.product_id)
		saved_lot_product= product_lot.product_id.name

		return saved_lot_product

	def _get_location(self, cr, uid, ids, field_name, arg, context={}):
		asset_id=contex.get('active_id')
		if asset_id != None:
			product_lot= self.pool.get('account.asset.asset').browse(cr, uid, asset_id)
			pprodlot = product_lot.prod_lot_id.id

			product_lot= self.pool.get('stock.production.lot').browse(cr, uid, pprodlot)
			debug(product_lot)
			#debug(product_lot.product_id)
			res = {}
			saved_move_location = 0
			saved_move_date = 0

			for move_object in product_lot.move_ids:
				#debug(key)
				#move_object= self.pool.get('stock.move').browse(cr, uid, key)
				debug(move_object)
				debug(move_object.date)
				temp_saved_move_date = parser.parse(move_object.date).date()
				if saved_move_date == 0:
					saved_move_location = move_object.location_dest_id.name
					saved_move_date = parser.parse(move_object.date).date()

				elif saved_move_date < temp_saved_move_date:
					saved_move_location = move_object.location_dest_id.name
					saved_move_date = parser.parse(move_object.date).date()

			return saved_move_location

	def _get_asset_product(self, cr, uid, ids, field_name, arg, context={}):
		asset_id=contex.get('active_id')
		if asset_id != None:
			product_lot= self.pool.get('account.asset.asset').browse(cr, uid, asset_id)

			debug(product_lot)
			#debug(product_lot.product_id)
			saved_lot_product= product_lot.product_id.name

			return saved_lot_product

	_columns = {
		'prod_lot_id': fields.many2one('stock.production.lot', 'Production Lot'), #, domain="[('company_id','=',product_id)]",
		'asset_product_id': fields.function(_get_asset_product,string='Product',type="char", store=False, method=True, readonly=True ),
		'location_id': fields.function(_get_location, string='Location', store=False, method=True,readonly=True ,type="char"),

	}
	_defaults = {
		'code': get_search,
		'partner_id': _getCompany
	}
	_sql_constraints = [
		('unique_asset_company', 'UNIQUE (partner_id,code)', 'You can not have two asset with the same code in the same partner !')
	]

	def on_change_search_location(self, cr, uid, ids, pprodlot):
		result = self.get_location1(cr,uid, ids,pprodlot)
		result2 = self.get_product1(cr,uid, ids,pprodlot)
		v = {}
		v['location_id'] = result
		v['asset_product_id'] = result2
		return {'value':v}

	def create(self, cr, uid, vals, context=None):
		sentCode = vals['code']
		debug(sentCode)
		if sentCode == (self.pool.get('ir.sequence').get_search(cr, uid, 'account.asset.asset')):
			codep=self.pool.get('ir.sequence').get(cr, uid, 'account.asset.asset')
		return super(ccorp_addons_account_assets, self).create(cr, uid, vals, context=None)

ccorp_addons_account_assets()

<commit_msg>[FIX] location in assets<commit_after>from osv import osv, fields
from tools import debug
import time
import pooler
from dateutil import parser
from datetime import date
from datetime import timedelta
import calendar
from tools.translate import _

class ccorp_addons_account_assets(osv.osv):
	_name = 'account.asset.asset'
	_inherit = 'account.asset.asset'

	def _process(self, s):
		return (s or '') % {
			'year':time.strftime('%Y'),
			'month': time.strftime('%m'),
			'day':time.strftime('%d'),
			'y': time.strftime('%y'),
			'doy': time.strftime('%j'),
			'woy': time.strftime('%W'),
			'weekday': time.strftime('%w'),
			'h24': time.strftime('%H'),
			'h12': time.strftime('%I'),
			'min': time.strftime('%M'),
			'sec': time.strftime('%S'),
		}

	def _getCompany(self, cr, uid, vals, context=None):
		user = pooler.get_pool(cr.dbname).get('res.users').browse(cr, uid, [uid], context=context)[0]
		if user.company_id:
			return user.company_id.id

	def get_location1(self, cr, uid, ids, pprodlot, context=None):
		product_lot= self.pool.get('stock.production.lot').browse(cr, uid, pprodlot)
		debug(product_lot)
		#debug(product_lot.product_id)
		res = {}
		saved_move_location = 0
		saved_move_date = 0

		for move_object in product_lot.move_ids:
			#debug(key)
			#move_object= self.pool.get('stock.move').browse(cr, uid, key)
			debug(move_object)
			debug(move_object.date)
			temp_saved_move_date = parser.parse(move_object.date).date()
			if saved_move_date == 0:
				saved_move_location = move_object.location_dest_id.name
				saved_move_date = parser.parse(move_object.date).date()

			elif saved_move_date < temp_saved_move_date:
				saved_move_location = move_object.location_dest_id.name
				saved_move_date = parser.parse(move_object.date).date()

		return saved_move_location

	def get_id_search(self, cr, uid, sequence_id, test='id', context=None):
		assert test in ('code','id')
		company_ids = self.pool.get('res.company').search(cr, uid, [], context=context)
		cr.execute('''SELECT id, number_next, prefix, suffix, padding
					FROM ir_sequence
					WHERE %s=%%s
					AND active=true
					AND (company_id in %%s or company_id is NULL)
					ORDER BY company_id, id
					FOR UPDATE NOWAIT''' % test,
					(sequence_id, tuple(company_ids)))
		res = cr.dictfetchone()
		if res:
			if res['number_next']:
				return self._process(res['prefix']) + '%%0%sd' % res['padding'] % res['number_next'] + self._process(res['suffix'])
			else:
				return self._process(res['prefix']) + self._process(res['suffix'])
		return False

	def get_search(self, cr, uid, context=None):
		code='account.asset.asset'
		return self.get_id_search(cr, uid, code, test='code')

	def get_product1(self, cr, uid, ids, pprodlot, context=None):
		product_lot= self.pool.get('stock.production.lot').browse(cr, uid, pprodlot)
		debug(product_lot)
		#debug(product_lot.product_id)
		saved_lot_product= product_lot.product_id.name

		return saved_lot_product

	def _get_location(self, cr, uid, ids, field_name, arg, context={}):
		asset_id=context.get('active_id')
		if asset_id != None:
			product_lot= self.pool.get('account.asset.asset').browse(cr, uid, asset_id)
			pprodlot = product_lot.prod_lot_id.id

			product_lot= self.pool.get('stock.production.lot').browse(cr, uid, pprodlot)
			debug(product_lot)
			#debug(product_lot.product_id)
			res = {}
			saved_move_location = 0
			saved_move_date = 0

			for move_object in product_lot.move_ids:
				#debug(key)
				#move_object= self.pool.get('stock.move').browse(cr, uid, key)
				debug(move_object)
				debug(move_object.date)
				temp_saved_move_date = parser.parse(move_object.date).date()
				if saved_move_date == 0:
					saved_move_location = move_object.location_dest_id.name
					saved_move_date = parser.parse(move_object.date).date()

				elif saved_move_date < temp_saved_move_date:
					saved_move_location = move_object.location_dest_id.name
					saved_move_date = parser.parse(move_object.date).date()

			return saved_move_location

	def _get_asset_product(self, cr, uid, ids, field_name, arg, context={}):
		asset_id=context.get('active_id')
		if asset_id != None:
			product_lot= self.pool.get('account.asset.asset').browse(cr, uid, asset_id)

			debug(product_lot)
			#debug(product_lot.product_id)
			saved_lot_product= product_lot.product_id.name

			return saved_lot_product

	_columns = {
		'prod_lot_id': fields.many2one('stock.production.lot', 'Production Lot'), #, domain="[('company_id','=',product_id)]",
		'asset_product_id': fields.function(_get_asset_product,string='Product',type="char", store=False, method=True, readonly=True ),
		'location_id': fields.function(_get_location, string='Location', store=False, method=True,readonly=True ,type="char"),

	}
	_defaults = {
		'code': get_search,
		'partner_id': _getCompany
	}
	_sql_constraints = [
		('unique_asset_company', 'UNIQUE (partner_id,code)', 'You can not have two asset with the same code in the same partner !')
	]

	def on_change_search_location(self, cr, uid, ids, pprodlot):
		result = self.get_location1(cr,uid, ids,pprodlot)
		result2 = self.get_product1(cr,uid, ids,pprodlot)
		v = {}
		v['location_id'] = result
		v['asset_product_id'] = result2
		return {'value':v}

	def create(self, cr, uid, vals, context=None):
		sentCode = vals['code']
		debug(sentCode)
		if sentCode == (self.pool.get('ir.sequence').get_search(cr, uid, 'account.asset.asset')):
			codep=self.pool.get('ir.sequence').get(cr, uid, 'account.asset.asset')
		return super(ccorp_addons_account_assets, self).create(cr, uid, vals, context=None)

ccorp_addons_account_assets()

<|endoftext|>


<commit_before>from south.modelsinspector import add_introspection_rules
from django.conf import settings

if "tagging" in settings.INSTALLED_APPS:
    try:
        from tagging.fields import TagField
    except ImportError:
        pass
    else:
        rules = [
            (
                (TagField, ),
                [],
                {
                    "blank": ["blank", {"default": True}],
                    "max_length": ["max_length", {"default": 255}],
                },
            ),
        ]

        add_introspection_rules(rules, ["^tagging\.fields",])
<commit_msg>Add tagging autocomplete field<commit_after>from south.modelsinspector import add_introspection_rules
from django.conf import settings

if "tagging" in settings.INSTALLED_APPS:
    try:
        from tagging.fields import TagField
    except ImportError:
        pass
    else:
        rules = [
            (
                (TagField, ),
                [],
                {
                    "blank": ["blank", {"default": True}],
                    "max_length": ["max_length", {"default": 255}],
                },
            ),
        ]
        add_introspection_rules(rules, ["^tagging\.fields",])

if "tagging_autocomplete" in settings.INSTALLED_APPS:
    add_introspection_rules([], ["^tagging_autocomplete\.models\.TagAutocompleteField"])

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import numpy as np
import cv2, os, lda

if __name__ == "__main__":

	# set parameter for experiment
	nTopics = 8

	# create folder for saving result
	if not os.path.exists("result"):
		os.mkdir("result")

	# create folder for showing fitting process
	if not os.path.exists("visualization"):
		os.mkdir("visualization")

	# load image files (created by createImage.py)
	data = np.zeros((1000,16),dtype=np.uint8)
	for i in range(1000):
		image =  cv2.resize(cv2.imread("image/%d.jpg"%i,0),(4,4),interpolation=cv2.INTER_NEAREST)
		data[i,:] = image.reshape((16)).astype(np.uint8)

	# apply latent dirichlet allocation
	model = lda.LDA()
	model.setData(data)
	model.solve(nTopics=nTopics)

	# show topics obtained
	for i in range(8):
	    topic = model.qPhi[i,:]
	    topic = topic/topic.max()*255
	    topic = topic.reshape((4,4))

	    cv2.imwrite("result/%d.bmp"%i,cv2.resize(topic.astype(np.uint8),(200,200),interpolation=cv2.INTER_NEAREST))

<commit_msg>improved test script<commit_after># -*- coding: utf-8 -*-
import numpy as np
import cv2, os, lda

if __name__ == "__main__":

	# set parameter for experiment
	nTopics = 8

	# create folder for saving result
	if not os.path.exists("result"):
		os.mkdir("result")

	# create folder for showing fitting process
	if not os.path.exists("visualization"):
		os.mkdir("visualization")

	# load image files (created by createImage.py)
	data = np.zeros((1000,16),dtype=np.uint8)
	for i in range(1000):
		image =  cv2.resize(cv2.imread("image/%d.jpg"%i,0),(4,4),interpolation=cv2.INTER_NEAREST)
		data[i,:] = image.reshape((16)).astype(np.uint8)

	# apply latent dirichlet allocation
	model = lda.LDA()
	model.setData(data)
	model.solve(nTopics=nTopics)

	# show topics obtained
	for i in range(nTopics):
	    topic = model.qPhi[i,:]
	    topic = topic/topic.max()*255
	    topic = topic.reshape((4,4))

	    cv2.imwrite("result/%d.bmp"%i,cv2.resize(topic.astype(np.uint8),(200,200),interpolation=cv2.INTER_NEAREST))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2011 - 2013 Therp BV (<http://therp.nl>).
#
#    All other contributions are (C) by their respective contributors
#
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from openerp.tools.translate import _

"""
This module adds support for Direct debit orders as applicable
in the Netherlands. Debit orders are advanced in total by the bank.
Amounts that cannot be debited or are canceled by account owners are
credited afterwards. Such a creditation is called a storno.

Invoice workflow:

1 the sale leads to
   1300 Debtors 100
   8000 Sales       100

Balance:
  Debtors  2000 |
  Sales        | 2000

2 an external booking takes place
   1100 Bank    100
   1300 Debtors     100
   This booking is reconciled with [1]
   The invoice gets set to state 'paid', and 'reconciled' = True

Balance:
   Debtors 1900 |
   Bank     100 |
   Sales        | 2000

This module implements the following diversion:

2a the invoice is included in a direct debit order. When the order is
   confirmed, a move is created per invoice:

   2000 Transfer account 100 |
   1300 Debtors              | 100
   Reconciliation takes place between 1 and 2a.
   The invoice gets set to state 'paid', and 'reconciled' = True

Balance:
   Debtors             0 |
   Transfer account 2000 |
   Bank                0 |
   Sales                 | 2000

3a the direct debit order is booked on the bank account

Balance:
  1100 Bank             2000 |
  2000 Transfer account      | 2000
  Reconciliation takes place between 3a and 2a

Balance:
   Debtors             0 |
   Transfer account    0 |
   Bank             2000 |
   Sales                 | 2000

4 a storno from invoice [1] triggers a new booking on the bank account
  1300 Debtors 100 |
  1100 Bank        | 100

Balance:
   Debtors           100 |
   Transfer account    0 |
   Bank             1900 |
   Sales                 | 2000

   The reconciliation of 2a is undone. The booking of 2a is reconciled
   with the booking of 4 instead.
   The payment line attribute 'storno' is set to True and the invoice
   state is no longer 'paid'.

Two cases need to be distinguisted:
   1) If the storno is a manual storno from the partner, the invoice is set to
      state 'debit_denied', with 'reconciled' = False
      This module implements this option by allowing the bank module to call

          netsvc.LocalService("workflow").trg_validate(
              uid, 'account.invoice', ids, 'debit_denied', cr)

   2) If the storno is an error generated by the bank (assumingly non-fatal),
      the invoice is reopened for the next debit run. This is a call to existing

         netsvc.LocalService("workflow").trg_validate(
             uid, 'account.invoice', ids, 'open_test', cr)

      Should also be adding a log entry on the invoice for tracing purposes

         self._log_event(cr, uid, ids, -1.0, 'Debit denied')

      If not for that funny comment
      "#TODO: implement messages system"  in account/invoice.py

   Repeating non-fatal fatal errors need to be dealt with manually by checking
   open invoices with a matured invoice- or due date.
"""

class account_invoice(orm.Model):
    _inherit = "account.invoice"

    def __init__(self, pool, cr):
        """
        Adding a state to the hardcoded state list of the inherited
        model. The alternative is duplicating the field definition
        in columns but only one module can do that!

        Maybe apply a similar trick when overriding the buttons' 'states' attributes
        in the form view, manipulating the xml in fields_view_get().
        """
        super(account_invoice, self).__init__(pool, cr)
        invoice_obj = pool.get('account.invoice')
        invoice_obj._columns['state'].selection.append(
            ('debit_denied', 'Debit denied'))

    def action_debit_denied(self, cr, uid, ids, context=None):
        for invoice_id in ids:
            if self.test_paid(cr, uid, [invoice_id], context):
                number = self.read(
                    cr, uid, invoice_id, ['number'], context=context)['number']
                raise osv.except_osv(
                    _('Error !'),
                    _('You cannot set invoice \'%s\' to state \'debit denied\', ' +
                      'as it is still reconciled.') % number)
        self.write(cr, uid, ids, {'state':'debit_denied'}, context=context)
        for inv_id, name in self.name_get(cr, uid, ids, context=context):
            message = _("Invoice '%s': direct debit is denied.") % name
            self.log(cr, uid, inv_id, message)
        return True

    def test_undo_debit_denied(self, cr, uid, ids, context=None):
        """
        Called from the workflow. Used to unset paid state on
        invoices that were paid with bank transfers which are being cancelled
        """
        for invoice in self.read(cr, uid, ids, ['reconciled'], context):
            if not invoice['reconciled']:
                return False
        return True
<commit_msg>Fix a wrong import (osv was used but not imported !) Coding style fix<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2011 - 2013 Therp BV (<http://therp.nl>).
#
#    All other contributions are (C) by their respective contributors
#
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm
from openerp.tools.translate import _

"""
This module adds support for Direct debit orders as applicable
in the Netherlands. Debit orders are advanced in total by the bank.
Amounts that cannot be debited or are canceled by account owners are
credited afterwards. Such a creditation is called a storno.

Invoice workflow:

1 the sale leads to
   1300 Debtors 100
   8000 Sales       100

Balance:
  Debtors  2000 |
  Sales        | 2000

2 an external booking takes place
   1100 Bank    100
   1300 Debtors     100
   This booking is reconciled with [1]
   The invoice gets set to state 'paid', and 'reconciled' = True

Balance:
   Debtors 1900 |
   Bank     100 |
   Sales        | 2000

This module implements the following diversion:

2a the invoice is included in a direct debit order. When the order is
   confirmed, a move is created per invoice:

   2000 Transfer account 100 |
   1300 Debtors              | 100
   Reconciliation takes place between 1 and 2a.
   The invoice gets set to state 'paid', and 'reconciled' = True

Balance:
   Debtors             0 |
   Transfer account 2000 |
   Bank                0 |
   Sales                 | 2000

3a the direct debit order is booked on the bank account

Balance:
  1100 Bank             2000 |
  2000 Transfer account      | 2000
  Reconciliation takes place between 3a and 2a

Balance:
   Debtors             0 |
   Transfer account    0 |
   Bank             2000 |
   Sales                 | 2000

4 a storno from invoice [1] triggers a new booking on the bank account
  1300 Debtors 100 |
  1100 Bank        | 100

Balance:
   Debtors           100 |
   Transfer account    0 |
   Bank             1900 |
   Sales                 | 2000

   The reconciliation of 2a is undone. The booking of 2a is reconciled
   with the booking of 4 instead.
   The payment line attribute 'storno' is set to True and the invoice
   state is no longer 'paid'.

Two cases need to be distinguisted:
   1) If the storno is a manual storno from the partner, the invoice is set to
      state 'debit_denied', with 'reconciled' = False
      This module implements this option by allowing the bank module to call

          netsvc.LocalService("workflow").trg_validate(
              uid, 'account.invoice', ids, 'debit_denied', cr)

   2) If the storno is an error generated by the bank (assumingly non-fatal),
      the invoice is reopened for the next debit run. This is a call to existing

         netsvc.LocalService("workflow").trg_validate(
             uid, 'account.invoice', ids, 'open_test', cr)

      Should also be adding a log entry on the invoice for tracing purposes

         self._log_event(cr, uid, ids, -1.0, 'Debit denied')

      If not for that funny comment
      "#TODO: implement messages system"  in account/invoice.py

   Repeating non-fatal fatal errors need to be dealt with manually by checking
   open invoices with a matured invoice- or due date.
"""

class account_invoice(orm.Model):
    _inherit = "account.invoice"

    def __init__(self, pool, cr):
        """
        Adding a state to the hardcoded state list of the inherited
        model. The alternative is duplicating the field definition
        in columns but only one module can do that!

        Maybe apply a similar trick when overriding the buttons' 'states' attributes
        in the form view, manipulating the xml in fields_view_get().
        """
        super(account_invoice, self).__init__(pool, cr)
        invoice_obj = pool.get('account.invoice')
        invoice_obj._columns['state'].selection.append(
            ('debit_denied', 'Debit denied'))

    def action_debit_denied(self, cr, uid, ids, context=None):
        for invoice_id in ids:
            if self.test_paid(cr, uid, [invoice_id], context):
                number = self.read(
                    cr, uid, invoice_id, ['number'], context=context)['number']
                raise orm.except_orm(
                    _('Error !'),
                    _('You cannot set invoice \'%s\' to state \'debit denied\', ' +
                      'as it is still reconciled.') % number)
        self.write(cr, uid, ids, {'state': 'debit_denied'}, context=context)
        for inv_id, name in self.name_get(cr, uid, ids, context=context):
            message = _("Invoice '%s': direct debit is denied.") % name
            self.log(cr, uid, inv_id, message)
        return True

    def test_undo_debit_denied(self, cr, uid, ids, context=None):
        """
        Called from the workflow. Used to unset paid state on
        invoices that were paid with bank transfers which are being cancelled
        """
        for invoice in self.read(cr, uid, ids, ['reconciled'], context):
            if not invoice['reconciled']:
                return False
        return True
<|endoftext|>


<commit_before>def changeDataToList(matrix):
    returnList, returnList2 = [], []
    for i in matrix:
        for ii in i:
            returnList.append(ii)
    returnList = sorted(returnList)
    for i in returnList:
        if i not in returnList2:
            returnList2.append(i)
    returnList = returnList2
    return returnList

def findAround(matrix, row, col):
    try:
        if matrix[row][col]:
            return True
    except:
        return False

def pointMatrix(twoDimMatrix, rowNum, colNum):
    def _first(func):
        def _sencond():
            func
        return _sencond
    return _first

def genList(matrix, row, col, parentList):
    for changeNum in [-1, 1]:
        if findAround(matrix, row + changeNum, col) and \
           matrix[row][col] < matrix[row + changeNum][col]:
            print(matrix[row + changeNum][col])
        if findAround(matrix, row, col + changeNum) and \
           matrix[row][col] < matrix[row][col + changeNum]:
            print(matrix[row][col + changeNum])

def main(m):

    sortedList = changeDataToList(m)
    print(startIndex)
<commit_msg>finish part of<commit_after>"""
def changeDataToList(matrix):
    returnList, returnList2 = [], []
    for i in matrix:
        for ii in i:
            returnList.append(ii)
    returnList = sorted(returnList)
    for i in returnList:
        if i not in returnList2:
            returnList2.append(i)
    returnList = returnList2
    return returnList"""

def findAround(matrix, row, col):
    try:
        if matrix[row][col]:
            return True
    except:
        return False

"""
def pointMatrix(twoDimMatrix, rowNum, colNum):
    def _first(func):
        def _sencond():
            func
        return _sencond
    return _first"""

def genList(matrix, row, col, parentList):
    returnDict = {}

    for changeNum in [-1, 1]:
        if findAround(matrix, row + changeNum, col) and \
           matrix[row][col] < matrix[row + changeNum][col]:
            print(matrix[row + changeNum][col])
            pathList = parentList
            print(pathList + [matrix[row + changeNum][col]])
            returnDict[(row + changeNum, col)
                       ] = pathList + [matrix[row + changeNum][col]]
        if findAround(matrix, row, col + changeNum) and \
           matrix[row][col] < matrix[row][col + changeNum]:
            print(matrix[row][col + changeNum])
            pathList = parentList
            print(pathList + [matrix[row][col + changeNum]])
            returnDict[(row, col + changeNum)
                       ] = pathList + [matrix[row][col + changeNum]]
    return returnDict

def main(m):
    pass
<|endoftext|>


<commit_before>import os
import jinja2
import base64
import sendgrid
import re
from sendgrid.helpers.mail.mail import Email
from sendgrid.helpers.mail.mail import Content
from sendgrid.helpers.mail.mail import Mail
from sendgrid.helpers.mail.mail import Attachment
from sendgrid.helpers.mail.mail import Personalization

def send(address, subject, template_name, context, attachment=None, for_real=False):

    templateLoader = jinja2.FileSystemLoader(searchpath="templates")
    templateEnv = jinja2.Environment(loader=templateLoader)
    html_template = templateEnv.get_template(template_name + ".html")

    html_to_send = html_template.render(context)

    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    impactstory_email = Email("team@impactstory.org", "Impactstory Team")
    to_email = Email(address)
    content = Content("text/html", html_to_send)
    mail = Mail(impactstory_email, subject, to_email, content)
    personalization = Personalization()
    personalization.add_to(to_email)
    personalization.add_to(impactstory_email)
    mail.add_personalization(personalization)

    if attachment:
        my_attachment = Attachment()
        my_attachment.type = "application/csv"
        my_attachment.filename = "results.csv"
        my_attachment.disposition = "attachment"
        my_attachment.content_id = "results csv file"
        with open(attachment, 'rb') as f:
            data = f.read()

        #handle the spammer
        if re.findall(u"[a-z0-9]{7,15}@gmail.com", address):
            mail = Mail(impactstory_email, "Over limit. Please email us at team@impactstory.org for other data access options.", to_email, content)
            personalization = Personalization()
            personalization.add_to(to_email)
            personalization.add_to(impactstory_email)
            mail.add_personalization(personalization)
        else:
            my_attachment.content = base64.b64encode(data)
            mail.add_attachment(my_attachment)

    # if for_real:
    #     response = sg.client.mail.send.post(request_body=mail.get())
    #     print u"Sent an email to {}".format(address)
    # else:
    #     print u"Didn't really send"

<commit_msg>send message<commit_after>import os
import jinja2
import base64
import sendgrid
import re
from sendgrid.helpers.mail.mail import Email
from sendgrid.helpers.mail.mail import Content
from sendgrid.helpers.mail.mail import Mail
from sendgrid.helpers.mail.mail import Attachment
from sendgrid.helpers.mail.mail import Personalization

def send(address, subject, template_name, context, attachment=None, for_real=False):

    templateLoader = jinja2.FileSystemLoader(searchpath="templates")
    templateEnv = jinja2.Environment(loader=templateLoader)
    html_template = templateEnv.get_template(template_name + ".html")

    html_to_send = html_template.render(context)

    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    impactstory_email = Email("team@impactstory.org", "Impactstory Team")
    to_email = Email(address)
    content = Content("text/html", html_to_send)
    mail = Mail(impactstory_email, subject, to_email, content)
    personalization = Personalization()
    personalization.add_to(to_email)
    personalization.add_to(impactstory_email)
    mail.add_personalization(personalization)

    if attachment:
        my_attachment = Attachment()
        my_attachment.type = "application/csv"
        my_attachment.filename = "results.csv"
        my_attachment.disposition = "attachment"
        my_attachment.content_id = "results csv file"
        with open(attachment, 'rb') as f:
            data = f.read()

        #handle the spammer
        if re.findall(u"[a-z0-9]{7,15}@gmail.com", address):
            mail = Mail(impactstory_email, "Over limit. Please email us at team@impactstory.org for other data access options.", to_email, content)
            personalization = Personalization()
            personalization.add_to(to_email)
            personalization.add_to(impactstory_email)
            mail.add_personalization(personalization)
        else:
            my_attachment.content = base64.b64encode(data)
            mail.add_attachment(my_attachment)

    if for_real:
        response = sg.client.mail.send.post(request_body=mail.get())
        print u"Sent an email to {}".format(address)
    else:
        print u"Didn't really send"

<|endoftext|>