<commit_before>"""

"""
import abc
import re
import requests
from requests import RequestException
import six
from six.moves.urllib import parse
from jiracli.cli import colorfunc
from jiracli.utils import COLOR

@six.add_metaclass(abc.ABCMeta)
class JiraBridge(object):

    def __init__(self, base_url, config, persist=True):
        self.base_url = self._check_redirect(base_url)
        self.config = config
        self.persist = persist

    def _check_redirect(self, url):
        try:
            resp = requests.get( url, allow_redirects = False )
            if resp.status_code in [301,302]:
                return resp.headers['location']
        except RequestException:
            return None
        finally:
            return url

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.base_url)

    def format_issue(self, issue, mode=0, formatter=None, comments_only=False, last_comment=False):
        fields = {}
        status_color = "blue"
        status_string = JiraBridge.object_from_key(
            issue.setdefault('status', '1'),
            self.get_statuses
        )["name"]

        if status_string.lower() in ["resolved", "closed", "done"]:
            status_color = "green"
        elif status_string.lower() in ["open", "unassigned", "reopened", "to do"]:
            status_color = "red"

        special_fields = {
            "status": self.get_statuses,
            "priority": self.get_priorities,
            "type": self.get_issue_types
        }

        if formatter:
            groups = re.compile("(%([\w]+))").findall(formatter)
            ret_str = formatter.encode('utf-8')
            for k, v in groups:
                if v.lower() in special_fields.keys():
                    key=issue[v.lower()]
                    data = "" or JiraBridge.object_from_key(key, special_fields[v.lower()])["name"]
                    ret_str = ret_str.replace(k, data)
                else:
                    ret_str = ret_str.replace(k, issue.setdefault(v.lower(),"")).encode('utf-8')
            return ret_str

        if mode >= 0:
            # minimal
            fields["issue"] = issue["key"]
            fields["status"] = colorfunc(JiraBridge.object_from_key(issue["status"], self.get_statuses)["name"], status_color)
            fields["reporter"] = issue.setdefault("reporter","")
            fields["assignee"] = issue.setdefault("assignee","")
            fields["summary"] = issue.setdefault("summary","")
            fields["link"] = colorfunc(
                "%s/browse/%s" % (
                    self.base_url, issue["key"]
                ), "white", attrs=["underline"]
            )
        if mode == 1 or comments_only:
            fields["description"] = issue.setdefault("description","") or ""
            if not issue.get("priority", ""):
                self.fields["priority"] = ""
            else:
                fields["priority"] = JiraBridge.object_from_key(issue["priority"], self.get_priorities)["name"]
            fields["type"] = JiraBridge.object_from_key(
                issue["type"],
                self.get_issue_types if 'parent' not in issue else self.get_subtask_issue_types
            )["name"]
            fields["comments"] = "\n"
            comments = self.get_issue_comments(issue["key"])
            for comment in comments:
                comment_str =  comment["body"].strip()
                fields["comments"] += "%s %s : %s\n" % ( colorfunc(comment["created"], "blue"), colorfunc(comment["author"], "green"), comment_str )
        if mode == 1 or last_comment:
            fields["description"] = issue.setdefault("description","") or ""
            if not issue.get("priority", ""):
                self.fields["priority"] = ""
            else:
                fields["priority"] = JiraBridge.object_from_key(issue["priority"], self.get_priorities)["name"]
            fields["type"] = JiraBridge.object_from_key(
                issue["type"],
                self.get_issue_types if 'parent' not in issue else self.get_subtask_issue_types
            )["name"]
            fields["comments"] = "\n"
            comments = self.get_issue_comments(issue["key"])
            for comment in comments:
                comment_str =  comment["body"].strip()
                fields["comments"] = "%s %s : %s\n" % ( colorfunc(comment["created"], "blue"), colorfunc(comment["author"], "green"), comment_str )
        children_string = ""
        if mode > 1:
            description = (issue.setdefault("description", "") or "").split("\n")
            fields["description"] = "\n".join([description[0]] + [" "*23 + k for k in description[1:]])

            for child in self.search_issues_jql("parent=%s" % issue["key"]):
                child_type = JiraBridge.object_from_key(child["type"], self.get_subtask_issue_types)["name"].lower()
                key = ("%s" % child_type).ljust(20)
                value = "%s (%s) %s" % (
                    child["key"], child["summary"], colorfunc("%s/browse/%s" % (self.base_url, child["key"]), "white", attrs=['underline'])
                )
                children_string += "%s : %s\n" % (key, value)
        if comments_only or last_comment:
            return fields["comments"].strip()
        elif mode < 0:
            url_str = colorfunc(parse.urljoin(self.base_url, "/browse/%s" % (issue["key"])), "white", attrs=["underline"])
            ret_str = colorfunc(issue["key"], status_color) + " " + issue.setdefault("summary", "") + " " + url_str
            if not COLOR:
                ret_str += " [%s] " % status_string
            return ret_str
        for k, v in fields.items():
            if not v:
                fields[k] = ""
        formatted = "\n".join(" : ".join((k.ljust(20), v)) for k, v in fields.items() if not k == 'comments') + "\n"
        formatted += children_string
        if "comments" in fields:
            formatted += fields["comments"]
        return formatted

    @staticmethod
    def object_from_key(value, callable, key='id'):
        map = callable()
        for k,v in map.items():
            if v[key] == value:
                return v
        return None

    @abc.abstractmethod
    def login(self, username, password):
        raise NotImplementedError

    @abc.abstractmethod
    def ping(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_filters(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_projects(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_components(self, project):
        raise NotImplementedError

    @abc.abstractmethod
    def get_priorities(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issue_types(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_subtask_issue_types(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_statuses(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_resolutions(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issue(self, issue_id):
        raise NotImplementedError

    @abc.abstractmethod
    def update_issue(self, issue_id, **kwargs):
        raise NotImplementedError

    @abc.abstractmethod
    def search_issues(self, free_text, project=None, limit=100):
        raise NotImplementedError

    @abc.abstractmethod
    def search_issues_jql(self, query, limit=100, project=None):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issues_by_filter(self, *filters):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issue_comments(self, issue):
        raise NotImplementedError

    @abc.abstractmethod
    def add_comment(self, issue, comment):
        raise NotImplementedError

    @abc.abstractmethod
    def create_issue(self, project, type=0, summary="", description="", priority="minor", parent=None):
        raise NotImplementedError

    @abc.abstractmethod
    def get_available_transitions(self, issue):
        raise NotImplementedError

    @abc.abstractmethod
    def transition_issue(self, issue, transition, comment=""):
        raise NotImplementedError

    @abc.abstractmethod
    def assign_issue(self, issue, assignee):
        raise NotImplementedError

    @abc.abstractmethod
    def change_reporter(self, issue, reporter):
        raise NotImplementedError

    @abc.abstractmethod
    def add_versions(self, issue, versions, type):
        raise NotImplementedError

    @abc.abstractmethod
    def remove_versions(self, issue, versions, type):
        raise NotImplementedError

from .rest import JiraRestBridge
from .soap import JiraSoapBridge

def get_bridge(protocol):
    """
    simple factory to get the jira bridge based on the protocol
    """
    return {
        'soap': JiraSoapBridge,
        'rest': JiraRestBridge
    }[protocol]
<commit_msg>Fix Nontype error when the issue type is sub-task when viewing --last_comment<commit_after>"""

"""
import abc
import re
import requests
from requests import RequestException
import six
from six.moves.urllib import parse
from jiracli.cli import colorfunc
from jiracli.utils import COLOR

@six.add_metaclass(abc.ABCMeta)
class JiraBridge(object):

    def __init__(self, base_url, config, persist=True):
        self.base_url = self._check_redirect(base_url)
        self.config = config
        self.persist = persist

    def _check_redirect(self, url):
        try:
            resp = requests.get( url, allow_redirects = False )
            if resp.status_code in [301,302]:
                return resp.headers['location']
        except RequestException:
            return None
        finally:
            return url

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.base_url)

    def format_issue(self, issue, mode=0, formatter=None, comments_only=False, last_comment=False):
        fields = {}
        status_color = "blue"
        status_string = JiraBridge.object_from_key(
            issue.setdefault('status', '1'),
            self.get_statuses
        )["name"]

        if status_string.lower() in ["resolved", "closed", "done"]:
            status_color = "green"
        elif status_string.lower() in ["open", "unassigned", "reopened", "to do"]:
            status_color = "red"

        special_fields = {
            "status": self.get_statuses,
            "priority": self.get_priorities,
            "type": self.get_issue_types
        }

        if formatter:
            groups = re.compile("(%([\w]+))").findall(formatter)
            ret_str = formatter.encode('utf-8')
            for k, v in groups:
                if v.lower() in special_fields.keys():
                    key=issue[v.lower()]
                    data = "" or JiraBridge.object_from_key(key, special_fields[v.lower()])["name"]
                    ret_str = ret_str.replace(k, data)
                else:
                    ret_str = ret_str.replace(k, issue.setdefault(v.lower(),"")).encode('utf-8')
            return ret_str

        if mode >= 0:
            # minimal
            fields["issue"] = issue["key"]
            fields["status"] = colorfunc(JiraBridge.object_from_key(issue["status"], self.get_statuses)["name"], status_color)
            fields["reporter"] = issue.setdefault("reporter","")
            fields["assignee"] = issue.setdefault("assignee","")
            fields["summary"] = issue.setdefault("summary","")
            fields["link"] = colorfunc(
                "%s/browse/%s" % (
                    self.base_url, issue["key"]
                ), "white", attrs=["underline"]
            )
        if mode == 1 or comments_only:
            fields["description"] = issue.setdefault("description","") or ""
            if not issue.get("priority", ""):
                self.fields["priority"] = ""
            else:
                fields["priority"] = JiraBridge.object_from_key(issue["priority"], self.get_priorities)["name"]
            fields["type"] = JiraBridge.object_from_key(
                issue["type"],
                self.get_issue_types if 'parent' not in issue else self.get_subtask_issue_types
            )["name"]
            fields["comments"] = "\n"
            comments = self.get_issue_comments(issue["key"])
            for comment in comments:
                comment_str =  comment["body"].strip()
                fields["comments"] += "%s %s : %s\n" % ( colorfunc(comment["created"], "blue"), colorfunc(comment["author"], "green"), comment_str )
        if mode == 1 or last_comment:
            # fields["description"] = issue.setdefault("description","") or ""
            # if not issue.get("priority", ""):
            #     self.fields["priority"] = ""
            # else:
            #     fields["priority"] = JiraBridge.object_from_key(issue["priority"], self.get_priorities)["name"]
            # fields["type"] = JiraBridge.object_from_key(
            #     issue["type"],
            #     self.get_issue_types if 'parent' not in issue else self.get_subtask_issue_types
            # )["name"]
            fields["comments"] = "\n"
            comments = self.get_issue_comments(issue["key"])
            for comment in comments:
                comment_str =  comment["body"].strip()
                fields["comments"] = "%s %s : %s\n" % ( colorfunc(comment["created"], "blue"), colorfunc(comment["author"], "green"), comment_str )
        children_string = ""
        if mode > 1:
            description = (issue.setdefault("description", "") or "").split("\n")
            fields["description"] = "\n".join([description[0]] + [" "*23 + k for k in description[1:]])

            for child in self.search_issues_jql("parent=%s" % issue["key"]):
                child_type = JiraBridge.object_from_key(child["type"], self.get_subtask_issue_types)["name"].lower()
                key = ("%s" % child_type).ljust(20)
                value = "%s (%s) %s" % (
                    child["key"], child["summary"], colorfunc("%s/browse/%s" % (self.base_url, child["key"]), "white", attrs=['underline'])
                )
                children_string += "%s : %s\n" % (key, value)
        if comments_only or last_comment:
            return fields["comments"].strip()
        elif mode < 0:
            url_str = colorfunc(parse.urljoin(self.base_url, "/browse/%s" % (issue["key"])), "white", attrs=["underline"])
            ret_str = colorfunc(issue["key"], status_color) + " " + issue.setdefault("summary", "") + " " + url_str
            if not COLOR:
                ret_str += " [%s] " % status_string
            return ret_str
        for k, v in fields.items():
            if not v:
                fields[k] = ""
        formatted = "\n".join(" : ".join((k.ljust(20), v)) for k, v in fields.items() if not k == 'comments') + "\n"
        formatted += children_string
        if "comments" in fields:
            formatted += fields["comments"]
        return formatted

    @staticmethod
    def object_from_key(value, callable, key='id'):
        map = callable()
        for k,v in map.items():
            if v[key] == value:
                return v
        return None

    @abc.abstractmethod
    def login(self, username, password):
        raise NotImplementedError

    @abc.abstractmethod
    def ping(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_filters(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_projects(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_components(self, project):
        raise NotImplementedError

    @abc.abstractmethod
    def get_priorities(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issue_types(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_subtask_issue_types(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_statuses(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_resolutions(self):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issue(self, issue_id):
        raise NotImplementedError

    @abc.abstractmethod
    def update_issue(self, issue_id, **kwargs):
        raise NotImplementedError

    @abc.abstractmethod
    def search_issues(self, free_text, project=None, limit=100):
        raise NotImplementedError

    @abc.abstractmethod
    def search_issues_jql(self, query, limit=100, project=None):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issues_by_filter(self, *filters):
        raise NotImplementedError

    @abc.abstractmethod
    def get_issue_comments(self, issue):
        raise NotImplementedError

    @abc.abstractmethod
    def add_comment(self, issue, comment):
        raise NotImplementedError

    @abc.abstractmethod
    def create_issue(self, project, type=0, summary="", description="", priority="minor", parent=None):
        raise NotImplementedError

    @abc.abstractmethod
    def get_available_transitions(self, issue):
        raise NotImplementedError

    @abc.abstractmethod
    def transition_issue(self, issue, transition, comment=""):
        raise NotImplementedError

    @abc.abstractmethod
    def assign_issue(self, issue, assignee):
        raise NotImplementedError

    @abc.abstractmethod
    def change_reporter(self, issue, reporter):
        raise NotImplementedError

    @abc.abstractmethod
    def add_versions(self, issue, versions, type):
        raise NotImplementedError

    @abc.abstractmethod
    def remove_versions(self, issue, versions, type):
        raise NotImplementedError

from .rest import JiraRestBridge
from .soap import JiraSoapBridge

def get_bridge(protocol):
    """
    simple factory to get the jira bridge based on the protocol
    """
    return {
        'soap': JiraSoapBridge,
        'rest': JiraRestBridge
    }[protocol]
<|endoftext|>


<commit_before>'''
Created on 06/05/2011

@author: mikel
'''

#Why the hell "import spotify" does not work?
from spotify import image as _image, BulkConditionChecker, link, session, SampleType
import threading, time, StringIO, cherrypy, re, struct
from audio import BufferUnderrunError
from cherrypy import wsgiserver
from cherrypy.process import servers
import weakref
from datetime import datetime
import string, random

#TODO: urllib 3.x compatibility
import urllib2

class HTTPProxyError(Exception):
    pass

def format_http_date(dt):
    """
    As seen on SO, compatible with py2.4+:
    http://stackoverflow.com/questions/225086/rfc-1123-date-representation-in-python
    """
    """Return a string representation of a date according to RFC 1123
    (HTTP/1.1).

    The supplied date must be in UTC.

    """
    weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()]
    month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep",
             "Oct", "Nov", "Dec"][dt.month - 1]
    return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (weekday, dt.day, month,
        dt.year, dt.hour, dt.minute, dt.second)

def create_base_token(length=30):
    """
    Creates a random token with an optional length.
    Original from SO:
    http://stackoverflow.com/a/9011133/28581
    """
    pool = string.letters + string.digits
    return ''.join(random.choice(pool) for i in xrange(length))

def create_user_token(base_token, user_agent):
    return sha1sum(str.join('', [base_token, user_agent]))

def get_my_token(proxy_address, user_agent=None):
    header_data = {}
    if user_agent is not None:
        header_data['User-Agent'] = user_agent

    url = 'http://%s/token' % proxy_address
    req = urllib2.Request(url, headers=header_data)

    return urllib2.urlopen(req).read()

def sha1sum(data):
    #SHA1 lib 2.4 compatibility
    try:
        from hashlib import sha1
        hash_obj = sha1()
    except:
        import sha
        hash_obj = sha.new()

    hash_obj.update(data)
    return hash_obj.hexdigest()

class ImageCallbacks(_image.ImageCallbacks):
    __checker = None

    def __init__(self, checker):
        self.__checker = checker

    def image_loaded(self, image):
        self.__checker.check_conditions()

class Token:
    __base_token = None

    def __init__(self, base_token):
        self.__base_token = base_token

    @cherrypy.expose
    def default(self):
        if 'User-Agent' in cherrypy.request.headers:
            ua = cherrypy.request.headers['User-Agent']
        else:
            ua = None

        return create_user_token(self.__base_token, ua)

class Image:
    __session = None
    __last_modified = None

    def __init__(self, session):
        self.__session = session
        self.__last_modified = format_http_date(datetime.utcnow())

    def _get_clean_image_id(self, image_str):
        #Strip the optional extension...
        r = re.compile('\.jpg$', re.IGNORECASE)
        return re.sub(r, '', image_str)

    @cherrypy.expose
    def default(self, image_id, **kwargs):
        method = cherrypy.request.method.upper()

        #Fail for other methods than get or head
        if method not in ("GET", "HEAD"):
            raise cherrypy.HTTPError(405)

        clean_image_id = self._get_clean_image_id(image_id)
        img = _image.create(self.__session, clean_image_id)
        checker = BulkConditionChecker()
        checker.add_condition(img.is_loaded)
        img_cb = ImageCallbacks(checker)
        img.add_load_callback(img_cb)

        #Wait 30 secs or timeout
        checker.complete_wait(30)

        #Fail if image was not loaded or wrong format
        if not img.is_loaded() or img.format() != _image.ImageFormat.JPEG:
            raise cherrypy.HTTPError(500)

        else:
            cherrypy.response.headers["Content-Type"] = "image/jpeg"
            cherrypy.response.headers["Content-Length"] = len(img.data())
            cherrypy.response.headers["Last-Modified"] = self.__last_modified

            if method == 'GET':
                return img.data()

class Track:
    __session = None
    __audio_buffer = None
    __is_playing = None
    __base_token = None

    def __init__(self, session, audio_buffer, base_token):
        self.__session = session
        self.__audio_buffer = audio_buffer
        self.__base_token = base_token
        self.__is_playing = False

    def _get_clean_track_id(self, track_str):
        #Strip the optional extension...
        r = re.compile('\.wav$', re.IGNORECASE)
        return re.sub(r, '', track_str)

    def _write_wave_header(self, numsamples, channels, samplerate, bitspersample):
        file = StringIO.StringIO()

        #Generate format chunk
        format_chunk_spec = "<4sLHHLLHH"
        format_chunk = struct.pack(
            format_chunk_spec,
            "fmt ", #Chunk id
            16, #Size of this chunk (excluding chunk id and this field)
            1, #Audio format, 1 for PCM
            channels, #Number of channels
            samplerate, #Samplerate, 44100, 48000, etc.
            samplerate * channels * (bitspersample / 8), #Byterate
            channels * (bitspersample / 8), #Blockalign
            bitspersample, #16 bits for two byte samples, etc.
        )

        #Generate data chunk
        data_chunk_spec = "<4sL"
        datasize = numsamples * channels * (bitspersample / 8)
        data_chunk = struct.pack(
            data_chunk_spec,
            "data", #Chunk id
            int(datasize), #Chunk size (excluding chunk id and this field)
        )

        sum_items = [
            #"WAVE" string following size field
            4,

            #"fmt " + chunk size field + chunk size
            struct.calcsize(format_chunk_spec),

            #Size of data chunk spec + data size
            struct.calcsize(data_chunk_spec) + datasize
        ]

        #Generate main header
        all_cunks_size = int(sum(sum_items))
        main_header_spec = "<4sL4s"
        main_header = struct.pack(
            main_header_spec,
            "RIFF",
            all_cunks_size,
            "WAVE"
        )

        #Write all the contents in
        file.write(main_header)
        file.write(format_chunk)
        file.write(data_chunk)

        return file.getvalue(), all_cunks_size + 8

    def _get_sample_width(self, sample_type):
        if sample_type == SampleType.Int16NativeEndian:
            return 16

        else:
            return -1

    def _generate_file_header(self, buf):
        has_frames = True

        while has_frames:
            try:
                frame, has_frames = buf.get_frame(0)
                track = buf.get_track()

                #Current sample duration (ms)
                framelen_ms = frame.frame_time * 1000

                #Calculate number of samples
                num_samples = track.duration() * frame.num_samples / framelen_ms

                #Build the whole header
                return self._write_wave_header(
                    num_samples, frame.num_channels, frame.sample_rate,
                    self._get_sample_width(frame.sample_type)
                )

            #Wait a bit if we are ahead of the buffer
            except BufferUnderrunError:
                time.sleep(0.1)

    def _write_frame_group(self, buf, start_frame_id):
        pass

    def _write_file_content(self, buf, wave_header):
        #Write wave header
        yield wave_header

        has_frames = True
        frame_num = 0

        #Loop while buffer tells to do so
        while has_frames:
            counter = 0
            file = StringIO.StringIO()

            #Write 10 frames at a time ~88k
            #TODO: Should check written size, instead of a fixed frame num?
            while counter < 10 and has_frames:
                try:
                    frame_data, has_frames = buf.get_frame(frame_num)
                    file.write(frame_data.data)
                    counter += 1
                    frame_num += 1

                #We've gone ahead of the buffer, let's wait
                except BufferUnderrunError:
                    time.sleep(0.1)

            #Write the generated frame group
            yield file.getvalue()

    def _check_request(self):
        method = cherrypy.request.method.upper()
        headers = cherrypy.request.headers

        #Fail for other methods than get or head
        if method not in ("GET", "HEAD"):
            raise cherrypy.HTTPError(405)

        #Ranges? not yet!
        if 'Range' in headers:
            raise cherrypy.HTTPError(416)

        #Error if no token or user agent are provided
        if 'User-Agent' not in headers or 'X-Spotify-Token' not in headers:
            raise cherrypy.HTTPError(403)

        #Check that the supplied token is correct
        user_token = headers['X-Spotify-Token']
        user_agent = headers['User-Agent']
        correct_token = create_user_token(self.__base_token, user_agent)
        if user_token != correct_token:
            raise cherrypy.HTTPError(403)

        return method

    def _write_http_headers(self, filesize):
        cherrypy.response.headers['Content-Type'] = 'audio/x-wav'
        cherrypy.response.headers['Content-Length'] = filesize
        cherrypy.response.headers['Accept-Ranges'] = 'none'
        cherrypy.response.headers['Connection'] = 'close'

    @cherrypy.expose
    def default(self, track_str, **kwargs):
        method = self._check_request()

        #Ensure that the track object is loaded
        track_id = self._get_clean_track_id(track_str)
        #track = self._load_track(track_id)

        #Open the buffer
        buf = self.__audio_buffer.open(self.__session, track_id)

        #Calculate file size, and write headers (http and file)
        wave_header, filesize = self._generate_file_header(buf)
        self._write_http_headers(filesize)

        #Serve file contents if method was GET
        if method == 'GET':
            return self._write_file_content(buf, wave_header)

    default._cp_config = {'response.stream': True}

class Root:
    __session = None

    image = None
    track = None

    def __init__(self, session, audio_buffer, base_token):
        self.__session = session
        self.image = Image(session)
        self.track = Track(session, audio_buffer, base_token)
        self.token = Token(base_token)

class ProxyRunner(threading.Thread):
    __server = None
    __audio_buffer = None
    __base_token = None

    def _find_free_port(self, host, port_list):
        for port in port_list:
            try:
                servers.check_port(host, port, .1)
                return port
            except:
                pass

        list_str = ','.join([str(item) for item in port_list])
        raise HTTPProxyError("Cannot find a free port. Tried: %s" % list_str)

    def __init__(self, session, audio_buffer, host='localhost', try_ports=range(8080,8090)):
        port = self._find_free_port(host, try_ports)
        self.__audio_buffer = audio_buffer
        sess_ref = weakref.proxy(session)
        self.__base_token = create_base_token()
        app = cherrypy.tree.mount(Root(sess_ref, audio_buffer, self.__base_token), '/')
        self.__server = wsgiserver.CherryPyWSGIServer((host, port), app)
        threading.Thread.__init__(self)

    def run(self):
        self.__server.start()

    def get_port(self):
        return self.__server.bind_addr[1]

    def ready_wait(self):
        while not self.__server.ready:
            time.sleep(.1)

    def stop(self):
        self.__audio_buffer.stop()
        self.__server.stop()
<commit_msg>HTTPProxy: Temporary workaround for crashes with local files.<commit_after>'''
Created on 06/05/2011

@author: mikel
'''

#Why the hell "import spotify" does not work?
from spotify import image as _image, BulkConditionChecker, link, session, SampleType
import threading, time, StringIO, cherrypy, re, struct
from audio import BufferUnderrunError
from cherrypy import wsgiserver
from cherrypy.process import servers
import weakref
from datetime import datetime
import string, random

#TODO: urllib 3.x compatibility
import urllib2

class HTTPProxyError(Exception):
    pass

def format_http_date(dt):
    """
    As seen on SO, compatible with py2.4+:
    http://stackoverflow.com/questions/225086/rfc-1123-date-representation-in-python
    """
    """Return a string representation of a date according to RFC 1123
    (HTTP/1.1).

    The supplied date must be in UTC.

    """
    weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()]
    month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep",
             "Oct", "Nov", "Dec"][dt.month - 1]
    return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (weekday, dt.day, month,
        dt.year, dt.hour, dt.minute, dt.second)

def create_base_token(length=30):
    """
    Creates a random token with an optional length.
    Original from SO:
    http://stackoverflow.com/a/9011133/28581
    """
    pool = string.letters + string.digits
    return ''.join(random.choice(pool) for i in xrange(length))

def create_user_token(base_token, user_agent):
    return sha1sum(str.join('', [base_token, user_agent]))

def get_my_token(proxy_address, user_agent=None):
    header_data = {}
    if user_agent is not None:
        header_data['User-Agent'] = user_agent

    url = 'http://%s/token' % proxy_address
    req = urllib2.Request(url, headers=header_data)

    return urllib2.urlopen(req).read()

def sha1sum(data):
    #SHA1 lib 2.4 compatibility
    try:
        from hashlib import sha1
        hash_obj = sha1()
    except:
        import sha
        hash_obj = sha.new()

    hash_obj.update(data)
    return hash_obj.hexdigest()

class ImageCallbacks(_image.ImageCallbacks):
    __checker = None

    def __init__(self, checker):
        self.__checker = checker

    def image_loaded(self, image):
        self.__checker.check_conditions()

class Token:
    __base_token = None

    def __init__(self, base_token):
        self.__base_token = base_token

    @cherrypy.expose
    def default(self):
        if 'User-Agent' in cherrypy.request.headers:
            ua = cherrypy.request.headers['User-Agent']
        else:
            ua = None

        return create_user_token(self.__base_token, ua)

class Image:
    __session = None
    __last_modified = None

    def __init__(self, session):
        self.__session = session
        self.__last_modified = format_http_date(datetime.utcnow())

    def _get_clean_image_id(self, image_str):
        #Strip the optional extension...
        r = re.compile('\.jpg$', re.IGNORECASE)
        return re.sub(r, '', image_str)

    @cherrypy.expose
    def default(self, image_id, **kwargs):
        method = cherrypy.request.method.upper()

        #Fail for other methods than get or head
        if method not in ("GET", "HEAD"):
            raise cherrypy.HTTPError(405)

        clean_image_id = self._get_clean_image_id(image_id)
        img = _image.create(self.__session, clean_image_id)
        checker = BulkConditionChecker()
        checker.add_condition(img.is_loaded)
        img_cb = ImageCallbacks(checker)
        img.add_load_callback(img_cb)

        #Wait 30 secs or timeout
        checker.complete_wait(30)

        #Fail if image was not loaded or wrong format
        if not img.is_loaded() or img.format() != _image.ImageFormat.JPEG:
            raise cherrypy.HTTPError(500)

        else:
            cherrypy.response.headers["Content-Type"] = "image/jpeg"
            cherrypy.response.headers["Content-Length"] = len(img.data())
            cherrypy.response.headers["Last-Modified"] = self.__last_modified

            if method == 'GET':
                return img.data()

class Track:
    __session = None
    __audio_buffer = None
    __is_playing = None
    __base_token = None

    def __init__(self, session, audio_buffer, base_token):
        self.__session = session
        self.__audio_buffer = audio_buffer
        self.__base_token = base_token
        self.__is_playing = False

    def _get_clean_track_id(self, track_str):
        #Fail if it's not a valid track id
        r  = re.compile('[a-z0-9]{22}(\.wav)?$', re.IGNORECASE)
        if r.match(track_str) is None:
            raise cherrypy.HTTPError(404)

        #Strip the optional extension...
        r = re.compile('\.wav$', re.IGNORECASE)
        return re.sub(r, '', track_str)

    def _write_wave_header(self, numsamples, channels, samplerate, bitspersample):
        file = StringIO.StringIO()

        #Generate format chunk
        format_chunk_spec = "<4sLHHLLHH"
        format_chunk = struct.pack(
            format_chunk_spec,
            "fmt ", #Chunk id
            16, #Size of this chunk (excluding chunk id and this field)
            1, #Audio format, 1 for PCM
            channels, #Number of channels
            samplerate, #Samplerate, 44100, 48000, etc.
            samplerate * channels * (bitspersample / 8), #Byterate
            channels * (bitspersample / 8), #Blockalign
            bitspersample, #16 bits for two byte samples, etc.
        )

        #Generate data chunk
        data_chunk_spec = "<4sL"
        datasize = numsamples * channels * (bitspersample / 8)
        data_chunk = struct.pack(
            data_chunk_spec,
            "data", #Chunk id
            int(datasize), #Chunk size (excluding chunk id and this field)
        )

        sum_items = [
            #"WAVE" string following size field
            4,

            #"fmt " + chunk size field + chunk size
            struct.calcsize(format_chunk_spec),

            #Size of data chunk spec + data size
            struct.calcsize(data_chunk_spec) + datasize
        ]

        #Generate main header
        all_cunks_size = int(sum(sum_items))
        main_header_spec = "<4sL4s"
        main_header = struct.pack(
            main_header_spec,
            "RIFF",
            all_cunks_size,
            "WAVE"
        )

        #Write all the contents in
        file.write(main_header)
        file.write(format_chunk)
        file.write(data_chunk)

        return file.getvalue(), all_cunks_size + 8

    def _get_sample_width(self, sample_type):
        if sample_type == SampleType.Int16NativeEndian:
            return 16

        else:
            return -1

    def _generate_file_header(self, buf):
        has_frames = True

        while has_frames:
            try:
                frame, has_frames = buf.get_frame(0)
                track = buf.get_track()

                #Current sample duration (ms)
                framelen_ms = frame.frame_time * 1000

                #Calculate number of samples
                num_samples = track.duration() * frame.num_samples / framelen_ms

                #Build the whole header
                return self._write_wave_header(
                    num_samples, frame.num_channels, frame.sample_rate,
                    self._get_sample_width(frame.sample_type)
                )

            #Wait a bit if we are ahead of the buffer
            except BufferUnderrunError:
                time.sleep(0.1)

    def _write_frame_group(self, buf, start_frame_id):
        pass

    def _write_file_content(self, buf, wave_header):
        #Write wave header
        yield wave_header

        has_frames = True
        frame_num = 0

        #Loop while buffer tells to do so
        while has_frames:
            counter = 0
            file = StringIO.StringIO()

            #Write 10 frames at a time ~88k
            #TODO: Should check written size, instead of a fixed frame num?
            while counter < 10 and has_frames:
                try:
                    frame_data, has_frames = buf.get_frame(frame_num)
                    file.write(frame_data.data)
                    counter += 1
                    frame_num += 1

                #We've gone ahead of the buffer, let's wait
                except BufferUnderrunError:
                    time.sleep(0.1)

            #Write the generated frame group
            yield file.getvalue()

    def _check_request(self):
        method = cherrypy.request.method.upper()
        headers = cherrypy.request.headers

        #Fail for other methods than get or head
        if method not in ("GET", "HEAD"):
            raise cherrypy.HTTPError(405)

        #Ranges? not yet!
        if 'Range' in headers:
            raise cherrypy.HTTPError(416)

        #Error if no token or user agent are provided
        if 'User-Agent' not in headers or 'X-Spotify-Token' not in headers:
            raise cherrypy.HTTPError(403)

        #Check that the supplied token is correct
        user_token = headers['X-Spotify-Token']
        user_agent = headers['User-Agent']
        correct_token = create_user_token(self.__base_token, user_agent)
        if user_token != correct_token:
            raise cherrypy.HTTPError(403)

        return method

    def _write_http_headers(self, filesize):
        cherrypy.response.headers['Content-Type'] = 'audio/x-wav'
        cherrypy.response.headers['Content-Length'] = filesize
        cherrypy.response.headers['Accept-Ranges'] = 'none'
        cherrypy.response.headers['Connection'] = 'close'

    @cherrypy.expose
    def default(self, track_str, **kwargs):
        method = self._check_request()

        #Ensure that the track object is loaded
        track_id = self._get_clean_track_id(track_str)
        #track = self._load_track(track_id)

        #Open the buffer
        buf = self.__audio_buffer.open(self.__session, track_id)

        #Calculate file size, and write headers (http and file)
        wave_header, filesize = self._generate_file_header(buf)
        self._write_http_headers(filesize)

        #Serve file contents if method was GET
        if method == 'GET':
            return self._write_file_content(buf, wave_header)

    default._cp_config = {'response.stream': True}

class Root:
    __session = None

    image = None
    track = None

    def __init__(self, session, audio_buffer, base_token):
        self.__session = session
        self.image = Image(session)
        self.track = Track(session, audio_buffer, base_token)
        self.token = Token(base_token)

class ProxyRunner(threading.Thread):
    __server = None
    __audio_buffer = None
    __base_token = None

    def _find_free_port(self, host, port_list):
        for port in port_list:
            try:
                servers.check_port(host, port, .1)
                return port
            except:
                pass

        list_str = ','.join([str(item) for item in port_list])
        raise HTTPProxyError("Cannot find a free port. Tried: %s" % list_str)

    def __init__(self, session, audio_buffer, host='localhost', try_ports=range(8080,8090)):
        port = self._find_free_port(host, try_ports)
        self.__audio_buffer = audio_buffer
        sess_ref = weakref.proxy(session)
        self.__base_token = create_base_token()
        app = cherrypy.tree.mount(Root(sess_ref, audio_buffer, self.__base_token), '/')
        self.__server = wsgiserver.CherryPyWSGIServer((host, port), app)
        threading.Thread.__init__(self)

    def run(self):
        self.__server.start()

    def get_port(self):
        return self.__server.bind_addr[1]

    def ready_wait(self):
        while not self.__server.ready:
            time.sleep(.1)

    def stop(self):
        self.__audio_buffer.stop()
        self.__server.stop()
<|endoftext|>


<commit_before>#!/user/bin/python
import WalabotAPI as wala
import json

class Serializer(object):
    @staticmethod
    def encode_obj(obj):
        if type(obj).__name__ =='instance':
            return obj.__dict__

    @staticmethod
    def serialize(obj):
        return json.dumps(obj, default=Serializer.encode_obj)

class TargetPoint:
    def __init__(self, index, target):
        self.index = index
        self.x = target.xPosCm
        self.y = target.yPosCm
        self.z = target.zPosCm
    def toJson(self):
        return Serializer.serialize(self)
    def toString(self):
        return str('Target #{}x:{}y:{}z:{}').format(self.index + 1,
                self.x, self.y, self.z)

def TargetsToAzure(targets):
    for i, target in enumerate(targets):
       t = TargetPoint(i,target)
       print(t.toJson())

if __name__ == "__main__":

    debug = True

    walabot_hub = 'walabot'

#    key_name = 'RootManageSharedAccessKey' # SharedAccessKeyName from Azure portal
#    key_value = '' # SharedAccessKey from Azure portal
#    sbs = ServiceBusService(service_namespace,
                #            shared_access_key_name=key_name,
                 #           shared_access_key_value=key_value)

#    sbs.create_event_hub(walabot_hub)

    wala.Init()
    wala.SetSettingsFolder()

    try:
        wala.ConnectAny()
    except WalabotError as err:
        if err.message == wala.WALABOT_INSTRUMENT_NOT_FOUND:
            print('Please connect your Walabot')
        else:
            print(err.message)

    wala.SetProfile(wala.PROF_SENSOR)
    wala.SetDynamicImageFilter(wala.FILTER_TYPE_MTI)

    wala.Start()

    while True:
        wala.Trigger()

        targets = wala.GetSensorTargets()
        if targets:
            appStatus, calibrationProcess = wala.GetStatus()

            if debug :print (appStatus)
            if debug :print (calibrationProcess)

            TargetsToAzure(targets)
        else:
            print('No Target Detected')

    wala.Stop()
    wala.Disconnect()
<commit_msg>azure eventhub does not run :(<commit_after>#!/user/bin/python

from azure.servicebus import ServiceBusService
import WalabotAPI as wala
import json

walabot_hub = 'walabot'
debug = True

class Serializer(object):
    @staticmethod
    def encode_obj(obj):
        if type(obj).__name__ =='instance':
            return obj.__dict__

    @staticmethod
    def serialize(obj):
        return json.dumps(obj, default=Serializer.encode_obj)

class TargetPoint:
    def __init__(self, index, target):
        self.index = index
        self.x = target.xPosCm
        self.y = target.yPosCm
        self.z = target.zPosCm

    def toJson(self):
        return Serializer.serialize(self)

    def toString(self):
        return str('Target #{}x:{}y:{}z:{}').format(self.index + 1,
                self.x, self.y, self.z)

def TargetsToAzure(targets,sbs):
    for i, target in enumerate(targets):
       t = TargetPoint(i,target)
       if debug : print(t.toJson())
       sbs.send_event(walabot_hub, t.toJson())

if __name__ == "__main__":

    key_name = "RootManageSharedAccessKey" # SharedAccessKeyName from Azure portal

    sbs = ServiceBusService(service_namespace=sn,
                           shared_access_key_name=key_name,
                           shared_access_key_value=key_value)

    sbs.create_event_hub(walabot_hub)

    wala.Init()
    wala.SetSettingsFolder()

    try:
        wala.ConnectAny()
    except WalabotError as err:
        if err.message == wala.WALABOT_INSTRUMENT_NOT_FOUND:
            print('Please connect your Walabot')
        else:
            print(err.message)

    wala.SetProfile(wala.PROF_SENSOR)
    wala.SetDynamicImageFilter(wala.FILTER_TYPE_MTI)

    wala.Start()

    while True:
        wala.Trigger()

        targets = wala.GetSensorTargets()
        if targets:
            appStatus, calibrationProcess = wala.GetStatus()

            if debug :print (appStatus)
            if debug :print (calibrationProcess)

            TargetsToAzure(targets, sbs)
        else:
            print('No Target Detected')

    wala.Stop()
    wala.Disconnect()
<|endoftext|>


<commit_before>import logging
import time
import functools

from hazelcast.errors import create_error_from_message, HazelcastInstanceNotActiveError, is_retryable_error, \
    TargetDisconnectedError, HazelcastClientNotActiveError, TargetNotMemberError, \
    EXCEPTION_MESSAGE_TYPE, IndeterminateOperationStateError, OperationTimeoutError
from hazelcast.future import Future
from hazelcast.protocol.codec import client_local_backup_listener_codec
from hazelcast.util import AtomicInteger
from hazelcast import six

_logger = logging.getLogger(__name__)

def _no_op_response_handler(_):
    pass

class Invocation(object):
    __slots__ = ("request", "timeout", "partition_id", "uuid", "connection", "event_handler",
                 "future", "sent_connection", "urgent", "response_handler", "backup_acks_received",
                 "backup_acks_expected", "pending_response", "pending_response_received_time")

    def __init__(self, request, partition_id=-1, uuid=None, connection=None,
                 event_handler=None, urgent=False, timeout=None, response_handler=_no_op_response_handler):
        self.request = request
        self.partition_id = partition_id
        self.uuid = uuid
        self.connection = connection
        self.event_handler = event_handler
        self.urgent = urgent
        self.timeout = timeout
        self.future = Future()
        self.timeout = None
        self.sent_connection = None
        self.response_handler = response_handler
        self.backup_acks_received = 0
        self.backup_acks_expected = -1
        self.pending_response = None
        self.pending_response_received_time = -1

    def set_response(self, response):
        try:
            result = self.response_handler(response)
            self.future.set_result(result)
        except Exception as e:
            self.future.set_exception(e)

    def set_exception(self, exception, traceback=None):
        self.future.set_exception(exception, traceback)

class InvocationService(object):
    _CLEAN_RESOURCES_PERIOD = 0.1

    def __init__(self, client, reactor):
        config = client.config
        smart_routing = config.smart_routing
        if smart_routing:
            self.invoke = self._invoke_smart
        else:
            self.invoke = self._invoke_non_smart

        self._client = client
        self._reactor = reactor
        self._partition_service = None
        self._connection_manager = None
        self._listener_service = None
        self._check_invocation_allowed_fn = None
        self._pending = {}
        self._next_correlation_id = AtomicInteger(1)
        self._is_redo_operation = config.redo_operation
        self._invocation_timeout = config.invocation_timeout
        self._invocation_retry_pause = config.invocation_retry_pause
        self._backup_ack_to_client_enabled = smart_routing and config.backup_ack_to_client_enabled
        self._fail_on_indeterminate_state = config.fail_on_indeterminate_operation_state
        self._backup_timeout = config.operation_backup_timeout
        self._clean_resources_timer = None
        self._shutdown = False

    def init(self, partition_service, connection_manager, listener_service):
        self._partition_service = partition_service
        self._connection_manager = connection_manager
        self._listener_service = listener_service
        self._check_invocation_allowed_fn = connection_manager.check_invocation_allowed

    def start(self):
        self._start_clean_resources_timer()
        if self._backup_ack_to_client_enabled:
            self._register_backup_listener()

    def handle_client_message(self, message):
        correlation_id = message.get_correlation_id()

        start_frame = message.start_frame
        if start_frame.has_event_flag() or start_frame.has_backup_event_flag():
            self._listener_service.handle_client_message(message, correlation_id)
            return

        invocation = self._pending.get(correlation_id, None)
        if not invocation:
            _logger.warning("Got message with unknown correlation id: %s", message)
            return

        if message.get_message_type() == EXCEPTION_MESSAGE_TYPE:
            error = create_error_from_message(message)
            return self._notify_error(invocation, error)

        self._notify(invocation, message)

    def shutdown(self):
        if self._shutdown:
            return

        self._shutdown = True
        if self._clean_resources_timer:
            self._clean_resources_timer.cancel()
        for invocation in list(six.itervalues(self._pending)):
            self._notify_error(invocation, HazelcastClientNotActiveError())

    def _invoke_on_partition_owner(self, invocation, partition_id):
        owner_uuid = self._partition_service.get_partition_owner(partition_id)
        if not owner_uuid:
            _logger.debug("Partition owner is not assigned yet")
            return False
        return self._invoke_on_target(invocation, owner_uuid)

    def _invoke_on_target(self, invocation, owner_uuid):
        connection = self._connection_manager.get_connection(owner_uuid)
        if not connection:
            _logger.debug("Client is not connected to target: %s", owner_uuid)
            return False
        return self._send(invocation, connection)

    def _invoke_on_random_connection(self, invocation):
        connection = self._connection_manager.get_random_connection()
        if not connection:
            _logger.debug("No connection found to invoke")
            return False
        return self._send(invocation, connection)

    def _invoke_smart(self, invocation):
        if not invocation.timeout:
            invocation.timeout = self._invocation_timeout + time.time()

        try:
            if not invocation.urgent:
                self._check_invocation_allowed_fn()

            connection = invocation.connection
            if connection:
                invoked = self._send(invocation, connection)
                if not invoked:
                    self._notify_error(invocation, IOError("Could not invoke on connection %s" % connection))
                return

            if invocation.partition_id != -1:
                invoked = self._invoke_on_partition_owner(invocation, invocation.partition_id)
            elif invocation.uuid:
                invoked = self._invoke_on_target(invocation, invocation.uuid)
            else:
                invoked = self._invoke_on_random_connection(invocation)

            if not invoked:
                invoked = self._invoke_on_random_connection(invocation)

            if not invoked:
                self._notify_error(invocation, IOError("No connection found to invoke"))
        except Exception as e:
            self._notify_error(invocation, e)

    def _invoke_non_smart(self, invocation):
        if not invocation.timeout:
            invocation.timeout = self._invocation_timeout + time.time()

        try:
            if not invocation.urgent:
                self._check_invocation_allowed_fn()

            connection = invocation.connection
            if connection:
                invoked = self._send(invocation, connection)
                if not invoked:
                    self._notify_error(invocation, IOError("Could not invoke on connection %s" % connection))
                return

            if not self._invoke_on_random_connection(invocation):
                self._notify_error(invocation, IOError("No connection found to invoke"))
        except Exception as e:
            self._notify_error(invocation, e)

    def _send(self, invocation, connection):
        if self._shutdown:
            raise HazelcastClientNotActiveError()

        if self._backup_ack_to_client_enabled:
            invocation.request.set_backup_aware_flag()

        correlation_id = self._next_correlation_id.get_and_increment()
        message = invocation.request
        message.set_correlation_id(correlation_id)
        message.set_partition_id(invocation.partition_id)
        self._pending[correlation_id] = invocation

        if invocation.event_handler:
            self._listener_service.add_event_handler(correlation_id, invocation.event_handler)

        _logger.debug("Sending %s to %s", message, connection)

        if not connection.send_message(message):
            if invocation.event_handler:
                self._listener_service.remove_event_handler(correlation_id)
            return False

        invocation.sent_connection = connection
        return True

    def _notify_error(self, invocation, error):
        _logger.debug("Got exception for request %s, error: %s", invocation.request, error)

        if not self._client.lifecycle_service.is_running():
            self._complete_with_error(invocation, HazelcastClientNotActiveError())
            return

        if not self._should_retry(invocation, error):
            self._complete_with_error(invocation, error)
            return

        if invocation.timeout < time.time():
            _logger.debug("Error will not be retried because invocation timed out: %s", error)
            error = OperationTimeoutError("Request timed out because an error occurred "
                                          "after invocation timeout: %s" % error)
            self._complete_with_error(invocation, error)
            return

        invoke_func = functools.partial(self.invoke, invocation)
        self._reactor.add_timer(self._invocation_retry_pause, invoke_func)

    def _should_retry(self, invocation, error):
        if invocation.connection and isinstance(error, (IOError, TargetDisconnectedError)):
            return True

        if invocation.uuid and isinstance(error, TargetNotMemberError):
            return False

        if isinstance(error, (IOError, HazelcastInstanceNotActiveError)) or is_retryable_error(error):
            return True

        if isinstance(error, TargetDisconnectedError):
            return invocation.request.retryable or self._is_redo_operation

        return False

    def _complete_with_error(self, invocation, error):
        invocation.set_exception(error, None)
        correlation_id = invocation.request.get_correlation_id()
        self._pending.pop(correlation_id, None)

    def _register_backup_listener(self):
        codec = client_local_backup_listener_codec
        request = codec.encode_request()
        self._listener_service.register_listener(request,
                                                 codec.decode_response,
                                                 lambda reg_id: None,
                                                 lambda m: codec.handle(m, self._backup_event_handler))

    def _backup_event_handler(self, correlation_id):
        invocation = self._pending.get(correlation_id, None)
        if not invocation:
            _logger.debug("Invocation not found for backup event, invocation id %s", correlation_id)
            return
        self._notify_backup_complete(invocation)

    def _notify(self, invocation, client_message):
        expected_backups = client_message.get_number_of_backup_acks()
        if expected_backups > invocation.backup_acks_received:
            invocation.pending_response_received_time = time.time()
            invocation.backup_acks_expected = expected_backups
            invocation.pending_response = client_message
            return

        self._complete(invocation, client_message)

    def _notify_backup_complete(self, invocation):
        invocation.backup_acks_received += 1
        if not invocation.pending_response:
            return

        if invocation.backup_acks_expected != invocation.backup_acks_received:
            return

        self._complete(invocation, invocation.pending_response)

    def _complete(self, invocation, client_message):
        invocation.set_response(client_message)
        correlation_id = invocation.request.get_correlation_id()
        self._pending.pop(correlation_id, None)

    def _start_clean_resources_timer(self):
        def run():
            if self._shutdown:
                return

            now = time.time()
            for invocation in list(self._pending.values()):
                connection = invocation.sent_connection
                if not connection:
                    continue

                if not connection.live:
                    error = TargetDisconnectedError(connection.close_reason)
                    self._notify_error(invocation, error)

                if self._backup_ack_to_client_enabled:
                    self._detect_and_handle_backup_timeout(invocation, now)

            self._clean_resources_timer = self._reactor.add_timer(self._CLEAN_RESOURCES_PERIOD, run)

        self._clean_resources_timer = self._reactor.add_timer(self._CLEAN_RESOURCES_PERIOD, run)

    def _detect_and_handle_backup_timeout(self, invocation, now):
        if not invocation.pending_response:
            return

        if invocation.backup_acks_expected == invocation.backup_acks_received:
            return

        expiration_time = invocation.pending_response_received_time + self._backup_timeout
        timeout_reached = 0 < expiration_time < now
        if not timeout_reached:
            return

        if self._fail_on_indeterminate_state:
            error = IndeterminateOperationStateError("Invocation failed because the backup acks are missed")
            self._complete_with_error(invocation, error)
            return

        self._complete(invocation, invocation.pending_response)
<commit_msg>Remove logging on sending each client message (#243)<commit_after>import logging
import time
import functools

from hazelcast.errors import create_error_from_message, HazelcastInstanceNotActiveError, is_retryable_error, \
    TargetDisconnectedError, HazelcastClientNotActiveError, TargetNotMemberError, \
    EXCEPTION_MESSAGE_TYPE, IndeterminateOperationStateError, OperationTimeoutError
from hazelcast.future import Future
from hazelcast.protocol.codec import client_local_backup_listener_codec
from hazelcast.util import AtomicInteger
from hazelcast import six

_logger = logging.getLogger(__name__)

def _no_op_response_handler(_):
    pass

class Invocation(object):
    __slots__ = ("request", "timeout", "partition_id", "uuid", "connection", "event_handler",
                 "future", "sent_connection", "urgent", "response_handler", "backup_acks_received",
                 "backup_acks_expected", "pending_response", "pending_response_received_time")

    def __init__(self, request, partition_id=-1, uuid=None, connection=None,
                 event_handler=None, urgent=False, timeout=None, response_handler=_no_op_response_handler):
        self.request = request
        self.partition_id = partition_id
        self.uuid = uuid
        self.connection = connection
        self.event_handler = event_handler
        self.urgent = urgent
        self.timeout = timeout
        self.future = Future()
        self.timeout = None
        self.sent_connection = None
        self.response_handler = response_handler
        self.backup_acks_received = 0
        self.backup_acks_expected = -1
        self.pending_response = None
        self.pending_response_received_time = -1

    def set_response(self, response):
        try:
            result = self.response_handler(response)
            self.future.set_result(result)
        except Exception as e:
            self.future.set_exception(e)

    def set_exception(self, exception, traceback=None):
        self.future.set_exception(exception, traceback)

class InvocationService(object):
    _CLEAN_RESOURCES_PERIOD = 0.1

    def __init__(self, client, reactor):
        config = client.config
        smart_routing = config.smart_routing
        if smart_routing:
            self.invoke = self._invoke_smart
        else:
            self.invoke = self._invoke_non_smart

        self._client = client
        self._reactor = reactor
        self._partition_service = None
        self._connection_manager = None
        self._listener_service = None
        self._check_invocation_allowed_fn = None
        self._pending = {}
        self._next_correlation_id = AtomicInteger(1)
        self._is_redo_operation = config.redo_operation
        self._invocation_timeout = config.invocation_timeout
        self._invocation_retry_pause = config.invocation_retry_pause
        self._backup_ack_to_client_enabled = smart_routing and config.backup_ack_to_client_enabled
        self._fail_on_indeterminate_state = config.fail_on_indeterminate_operation_state
        self._backup_timeout = config.operation_backup_timeout
        self._clean_resources_timer = None
        self._shutdown = False

    def init(self, partition_service, connection_manager, listener_service):
        self._partition_service = partition_service
        self._connection_manager = connection_manager
        self._listener_service = listener_service
        self._check_invocation_allowed_fn = connection_manager.check_invocation_allowed

    def start(self):
        self._start_clean_resources_timer()
        if self._backup_ack_to_client_enabled:
            self._register_backup_listener()

    def handle_client_message(self, message):
        correlation_id = message.get_correlation_id()

        start_frame = message.start_frame
        if start_frame.has_event_flag() or start_frame.has_backup_event_flag():
            self._listener_service.handle_client_message(message, correlation_id)
            return

        invocation = self._pending.get(correlation_id, None)
        if not invocation:
            _logger.warning("Got message with unknown correlation id: %s", message)
            return

        if message.get_message_type() == EXCEPTION_MESSAGE_TYPE:
            error = create_error_from_message(message)
            return self._notify_error(invocation, error)

        self._notify(invocation, message)

    def shutdown(self):
        if self._shutdown:
            return

        self._shutdown = True
        if self._clean_resources_timer:
            self._clean_resources_timer.cancel()
        for invocation in list(six.itervalues(self._pending)):
            self._notify_error(invocation, HazelcastClientNotActiveError())

    def _invoke_on_partition_owner(self, invocation, partition_id):
        owner_uuid = self._partition_service.get_partition_owner(partition_id)
        if not owner_uuid:
            _logger.debug("Partition owner is not assigned yet")
            return False
        return self._invoke_on_target(invocation, owner_uuid)

    def _invoke_on_target(self, invocation, owner_uuid):
        connection = self._connection_manager.get_connection(owner_uuid)
        if not connection:
            _logger.debug("Client is not connected to target: %s", owner_uuid)
            return False
        return self._send(invocation, connection)

    def _invoke_on_random_connection(self, invocation):
        connection = self._connection_manager.get_random_connection()
        if not connection:
            _logger.debug("No connection found to invoke")
            return False
        return self._send(invocation, connection)

    def _invoke_smart(self, invocation):
        if not invocation.timeout:
            invocation.timeout = self._invocation_timeout + time.time()

        try:
            if not invocation.urgent:
                self._check_invocation_allowed_fn()

            connection = invocation.connection
            if connection:
                invoked = self._send(invocation, connection)
                if not invoked:
                    self._notify_error(invocation, IOError("Could not invoke on connection %s" % connection))
                return

            if invocation.partition_id != -1:
                invoked = self._invoke_on_partition_owner(invocation, invocation.partition_id)
            elif invocation.uuid:
                invoked = self._invoke_on_target(invocation, invocation.uuid)
            else:
                invoked = self._invoke_on_random_connection(invocation)

            if not invoked:
                invoked = self._invoke_on_random_connection(invocation)

            if not invoked:
                self._notify_error(invocation, IOError("No connection found to invoke"))
        except Exception as e:
            self._notify_error(invocation, e)

    def _invoke_non_smart(self, invocation):
        if not invocation.timeout:
            invocation.timeout = self._invocation_timeout + time.time()

        try:
            if not invocation.urgent:
                self._check_invocation_allowed_fn()

            connection = invocation.connection
            if connection:
                invoked = self._send(invocation, connection)
                if not invoked:
                    self._notify_error(invocation, IOError("Could not invoke on connection %s" % connection))
                return

            if not self._invoke_on_random_connection(invocation):
                self._notify_error(invocation, IOError("No connection found to invoke"))
        except Exception as e:
            self._notify_error(invocation, e)

    def _send(self, invocation, connection):
        if self._shutdown:
            raise HazelcastClientNotActiveError()

        if self._backup_ack_to_client_enabled:
            invocation.request.set_backup_aware_flag()

        correlation_id = self._next_correlation_id.get_and_increment()
        message = invocation.request
        message.set_correlation_id(correlation_id)
        message.set_partition_id(invocation.partition_id)
        self._pending[correlation_id] = invocation

        if invocation.event_handler:
            self._listener_service.add_event_handler(correlation_id, invocation.event_handler)

        if not connection.send_message(message):
            if invocation.event_handler:
                self._listener_service.remove_event_handler(correlation_id)
            return False

        invocation.sent_connection = connection
        return True

    def _notify_error(self, invocation, error):
        _logger.debug("Got exception for request %s, error: %s", invocation.request, error)

        if not self._client.lifecycle_service.is_running():
            self._complete_with_error(invocation, HazelcastClientNotActiveError())
            return

        if not self._should_retry(invocation, error):
            self._complete_with_error(invocation, error)
            return

        if invocation.timeout < time.time():
            _logger.debug("Error will not be retried because invocation timed out: %s", error)
            error = OperationTimeoutError("Request timed out because an error occurred "
                                          "after invocation timeout: %s" % error)
            self._complete_with_error(invocation, error)
            return

        invoke_func = functools.partial(self.invoke, invocation)
        self._reactor.add_timer(self._invocation_retry_pause, invoke_func)

    def _should_retry(self, invocation, error):
        if invocation.connection and isinstance(error, (IOError, TargetDisconnectedError)):
            return True

        if invocation.uuid and isinstance(error, TargetNotMemberError):
            return False

        if isinstance(error, (IOError, HazelcastInstanceNotActiveError)) or is_retryable_error(error):
            return True

        if isinstance(error, TargetDisconnectedError):
            return invocation.request.retryable or self._is_redo_operation

        return False

    def _complete_with_error(self, invocation, error):
        invocation.set_exception(error, None)
        correlation_id = invocation.request.get_correlation_id()
        self._pending.pop(correlation_id, None)

    def _register_backup_listener(self):
        codec = client_local_backup_listener_codec
        request = codec.encode_request()
        self._listener_service.register_listener(request,
                                                 codec.decode_response,
                                                 lambda reg_id: None,
                                                 lambda m: codec.handle(m, self._backup_event_handler))

    def _backup_event_handler(self, correlation_id):
        invocation = self._pending.get(correlation_id, None)
        if not invocation:
            _logger.debug("Invocation not found for backup event, invocation id %s", correlation_id)
            return
        self._notify_backup_complete(invocation)

    def _notify(self, invocation, client_message):
        expected_backups = client_message.get_number_of_backup_acks()
        if expected_backups > invocation.backup_acks_received:
            invocation.pending_response_received_time = time.time()
            invocation.backup_acks_expected = expected_backups
            invocation.pending_response = client_message
            return

        self._complete(invocation, client_message)

    def _notify_backup_complete(self, invocation):
        invocation.backup_acks_received += 1
        if not invocation.pending_response:
            return

        if invocation.backup_acks_expected != invocation.backup_acks_received:
            return

        self._complete(invocation, invocation.pending_response)

    def _complete(self, invocation, client_message):
        invocation.set_response(client_message)
        correlation_id = invocation.request.get_correlation_id()
        self._pending.pop(correlation_id, None)

    def _start_clean_resources_timer(self):
        def run():
            if self._shutdown:
                return

            now = time.time()
            for invocation in list(self._pending.values()):
                connection = invocation.sent_connection
                if not connection:
                    continue

                if not connection.live:
                    error = TargetDisconnectedError(connection.close_reason)
                    self._notify_error(invocation, error)

                if self._backup_ack_to_client_enabled:
                    self._detect_and_handle_backup_timeout(invocation, now)

            self._clean_resources_timer = self._reactor.add_timer(self._CLEAN_RESOURCES_PERIOD, run)

        self._clean_resources_timer = self._reactor.add_timer(self._CLEAN_RESOURCES_PERIOD, run)

    def _detect_and_handle_backup_timeout(self, invocation, now):
        if not invocation.pending_response:
            return

        if invocation.backup_acks_expected == invocation.backup_acks_received:
            return

        expiration_time = invocation.pending_response_received_time + self._backup_timeout
        timeout_reached = 0 < expiration_time < now
        if not timeout_reached:
            return

        if self._fail_on_indeterminate_state:
            error = IndeterminateOperationStateError("Invocation failed because the backup acks are missed")
            self._complete_with_error(invocation, error)
            return

        self._complete(invocation, invocation.pending_response)
<|endoftext|>


<commit_before>import re

from django.contrib.gis.measure import D
from tastypie import fields
from tastypie.serializers import Serializer

from boundaryservice.authentication import NoOpApiKeyAuthentication
from boundaryservice.models import BoundarySet, Boundary
from boundaryservice.tastyhacks import SluggedResource
from boundaryservice.throttle import AnonymousThrottle

class BoundarySetResource(SluggedResource):
    boundaries = fields.ToManyField('boundaryservice.resources.BoundaryResource', 'boundaries')

    class Meta:
        queryset = BoundarySet.objects.all()
        serializer = Serializer(formats=['json', 'jsonp'], content_types = {'json': 'application/json', 'jsonp': 'text/javascript'})
        resource_name = 'boundary-set'
        excludes = ['id', 'singular', 'kind_first']
        allowed_methods = ['get']
        authentication = NoOpApiKeyAuthentication()
        #throttle = AnonymousThrottle(throttle_at=100)

class BoundaryResource(SluggedResource):
    set = fields.ForeignKey(BoundarySetResource, 'set')

    class Meta:
        queryset = Boundary.objects.all()
        serializer = Serializer(formats=['json', 'jsonp'], content_types = {'json': 'application/json', 'jsonp': 'text/javascript'})
        resource_name = 'boundary'
        excludes = ['id', 'display_name', 'shape']
        allowed_methods = ['get']
        authentication = NoOpApiKeyAuthentication()
        #throttle = AnonymousThrottle(throttle_at=100)

    def build_filters(self, filters=None):
        """
        Override build_filters to support geoqueries.
        """
        if filters is None:
            filters = {}

        orm_filters = super(BoundaryResource, self).build_filters(filters)

        if 'sets' in filters:
            sets = filters['sets'].split(',')

            orm_filters.update({'set__slug__in': sets})

        if 'contains' in filters:
            lat, lon = filters['contains'].split(',')
            wkt_pt = 'POINT(%s %s)' % (lon, lat)

            orm_filters.update({'shape__contains': wkt_pt})

        if 'near' in filters:
            lat, lon, range = filters['near'].split(',')
            wkt_pt = 'POINT(%s %s)' % (lon, lat)
            numeral = re.match('([0-9]+)', range).group(1)
            unit = range[len(numeral):]
            numeral = int(numeral)
            kwargs = {unit: numeral}

            orm_filters.update({'shape__distance_lte': (wkt_pt, D(**kwargs))})

        if 'intersects' in filters:
            slug = filters['intersects']
            bounds = Boundary.objects.get(slug=slug)

            orm_filters.update({'shape__intersects': bounds.shape})

        return orm_filters
<commit_msg>Shape types can now be selected using the shape_type querystring parameter. Closes #3.<commit_after>import re

from django.contrib.gis.measure import D
from tastypie import fields
from tastypie.serializers import Serializer

from boundaryservice.authentication import NoOpApiKeyAuthentication
from boundaryservice.models import BoundarySet, Boundary
from boundaryservice.tastyhacks import SluggedResource
from boundaryservice.throttle import AnonymousThrottle

class BoundarySetResource(SluggedResource):
    boundaries = fields.ToManyField('boundaryservice.resources.BoundaryResource', 'boundaries')

    class Meta:
        queryset = BoundarySet.objects.all()
        serializer = Serializer(formats=['json', 'jsonp'], content_types = {'json': 'application/json', 'jsonp': 'text/javascript'})
        resource_name = 'boundary-set'
        excludes = ['id', 'singular', 'kind_first']
        allowed_methods = ['get']
        authentication = NoOpApiKeyAuthentication()
        #throttle = AnonymousThrottle(throttle_at=100)

class BoundaryResource(SluggedResource):
    set = fields.ForeignKey(BoundarySetResource, 'set')

    class Meta:
        queryset = Boundary.objects.all()
        serializer = Serializer(formats=['json', 'jsonp'], content_types = {'json': 'application/json', 'jsonp': 'text/javascript'})
        resource_name = 'boundary'
        excludes = ['id', 'display_name']
        allowed_methods = ['get']
        authentication = NoOpApiKeyAuthentication()
        #throttle = AnonymousThrottle(throttle_at=100)

    def alter_list_data_to_serialize(self, request, data):
        """
        Allow the selection of simple, full or no shapes using a query parameter.
        """
        shape_type = request.GET.get('shape_type', 'simple')

        for obj in data['objects']:
            if shape_type != 'simple':
                del obj.data['simple_shape']

            if shape_type != 'full':
                del obj.data['shape']

        return data

    def alter_detail_data_to_serialize(self, request, bundle):
        """
        Allow the selection of simple, full or no shapes using a query parameter.
        """
        shape_type = request.GET.get('shape_type', 'simple')

        if shape_type != 'simple':
            del bundle.data['simple_shape']

        if shape_type != 'full':
            del bundle.data['shape']

        return bundle

    def build_filters(self, filters=None):
        """
        Override build_filters to support geoqueries.
        """
        if filters is None:
            filters = {}

        orm_filters = super(BoundaryResource, self).build_filters(filters)

        if 'sets' in filters:
            sets = filters['sets'].split(',')

            orm_filters.update({'set__slug__in': sets})

        if 'contains' in filters:
            lat, lon = filters['contains'].split(',')
            wkt_pt = 'POINT(%s %s)' % (lon, lat)

            orm_filters.update({'shape__contains': wkt_pt})

        if 'near' in filters:
            lat, lon, range = filters['near'].split(',')
            wkt_pt = 'POINT(%s %s)' % (lon, lat)
            numeral = re.match('([0-9]+)', range).group(1)
            unit = range[len(numeral):]
            numeral = int(numeral)
            kwargs = {unit: numeral}

            orm_filters.update({'shape__distance_lte': (wkt_pt, D(**kwargs))})

        if 'intersects' in filters:
            slug = filters['intersects']
            bounds = Boundary.objects.get(slug=slug)

            orm_filters.update({'shape__intersects': bounds.shape})

        return orm_filters
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 26-10-2017 9:50
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Scratchpad / playground for the 4th chapter on the book
"""

import time
import pylab
import random

# Seed pseudo-random number generator
random.seed(time.time())

print("+++> Create a random 100x5 Matrix")
features = pylab.rand(100, 5)
print("+++> Create the prediction target")
target = pylab.rand(100) > 0.5<commit_msg>section on holdout method<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 26-10-2017 9:50
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Scratchpad / playground for the 4th chapter on the book
"""

import time
import pylab
import random

# Seed pseudo-random number generator
random.seed(time.time())

print("+++> Create a random 100x5 Matrix")
features = pylab.rand(100, 5)
print("+++> Create the prediction target")
target = pylab.rand(100) > 0.5

# First we're going to try the holdout method
<|endoftext|>


<commit_before>import sys
from collections import OrderedDict
from multiprocessing import cpu_count

import joblib

from .crem import grow_mol2, mutate_mol2

def __get_child_added_atom_ids(child_mol):
    '''
    Returns ids of atoms in a current mol which were added upon grow/mutation procedure
    # After RDKit reaction procedure there is a field <react_atom_idx> with initial parent atom idx in a child mol
    '''
    added_mol_ids = []
    for a in child_mol.GetAtoms():
        if not a.HasProp('react_atom_idx'):
            added_mol_ids.append(a.GetIdx())
    return sorted(added_mol_ids)

def __get_child_protected_atom_ids(mol, protected_parent_ids):
    '''

    :param mol:
    :param protected_parent_ids: ids of a parent molecule which were protected and should be transferred to the
                                 current molecule
    :type  protected_parent_ids: list[int]
    :return: sorted list of integers
    '''
    # After RDKit reaction procedure there is a field <react_atom_idx> with initial parent atom idx in product mol
    protected_mol_ids = []
    for a in mol.GetAtoms():
        if a.HasProp('react_atom_idx') and int(a.GetProp('react_atom_idx')) in protected_parent_ids:
            protected_mol_ids.append(a.GetIdx())
    return sorted(protected_mol_ids)

def __mol_with_atom_index(mol):
    atoms = mol.GetNumAtoms()
    for idx in range(atoms):
        mol.GetAtomWithIdx(idx).SetProp('molAtomMapNumber', str(mol.GetAtomWithIdx(idx).GetIdx()))
    return mol

def enumerate_compounds(mol, db_fname, mode='scaffold', n_iterations=1, radius=3, max_replacements=None,
                        min_freq=0, protected_ids=None, replace_ids=None, protect_added_frag=False, ncpu=None,
                        **kwargs):
    '''
    Convenience function to perform scaffold decoration or enumeration of analog series. This performs in multiple
    iterations by modification of compounds enumerated on the previous iteration. May result in combinatorial explosion.

    Scaffold decoration uses grow procedure. Hydrogens will be added to the supplied molecule and replaced with
    fragments from the database. A user can protect particular positions from expansion by setting protect_ids argument.
    Note: The value of protect_added_frag parameter is ignored. New fragments cannot be attached to previously
    added fragments.

    Enumeration of analog series uses mutate procedure. The molecule should be supplied with explicit hydrogens if one
    wants to replace them as well, otherwise only heavy atoms will be considered for replacements. It is recommended to
    set an altered part of a molecule not too large to obtain reasonable suggestions.

    :param mol:
    :param db_fname: path to DB file with fragment replacements.
    :param mode: 'scaffold' decoration or 'analogs' enumeration. In 'scaffold' mode the supplied molecule will be
                 substituted with fragments from a database. In 'analogs' mode the supplied molecule wil be mutated.
                 Default: scaffold.
    :param n_iterations: the number of rounds of generation. Molecules generated on the previous round are supplied
                         to the next iteration. Be careful setting this parameter as it may cause combinatorial
                         explosion. Default: 1.
    :param radius: radius of context which will be considered for replacement. Default: 3.
    :param max_replacements: maximum number of replacements to make for each molecule on each iteration. None will
                             result in all possible replacements or it is possible to set a randomly chosen number of
                             replacements. Default: None.
    :param protected_ids: iterable with atom ids which will not be altered. In 'scaffold' mode this can be ids of
                          hydrogens or heavy atoms whose hydrogens should be protected from expansion. In 'analogs' mode
                          these are ids of hydrogens and heavy atoms. Please note, a molecule should be supplied with
                          explicit hydrogens in 'analogs' mode if one wants to replace them. Default: None.
    :param replace_ids: iterable with atom ids to replace, it has lower priority then `protected_ids`. Default: None.
    :param protect_added_frag: True or False. If set True new fragments cannot be attached/replace fragments added on
                               previous iterations. Applicable only in 'analogs' mode. In 'scaffold' mode user input is
                               ignored and the argument internally set to True. Default: False
    :param ncpu: number of cores. None means all cpus.

    *Common grow/mutate arguments*
    :param min_freq: minimum occurrence of fragments in DB for replacement. Default: 0.

    :param kwargs: these arguments will be passed to grow_mol (for 'scaffold' mode) and mutate_mol ('analogs' mode)
    functions.

    arguments relevant to 'scaffold' mode
    min_atoms: minimum number of atoms in the fragment which will replace H
    max_atoms: maximum number of atoms in the fragment which will replace H

    arguments relevant to 'analogs' mode
    :min_size: minimum number of heavy atoms in a fragment to replace. If 0 - hydrogens will be replaced
                     (if they are explicit). Default: 0.
    :max_size: maximum number of heavy atoms in a fragment to replace. Default: 10.
    :min_inc: minimum change of a number of heavy atoms in replacing fragments to a number of heavy atoms in
                    replaced one. Negative value means that the replacing fragments would be smaller than the replaced
                    one on a specified number of heavy atoms. Default: -2.
    :max_inc: maximum change of a number of heavy atoms in replacing fragments to a number of heavy atoms in
                    replaced one. Default: 2.
    :replace_cycles: looking for replacement of a fragment containing cycles irrespectively of the fragment size.
                    Default: False.

    '''

    if mode not in ['scaffold', 'analogs']:
        print('Wrong mode. Please choose one from the list - "analogs","scaffold"')
        return None

    if ncpu is None:
        ncpu = cpu_count()
    else:
        ncpu = max(1, min(int(ncpu), cpu_count()))
    pool = joblib.Parallel(n_jobs=ncpu)

    # to check if the statical arguments are in the kwargs dict
    for kw in ['return_mol', 'return_rxn', 'return_rxn_freq', 'ncores']:
        if kw in kwargs:
            kwargs.pop(kw)

    if mode == 'scaffold':
        protect_added_frag = True

    if protected_ids is None and replace_ids is not None:
        protected_ids = list(set(a.GetIdx() for a in mol.GetAtoms()).difference(replace_ids))
    if protected_ids is None:
        protected_ids = []

    start_mols = {mol: protected_ids}
    # to get results ordered by iterations
    generated_mols = OrderedDict()
    n = 0

    for n in range(n_iterations):
        new_mols = ()
        if mode == 'scaffold':
            new_mols = pool(joblib.delayed(grow_mol2)(m, db_name=db_fname, protected_ids=prot_ids,
                                                      min_freq=min_freq, radius=radius,
                                                      max_replacements=max_replacements,
                                                      return_mol=True, return_rxn=False, return_rxn_freq=False,
                                                      ncores=1 if len(start_mols) > 1 else ncpu, **kwargs)
                            for m, prot_ids in start_mols.items())
        if mode == 'analogs':
            new_mols = pool(joblib.delayed(mutate_mol2)(m, db_name=db_fname, protected_ids=prot_ids,
                                                        min_freq=0, radius=radius, max_replacements=max_replacements,
                                                        return_mol=True, return_rxn=False, return_rxn_freq=False,
                                                        ncores=1 if len(start_mols) > 1 else ncpu, **kwargs)
                            for m, prot_ids in start_mols.items())

        parent_protected_ids_list = start_mols.values()
        start_mols = OrderedDict()
        for childs, parent_protected_ids in zip(new_mols, parent_protected_ids_list):
            for items in childs:
                if items[0] not in generated_mols:
                    protected_ids = __get_child_protected_atom_ids(items[1], parent_protected_ids)
                    if protect_added_frag:
                        protect_added_ids = __get_child_added_atom_ids(items[1])
                        protected_ids = set(protected_ids + protect_added_ids)

                    generated_mols[items[0]] = items[1]
                    start_mols[items[1]] = protected_ids

        if not start_mols:
            break

    if n + 1 < n_iterations:
        sys.stderr.write(f'INFO. Procedure is finished after {n + 1} iterations instead of {n_iterations}\n')

    return generated_mols
<commit_msg>Raise ValueError if wrong mode was supplied in enumerate_compounds<commit_after>import sys
from collections import OrderedDict
from multiprocessing import cpu_count

import joblib

from .crem import grow_mol2, mutate_mol2

def __get_child_added_atom_ids(child_mol):
    '''
    Returns ids of atoms in a current mol which were added upon grow/mutation procedure
    # After RDKit reaction procedure there is a field <react_atom_idx> with initial parent atom idx in a child mol
    '''
    added_mol_ids = []
    for a in child_mol.GetAtoms():
        if not a.HasProp('react_atom_idx'):
            added_mol_ids.append(a.GetIdx())
    return sorted(added_mol_ids)

def __get_child_protected_atom_ids(mol, protected_parent_ids):
    '''

    :param mol:
    :param protected_parent_ids: ids of a parent molecule which were protected and should be transferred to the
                                 current molecule
    :type  protected_parent_ids: list[int]
    :return: sorted list of integers
    '''
    # After RDKit reaction procedure there is a field <react_atom_idx> with initial parent atom idx in product mol
    protected_mol_ids = []
    for a in mol.GetAtoms():
        if a.HasProp('react_atom_idx') and int(a.GetProp('react_atom_idx')) in protected_parent_ids:
            protected_mol_ids.append(a.GetIdx())
    return sorted(protected_mol_ids)

def __mol_with_atom_index(mol):
    atoms = mol.GetNumAtoms()
    for idx in range(atoms):
        mol.GetAtomWithIdx(idx).SetProp('molAtomMapNumber', str(mol.GetAtomWithIdx(idx).GetIdx()))
    return mol

def enumerate_compounds(mol, db_fname, mode='scaffold', n_iterations=1, radius=3, max_replacements=None,
                        min_freq=0, protected_ids=None, replace_ids=None, protect_added_frag=False, ncpu=None,
                        **kwargs):
    '''
    Convenience function to perform scaffold decoration or enumeration of analog series. This performs in multiple
    iterations by modification of compounds enumerated on the previous iteration. May result in combinatorial explosion.

    Scaffold decoration uses grow procedure. Hydrogens will be added to the supplied molecule and replaced with
    fragments from the database. A user can protect particular positions from expansion by setting protect_ids argument.
    Note: The value of protect_added_frag parameter is ignored. New fragments cannot be attached to previously
    added fragments.

    Enumeration of analog series uses mutate procedure. The molecule should be supplied with explicit hydrogens if one
    wants to replace them as well, otherwise only heavy atoms will be considered for replacements. It is recommended to
    set an altered part of a molecule not too large to obtain reasonable suggestions.

    :param mol:
    :param db_fname: path to DB file with fragment replacements.
    :param mode: 'scaffold' decoration or 'analogs' enumeration. In 'scaffold' mode the supplied molecule will be
                 substituted with fragments from a database. In 'analogs' mode the supplied molecule wil be mutated.
                 Default: scaffold.
    :param n_iterations: the number of rounds of generation. Molecules generated on the previous round are supplied
                         to the next iteration. Be careful setting this parameter as it may cause combinatorial
                         explosion. Default: 1.
    :param radius: radius of context which will be considered for replacement. Default: 3.
    :param max_replacements: maximum number of replacements to make for each molecule on each iteration. None will
                             result in all possible replacements or it is possible to set a randomly chosen number of
                             replacements. Default: None.
    :param protected_ids: iterable with atom ids which will not be altered. In 'scaffold' mode this can be ids of
                          hydrogens or heavy atoms whose hydrogens should be protected from expansion. In 'analogs' mode
                          these are ids of hydrogens and heavy atoms. Please note, a molecule should be supplied with
                          explicit hydrogens in 'analogs' mode if one wants to replace them. Default: None.
    :param replace_ids: iterable with atom ids to replace, it has lower priority then `protected_ids`. Default: None.
    :param protect_added_frag: True or False. If set True new fragments cannot be attached/replace fragments added on
                               previous iterations. Applicable only in 'analogs' mode. In 'scaffold' mode user input is
                               ignored and the argument internally set to True. Default: False
    :param ncpu: number of cores. None means all cpus.

    *Common grow/mutate arguments*
    :param min_freq: minimum occurrence of fragments in DB for replacement. Default: 0.

    :param kwargs: these arguments will be passed to grow_mol (for 'scaffold' mode) and mutate_mol ('analogs' mode)
    functions.

    arguments relevant to 'scaffold' mode
    min_atoms: minimum number of atoms in the fragment which will replace H
    max_atoms: maximum number of atoms in the fragment which will replace H

    arguments relevant to 'analogs' mode
    :min_size: minimum number of heavy atoms in a fragment to replace. If 0 - hydrogens will be replaced
                     (if they are explicit). Default: 0.
    :max_size: maximum number of heavy atoms in a fragment to replace. Default: 10.
    :min_inc: minimum change of a number of heavy atoms in replacing fragments to a number of heavy atoms in
                    replaced one. Negative value means that the replacing fragments would be smaller than the replaced
                    one on a specified number of heavy atoms. Default: -2.
    :max_inc: maximum change of a number of heavy atoms in replacing fragments to a number of heavy atoms in
                    replaced one. Default: 2.
    :replace_cycles: looking for replacement of a fragment containing cycles irrespectively of the fragment size.
                    Default: False.

    '''

    if mode not in ['scaffold', 'analogs']:
        raise ValueError('Wrong mode. Please choose one from the list - "analogs","scaffold"')

    if ncpu is None:
        ncpu = cpu_count()
    else:
        ncpu = max(1, min(int(ncpu), cpu_count()))
    pool = joblib.Parallel(n_jobs=ncpu)

    # to check if the statical arguments are in the kwargs dict
    for kw in ['return_mol', 'return_rxn', 'return_rxn_freq', 'ncores']:
        if kw in kwargs:
            kwargs.pop(kw)

    if mode == 'scaffold':
        protect_added_frag = True

    if protected_ids is None and replace_ids is not None:
        protected_ids = list(set(a.GetIdx() for a in mol.GetAtoms()).difference(replace_ids))
    if protected_ids is None:
        protected_ids = []

    start_mols = {mol: protected_ids}
    # to get results ordered by iterations
    generated_mols = OrderedDict()
    n = 0

    for n in range(n_iterations):
        new_mols = ()
        if mode == 'scaffold':
            new_mols = pool(joblib.delayed(grow_mol2)(m, db_name=db_fname, protected_ids=prot_ids,
                                                      min_freq=min_freq, radius=radius,
                                                      max_replacements=max_replacements,
                                                      return_mol=True, return_rxn=False, return_rxn_freq=False,
                                                      ncores=1 if len(start_mols) > 1 else ncpu, **kwargs)
                            for m, prot_ids in start_mols.items())
        if mode == 'analogs':
            new_mols = pool(joblib.delayed(mutate_mol2)(m, db_name=db_fname, protected_ids=prot_ids,
                                                        min_freq=0, radius=radius, max_replacements=max_replacements,
                                                        return_mol=True, return_rxn=False, return_rxn_freq=False,
                                                        ncores=1 if len(start_mols) > 1 else ncpu, **kwargs)
                            for m, prot_ids in start_mols.items())

        parent_protected_ids_list = start_mols.values()
        start_mols = OrderedDict()
        for childs, parent_protected_ids in zip(new_mols, parent_protected_ids_list):
            for items in childs:
                if items[0] not in generated_mols:
                    protected_ids = __get_child_protected_atom_ids(items[1], parent_protected_ids)
                    if protect_added_frag:
                        protect_added_ids = __get_child_added_atom_ids(items[1])
                        protected_ids = set(protected_ids + protect_added_ids)

                    generated_mols[items[0]] = items[1]
                    start_mols[items[1]] = protected_ids

        if not start_mols:
            break

    if n + 1 < n_iterations:
        sys.stderr.write(f'INFO. Procedure is finished after {n + 1} iterations instead of {n_iterations}\n')

    return generated_mols
<|endoftext|>