<commit_before># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
from StringIO import StringIO

from fabric.context_managers import settings, hide
from fabric.operations import get, run
from fabric.state import env
from fabric.utils import error

from prestoadmin.config import get_conf_from_properties_data
from prestoadmin.util.constants import REMOTE_CONF_DIR, CONFIG_PROPERTIES

HTTP_ENABLED_KEY = 'http-server.http.enabled'
HTTPS_ENABLED_KEY = 'http-server.https.enabled'
HTTP_PORT_KEY = 'http-server.http.port'
HTTPS_PORT_KEY = 'http-server.https.port'
CLIENT_KEYSTORE_PATH_KEY = 'internal-communication.https.keystore.path'
CLIENT_KEYSTORE_PASSWORD_KEY = 'internal-communication.https.keystore.key'
AUTHENTICATION_KEY = 'http-server.authentication.type'
LDAP_CLIENT_USER_KEY = 'internal-communication.authentication.ldap.user'
LDAP_CLIENT_PASSWORD_KEY = 'internal-communication.authentication.ldap.password'

_LOGGER = logging.getLogger(__name__)
# properties file literals
PROPERTIES_TRUE = 'true'
PROPERTIES_FALSE = 'false'

class PrestoConfig:
    # Defaults from Presto
    default_config = {
        HTTP_ENABLED_KEY: PROPERTIES_TRUE,
        HTTPS_ENABLED_KEY: PROPERTIES_FALSE,
        HTTP_PORT_KEY: '8080',
        HTTPS_PORT_KEY: '8443',
        CLIENT_KEYSTORE_PATH_KEY: None,
        CLIENT_KEYSTORE_PASSWORD_KEY: None,
        LDAP_CLIENT_USER_KEY: None,
        LDAP_CLIENT_PASSWORD_KEY: None
    }

    def __init__(self, config_properties, config_path, config_host):
        self.config_path = config_path
        self.config_host = config_host
        if not config_properties:
            self.config_properties = self.default_config
        else:
            self.config_properties = config_properties

    @staticmethod
    def from_file(config, config_path=None, config_host=None):
        presto_config_dict = get_conf_from_properties_data(config)
        return PrestoConfig(presto_config_dict, config_path, config_host)

    @staticmethod
    def coordinator_config():
        config_path = os.path.join(REMOTE_CONF_DIR, CONFIG_PROPERTIES)
        config_host = env.roledefs['coordinator'][0]
        try:
            data = StringIO()
            with settings(host_string='%s@%s' % (env.user, config_host)):
                with hide('stderr', 'stdout'):
                    temp_dir = run('mktemp -d /tmp/prestoadmin.XXXXXXXXXXXXXX')
                try:
                    get(config_path, data, use_sudo=True, temp_dir=temp_dir)
                finally:
                    run('rm -r %s' % temp_dir)

            data.seek(0)
            return PrestoConfig.from_file(data, config_path, config_host)
        except:
            _LOGGER.info('Could not find Presto config.')
            return PrestoConfig(None, config_path, config_host)

    def _lookup(self, key):
        result = self.config_properties.get(key, self.default_config[key])
        if not result:
            error(
                    "Key %s is not configured in coordinator configuration"
                    "%s on host %s and has no default" %
                    (key, self.config_host, self.config_path))
        return result

    def use_https(self):
        http_enabled = self._lookup(HTTP_ENABLED_KEY) == PROPERTIES_TRUE
        https_enabled = self._lookup(HTTPS_ENABLED_KEY) == PROPERTIES_TRUE

        return https_enabled and not http_enabled

    def get_client_keystore_path(self):
        return self._lookup(CLIENT_KEYSTORE_PATH_KEY)

    def get_client_keystore_password(self):
        return self._lookup(CLIENT_KEYSTORE_PASSWORD_KEY)

    def get_https_port(self):
        return int(self._lookup(HTTPS_PORT_KEY))

    def get_http_port(self):
        return int(self._lookup(HTTP_PORT_KEY))

    def use_ldap(self):
        if not self.use_https():
            return False

        if AUTHENTICATION_KEY in self.config_properties:
            return self.config_properties[AUTHENTICATION_KEY] == 'LDAP'
        return False

    def get_ldap_user(self):
        return self._lookup(LDAP_CLIENT_USER_KEY)

    def get_ldap_password(self):
        return self._lookup(LDAP_CLIENT_PASSWORD_KEY)
<commit_msg>Remove temp_dir with force<commit_after># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
from StringIO import StringIO

from fabric.context_managers import settings, hide
from fabric.operations import get, run
from fabric.state import env
from fabric.utils import error

from prestoadmin.config import get_conf_from_properties_data
from prestoadmin.util.constants import REMOTE_CONF_DIR, CONFIG_PROPERTIES

HTTP_ENABLED_KEY = 'http-server.http.enabled'
HTTPS_ENABLED_KEY = 'http-server.https.enabled'
HTTP_PORT_KEY = 'http-server.http.port'
HTTPS_PORT_KEY = 'http-server.https.port'
CLIENT_KEYSTORE_PATH_KEY = 'internal-communication.https.keystore.path'
CLIENT_KEYSTORE_PASSWORD_KEY = 'internal-communication.https.keystore.key'
AUTHENTICATION_KEY = 'http-server.authentication.type'
LDAP_CLIENT_USER_KEY = 'internal-communication.authentication.ldap.user'
LDAP_CLIENT_PASSWORD_KEY = 'internal-communication.authentication.ldap.password'

_LOGGER = logging.getLogger(__name__)
# properties file literals
PROPERTIES_TRUE = 'true'
PROPERTIES_FALSE = 'false'

class PrestoConfig:
    # Defaults from Presto
    default_config = {
        HTTP_ENABLED_KEY: PROPERTIES_TRUE,
        HTTPS_ENABLED_KEY: PROPERTIES_FALSE,
        HTTP_PORT_KEY: '8080',
        HTTPS_PORT_KEY: '8443',
        CLIENT_KEYSTORE_PATH_KEY: None,
        CLIENT_KEYSTORE_PASSWORD_KEY: None,
        LDAP_CLIENT_USER_KEY: None,
        LDAP_CLIENT_PASSWORD_KEY: None
    }

    def __init__(self, config_properties, config_path, config_host):
        self.config_path = config_path
        self.config_host = config_host
        if not config_properties:
            self.config_properties = self.default_config
        else:
            self.config_properties = config_properties

    @staticmethod
    def from_file(config, config_path=None, config_host=None):
        presto_config_dict = get_conf_from_properties_data(config)
        return PrestoConfig(presto_config_dict, config_path, config_host)

    @staticmethod
    def coordinator_config():
        config_path = os.path.join(REMOTE_CONF_DIR, CONFIG_PROPERTIES)
        config_host = env.roledefs['coordinator'][0]
        try:
            data = StringIO()
            with settings(host_string='%s@%s' % (env.user, config_host)):
                with hide('stderr', 'stdout'):
                    temp_dir = run('mktemp -d /tmp/prestoadmin.XXXXXXXXXXXXXX')
                try:
                    get(config_path, data, use_sudo=True, temp_dir=temp_dir)
                finally:
                    run('rm -rf %s' % temp_dir)

            data.seek(0)
            return PrestoConfig.from_file(data, config_path, config_host)
        except:
            _LOGGER.info('Could not find Presto config.')
            return PrestoConfig(None, config_path, config_host)

    def _lookup(self, key):
        result = self.config_properties.get(key, self.default_config[key])
        if not result:
            error(
                    "Key %s is not configured in coordinator configuration"
                    "%s on host %s and has no default" %
                    (key, self.config_host, self.config_path))
        return result

    def use_https(self):
        http_enabled = self._lookup(HTTP_ENABLED_KEY) == PROPERTIES_TRUE
        https_enabled = self._lookup(HTTPS_ENABLED_KEY) == PROPERTIES_TRUE

        return https_enabled and not http_enabled

    def get_client_keystore_path(self):
        return self._lookup(CLIENT_KEYSTORE_PATH_KEY)

    def get_client_keystore_password(self):
        return self._lookup(CLIENT_KEYSTORE_PASSWORD_KEY)

    def get_https_port(self):
        return int(self._lookup(HTTPS_PORT_KEY))

    def get_http_port(self):
        return int(self._lookup(HTTP_PORT_KEY))

    def use_ldap(self):
        if not self.use_https():
            return False

        if AUTHENTICATION_KEY in self.config_properties:
            return self.config_properties[AUTHENTICATION_KEY] == 'LDAP'
        return False

    def get_ldap_user(self):
        return self._lookup(LDAP_CLIENT_USER_KEY)

    def get_ldap_password(self):
        return self._lookup(LDAP_CLIENT_PASSWORD_KEY)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from socket import (socket, AF_INET, SOCK_STREAM,
                    gethostbyname, setdefaulttimeout)
from time import time
from re import search
from util_funcs import errorExit, getHTML
try:
    from bs4 import BeautifulSoup
except ImportError as err:
    errorExit(
        (
            "%s\n"
            "Try 'sudo apt-get install python-bs4' "
            "or 'sudo apt-get install python3-bs4'" % err
        ),
        1
    )

class RoundTrip:
    def __init__(self, url):
        self.url = url
        try:
            self.addr = gethostbyname(self.url)
        except IOError as err:
            print("Could not resolve hostname\n%s" % err)
            return

    def __tcpPing(self):
        """Return latency to url's resolved IP address"""
        port = 80
        setdefaulttimeout(2.5)
        s = socket(AF_INET, SOCK_STREAM)
        send_tstamp = time()*1000
        try:
            s.connect((self.addr, port))
        except IOError:
            return

        recv_tstamp = time()*1000
        rtt = recv_tstamp - send_tstamp
        s.close()
        return rtt

    def minRTT(self):
        """Return lowest rtt"""
        rtts = []
        for i in range(3):
            rtt = self.__tcpPing()
            if rtt:
                rtts.append(rtt)
            else:
                rtts = None
                break

        if rtts:
            return round(min(rtts))
        else:
            return

# Possible statuses from mirror launchpad sites
statuses = (
    "unknown",
    "One week behind",
    "Two days behind",
    "One day behind",
    "Up to date"
)

class Data:
    def __init__(self, url, launch_url, codename, hardware, min_status=None):
        self.url = url
        self.launch_url = launch_url
        self.codename = codename
        self.hardware = hardware
        global statuses

        if min_status:
            min_index = statuses.index(min_status)
            statuses = statuses[min_index:]

        self.regex = (
            (
                r'Version\nArchitecture\nStatus\n[\w\s]+'
                'The\s%s\s\w+\n%s\n(.*)\n' % (self.codename, self.hardware)
            ),
            r'Speed:\n([0-9]{1,3}\s\w+)'
        )

    def __reFind(self, regex, string):
        """Find and return regex match"""
        match = search(regex, string)
        try:
            match = match.group(1)
        except AttributeError:
            pass

        return match

    def getInfo(self):
        """Return mirror status and bandwidth"""
        launch_html = getHTML(self.launch_url)
        if not launch_html:
            return

        text = BeautifulSoup(launch_html).get_text()
        status = self.__reFind(self.regex[0], text)
        if not status:
            print("Unable to parse status info from %s", self.launch_url)
            return

        if "unknown" in status:
            status = "unknown"

        if status not in statuses:
            return

        speed = self.__reFind(self.regex[1], text)
        if not speed:
            return
        else:
            return (self.url, (status, speed))

<commit_msg>start test only if address resolves<commit_after>#!/usr/bin/env python

from socket import (socket, AF_INET, SOCK_STREAM,
                    gethostbyname, setdefaulttimeout)
from time import time
from re import search
from util_funcs import errorExit, getHTML
try:
    from bs4 import BeautifulSoup
except ImportError as err:
    errorExit(
        (
            "%s\n"
            "Try 'sudo apt-get install python-bs4' "
            "or 'sudo apt-get install python3-bs4'" % err
        ),
        1
    )

class RoundTrip:
    def __init__(self, url):
        self.url = url
        try:
            self.addr = gethostbyname(self.url)
        except IOError as err:
            print("\nCould not resolve hostname %s\n%s" % (self.url, err))
            self.addr = None

    def __tcpPing(self):
        """Return latency to url's resolved IP address"""
        port = 80
        setdefaulttimeout(2.5)
        s = socket(AF_INET, SOCK_STREAM)
        send_tstamp = time()*1000
        try:
            s.connect((self.addr, port))
        except IOError:
            return

        recv_tstamp = time()*1000
        rtt = recv_tstamp - send_tstamp
        s.close()
        return rtt

    def minRTT(self):
        """Return lowest rtt"""
        if not self.addr:
            return

        rtts = []
        for i in range(3):
            rtt = self.__tcpPing()
            if rtt:
                rtts.append(rtt)
            else:
                rtts = None
                break

        if rtts:
            return round(min(rtts))
        else:
            return

# Possible statuses from mirror launchpad sites
statuses = (
    "unknown",
    "One week behind",
    "Two days behind",
    "One day behind",
    "Up to date"
)

class Data:
    def __init__(self, url, launch_url, codename, hardware, min_status=None):
        self.url = url
        self.launch_url = launch_url
        self.codename = codename
        self.hardware = hardware
        global statuses

        if min_status:
            min_index = statuses.index(min_status)
            statuses = statuses[min_index:]

        self.regex = (
            (
                r'Version\nArchitecture\nStatus\n[\w\s]+'
                'The\s%s\s\w+\n%s\n(.*)\n' % (self.codename, self.hardware)
            ),
            r'Speed:\n([0-9]{1,3}\s\w+)'
        )

    def __reFind(self, regex, string):
        """Find and return regex match"""
        match = search(regex, string)
        try:
            match = match.group(1)
        except AttributeError:
            pass

        return match

    def getInfo(self):
        """Return mirror status and bandwidth"""
        launch_html = getHTML(self.launch_url)
        if not launch_html:
            return

        text = BeautifulSoup(launch_html).get_text()
        status = self.__reFind(self.regex[0], text)
        if not status:
            print("Unable to parse status info from %s", self.launch_url)
            return

        if "unknown" in status:
            status = "unknown"

        if status not in statuses:
            return

        speed = self.__reFind(self.regex[1], text)
        if not speed:
            return
        else:
            return (self.url, (status, speed))

<|endoftext|>


<commit_before>__version__='0.22.1'
<commit_msg>Version -> 0.23<commit_after>__version__='0.23'
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""The mirrors module defines classes and methods for Ubuntu archive mirrors.

   Provides latency testing and mirror attribute getting from Launchpad."""

from sys import stderr
from socket import (socket, AF_INET, SOCK_STREAM,
                    gethostbyname, setdefaulttimeout,
                    error, timeout, gaierror)
from time import time
from util_funcs import get_html, HTMLGetError, progress_msg
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

from threading import Thread

try:
    from queue import Queue, Empty
except ImportError:
    from Queue import Queue, Empty

try:
    from bs4 import BeautifulSoup, FeatureNotFound
except ImportError as err:
    exit((
        "%s\n"
        "Try 'sudo apt-get install python-bs4' "
        "or 'pip install beautifulsoup4'" % err
    ))
else:
    PARSER = "lxml"
    try:
        BeautifulSoup("", PARSER)
    except FeatureNotFound:
        PARSER = "html.PARSER"

class ConnectError(Exception):
    """Socket connection errors"""
    pass

class Mirrors(object):
    """Base for collection of archive mirrors"""

    def __init__(self, url_list, flag_ping, flag_status):
        self.ranked = []
        self.urls = {}
        self.url_list = url_list
        self.num_trips = 0
        self.got = {"ping": 0, "data": 0}
        self.ranked = []
        self.top_list = []
        self.trip_queue = Queue()
        if not flag_ping:
            self.launchpad_base = "https://launchpad.net"
            self.launchpad_url = self.launchpad_base + "/ubuntu/+archivemirrors"
            self.launchpad_html = ""
            self.abort_launch = False
            self.status_opts = (
                "unknown",
                "One week behind",
                "Two days behind",
                "One day behind",
                "Up to date"
            )
            index = self.status_opts.index(flag_status)
            self.status_opts = self.status_opts[index:]
            # Default to top
            self.status_num = 1

    def get_launchpad_urls(self):
        """Obtain mirrors' corresponding launchpad URLs"""
        stderr.write("Getting list of launchpad URLs...")
        try:
            self.launchpad_html = get_html(self.launchpad_url)
        except HTMLGetError as err:
            stderr.write((
                "%s: %s\nUnable to retrieve list of launchpad sites\n"
                "Reverting to latency only" % (self.launchpad_url, err)
            ))
            self.abort_launch = True
        else:
            stderr.write("done.\n")
            self.__parse_launchpad_list()

    def __parse_launchpad_list(self):
            """Parse Launchpad's list page to find each mirror's Official page"""
            soup = BeautifulSoup(self.launchpad_html, PARSER)
            prev = ""
            for element in soup.table.descendants:
                try:
                    url = element.a
                except AttributeError:
                    pass
                else:
                    try:
                        url = url["href"]
                    except TypeError:
                        pass
                    else:
                        if url in self.urls:
                            self.urls[url]["Launchpad"] = self.launchpad_base + prev

                        if url.startswith("/ubuntu/+mirror/"):
                            prev = url

    def __kickoff_trips(self):
        """Instantiate round trips class for all, initiating queued threads"""

        for url in self.url_list:
            host = urlparse(url).netloc
            try:
                thread = Thread(
                    target=_RoundTrip(url, host, self.trip_queue).min_rtt
                )
            except gaierror as err:
                stderr.write("%s: %s ignored\n" % (err, url))
            else:
                self.urls[url] = {"Host": host}
                thread.daemon = True
                thread.start()
                self.num_trips += 1

    def get_rtts(self):
        """Test latency to all mirrors"""

        stderr.write("Testing latency to mirror(s)\n")
        self.__kickoff_trips()

        processed = 0
        progress_msg(processed, self.num_trips)
        for _ in range(self.num_trips):
            try:
                min_rtt = self.trip_queue.get(block=True)
            except Empty:
                pass
            else:
                # we can ignore empty rtt results (None) from the queue
                # as in this case ConnectError was already raised
                if min_rtt:
                    self.trip_queue.task_done()
                    self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
                    self.got["ping"] += 1

            processed += 1
            progress_msg(processed, self.num_trips)

        stderr.write('\n')
        # Mirrors without latency info are removed
        self.urls = {
            key: val for key, val in self.urls.items() if "Latency" in val
        }

        self.ranked = sorted(
            self.urls, key=lambda x: self.urls[x]["Latency"]
        )

    def __queue_lookups(self, codename, arch, data_queue):
        """Queue threads for data retrieval from launchpad.net

           Returns number of threads started to fulfill number of
           requested statuses"""
        num_threads = 0
        for url in self.ranked:
            try:
                launch_url = self.urls[url]["Launchpad"]
            except KeyError:
                pass
            else:
                thread = Thread(
                    target=_LaunchData(
                        url, launch_url, codename, arch, data_queue
                    ).get_info
                )
                thread.daemon = True
                thread.start()

                num_threads += 1

            # We expect number of retrieved status requests may already
            # be greater than 0.  This would be the case anytime an initial
            # pass ran into errors.
            if num_threads == (self.status_num - self.got["data"]):
                break

        return num_threads

    def lookup_statuses(self, min_status, codename, arch):
        """Scrape statuses/info in from launchpad.net mirror pages"""
        while (self.got["data"] < self.status_num) and self.ranked:
            data_queue = Queue()
            num_threads = self.__queue_lookups(codename, arch, data_queue)
            if num_threads == 0:
                break
            # Get output of all started thread methods from queue
            progress_msg(self.got["data"], self.status_num)
            for _ in range(num_threads):
                try:
                    # We don't care about timeouts longer than 7 seconds as
                    # we're only getting 16 KB
                    info = data_queue.get(block=True, timeout=7)
                except Empty:
                    pass
                else:
                    data_queue.task_done()
                    if info[1] and info[1]["Status"] in self.status_opts:
                        self.urls[info[0]].update(info[1])
                        self.got["data"] += 1
                        self.top_list.append(info[0])
                        progress_msg(self.got["data"], self.status_num)

                    # Eliminate the url from the ranked list as long as
                    # something is received from the queue (for selective
                    # iteration if another queue needs to be built)
                    self.ranked.remove(info[0])

                if (self.got["data"] == self.status_num):
                    break

            # Reorder by latency as queue returns vary building final list
            self.top_list = sorted(
                self.top_list, key=lambda x: self.urls[x]["Latency"]
            )

            data_queue.join()

class _RoundTrip(object):
    """Socket connections for latency reporting"""

    def __init__(self, url, host, trip_queue):
        self.url = url
        self.host = host
        self.trip_queue = trip_queue
        try:
            self.addr = gethostbyname(host)
        except gaierror as err:
            raise gaierror(err)

    def __tcp_ping(self):
        """Return socket latency to host's resolved IP address"""
        port = 80
        setdefaulttimeout(2.5)
        sock = socket(AF_INET, SOCK_STREAM)
        send_tstamp = time()*1000
        try:
            sock.connect((self.addr, port))
        except (timeout, error) as err:
            raise ConnectError(err)

        recv_tstamp = time()*1000
        rtt = recv_tstamp - send_tstamp
        sock.close()
        return rtt

    def min_rtt(self):
        """Return lowest rtt"""
        rtts = []
        for _ in range(3):
            try:
                rtt = self.__tcp_ping()
            except ConnectError as err:
                stderr.write("\n\tconnection to %s: %s\n" % (self.host, err))
                self.trip_queue.put_nowait(None)
                return
            else:
                rtts.append(rtt)

        self.trip_queue.put((self.url, round(min(rtts))))

class _LaunchData(object):
    def __init__(self, url, launch_url, codename, arch, data_queue):
        self.url = url
        self.launch_url = launch_url
        self.codename = codename
        self.arch = arch
        self.data_queue = data_queue

    def __parse_mirror_html(self, launch_html):
        info = {}
        soup = BeautifulSoup(launch_html, PARSER)
        # Find elements of the ids we need
        for line in soup.find_all(id=['arches', 'speed', 'organisation']):
            if line.name == 'table':
                # Status information lives in a table column alongside
                # series name and machine architecture
                for tr in line.find('tbody').find_all('tr'):
                    arches = [x.get_text() for x in tr.find_all('td')]
                    if (self.codename in arches[0] and
                            arches[1] == self.arch):
                        info.update({"Status": arches[2]})
            else:
                # "Speed" lives in a dl, and we use the key -> value as such
                info.update({
                    line.dt.get_text().strip(':'): line.dd.get_text()
                })

        return info

    def get_info(self):
        """Parse launchpad page HTML for mirror information

        Ideally, launchpadlib would be used to get mirror information, but the
        Launchpad API doesn't support access to archivemirror statuses."""

        try:
            launch_html = get_html(self.launch_url)
        except HTMLGetError as err:
            stderr.write("connection to %s: %s" % (self.launch_url, err))
            self.data_queue.put_nowait((self.url, None))
        else:
            info = self.__parse_mirror_html(launch_html)
            if "Status" not in info:
                stderr.write((
                    "Unable to parse status info from %s" % self.launch_url
                ))
                self.data_queue.put_nowait((self.url, None))
                return

            # Launchpad has more descriptive "unknown" status.
            # It's trimmed here to match statuses list
            if "unknown" in info["Status"]:
                info["Status"] = "unknown"

            self.data_queue.put((self.url, info))
<commit_msg>More consistent newlines for handled error messages<commit_after>#!/usr/bin/env python
"""The mirrors module defines classes and methods for Ubuntu archive mirrors.

   Provides latency testing and mirror attribute getting from Launchpad."""

from sys import stderr
from socket import (socket, AF_INET, SOCK_STREAM,
                    gethostbyname, setdefaulttimeout,
                    error, timeout, gaierror)
from time import time
from util_funcs import get_html, HTMLGetError, progress_msg
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

from threading import Thread

try:
    from queue import Queue, Empty
except ImportError:
    from Queue import Queue, Empty

try:
    from bs4 import BeautifulSoup, FeatureNotFound
except ImportError as err:
    exit((
        "%s\n"
        "Try 'sudo apt-get install python-bs4' "
        "or 'pip install beautifulsoup4'" % err
    ))
else:
    PARSER = "lxml"
    try:
        BeautifulSoup("", PARSER)
    except FeatureNotFound:
        PARSER = "html.PARSER"

class ConnectError(Exception):
    """Socket connection errors"""
    pass

class Mirrors(object):
    """Base for collection of archive mirrors"""

    def __init__(self, url_list, flag_ping, flag_status):
        self.ranked = []
        self.urls = {}
        self.url_list = url_list
        self.num_trips = 0
        self.got = {"ping": 0, "data": 0}
        self.ranked = []
        self.top_list = []
        self.trip_queue = Queue()
        if not flag_ping:
            self.launchpad_base = "https://launchpad.net"
            self.launchpad_url = self.launchpad_base + "/ubuntu/+archivemirrors"
            self.launchpad_html = ""
            self.abort_launch = False
            self.status_opts = (
                "unknown",
                "One week behind",
                "Two days behind",
                "One day behind",
                "Up to date"
            )
            index = self.status_opts.index(flag_status)
            self.status_opts = self.status_opts[index:]
            # Default to top
            self.status_num = 1

    def get_launchpad_urls(self):
        """Obtain mirrors' corresponding launchpad URLs"""
        stderr.write("Getting list of launchpad URLs...")
        try:
            self.launchpad_html = get_html(self.launchpad_url)
        except HTMLGetError as err:
            stderr.write((
                "%s: %s\nUnable to retrieve list of launchpad sites\n"
                "Reverting to latency only" % (self.launchpad_url, err)
            ))
            self.abort_launch = True
        else:
            stderr.write("done.\n")
            self.__parse_launchpad_list()

    def __parse_launchpad_list(self):
            """Parse Launchpad's list page to find each mirror's Official page"""
            soup = BeautifulSoup(self.launchpad_html, PARSER)
            prev = ""
            for element in soup.table.descendants:
                try:
                    url = element.a
                except AttributeError:
                    pass
                else:
                    try:
                        url = url["href"]
                    except TypeError:
                        pass
                    else:
                        if url in self.urls:
                            self.urls[url]["Launchpad"] = self.launchpad_base + prev

                        if url.startswith("/ubuntu/+mirror/"):
                            prev = url

    def __kickoff_trips(self):
        """Instantiate round trips class for all, initiating queued threads"""

        for url in self.url_list:
            host = urlparse(url).netloc
            try:
                thread = Thread(
                    target=_RoundTrip(url, host, self.trip_queue).min_rtt
                )
            except gaierror as err:
                stderr.write("%s: %s ignored\n" % (err, url))
            else:
                self.urls[url] = {"Host": host}
                thread.daemon = True
                thread.start()
                self.num_trips += 1

    def get_rtts(self):
        """Test latency to all mirrors"""

        stderr.write("Testing latency to mirror(s)\n")
        self.__kickoff_trips()

        processed = 0
        progress_msg(processed, self.num_trips)
        for _ in range(self.num_trips):
            try:
                min_rtt = self.trip_queue.get(block=True)
            except Empty:
                pass
            else:
                # we can ignore empty rtt results (None) from the queue
                # as in this case ConnectError was already raised
                if min_rtt:
                    self.trip_queue.task_done()
                    self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
                    self.got["ping"] += 1

            processed += 1
            progress_msg(processed, self.num_trips)

        stderr.write('\n')
        # Mirrors without latency info are removed
        self.urls = {
            key: val for key, val in self.urls.items() if "Latency" in val
        }

        self.ranked = sorted(
            self.urls, key=lambda x: self.urls[x]["Latency"]
        )

    def __queue_lookups(self, codename, arch, data_queue):
        """Queue threads for data retrieval from launchpad.net

           Returns number of threads started to fulfill number of
           requested statuses"""
        num_threads = 0
        for url in self.ranked:
            try:
                launch_url = self.urls[url]["Launchpad"]
            except KeyError:
                pass
            else:
                thread = Thread(
                    target=_LaunchData(
                        url, launch_url, codename, arch, data_queue
                    ).get_info
                )
                thread.daemon = True
                thread.start()

                num_threads += 1

            # We expect number of retrieved status requests may already
            # be greater than 0.  This would be the case anytime an initial
            # pass ran into errors.
            if num_threads == (self.status_num - self.got["data"]):
                break

        return num_threads

    def lookup_statuses(self, min_status, codename, arch):
        """Scrape statuses/info in from launchpad.net mirror pages"""
        while (self.got["data"] < self.status_num) and self.ranked:
            data_queue = Queue()
            num_threads = self.__queue_lookups(codename, arch, data_queue)
            if num_threads == 0:
                break
            # Get output of all started thread methods from queue
            progress_msg(self.got["data"], self.status_num)
            for _ in range(num_threads):
                try:
                    # We don't care about timeouts longer than 7 seconds as
                    # we're only getting 16 KB
                    info = data_queue.get(block=True, timeout=7)
                except Empty:
                    pass
                else:
                    data_queue.task_done()
                    if info[1] and info[1]["Status"] in self.status_opts:
                        self.urls[info[0]].update(info[1])
                        self.got["data"] += 1
                        self.top_list.append(info[0])
                        progress_msg(self.got["data"], self.status_num)

                    # Eliminate the url from the ranked list as long as
                    # something is received from the queue (for selective
                    # iteration if another queue needs to be built)
                    self.ranked.remove(info[0])

                if (self.got["data"] == self.status_num):
                    break

            # Reorder by latency as queue returns vary building final list
            self.top_list = sorted(
                self.top_list, key=lambda x: self.urls[x]["Latency"]
            )

            data_queue.join()

class _RoundTrip(object):
    """Socket connections for latency reporting"""

    def __init__(self, url, host, trip_queue):
        self.url = url
        self.host = host
        self.trip_queue = trip_queue
        try:
            self.addr = gethostbyname(host)
        except gaierror as err:
            raise gaierror(err)

    def __tcp_ping(self):
        """Return socket latency to host's resolved IP address"""
        port = 80
        setdefaulttimeout(2.5)
        sock = socket(AF_INET, SOCK_STREAM)
        send_tstamp = time()*1000
        try:
            sock.connect((self.addr, port))
        except (timeout, error) as err:
            raise ConnectError(err)

        recv_tstamp = time()*1000
        rtt = recv_tstamp - send_tstamp
        sock.close()
        return rtt

    def min_rtt(self):
        """Return lowest rtt"""
        rtts = []
        for _ in range(3):
            try:
                rtt = self.__tcp_ping()
            except ConnectError as err:
                stderr.write("\tconnection to %s: %s\n" % (self.host, err))
                self.trip_queue.put_nowait(None)
                return
            else:
                rtts.append(rtt)

        self.trip_queue.put((self.url, round(min(rtts))))

class _LaunchData(object):
    def __init__(self, url, launch_url, codename, arch, data_queue):
        self.url = url
        self.launch_url = launch_url
        self.codename = codename
        self.arch = arch
        self.data_queue = data_queue

    def __parse_mirror_html(self, launch_html):
        info = {}
        soup = BeautifulSoup(launch_html, PARSER)
        # Find elements of the ids we need
        for line in soup.find_all(id=['arches', 'speed', 'organisation']):
            if line.name == 'table':
                # Status information lives in a table column alongside
                # series name and machine architecture
                for tr in line.find('tbody').find_all('tr'):
                    arches = [x.get_text() for x in tr.find_all('td')]
                    if (self.codename in arches[0] and
                            arches[1] == self.arch):
                        info.update({"Status": arches[2]})
            else:
                # "Speed" lives in a dl, and we use the key -> value as such
                info.update({
                    line.dt.get_text().strip(':'): line.dd.get_text()
                })

        return info

    def get_info(self):
        """Parse launchpad page HTML for mirror information

        Ideally, launchpadlib would be used to get mirror information, but the
        Launchpad API doesn't support access to archivemirror statuses."""

        try:
            launch_html = get_html(self.launch_url)
        except HTMLGetError as err:
            stderr.write("connection to %s: %s\n" % (self.launch_url, err))
            self.data_queue.put_nowait((self.url, None))
        else:
            info = self.__parse_mirror_html(launch_html)
            if "Status" not in info:
                stderr.write((
                    "Unable to parse status info from %s\n" % self.launch_url
                ))
                self.data_queue.put_nowait((self.url, None))
                return

            # Launchpad has more descriptive "unknown" status.
            # It's trimmed here to match statuses list
            if "unknown" in info["Status"]:
                info["Status"] = "unknown"

            self.data_queue.put((self.url, info))
<|endoftext|>


<commit_before>from __future__ import absolute_import

VERSION = (0, 9, 0)
__version__ = '.'.join(map(str, VERSION)) + '-dev'
<commit_msg>Set development version<commit_after>from __future__ import absolute_import

VERSION = (1, 0, 0)
__version__ = '.'.join(map(str, VERSION)) + '-dev'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    pyFlyDoc, python library for FlyDoc webservices
#    Copyright (C) 2014 SYLEAM Info Services (<http://www.Syleam.fr/>)
#              Sylvain Garancher <sylvain.garancher@syleam.fr>
#
#    This file is a part of pyFlyDoc
#
#    pyFlyDoc is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    pyFlyDoc is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from suds.client import Client

class FlyDocService(object):
    """
    Base for FlyDoc services classes
    Loads the WSDL file at instanciation and defines some helpers to simplify written code
    """
    def __init__(self, wsdlFile):
        self.client = Client(wsdlFile)

class FlyDocSessionService(FlyDocService):
    """
    Session Service class
    """
    pass

class FlyDocSubmissionService(FlyDocService):
    """
    Submission Service class
    """
    def __init__(self, wsdlFile):
        super(FlyDocSubmissionService, self).__init__(wsdlFile)

        # Initialize enumeration constants
        self.ATTACHMENTS_FILTER = self.client.factory.create('ATTACHMENTS_FILTER')
        self.RESOURCE_TYPE = self.client.factory.create('RESOURCE_TYPE')
        self.WSFILE_MODE = self.client.factory.create('WSFILE_MODE')
        self.VAR_TYPE = self.client.factory.create('VAR_TYPE')

class FlyDocQueryService(FlyDocService):
    """
    Query Service class
    """
    def __init__(self, wsdlFile):
        super(FlyDocQueryService, self).__init__(wsdlFile)

        # Initialize enumeration constants
        self.ATTACHMENTS_FILTER = self.client.factory.create('ATTACHMENTS_FILTER')
        self.WSFILE_MODE = self.client.factory.create('WSFILE_MODE')
        self.VAR_TYPE = self.client.factory.create('VAR_TYPE')

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP] Add base class for global access<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    pyFlyDoc, python library for FlyDoc webservices
#    Copyright (C) 2014 SYLEAM Info Services (<http://www.Syleam.fr/>)
#              Sylvain Garancher <sylvain.garancher@syleam.fr>
#
#    This file is a part of pyFlyDoc
#
#    pyFlyDoc is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    pyFlyDoc is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import pkg_resources
from suds.client import Client

class FlyDocService(object):
    """
    Base for FlyDoc services classes
    Loads the WSDL file at instanciation and defines some helpers to simplify written code
    """
    def __init__(self, wsdlFile):
        self.client = Client(wsdlFile)

class FlyDocSessionService(FlyDocService):
    """
    Session Service class
    """
    pass

class FlyDocSubmissionService(FlyDocService):
    """
    Submission Service class
    """
    def __init__(self, wsdlFile):
        super(FlyDocSubmissionService, self).__init__(wsdlFile)

        # Initialize enumeration constants
        self.ATTACHMENTS_FILTER = self.client.factory.create('ATTACHMENTS_FILTER')
        self.RESOURCE_TYPE = self.client.factory.create('RESOURCE_TYPE')
        self.WSFILE_MODE = self.client.factory.create('WSFILE_MODE')
        self.VAR_TYPE = self.client.factory.create('VAR_TYPE')

class FlyDocQueryService(FlyDocService):
    """
    Query Service class
    """
    def __init__(self, wsdlFile):
        super(FlyDocQueryService, self).__init__(wsdlFile)

        # Initialize enumeration constants
        self.ATTACHMENTS_FILTER = self.client.factory.create('ATTACHMENTS_FILTER')
        self.WSFILE_MODE = self.client.factory.create('WSFILE_MODE')
        self.VAR_TYPE = self.client.factory.create('VAR_TYPE')

class FlyDoc(object):
    """
    General FlyDoc class
    """
    def __init__(self, sessionServiceWsdlFile=None, submissionServiceWsdlFile=None, queryServiceWsdlFile=None):
        """
        Initialize services instances from WSDL files
        """
        if sessionServiceWsdlFile is None:
            sessionServiceWsdlFile = 'file://' + pkg_resources.resource_filename('flydoc', 'WSDL/SessionService.wsdl')
        if submissionServiceWsdlFile is None:
            submissionServiceWsdlFile = 'file://' + pkg_resources.resource_filename('flydoc', 'WSDL/SubmissionService.wsdl')
        if queryServiceWsdlFile is None:
            queryServiceWsdlFile = 'file://' + pkg_resources.resource_filename('flydoc', 'WSDL/QueryService.wsdl')

        self.sessionService = FlyDocSessionService(sessionServiceWsdlFile)
        self.submissionService = FlyDocSubmissionService(submissionServiceWsdlFile)
        self.queryService = FlyDocQueryService(queryServiceWsdlFile)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>""" Path manipulation utilities """

from pathlib import (
        Path,
        PurePath,
        )
from shutil import copyfile
from typing import (
        Dict,
        Iterable,
        List,
        )

from fmbiopy.fmlist import not_empty

def absglob(directory: Path, pattern: str)-> List[Path]:
    """Like normal glob except absolute paths are returned"""
    relative = sorted(list(directory.glob(pattern)))
    absolute = [path.resolve() for path in relative]
    return absolute

def add_suffix(path: PurePath, suffix: str)-> Path:
    """Append a suffix to a path"""
    return Path(str(path) + suffix)

@not_empty
def all_absolute(paths: Iterable[Path])-> bool:
    """Return True if all paths in `paths` are absolute"""
    return all([p.is_absolute() for p in paths])

@not_empty
def all_exist(paths: Iterable[Path])-> bool:
    """Return True if all paths in list exist """
    return all(apply_exists(paths))

@not_empty
def any_dont_exist(paths: Iterable[Path]) -> bool:
    """Return True if any path in list does not exist """
    return not all(apply_exists(paths))

@not_empty
def any_exist(paths: Iterable[Path]) -> bool:
    """Return True if any path in list exists """
    return any(apply_exists(paths))

@not_empty
def all_empty(paths: Iterable[Path]) -> bool:
    """Return True if all paths in list exist, and are non-empty"""
    check_all_exist(paths)
    return all(apply_is_empty(paths))

def apply_is_empty(paths: Iterable[Path]) -> List[bool]:
    """Check whether each file in list has a nonzero file size.

    Returns
    -------
    List[bool]
        True if empty for each path"""
    return [is_empty(p) for p in paths]

def apply_exists(paths: Iterable[Path])-> List[bool]:
    """Apply os.path.exists to a list of paths"""
    return [p.exists() for p in paths]

def as_dict(directory: Path) -> Dict[str, List[Path]]:
    """Represent the contents of a directory as a dictionary

    Parameters
    ----------
    directory
        A directory

    Returns
    -------
    Dict[str, List[Path]]
        Output dictionary is of the form Dict[x, y] where x is a subdirectory
        of `direc` and and y is the contents of x as a list.
    """
    subdirs = listdirs(directory)
    output = {}
    for d in subdirs:
        output[d.name] = absglob(d, '*')
    return output

def as_paths(
        strs: Iterable[str],
        absolute: bool = True,
        )-> List[Path]:
    """Convert a list of `str` to `pathlib.Path`

    Parameters
    ----------
    absolute
        If True, an absolute path is returned
    strict
        Passed to `pathlib.Path.resolve()`. Does nothing if `absolute` is
        False.

    Returns
    -------
    List[Path]
        The list as paths
    """
    paths = [Path(string) for string in strs]
    if absolute:
        paths = [path.absolute() for path in paths]
    return paths

def as_strs(paths: Iterable[PurePath])-> List[str]:
    """Convert a list of `pathlib.Path` to string"""
    return [str(path) for path in paths]

def check_all_exist(paths: Iterable[Path])-> None:
    """Raise OSError if any paths in list do not exist """
    if not all_exist(paths):
        raise FileNotFoundError(
                "Not all paths exist: \n" + ' '.join(as_strs(paths)))

def create_all(paths: Iterable[Path])-> None:
    """Given a list of nonexistant paths, create them"""
    for path in paths:
        path.touch(exist_ok=False)

def extension_without_gz(path: Path)-> str:
    """Get the file extension ignoring .gz if present"""
    suffix = path.suffix
    if suffix == '.gz':
        return Path(path.stem).suffix
    return suffix

def find(
        directory: Path,
        extensions: Iterable[str] = None,
        substring: str = None)-> List[Path]:
    """List all files with a given file extension and/or substring

    Parameters
    ----------
    directory
        The name of the directory to search in
    extensions, optional
        List of target file extensions (e.g py, txt)
    substring, optional
        Target substring

    Returns
    -------
    List[Path]
        List of files with the given extensions and substrings"""
    # Filter by file extension
    hits: List[Path] = []
    if extensions is not None:
        for ext in extensions:
            hits += absglob(directory, '*' + ext)
    else:
        hits = absglob(directory, '*')

    # Filter by substring
    if substring is not None:
        out: List[Path] = []
        for hit in hits:
            if substring in hit.name:
                out.append(hit)
    else:
        out = hits
    return sorted(out)

def get_bowtie2_indices(prefix: str)-> List[Path]:
    """Given the bowtie2 index prefix, return the bowtie2 indices"""
    bowtie_suffixes = ['.1.bt2', '.2.bt2', '.3.bt2', '.4.bt2', '.rev.1.bt2',
                       '.rev.2.bt2']
    paths = [Path(prefix + suffix) for suffix in bowtie_suffixes]
    return paths

def is_empty(path: Path)-> bool:
    """Return True if `path` is empty"""
    return path.stat().st_size < 1

def listdirs(directory: Path) -> List[Path]:
    """List all the subdirectories of a directory"""
    contents = absglob(directory, '*')
    out = []
    for path in contents:
        if path.is_dir():
            out.append(path)
    return out

def move(src: Path, dest: Path)-> None:
    """Move a file

    Differs from `shutil.move` in that when a symlink is encountered, the
    contents are copied to the new location and the link is destroyed"""
    if src.is_symlink():
        copyfile(str(src), str(dest))
        src.unlink()
    else:
        src.rename(dest)

def prefix(path: PurePath)-> str:
    """Get the part of a string before the first dot"""
    return path.name.split('.')[0]

def rm_gz_suffix(path: PurePath)-> PurePath:
    """Gzip aware suffix removal

    If .gz is the final path extension then two extensions are removed."""
    if path.suffix == '.gz':
        prefix = path.parent / Path(path.stem).stem
    else:
        prefix = path.parent / path.stem
    return prefix

def root(path: Path)-> Path:
    """Return the root name of a path (with directory included)"""
    while '.' in path.name:
        path = path.parent / path.stem
    return path

<commit_msg>Added size function<commit_after>""" Path manipulation utilities """

from pathlib import (
        Path,
        PurePath,
        )
from shutil import copyfile
from typing import (
        Dict,
        Iterable,
        List,
        )

from fmbiopy.fmlist import not_empty

def absglob(directory: Path, pattern: str)-> List[Path]:
    """Like normal glob except absolute paths are returned"""
    relative = sorted(list(directory.glob(pattern)))
    absolute = [path.resolve() for path in relative]
    return absolute

def add_suffix(path: PurePath, suffix: str)-> Path:
    """Append a suffix to a path"""
    return Path(str(path) + suffix)

@not_empty
def all_absolute(paths: Iterable[Path])-> bool:
    """Return True if all paths in `paths` are absolute"""
    return all([p.is_absolute() for p in paths])

@not_empty
def all_exist(paths: Iterable[Path])-> bool:
    """Return True if all paths in list exist """
    return all(apply_exists(paths))

@not_empty
def any_dont_exist(paths: Iterable[Path]) -> bool:
    """Return True if any path in list does not exist """
    return not all(apply_exists(paths))

@not_empty
def any_exist(paths: Iterable[Path]) -> bool:
    """Return True if any path in list exists """
    return any(apply_exists(paths))

@not_empty
def all_empty(paths: Iterable[Path]) -> bool:
    """Return True if all paths in list exist, and are non-empty"""
    check_all_exist(paths)
    return all(apply_is_empty(paths))

def apply_is_empty(paths: Iterable[Path]) -> List[bool]:
    """Check whether each file in list has a nonzero file size.

    Returns
    -------
    List[bool]
        True if empty for each path"""
    return [is_empty(p) for p in paths]

def apply_exists(paths: Iterable[Path])-> List[bool]:
    """Apply os.path.exists to a list of paths"""
    return [p.exists() for p in paths]

def as_dict(directory: Path) -> Dict[str, List[Path]]:
    """Represent the contents of a directory as a dictionary

    Parameters
    ----------
    directory
        A directory

    Returns
    -------
    Dict[str, List[Path]]
        Output dictionary is of the form Dict[x, y] where x is a subdirectory
        of `direc` and and y is the contents of x as a list.
    """
    subdirs = listdirs(directory)
    output = {}
    for d in subdirs:
        output[d.name] = absglob(d, '*')
    return output

def as_paths(
        strs: Iterable[str],
        absolute: bool = True,
        )-> List[Path]:
    """Convert a list of `str` to `pathlib.Path`

    Parameters
    ----------
    absolute
        If True, an absolute path is returned
    strict
        Passed to `pathlib.Path.resolve()`. Does nothing if `absolute` is
        False.

    Returns
    -------
    List[Path]
        The list as paths
    """
    paths = [Path(string) for string in strs]
    if absolute:
        paths = [path.absolute() for path in paths]
    return paths

def as_strs(paths: Iterable[PurePath])-> List[str]:
    """Convert a list of `pathlib.Path` to string"""
    return [str(path) for path in paths]

def check_all_exist(paths: Iterable[Path])-> None:
    """Raise OSError if any paths in list do not exist """
    if not all_exist(paths):
        raise FileNotFoundError(
                "Not all paths exist: \n" + ' '.join(as_strs(paths)))

def create_all(paths: Iterable[Path])-> None:
    """Given a list of nonexistant paths, create them"""
    for path in paths:
        path.touch(exist_ok=False)

def extension_without_gz(path: Path)-> str:
    """Get the file extension ignoring .gz if present"""
    suffix = path.suffix
    if suffix == '.gz':
        return Path(path.stem).suffix
    return suffix

def find(
        directory: Path,
        extensions: Iterable[str] = None,
        substring: str = None)-> List[Path]:
    """List all files with a given file extension and/or substring

    Parameters
    ----------
    directory
        The name of the directory to search in
    extensions, optional
        List of target file extensions (e.g py, txt)
    substring, optional
        Target substring

    Returns
    -------
    List[Path]
        List of files with the given extensions and substrings"""
    # Filter by file extension
    hits: List[Path] = []
    if extensions is not None:
        for ext in extensions:
            hits += absglob(directory, '*' + ext)
    else:
        hits = absglob(directory, '*')

    # Filter by substring
    if substring is not None:
        out: List[Path] = []
        for hit in hits:
            if substring in hit.name:
                out.append(hit)
    else:
        out = hits
    return sorted(out)

def get_bowtie2_indices(prefix: str)-> List[Path]:
    """Given the bowtie2 index prefix, return the bowtie2 indices"""
    bowtie_suffixes = ['.1.bt2', '.2.bt2', '.3.bt2', '.4.bt2', '.rev.1.bt2',
                       '.rev.2.bt2']
    paths = [Path(prefix + suffix) for suffix in bowtie_suffixes]
    return paths

def is_empty(path: Path)-> bool:
    """Return True if `path` is empty"""
    return size(path) < 1

def listdirs(directory: Path) -> List[Path]:
    """List all the subdirectories of a directory"""
    contents = absglob(directory, '*')
    out = []
    for path in contents:
        if path.is_dir():
            out.append(path)
    return out

def move(src: Path, dest: Path)-> None:
    """Move a file

    Differs from `shutil.move` in that when a symlink is encountered, the
    contents are copied to the new location and the link is destroyed"""
    if src.is_symlink():
        copyfile(str(src), str(dest))
        src.unlink()
    else:
        src.rename(dest)

def prefix(path: PurePath)-> str:
    """Get the part of a string before the first dot"""
    return path.name.split('.')[0]

def rm_gz_suffix(path: PurePath)-> PurePath:
    """Gzip aware suffix removal

    If .gz is the final path extension then two extensions are removed."""
    if path.suffix == '.gz':
        prefix = path.parent / Path(path.stem).stem
    else:
        prefix = path.parent / path.stem
    return prefix

def root(path: Path)-> Path:
    """Return the root name of a path (with directory included)"""
    while '.' in path.name:
        path = path.parent / path.stem
    return path

def size(path: Path)-> int:
    """Return the filesize of a path in bytes"""
    return path.stat().st_size
<|endoftext|>


<commit_before>import abc
import logging
import re

from datetime import datetime, timedelta

import foomodules.Base as Base
import foomodules.URLLookup as URLLookup

class Pong(Base.MessageHandler):
    def __call__(self, msg, errorSink=None):
        if msg["body"].strip().lower() == "ping":
            self.reply(msg, "pong")
            return True

class IgnoreList(Base.MessageHandler):
    def __init__(self, message="I will ignore you.",
                 initial=[], **kwargs):
        super().__init__(**kwargs)
        self.message = message
        self.ignoredJids = set(initial)

    def __call__(self, msg, errorSink=None):
        bare = str(msg["from"].bare)
        if bare in self.ignoredJids:
            return
        self.ignoredJids.add(bare)
        self.reply(msg, self.message)

class NumericDocumentMatcher(Base.MessageHandler):
    def __init__(self, document_regexes, url_lookup, **kwargs):
        super().__init__(**kwargs)
        self.document_regexes = list(map(self._complete_docex, document_regexes))
        self.url_lookup = url_lookup

    @staticmethod
    def _complete_docex(docex):
        if len(docex) == 2:
            return docex[0], docex[1], lambda x, y: (x, y)
        else:
            return docex

    def __call__(self, msg, errorSink=None):
        contents = msg["body"]

        for regex, document_format, converter in self.document_regexes:
            for match in regex.finditer(contents):
                groups = match.groups()
                groupdict = match.groupdict()
                groups, groupdict = converter(groups, groupdict)
                document_url = document_format.format(*groups, **groupdict)

                try:
                    iterable = iter(self.url_lookup.processURL(document_url))
                    first_line = next(iterable)
                    self.reply(msg, "<{0}>: {1}".format(document_url, first_line))
                    for line in iterable:
                        self.reply(msg, line)
                except URLLookup.URLLookupError as err:
                    self.reply(msg, "<{0}>: sorry, couldn't look it up: {1}".format(document_url, str(err)))
                    pass

class UnicodeMatcher(Base.MessageHandler):
    uniid_re = re.compile(r"u\+([0-9a-f]{2,8})", re.I)

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _format_unichar(self, codepoint):
        import unicodedata
        character = chr(codepoint)
        try:
            name = unicodedata.name(character)
        except ValueError:
            name = "unnamed"
        character_vis = character
        if unicodedata.category(character).startswith("C"):
            character_vis = "⟨non-character⟩"

        try:
            utf8str = " ".join(map("0x{:02x}".format, character.encode("utf8")))
        except UnicodeEncodeError:
            utf8str = "⟨not representable in UTF-8⟩"

        return "{chr!s}: U+{codepoint:04X} {name}, dec: {codepoint:d}, UTF-8: {utf8str}".format(
            codepoint=codepoint,
            name=name,
            chr=character_vis,
            utf8str=utf8str)

    def __call__(self, msg, errorSink=None):
        contents = msg["body"]
        for match in self.uniid_re.finditer(contents):
            value = int(match.group(1), 16)
            self.reply(msg, self._format_unichar(value))

class BroadcastMessage(Base.XMPPObject, metaclass=abc.ABCMeta):
    def __init__(self, targets, mtype="chat", **kwargs):
        super().__init__(**kwargs)
        self.targets = targets
        self.mtype = mtype

    @abc.abstractmethod
    def _get_message(self, target):
        pass

    def _send_message(self, target):
        text = self._get_message(target)
        logging.info("sending %s to %s with mtype %s", text, target, self.mtype)
        self.xmpp.send_message(target, text, mtype=self.mtype)

    def __call__(self):
        for target in self.targets:
            self._send_message(target)

class BroadcastDynamicMessage(BroadcastMessage):
    def __init__(self, targets, mgen, **kwargs):
        super().__init__(targets, **kwargs)
        self.mgen = mgen

    def _get_message(self, target):
        return self.mgen(target)

class BroadcastStaticMessage(BroadcastMessage):
    def __init__(self, targets, text, **kwargs):
        super().__init__(targets, **kwargs)
        self.text = text

    def _get_message(self, target):
        return self.text

class SYNACK(Base.MessageHandler):
    def __init__(self,
            timeout=10,
            timeout_message="handshake timed out",
            abort=True,
            send_rst=True,
            process_fin=True,
            **kwargs):
        super().__init__(**kwargs)
        self.timeout = float(timeout)
        self.timeout_message = timeout_message
        self.abort = abort
        self.pending_acks = {}
        self.process_fin = process_fin
        self.established = set()
        self.send_rst = send_rst

    def _get_uid(self, jid):
        return "{0!r}.{1!s}".format(self, jid)

    def _syn(self, msg):
        jid = msg["from"]
        try:
            uid, mode = self.pending_acks[str(jid)]
            self.xmpp.scheduler.remove(uid)
        except KeyError:
            uid = self._get_uid(jid)
            mode = "syn"
            self.pending_acks[str(jid)] = (uid, mode)

        if mode == "fin":
            self.reply(msg, "RST")
            del self.established[str(jid)]
            del self.pending_acks[str(jid)]
            return

        self.xmpp.scheduler.add(
            uid,
            self.timeout,
            lambda: self._timeout_syn(msg)
        )
        self.reply(msg, "SYN ACK")

    def _ack(self, msg):
        jid = msg["from"]
        jidstr = str(jid)
        try:
            uid, mode = self.pending_acks.pop(jidstr)
        except KeyError:
            if not jidstr in self.established:
                self.reply(msg, "RST")
            # no syn before
            return

        if mode == "rst":
            self.xmpp.scheduler.remove(uid)
            self.established.remove(jidstr)
        elif mode == "syn":
            self.xmpp.scheduler.remove(uid)
            if self.process_fin:
                self.established.add(jidstr)

    def _fin(self, msg):
        if not self.process_fin:
            return
        jid = msg["from"]
        if not jid in self.established:
            self.reply(msg, "RST")
            return

        uid = self._get_uid(jid)
        self.pending_acks[str(jid)] = uid, "rst"
        self.reply(msg, "FIN ACK")
        self.xmpp.scheduler.add(
            uid,
            self.timeout,
            lambda: self._timeout_fin(msg)
        )

    def _timeout_syn(self, msg):
        self.prefixed_reply(msg, self.timeout_message)
        del self.pending_acks[str(msg["from"])]

    def _timeout_fin(self, msg):
        self.reply(msg, "FIN ACK")

    def __call__(self, msg, errorSink=None):
        body = msg["body"].strip().lower()
        if body == "syn":
            self._syn(msg)
            return self.abort
        elif body == "ack":
            self._ack(msg)
            return self.abort
        elif body == "fin":
            self._fin(msg)
            return self.abort
        return False

class CTCP(Base.MessageHandler):
    def __init__(self,
            versionstr,
            dateformat="%a %d %b %Y %H:%M:%S UTC",
            **kwargs):
        super().__init__(**kwargs)
        self._versionstr = versionstr
        self._dateformat = dateformat

    def __call__(self, msg, errorSink=None):
        if msg["mtype"] != "chat":
            return False

        body = msg["body"].strip()
        if not body.startswith("\u0001") and not body.startswith("CTCP"):
            return False

        #body = body[5:]
        #if body.startswith("VERSION"):
        #    self.reply(msg, self._versionstr)
        #elif body.startswith("TIME"):
        #    self.reply(msg, datetime.utcnow().strftime(self._dateformat))
        #elif body.startswith("PING"):
        #    self.reply(msg, body)

        # Until we have found out on how to properly send NOTICEs, we'll
        # just ignore these messages

        return True
<commit_msg>Silently swallow codepoints out of unicode range<commit_after>import abc
import logging
import re

from datetime import datetime, timedelta

import foomodules.Base as Base
import foomodules.URLLookup as URLLookup

class Pong(Base.MessageHandler):
    def __call__(self, msg, errorSink=None):
        if msg["body"].strip().lower() == "ping":
            self.reply(msg, "pong")
            return True

class IgnoreList(Base.MessageHandler):
    def __init__(self, message="I will ignore you.",
                 initial=[], **kwargs):
        super().__init__(**kwargs)
        self.message = message
        self.ignoredJids = set(initial)

    def __call__(self, msg, errorSink=None):
        bare = str(msg["from"].bare)
        if bare in self.ignoredJids:
            return
        self.ignoredJids.add(bare)
        self.reply(msg, self.message)

class NumericDocumentMatcher(Base.MessageHandler):
    def __init__(self, document_regexes, url_lookup, **kwargs):
        super().__init__(**kwargs)
        self.document_regexes = list(map(self._complete_docex, document_regexes))
        self.url_lookup = url_lookup

    @staticmethod
    def _complete_docex(docex):
        if len(docex) == 2:
            return docex[0], docex[1], lambda x, y: (x, y)
        else:
            return docex

    def __call__(self, msg, errorSink=None):
        contents = msg["body"]

        for regex, document_format, converter in self.document_regexes:
            for match in regex.finditer(contents):
                groups = match.groups()
                groupdict = match.groupdict()
                groups, groupdict = converter(groups, groupdict)
                document_url = document_format.format(*groups, **groupdict)

                try:
                    iterable = iter(self.url_lookup.processURL(document_url))
                    first_line = next(iterable)
                    self.reply(msg, "<{0}>: {1}".format(document_url, first_line))
                    for line in iterable:
                        self.reply(msg, line)
                except URLLookup.URLLookupError as err:
                    self.reply(msg, "<{0}>: sorry, couldn't look it up: {1}".format(document_url, str(err)))
                    pass

class UnicodeMatcher(Base.MessageHandler):
    uniid_re = re.compile(r"u\+([0-9a-f]{2,8})", re.I)

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _format_unichar(self, codepoint):
        import unicodedata
        try:
            character = chr(codepoint)
        except OverflowError:
            return

        try:
            name = unicodedata.name(character)
        except ValueError:
            name = "unnamed"
        character_vis = character
        if unicodedata.category(character).startswith("C"):
            character_vis = "⟨non-character⟩"

        try:
            utf8str = " ".join(map("0x{:02x}".format, character.encode("utf8")))
        except UnicodeEncodeError:
            utf8str = "⟨not representable in UTF-8⟩"

        return "{chr!s}: U+{codepoint:04X} {name}, dec: {codepoint:d}, UTF-8: {utf8str}".format(
            codepoint=codepoint,
            name=name,
            chr=character_vis,
            utf8str=utf8str)

    def __call__(self, msg, errorSink=None):
        contents = msg["body"]
        for match in self.uniid_re.finditer(contents):
            value = int(match.group(1), 16)
            self.reply(msg, self._format_unichar(value))

class BroadcastMessage(Base.XMPPObject, metaclass=abc.ABCMeta):
    def __init__(self, targets, mtype="chat", **kwargs):
        super().__init__(**kwargs)
        self.targets = targets
        self.mtype = mtype

    @abc.abstractmethod
    def _get_message(self, target):
        pass

    def _send_message(self, target):
        text = self._get_message(target)
        logging.info("sending %s to %s with mtype %s", text, target, self.mtype)
        self.xmpp.send_message(target, text, mtype=self.mtype)

    def __call__(self):
        for target in self.targets:
            self._send_message(target)

class BroadcastDynamicMessage(BroadcastMessage):
    def __init__(self, targets, mgen, **kwargs):
        super().__init__(targets, **kwargs)
        self.mgen = mgen

    def _get_message(self, target):
        return self.mgen(target)

class BroadcastStaticMessage(BroadcastMessage):
    def __init__(self, targets, text, **kwargs):
        super().__init__(targets, **kwargs)
        self.text = text

    def _get_message(self, target):
        return self.text

class SYNACK(Base.MessageHandler):
    def __init__(self,
            timeout=10,
            timeout_message="handshake timed out",
            abort=True,
            send_rst=True,
            process_fin=True,
            **kwargs):
        super().__init__(**kwargs)
        self.timeout = float(timeout)
        self.timeout_message = timeout_message
        self.abort = abort
        self.pending_acks = {}
        self.process_fin = process_fin
        self.established = set()
        self.send_rst = send_rst

    def _get_uid(self, jid):
        return "{0!r}.{1!s}".format(self, jid)

    def _syn(self, msg):
        jid = msg["from"]
        try:
            uid, mode = self.pending_acks[str(jid)]
            self.xmpp.scheduler.remove(uid)
        except KeyError:
            uid = self._get_uid(jid)
            mode = "syn"
            self.pending_acks[str(jid)] = (uid, mode)

        if mode == "fin":
            self.reply(msg, "RST")
            del self.established[str(jid)]
            del self.pending_acks[str(jid)]
            return

        self.xmpp.scheduler.add(
            uid,
            self.timeout,
            lambda: self._timeout_syn(msg)
        )
        self.reply(msg, "SYN ACK")

    def _ack(self, msg):
        jid = msg["from"]
        jidstr = str(jid)
        try:
            uid, mode = self.pending_acks.pop(jidstr)
        except KeyError:
            if not jidstr in self.established:
                self.reply(msg, "RST")
            # no syn before
            return

        if mode == "rst":
            self.xmpp.scheduler.remove(uid)
            self.established.remove(jidstr)
        elif mode == "syn":
            self.xmpp.scheduler.remove(uid)
            if self.process_fin:
                self.established.add(jidstr)

    def _fin(self, msg):
        if not self.process_fin:
            return
        jid = msg["from"]
        if not jid in self.established:
            self.reply(msg, "RST")
            return

        uid = self._get_uid(jid)
        self.pending_acks[str(jid)] = uid, "rst"
        self.reply(msg, "FIN ACK")
        self.xmpp.scheduler.add(
            uid,
            self.timeout,
            lambda: self._timeout_fin(msg)
        )

    def _timeout_syn(self, msg):
        self.prefixed_reply(msg, self.timeout_message)
        del self.pending_acks[str(msg["from"])]

    def _timeout_fin(self, msg):
        self.reply(msg, "FIN ACK")

    def __call__(self, msg, errorSink=None):
        body = msg["body"].strip().lower()
        if body == "syn":
            self._syn(msg)
            return self.abort
        elif body == "ack":
            self._ack(msg)
            return self.abort
        elif body == "fin":
            self._fin(msg)
            return self.abort
        return False

class CTCP(Base.MessageHandler):
    def __init__(self,
            versionstr,
            dateformat="%a %d %b %Y %H:%M:%S UTC",
            **kwargs):
        super().__init__(**kwargs)
        self._versionstr = versionstr
        self._dateformat = dateformat

    def __call__(self, msg, errorSink=None):
        if msg["mtype"] != "chat":
            return False

        body = msg["body"].strip()
        if not body.startswith("\u0001") and not body.startswith("CTCP"):
            return False

        #body = body[5:]
        #if body.startswith("VERSION"):
        #    self.reply(msg, self._versionstr)
        #elif body.startswith("TIME"):
        #    self.reply(msg, datetime.utcnow().strftime(self._dateformat))
        #elif body.startswith("PING"):
        #    self.reply(msg, body)

        # Until we have found out on how to properly send NOTICEs, we'll
        # just ignore these messages

        return True
<|endoftext|>


<commit_before>import nflgame # https://github.com/BurntSushi/nflgame
from football import stats, bye_week

def get_stats(name, year, wks):
    player = nflgame.find(name, team=None)[0]
    byeweek = bye_week.find(player.team, year)
    player_stats = {}
    for x in range(0, len(wks)):
        dictprop = 'week_'+str(wks[x])
        if wks[x] == byeweek:
            player_stats[dictprop]=None
        else:
            player_stats[dictprop]=(stats.player_stats(name, year, wks[x]))<commit_msg>Points issues fixed<commit_after>import nflgame # https://github.com/BurntSushi/nflgame
from football import stats, bye_week

def get_stats(name, year, wks):
    # gets stats for a player
    # note: weeks must be defined as a list
    player = nflgame.find(name, team=None)[0]
    byeweek = bye_week.find(player.team, year)
    player_stats = {}
    for x in range(0, len(wks)):
        dictprop = 'week_'+str(wks[x])
        player_stats[dictprop]=(stats.player_stats(name, year, wks[x]))
        if wks[x] == byeweek:
            player_stats[dictprop]['bye_week'] = True
        else:
            player_stats[dictprop]['bye_week'] = False
    return player_stats

def standard_player_points(name, year, wks):
    player_stats = get_stats(name, year, wks)
    points = {}
    # unfinished method<|endoftext|>


<commit_before>from fp.monads.monad import Monad, MonadIter, MonadPlus

class Maybe(Monad, MonadIter, MonadPlus):
    """
    A Maybe monad.
    """

    ##=====================================================================
    ## Maybe methods
    ##=====================================================================
    def __init__(self, value):
        self.__value = value

    def __eq__(self, other):
        return self.__value == other.__value

    def __repr__(self):
        return "Maybe({0!r})".format(self.__value)

    @property
    def is_just(self):
        """
        Tests if the Maybe is just a value

        >>> Maybe(1).is_just
        True

        >>> Maybe(None).is_just
        False
        """
        return self.__value is not None

    @property
    def is_nothing(self):
        """
        Tests if the Maybe is Nothing

        >>> Maybe(None).is_nothing
        True

        >>> Maybe(1).is_nothing
        False
        """
        return not self.is_just

    @property
    def from_just(self):
        """
        Extracts the just value from the Maybe; raises a ValueError
        if the value is None

        >>> Maybe(1).from_just
        1

        >>> Maybe(None).from_just
        Traceback (most recent call last):
            ...
        ValueError: Maybe.from_just called on Nothing
        """
        if self.is_just:
            return self.__value
        else:
            raise ValueError("Maybe.from_just called on Nothing")

    def default(self, default_value):
        """
        Returns the just value or the default value if Nothing

        >>> Maybe(1).default(-1)
        1

        >>> Maybe(None).default(-1)
        -1
        """
        if self.is_just:
            return self.__value
        else:
            return default_value

    @classmethod
    def cat_maybes(cls, maybes):
        """
        Returns an iterator of the Just values

        >>> list(Maybe.cat_maybes([
        ...     Maybe(1), Maybe(None), Maybe(2)
        ... ]))
        [1, 2]
        """
        for maybe in maybes:
            if maybe.is_just:
                yield maybe.__value

    @classmethod
    def map_maybes(cls, f, xs):
        """
        maps a list over a Maybe arrow

        >>> def maybe_even(x):
        ...     if x % 2 == 0:
        ...         return Maybe(x)
        ...     else:
        ...         return Maybe(None)
        >>> list(Maybe.map_maybes(maybe_even, [1,2,3,4]))
        [2, 4]
        """
        for x in xs:
            maybe = f(x)
            if maybe.is_just:
                yield maybe.__value

    @staticmethod
    def error_to_nothing(f):
        """
        Converts a function that raises an exception of any kind to
        Maybe(None).
        """
        def inner(*args, **kwargs):
            try:
                return Maybe(f(*args, **kwargs))
            except:
                return Maybe(None)
        return inner

    ##=====================================================================
    ## Monad methods
    ##=====================================================================
    def bind(self, f):
        """
        The Maybe's bind function

        >>> Maybe(1).bind(lambda x: Maybe(x)).from_just
        1

        >>> def crashy(x):
        ...     assert False, "bind will not call me"
        >>> Maybe(None).bind(crashy).is_nothing
        1
        """
        if self.is_just:
            return f(self.__value)
        else:
            return self

    ##=====================================================================
    ## MonadIter methods
    ##=====================================================================
    def __iter__(self):
        """
        The MonadIter.__iter__ function to enable monadic exploitation
        of list generators:

        >>> list(Maybe(1))
        [1]

        >>> list(Maybe(None))
        []

        >>> def lookup(d, x):
        ...     return Maybe(d.get(x))
        >>> data = {"foo": {"bar": "baz"}}

        >>> Maybe.from_iterable(
        ...    val
        ...    for inner in lookup(data, "foo")
        ...    for val   in lookup(inner, "bar")
        ... )
        Maybe('baz')

        Failure on the first key:

        >>> Maybe.from_iterable(
        ...    val
        ...    for inner in lookup(data, "fud")
        ...    for val   in lookup(inner, "baz")
        ... )
        Maybe(None)

        Failure on the second key:

        >>> Maybe.from_iterable(
        ...    val
        ...    for inner in lookup(data, "foo")
        ...    for val   in lookup(inner, "bong")
        ... )
        Maybe(None)
        """
        if self.is_just:
            return iter([self.__value])
        else:
            return iter([])

    @classmethod
    def from_iterable(cls, iterable):
        for x in iterable:
            return Maybe(x)
        return Maybe(None)

    ##=====================================================================
    ## MonadPlus methods
    ##=====================================================================
    def mplus(self, y):
        """
        An associative operation.  We're not
        >>> Maybe(1).mplus(Maybe.mzero)
        Maybe(1)

        >>> Maybe.mzero.mplus(Maybe(2))
        Maybe(2)

        >>> Maybe.msum([Maybe(1), Maybe(None), Maybe(2)])
        Maybe(1)

        >>> Maybe.msum([Maybe(None), Maybe(None), Maybe(2)])
        Maybe(2)

        >>> Maybe.msum([Maybe(None), Maybe(None), Maybe(None)])
        Maybe(None)

        """
        if self.is_just:
            return self
        else:
            return y

Maybe.mzero = Maybe(None)

if __name__ == '__main__':
    import pytest, sys
    sys.exit(pytest.main(["--doctest-modules", __file__]))
<commit_msg>Added some doctests Aliased Maybe to Just and Nothing and changed the __repr__ to reflect that<commit_after>from fp.monads.monad import Monad, MonadIter, MonadPlus

class Maybe(Monad, MonadIter, MonadPlus):
    """
    A Maybe monad.

    Examples:

    >>> Maybe.sequence([Just(1), Nothing, Just(2)])
    Nothing

    >>> Maybe.sequence([Just(1), Just(2)])
    Just([1, 2])

    >>> Maybe.sequence_dict({"foo": Just(1), "bar": Just(2)})
    Just({'foo': 1, 'bar': 2})

    >>> maybe_int = Maybe.error_to_nothing(int)
    >>> Maybe.mapM(maybe_int, ["1", "2"])
    Just([1, 2])

    >>> Maybe.mapM(maybe_int, ["1", "a"])
    Nothing

    Left to Right arrow composition.  maybe cast the string as an int, then maybe add 1

    >>> string_to_plus1 = Maybe.arrow_cl(maybe_int, lambda x: Maybe(x+1))
    >>> string_to_plus1("1")
    Just(2)

    Right to Left arrow composition.  maybe cast the string as an int, then maybe add 1

    >>> string_to_plus1 = Maybe.arrow_cr(lambda x: Maybe(x+1), maybe_int)
    >>> string_to_plus1("1")
    Just(2)

    >>> Maybe.ap(lambda x: x+1, Just(1))
    Just(2)

    >>> Maybe.ap(lambda x: x+1, Nothing)
    Nothing

    >>> import operator
    >>> Maybe.ap(operator.add, Just(1), Just(2))
    Just(3)

    >>> Maybe.ap(operator.add, Nothing, Just(2))
    Nothing

    >>> Maybe.ap(operator.add, Just(1), Nothing)
    Nothing

    >>> from datetime import timedelta
    >>> Maybe.ap(timedelta, days=Just(1), seconds=Just(60))
    Just(datetime.timedelta(1, 60))

    >>> from fp.monads.maybe import Maybe
    >>> from fp import even, c, p
    >>> maybeInt = Maybe.error_to_nothing(int) # convert int to a Maybe arrow
    >>> maybeEven = p(Maybe.ap, even) # lift even into Maybe
    >>> Maybe.filterM(c(maybeEven, maybeInt), ["x","1","2","3","4"])
    Just(['2', '4'])

    >>> Maybe.msum([Maybe(1), Nothing, Maybe(2)])
    Just(1)

    >>> Maybe.msum([Nothing, Nothing, Maybe(2)])
    Just(2)

    >>> Maybe.msum([Nothing, Nothing, Nothing])
    Nothing

    """

    ##=====================================================================
    ## Maybe methods
    ##=====================================================================
    def __init__(self, value):
        self.__value = value

    def __eq__(self, other):
        return self.__value == other.__value

    def __repr__(self):
        if self.is_just:
            return "Just({0!r})".format(self.__value)
        else:
            return "Nothing"

    @property
    def is_just(self):
        """
        Tests if the Maybe is just a value

        >>> Just(1).is_just
        True

        >>> Nothing.is_just
        False
        """
        return self.__value is not None

    @property
    def is_nothing(self):
        """
        Tests if the Maybe is Nothing

        >>> Nothing.is_nothing
        True

        >>> Just(1).is_nothing
        False
        """
        return not self.is_just

    @property
    def from_just(self):
        """
        Extracts the just value from the Maybe; raises a ValueError
        if the value is None

        >>> Just(1).from_just
        1

        >>> Nothing.from_just
        Traceback (most recent call last):
            ...
        ValueError: Maybe.from_just called on Nothing
        """
        if self.is_just:
            return self.__value
        else:
            raise ValueError("Maybe.from_just called on Nothing")

    def default(self, default_value):
        """
        Returns the just value or the default value if Nothing

        >>> Just(1).default(-1)
        1

        >>> Nothing.default(-1)
        -1
        """
        if self.is_just:
            return self.__value
        else:
            return default_value

    @classmethod
    def cat_maybes(cls, maybes):
        """
        Returns an iterator of the Just values

        >>> list(Maybe.cat_maybes([
        ...     Just(1), Nothing, Just(2)
        ... ]))
        [1, 2]
        """
        for maybe in maybes:
            if maybe.is_just:
                yield maybe.__value

    @classmethod
    def map_maybes(cls, f, xs):
        """
        maps a list over a Maybe arrow

        >>> def maybe_even(x):
        ...     if x % 2 == 0:
        ...         return Just(x)
        ...     else:
        ...         return Nothing
        >>> list(Maybe.map_maybes(maybe_even, [1,2,3,4]))
        [2, 4]
        """
        for x in xs:
            maybe = f(x)
            if maybe.is_just:
                yield maybe.__value

    @staticmethod
    def error_to_nothing(f):
        """
        Converts a function that raises an exception of any kind to
        Nothing.
        """
        def inner(*args, **kwargs):
            try:
                return Just(f(*args, **kwargs))
            except:
                return Nothing
        return inner

    ##=====================================================================
    ## Monad methods
    ##=====================================================================
    def bind(self, f):
        """
        The Maybe's bind function

        >>> Just(1).bind(lambda x: Maybe(x)).from_just
        1

        >>> def crashy(x):
        ...     assert False, "bind will not call me"
        >>> Nothing.bind(crashy).is_nothing
        1
        """
        if self.is_just:
            return f(self.__value)
        else:
            return self

    ##=====================================================================
    ## MonadIter methods
    ##=====================================================================
    def __iter__(self):
        """
        The MonadIter.__iter__ function to enable monadic exploitation
        of list generators:

        >>> list(Just(1))
        [1]

        >>> list(Nothing)
        []

        >>> def lookup(d, x):
        ...     return Maybe(d.get(x))
        >>> data = {"foo": {"bar": "baz"}}

        >>> Maybe.from_iterable(
        ...    val
        ...    for inner in lookup(data, "foo")
        ...    for val   in lookup(inner, "bar")
        ... )
        Just('baz')

        Failure on the first key:

        >>> Maybe.from_iterable(
        ...    val
        ...    for inner in lookup(data, "fud")
        ...    for val   in lookup(inner, "baz")
        ... )
        Nothing

        Failure on the second key:

        >>> Maybe.from_iterable(
        ...    val
        ...    for inner in lookup(data, "foo")
        ...    for val   in lookup(inner, "bong")
        ... )
        Nothing
        """
        if self.is_just:
            return iter([self.__value])
        else:
            return iter([])

    @classmethod
    def from_iterable(cls, iterable):
        for x in iterable:
            return Maybe(x)
        return Nothing

    ##=====================================================================
    ## MonadPlus methods
    ##=====================================================================
    def mplus(self, y):
        """
        An associative operation.  We're not
        >>> Maybe(1).mplus(Maybe.mzero)
        Just(1)

        >>> Maybe.mzero.mplus(Maybe(2))
        Just(2)
        """
        if self.is_just:
            return self
        else:
            return y

Just = Maybe
Nothing = Maybe(None)

Maybe.mzero = Nothing

if __name__ == '__main__':
    import pytest, sys
    sys.exit(pytest.main(["--doctest-modules", __file__]))
<|endoftext|>