<commit_before># (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################

import fnmatch
import os
import re

import subprocess
import ansible.constants as C
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible.inventory.group import Group
from ansible.inventory.host import Host
from ansible import errors
from ansible import utils

class Inventory(object):
    """
    Host inventory for ansible.
    """

    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', '_is_script',
                  'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list']

    def __init__(self, host_list=C.DEFAULT_HOST_LIST):

        # the host file file, or script path, or list of hosts
        # if a list, inventory data will NOT be loaded
        self.host_list = host_list

        # caching to avoid repeated calculations, particularly with
        # external inventory scripts.

        self._vars_per_host  = {}
        self._vars_per_group = {}
        self._hosts_cache    = {}
        self._groups_list    = {}

        # the inventory object holds a list of groups
        self.groups = []

        # a list of host(names) to contain current inquiries to
        self._restriction = None
        self._also_restriction = None
        self._subset = None

        # whether the inventory file is a script
        self._is_script = False

        if type(host_list) in [ str, unicode ]:
            if host_list.find(",") != -1:
                host_list = host_list.split(",")
                host_list = [ h for h in host_list if h and h.strip() ]

        else:
            utils.plugins.vars_loader.add_directory(self.basedir())

        if type(host_list) == list:
            all = Group('all')
            self.groups = [ all ]
            for x in host_list:
                if x.find(":") != -1:
                    tokens = x.split(":",1)
                    all.add_host(Host(tokens[0], tokens[1]))
                else:
                    all.add_host(Host(x))
        elif utils.is_executable(host_list):
            self._is_script = True
            self.parser = InventoryScript(filename=host_list)
            self.groups = self.parser.groups.values()
        else:
            data = file(host_list).read()
            if not data.startswith("---"):
                self.parser = InventoryParser(filename=host_list)
                self.groups = self.parser.groups.values()
            else:
                raise errors.AnsibleError("YAML inventory support is deprecated in 0.6 and removed in 0.7, see the migration script in examples/scripts in the git checkout")

    def _match(self, str, pattern_str):
        if pattern_str.startswith('~'):
            return re.search(pattern_str[1:], str)
        else:
            return fnmatch.fnmatch(str, pattern_str)

    def get_hosts(self, pattern="all"):
        """
        find all host names matching a pattern string, taking into account any inventory restrictions or
        applied subsets.
        """

        # process patterns
        if isinstance(pattern, list):
            pattern = ';'.join(pattern)
        patterns = pattern.replace(";",":").split(":")
        hosts = self._get_hosts(patterns)

        # exclude hosts not in a subset, if defined
        if self._subset:
            subset = self._get_hosts(self._subset)
            hosts.intersection_update(subset)

        # exclude hosts mentioned in any restriction (ex: failed hosts)
        if self._restriction is not None:
            hosts = [ h for h in hosts if h.name in self._restriction ]
        if self._also_restriction is not None:
            hosts = [ h for h in hosts if h.name in self._also_restriction ]

        return sorted(hosts, key=lambda x: x.name)

    def _get_hosts(self, patterns):
        """
        finds hosts that match a list of patterns. Handles negative
        matches as well as intersection matches.
        """

        hosts = set()
        for p in patterns:
            if p.startswith("!"):
                # Discard excluded hosts
                hosts.difference_update(self.__get_hosts(p))
            elif p.startswith("&"):
                # Only leave the intersected hosts
                hosts.intersection_update(self.__get_hosts(p))
            else:
                # Get all hosts from both patterns
                hosts.update(self.__get_hosts(p))
        return hosts

    def __get_hosts(self, pattern):
        """
        finds hosts that postively match a particular pattern.  Does not
        take into account negative matches.
        """

        (name, enumeration_details) = self._enumeration_info(pattern)
        hpat = self._hosts_in_unenumerated_pattern(name)
        hpat = sorted(hpat, key=lambda x: x.name)

        return set(self._apply_ranges(pattern, hpat))

    def _enumeration_info(self, pattern):
        """
        returns (pattern, limits) taking a regular pattern and finding out
        which parts of it correspond to start/stop offsets.  limits is
        a tuple of (start, stop) or None
        """

        if not "[" in pattern or pattern.startswith('~'):
            return (pattern, None)
        (first, rest) = pattern.split("[")
        rest = rest.replace("]","")
        if not "-" in rest:
            raise errors.AnsibleError("invalid pattern: %s" % pattern)
        (left, right) = rest.split("-",1)
        return (first, (left, right))

    def _apply_ranges(self, pat, hosts):
        """
        given a pattern like foo, that matches hosts, return all of hosts
        given a pattern like foo[0:5], where foo matches hosts, return the first 6 hosts
        """

        (loose_pattern, limits) = self._enumeration_info(pat)
        if not limits:
            return hosts

        (left, right) = limits
        enumerated = enumerate(hosts)
        if left == '':
            left = 0
        if right == '':
            right = 0
        left=int(left)
        right=int(right)
        enumerated = [ h for (i,h) in enumerated if i>=left and i<=right ]
        return enumerated

    # TODO: cache this logic so if called a second time the result is not recalculated
    def _hosts_in_unenumerated_pattern(self, pattern):
        """ Get all host names matching the pattern """

        hosts = {}
        # ignore any negative checks here, this is handled elsewhere
        pattern = pattern.replace("!","").replace("&", "")

        groups = self.get_groups()
        for group in groups:
            for host in group.get_hosts():
                if pattern == 'all' or self._match(group.name, pattern) or self._match(host.name, pattern):
                    hosts[host.name] = host
        return sorted(hosts.values(), key=lambda x: x.name)

    def groups_for_host(self, host):
        results = []
        groups = self.get_groups()
        for group in groups:
            for hostn in group.get_hosts():
                if host == hostn.name:
                    results.append(group)
                    continue
        return results

    def groups_list(self):
        if not self._groups_list:
            groups = {}
            for g in self.groups:
                groups[g.name] = [h.name for h in g.get_hosts()]
                ancestors = g.get_ancestors()
                for a in ancestors:
                    groups[a.name] = [h.name for h in a.get_hosts()]
            self._groups_list = groups
        return self._groups_list

    def get_groups(self):
        return self.groups

    def get_host(self, hostname):
        if hostname not in self._hosts_cache:
            self._hosts_cache[hostname] = self._get_host(hostname)
        return self._hosts_cache[hostname]

    def _get_host(self, hostname):
        for group in self.groups:
            for host in group.get_hosts():
                if hostname == host.name:
                    return host
        return None

    def get_group(self, groupname):
        for group in self.groups:
            if group.name == groupname:
                return group
        return None

    def get_group_variables(self, groupname):
        if groupname not in self._vars_per_group:
            self._vars_per_group[groupname] = self._get_group_variables(groupname)
        return self._vars_per_group[groupname]

    def _get_group_variables(self, groupname):
        group = self.get_group(groupname)
        if group is None:
            raise Exception("group not found: %s" % groupname)
        return group.get_variables()

    def get_variables(self, hostname):
        if hostname not in self._vars_per_host:
            self._vars_per_host[hostname] = self._get_variables(hostname)
        return self._vars_per_host[hostname]

    def _get_variables(self, hostname):

        host = self.get_host(hostname)
        if host is None:
            raise errors.AnsibleError("host not found: %s" % hostname)

        vars = {}
        for updated in map(lambda x: x.run(host), utils.plugins.vars_loader.all(self)):
            if updated is not None:
                vars.update(updated)

        vars.update(host.get_variables())
        if self._is_script:
            cmd = [self.host_list,"--host",hostname]
            try:
                sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            except OSError, e:
                raise errors.AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
            (out, err) = sp.communicate()
            results = utils.parse_json(out)

            vars.update(results)
        return vars

    def add_group(self, group):
        self.groups.append(group)

    def list_hosts(self, pattern="all"):
        return [ h.name for h in self.get_hosts(pattern) ]

    def list_groups(self):
        return sorted([ g.name for g in self.groups ], key=lambda x: x.name)

    # TODO: remove this function
    def get_restriction(self):
        return self._restriction

    def restrict_to(self, restriction):
        """
        Restrict list operations to the hosts given in restriction.  This is used
        to exclude failed hosts in main playbook code, don't use this for other
        reasons.
        """
        if type(restriction) != list:
            restriction = [ restriction ]
        self._restriction = restriction

    def also_restrict_to(self, restriction):
        """
        Works like restict_to but offers an additional restriction.  Playbooks use this
        to implement serial behavior.
        """
        if type(restriction) != list:
            restriction = [ restriction ]
        self._also_restriction = restriction

    def subset(self, subset_pattern):
        """
        Limits inventory results to a subset of inventory that matches a given
        pattern, such as to select a given geographic of numeric slice amongst
        a previous 'hosts' selection that only select roles, or vice versa.
        Corresponds to --limit parameter to ansible-playbook
        """
        if subset_pattern is None:
            self._subset = None
        else:
            subset_pattern = subset_pattern.replace(',',':')
            self._subset = subset_pattern.replace(";",":").split(":")

    def lift_restriction(self):
        """ Do not restrict list operations """
        self._restriction = None

    def lift_also_restriction(self):
        """ Clears the also restriction """
        self._also_restriction = None

    def is_file(self):
        """ did inventory come from a file? """
        if not isinstance(self.host_list, basestring):
            return False
        return os.path.exists(self.host_list)

    def basedir(self):
        """ if inventory came from a file, what's the directory? """
        if not self.is_file():
            return None
        return os.path.dirname(self.host_list)
<commit_msg>Allow pattern[0] to address a single node<commit_after># (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################

import fnmatch
import os
import re

import subprocess
import ansible.constants as C
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible.inventory.group import Group
from ansible.inventory.host import Host
from ansible import errors
from ansible import utils

class Inventory(object):
    """
    Host inventory for ansible.
    """

    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', '_is_script',
                  'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list']

    def __init__(self, host_list=C.DEFAULT_HOST_LIST):

        # the host file file, or script path, or list of hosts
        # if a list, inventory data will NOT be loaded
        self.host_list = host_list

        # caching to avoid repeated calculations, particularly with
        # external inventory scripts.

        self._vars_per_host  = {}
        self._vars_per_group = {}
        self._hosts_cache    = {}
        self._groups_list    = {}

        # the inventory object holds a list of groups
        self.groups = []

        # a list of host(names) to contain current inquiries to
        self._restriction = None
        self._also_restriction = None
        self._subset = None

        # whether the inventory file is a script
        self._is_script = False

        if type(host_list) in [ str, unicode ]:
            if host_list.find(",") != -1:
                host_list = host_list.split(",")
                host_list = [ h for h in host_list if h and h.strip() ]

        else:
            utils.plugins.vars_loader.add_directory(self.basedir())

        if type(host_list) == list:
            all = Group('all')
            self.groups = [ all ]
            for x in host_list:
                if x.find(":") != -1:
                    tokens = x.split(":",1)
                    all.add_host(Host(tokens[0], tokens[1]))
                else:
                    all.add_host(Host(x))
        elif utils.is_executable(host_list):
            self._is_script = True
            self.parser = InventoryScript(filename=host_list)
            self.groups = self.parser.groups.values()
        else:
            data = file(host_list).read()
            if not data.startswith("---"):
                self.parser = InventoryParser(filename=host_list)
                self.groups = self.parser.groups.values()
            else:
                raise errors.AnsibleError("YAML inventory support is deprecated in 0.6 and removed in 0.7, see the migration script in examples/scripts in the git checkout")

    def _match(self, str, pattern_str):
        if pattern_str.startswith('~'):
            return re.search(pattern_str[1:], str)
        else:
            return fnmatch.fnmatch(str, pattern_str)

    def get_hosts(self, pattern="all"):
        """
        find all host names matching a pattern string, taking into account any inventory restrictions or
        applied subsets.
        """

        # process patterns
        if isinstance(pattern, list):
            pattern = ';'.join(pattern)
        patterns = pattern.replace(";",":").split(":")
        hosts = self._get_hosts(patterns)

        # exclude hosts not in a subset, if defined
        if self._subset:
            subset = self._get_hosts(self._subset)
            hosts.intersection_update(subset)

        # exclude hosts mentioned in any restriction (ex: failed hosts)
        if self._restriction is not None:
            hosts = [ h for h in hosts if h.name in self._restriction ]
        if self._also_restriction is not None:
            hosts = [ h for h in hosts if h.name in self._also_restriction ]

        return sorted(hosts, key=lambda x: x.name)

    def _get_hosts(self, patterns):
        """
        finds hosts that match a list of patterns. Handles negative
        matches as well as intersection matches.
        """

        hosts = set()
        for p in patterns:
            if p.startswith("!"):
                # Discard excluded hosts
                hosts.difference_update(self.__get_hosts(p))
            elif p.startswith("&"):
                # Only leave the intersected hosts
                hosts.intersection_update(self.__get_hosts(p))
            else:
                # Get all hosts from both patterns
                hosts.update(self.__get_hosts(p))
        return hosts

    def __get_hosts(self, pattern):
        """
        finds hosts that postively match a particular pattern.  Does not
        take into account negative matches.
        """

        (name, enumeration_details) = self._enumeration_info(pattern)
        hpat = self._hosts_in_unenumerated_pattern(name)
        hpat = sorted(hpat, key=lambda x: x.name)

        return set(self._apply_ranges(pattern, hpat))

    def _enumeration_info(self, pattern):
        """
        returns (pattern, limits) taking a regular pattern and finding out
        which parts of it correspond to start/stop offsets.  limits is
        a tuple of (start, stop) or None
        """

        if not "[" in pattern or pattern.startswith('~'):
            return (pattern, None)
        (first, rest) = pattern.split("[")
        rest = rest.replace("]","")
        if "-" in rest:
            (left, right) = rest.split("-",1)
            return (first, (left, right))
        else:
            return (first, (rest, rest))

    def _apply_ranges(self, pat, hosts):
        """
        given a pattern like foo, that matches hosts, return all of hosts
        given a pattern like foo[0:5], where foo matches hosts, return the first 6 hosts
        """

        (loose_pattern, limits) = self._enumeration_info(pat)
        if not limits:
            return hosts

        (left, right) = limits
        enumerated = enumerate(hosts)
        if left == '':
            left = 0
        if right == '':
            right = 0
        left=int(left)
        right=int(right)
        enumerated = [ h for (i,h) in enumerated if i>=left and i<=right ]
        return enumerated

    # TODO: cache this logic so if called a second time the result is not recalculated
    def _hosts_in_unenumerated_pattern(self, pattern):
        """ Get all host names matching the pattern """

        hosts = {}
        # ignore any negative checks here, this is handled elsewhere
        pattern = pattern.replace("!","").replace("&", "")

        groups = self.get_groups()
        for group in groups:
            for host in group.get_hosts():
                if pattern == 'all' or self._match(group.name, pattern) or self._match(host.name, pattern):
                    hosts[host.name] = host
        return sorted(hosts.values(), key=lambda x: x.name)

    def groups_for_host(self, host):
        results = []
        groups = self.get_groups()
        for group in groups:
            for hostn in group.get_hosts():
                if host == hostn.name:
                    results.append(group)
                    continue
        return results

    def groups_list(self):
        if not self._groups_list:
            groups = {}
            for g in self.groups:
                groups[g.name] = [h.name for h in g.get_hosts()]
                ancestors = g.get_ancestors()
                for a in ancestors:
                    groups[a.name] = [h.name for h in a.get_hosts()]
            self._groups_list = groups
        return self._groups_list

    def get_groups(self):
        return self.groups

    def get_host(self, hostname):
        if hostname not in self._hosts_cache:
            self._hosts_cache[hostname] = self._get_host(hostname)
        return self._hosts_cache[hostname]

    def _get_host(self, hostname):
        for group in self.groups:
            for host in group.get_hosts():
                if hostname == host.name:
                    return host
        return None

    def get_group(self, groupname):
        for group in self.groups:
            if group.name == groupname:
                return group
        return None

    def get_group_variables(self, groupname):
        if groupname not in self._vars_per_group:
            self._vars_per_group[groupname] = self._get_group_variables(groupname)
        return self._vars_per_group[groupname]

    def _get_group_variables(self, groupname):
        group = self.get_group(groupname)
        if group is None:
            raise Exception("group not found: %s" % groupname)
        return group.get_variables()

    def get_variables(self, hostname):
        if hostname not in self._vars_per_host:
            self._vars_per_host[hostname] = self._get_variables(hostname)
        return self._vars_per_host[hostname]

    def _get_variables(self, hostname):

        host = self.get_host(hostname)
        if host is None:
            raise errors.AnsibleError("host not found: %s" % hostname)

        vars = {}
        for updated in map(lambda x: x.run(host), utils.plugins.vars_loader.all(self)):
            if updated is not None:
                vars.update(updated)

        vars.update(host.get_variables())
        if self._is_script:
            cmd = [self.host_list,"--host",hostname]
            try:
                sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            except OSError, e:
                raise errors.AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
            (out, err) = sp.communicate()
            results = utils.parse_json(out)

            vars.update(results)
        return vars

    def add_group(self, group):
        self.groups.append(group)

    def list_hosts(self, pattern="all"):
        return [ h.name for h in self.get_hosts(pattern) ]

    def list_groups(self):
        return sorted([ g.name for g in self.groups ], key=lambda x: x.name)

    # TODO: remove this function
    def get_restriction(self):
        return self._restriction

    def restrict_to(self, restriction):
        """
        Restrict list operations to the hosts given in restriction.  This is used
        to exclude failed hosts in main playbook code, don't use this for other
        reasons.
        """
        if type(restriction) != list:
            restriction = [ restriction ]
        self._restriction = restriction

    def also_restrict_to(self, restriction):
        """
        Works like restict_to but offers an additional restriction.  Playbooks use this
        to implement serial behavior.
        """
        if type(restriction) != list:
            restriction = [ restriction ]
        self._also_restriction = restriction

    def subset(self, subset_pattern):
        """
        Limits inventory results to a subset of inventory that matches a given
        pattern, such as to select a given geographic of numeric slice amongst
        a previous 'hosts' selection that only select roles, or vice versa.
        Corresponds to --limit parameter to ansible-playbook
        """
        if subset_pattern is None:
            self._subset = None
        else:
            subset_pattern = subset_pattern.replace(',',':')
            self._subset = subset_pattern.replace(";",":").split(":")

    def lift_restriction(self):
        """ Do not restrict list operations """
        self._restriction = None

    def lift_also_restriction(self):
        """ Clears the also restriction """
        self._also_restriction = None

    def is_file(self):
        """ did inventory come from a file? """
        if not isinstance(self.host_list, basestring):
            return False
        return os.path.exists(self.host_list)

    def basedir(self):
        """ if inventory came from a file, what's the directory? """
        if not self.is_file():
            return None
        return os.path.dirname(self.host_list)
<|endoftext|>


<commit_before>#Author: Yue-Wen FANG at Kyoto University
#Contact: fyuewen@gmail.com
######################################################
#Purpose: This is a python crawler/spider to download#
#the phonon database maintained by Togo-sensei       #
#python 2.x should be used.
import urllib2
import re
import urllib

#define url
url='http://phonondb.mtl.kyoto-u.ac.jp/raw_data/'
website = urllib2.urlopen(url)
#read html code
html = website.read()
#use re.findall to get all the links
links = re.findall('"((http)s?://.*.tar.lzma)"', html)

pattern = re.compile(r'mp-\d+.tar.lzma')  #here, mp-\d+.tar.lzma is the pattern
matchlist = pattern.findall(html)  #note in this matchlist, the comound ID is printed twice.
new_matchlist = matchlist[::2] #Here, I selected all the matched iterms, and discarded the duplicated ones
print(new_matchlist[:10])
print(type(new_matchlist))
#print(' '.join(matchlist))  #through joinining, the quotations are removed.

for i in new_matchlist:
    file_url = 'http://phonondb.mtl.kyoto-u.ac.jp/raw_data/' + str(i)
    urllib.urlretrieve(file_url, i) #download files with the name of i.
    print(file_url)
#usage of urllib.urlretrieve
#urllib.urlretrieve(url, filename) #here, filename should be a string
<commit_msg>add usage<commit_after>#Author: Yue-Wen FANG at Kyoto University
#Contact: fyuewen@gmail.com
######################################################
#Purpose: This is a python crawler/spider to download#
#the phonon database maintained by Togo-sensei       #
#python 2.x should be used.
#usage: python spider.py
import urllib2
import re
import urllib

#define url
url='http://phonondb.mtl.kyoto-u.ac.jp/raw_data/'
website = urllib2.urlopen(url)
#read html code
html = website.read()
#use re.findall to get all the links
links = re.findall('"((http)s?://.*.tar.lzma)"', html)

pattern = re.compile(r'mp-\d+.tar.lzma')  #here, mp-\d+.tar.lzma is the pattern
matchlist = pattern.findall(html)  #note in this matchlist, the comound ID is printed twice.
new_matchlist = matchlist[::2] #Here, I selected all the matched iterms, and discarded the duplicated ones
print(new_matchlist[:10])
print(type(new_matchlist))
#print(' '.join(matchlist))  #through joinining, the quotations are removed.

for i in new_matchlist:
    file_url = 'http://phonondb.mtl.kyoto-u.ac.jp/raw_data/' + str(i)
    urllib.urlretrieve(file_url, i) #download files with the name of i.
    print(file_url)
#usage of urllib.urlretrieve
#urllib.urlretrieve(url, filename) #here, filename should be a string
<|endoftext|>


<commit_before>import json
import xml.etree.ElementTree as ET
import re
import os
from cStringIO import StringIO
import time

import tornado.web
from tornado import gen

from epub import EPUB
from epub.utils import listFiles

from data.utils import opendb
from data.data import DatabaseConnection
from data import opds

from urlparse import parse_qs as parse_querystring

def accepted_formats(header):
    header_list = header.split(",")
    for i, v in enumerate(header_list):
        header_list[i] = v.split(";")[0].strip()
    return header_list

def user_real_ip(request):
    if "X-Real-IP" in request.headers:
        return request.headers.get("X-Real-IP")
    else:
        return request.remote_ip

class GeneralErrorHandler(tornado.web.RequestHandler):
    def __init__(self, application, request, status_code):
        tornado.web.RequestHandler.__init__(self, application, request)
        self.set_status(status_code)

    def write_error(self, status, **kwargs):
        self.write("Bad request")

    def prepare(self):
        raise tornado.web.HTTPError(400)

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.render("hello.html", title="Welcome!", user=user_real_ip(self.request))

class GetInfo(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, filename):
        if filename:
            response = yield gen.Task(self.querydb, filename)
            if "text/html" in accepted_formats(self.request.headers.get("accept")):
                self.render("info.html",
                            title=response["metadata"]["title"],
                            id=response["id"],
                            meta=response["metadata"],
                            contents=response["toc"],
                            manifest=response["manifest"],
                            cover=response["cover"]
                            )
            else:
                output = response
                output["cover"] = "/book/{0}/manifest/{1}".format(response["id"], response["cover"])
                self.write(json.dumps(output, indent=4))
                self.finish()
        else:
            raise tornado.web.HTTPError(400)

    def querydb(self, isbn, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ? ", (isbn,)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        epubfile = EPUB(path)
        output = epubfile.info
        output["cover"] = epubfile.cover
        output["id"] = epubfile.id
        output["toc"] = epubfile.contents
        return callback(output)

class ShowManifest(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, filename):
        if filename:
            response = yield gen.Task(self.querydb, filename)
            self.set_header("Content-Type", "application/json")
            self.set_header("Charset", "UTF-8")
            self.write(json.dumps(response, indent=4))
            self.finish()
        else:
            raise tornado.web.HTTPError(400)

    def querydb(self, isbn, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ? ", (isbn,)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        epubfile = EPUB(path)
        output = epubfile.info["manifest"]
        return callback(output)

class ListFiles(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self):
        response = yield gen.Task(self.cataloguedump)
        if "text/html" in accepted_formats(self.request.headers.get("accept")):
            self.render("catalogue.html",
                        output=response, search=False)
        else:
            self.set_header("Content-Type", "application/json")
            self.set_header("Charset", "UTF-8")
            self.write(json.dumps(response, indent=4))
            self.finish()

    def cataloguedump(self, callback):
        response = listFiles()
        return callback(response)

class ShowFileToc(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier):
        if identifier:
            try:
                output = yield gen.Task(self.queryToc, identifier)
                self.set_header("Content-Type", "application/json")
                self.write(json.dumps(output, indent=4))
                self.finish()
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(400)

    def queryToc(self, identifier, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ?", (identifier, )).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        output = EPUB(path).contents
        return callback(output)

class GetFilePart(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier, part, section=False):
        if identifier and part and not section:
            try:
                output = yield gen.Task(self.perform, identifier, part, section=False)
            except IOError:
                raise tornado.web.HTTPError(404)
        elif identifier and part and section:
            try:
                output = yield gen.Task(self.perform, identifier, part, section)
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(405)

        self.set_header("Content-Type", "text/html")
        self.set_header("Charset", "UTF-8")
        self.write(output)
        self.finish()

    def perform(self, identifier, part, section, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ?", (identifier, )).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        try:
            epub = EPUB(path)
            part_path = ""
            for i in epub.contents:
                if part == i.get("id"):
                    part_path = i.get("src")
            output = epub.read(re.sub(r"#(.*)", "", part_path))  # strip fragment id.

            output = re.sub(r'(href|src)="(\.\./)?(.*?)"', '\g<1>="/getpath/{0}/\g<3>"'.format(identifier), output)
            output = re.sub(r"(href|src)='(\.\./)?(.*?)'", '\g<1>="/getpath/{0}/\g<3>"'.format(identifier), output)

            if section:
                try:
                    from htmlentitydefs import entitydefs
                    parser = ET.XMLParser()
                    parser.parser.UseForeignDTD(True)
                    parser.entity.update(entitydefs)
                    source = StringIO(output)
                    root = ET.parse(source, parser)
                    section = int(section) - 1
                    name = root.find(".//{http://www.w3.org/1999/xhtml}body")[section]
                    output = " ".join([t for t in list(name.itertext())])
                except Exception, e:
                    print e
                    raise tornado.web.HTTPError(404)

        except KeyError:
            raise tornado.web.HTTPError(404)

        return callback(output)

class GetFilePath(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier, part):
        if identifier and part:
            try:
                output = yield gen.Task(self.perform, identifier, part)
                self.set_header("Content-Type", "text/html")
                self.write(output)
                self.finish()
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(400)

    def perform(self, identifier, part, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = '{0}'".format(identifier)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        filepath = ""
        try:
            epub = EPUB(path)
            for i in epub.namelist():
                if i.endswith(part):
                    filepath = i
            output = epub.read(filepath)

        except KeyError:
            output = "Nope."
            pass
        return callback(output)

class GetResource(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier, manifest_id):
        if identifier and manifest_id:
            try:
                output, mimetype = yield gen.Task(self.perform, identifier, manifest_id)
                self.set_header("Content-Type", mimetype)
                self.write(output)
                self.finish()
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(400)

    def perform(self, identifier, toc_id, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = '{0}'".format(identifier)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        filepath = ""
        mimetype = ""
        try:
            epub = EPUB(path)
            for i in epub.info["manifest"]:
                if i["id"] == toc_id:
                    filepath = i["href"]
                    mimetype = i["mimetype"]
            if not mimetype:
                mimetype = "text/html"
            output = epub.read(os.path.join(epub.root_folder, filepath)), mimetype

        except KeyError:
            output = "KEY ERROR"
            pass
        return callback(output)

class DownloadPublication(tornado.web.RequestHandler):

    def get(self, filename):
        if filename:
            database, conn = opendb()
            try:
                path = database.execute("SELECT path FROM books WHERE isbn = '{0}' ".format(filename)).fetchone()["path"]
            except TypeError:
                raise tornado.web.HTTPError(404)
            finally:
                conn.close()
            output = open(path, "r")
            self.set_header('Content-Type', 'application/zip')
            self.set_header('Content-Disposition', 'attachment; filename='+os.path.basename(path)+'')
            self.write(output.read())
        else:
            raise tornado.web.HTTPError(404)

class OPDSCatalogue(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self):
        if os.path.exists("feed.xml") and (time.time() - os.path.getmtime("feed.xml")) < 200:
            self.set_header("Content-Type", "application/atom+xml")
            with open("feed.xml", "r") as f:
                self.write(f.read())
                self.finish()
        else:
            catalogue = yield gen.Task(self.perform, )
            self.set_header("Content-Type", "application/atom+xml")
            self.write(catalogue)
            self.finish()

    def perform(self, callback):
        catalogue = opds.generateCatalogRoot()
        #  Raw cache. TODO: better cache handling
        with open("feed.xml", "w") as f:
            f.write(catalogue)
        return callback(catalogue)

class MainQuery(tornado.web.RequestHandler):

    def get(self):
        query = parse_querystring(self.request.query)
        if not query:
            self.redirect("/catalogue")
        connessione = DatabaseConnection()
        result = connessione.query(query.keys()[0], query.values()[0][0])
        meta = {}
        for entry in result:
            meta[entry["isbn"]] = [entry["title"], os.path.basename(entry["path"]), entry["author"]]
        connessione.exit()
        if "text/html" in accepted_formats(self.request.headers.get("accept")):
            self.render("catalogue.html", output=meta, search=query.values()[0][0])
        else:
            self.write(meta)
<commit_msg>Default request handler<commit_after>import json
import xml.etree.ElementTree as ET
import re
import os
import time
from StringIO import StringIO
from urlparse import parse_qs as parse_querystring

import tornado.web
from tornado import gen

from epub import EPUB
from epub.utils import listFiles

from data.utils import opendb
from data.data import DatabaseConnection
from data import opds

def accepted_formats(header):
    """
    Returns a list of accepted formats in HTTP request header

    :type header: tornado.web.RequestHandler.request.headers
    :param header: HTTP Request Header
    """
    header_list = header.split(",")
    for i, v in enumerate(header_list):
        header_list[i] = v.split(";")[0].strip()
    return header_list

def user_real_ip(request):
    """
    Tornado is meant to be deployed behind a proxy. The official documentation says
    remote_ip is to check if a X-Real-IP exists, but apparently that's not the case.

    :type request: tornado.web.RequestHandler.request
    :param request: HTTP Request
    """
    if "X-Real-IP" in request.headers:
        return request.headers.get("X-Real-IP")
    else:
        return request.remote_ip

class GeneralErrorHandler(tornado.web.RequestHandler):
    def __init__(self, application, request, status_code):
        tornado.web.RequestHandler.__init__(self, application, request)
        self.set_status(status_code)

    def write_error(self, status, **kwargs):
        self.write("Bad request")

    def prepare(self):
        raise tornado.web.HTTPError(400)

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.render("hello.html", title="Welcome!", user=user_real_ip(self.request))

class GetInfo(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, filename):
        if filename:
            response = yield gen.Task(self.querydb, filename)
            if "text/html" in accepted_formats(self.request.headers.get("accept")):
                self.render("info.html",
                            title=response["metadata"]["title"],
                            id=response["id"],
                            meta=response["metadata"],
                            contents=response["toc"],
                            manifest=response["manifest"],
                            cover=response["cover"]
                            )
            else:
                output = response
                output["cover"] = "/book/{0}/manifest/{1}".format(response["id"], response["cover"])
                self.write(json.dumps(output, indent=4))
                self.finish()
        else:
            raise tornado.web.HTTPError(400)

    def querydb(self, isbn, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ? ", (isbn,)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        epubfile = EPUB(path)
        output = epubfile.info
        output["cover"] = epubfile.cover
        output["id"] = epubfile.id
        output["toc"] = epubfile.contents
        return callback(output)

class ShowManifest(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, filename):
        if filename:
            response = yield gen.Task(self.querydb, filename)
            self.set_header("Content-Type", "application/json")
            self.set_header("Charset", "UTF-8")
            self.write(json.dumps(response, indent=4))
            self.finish()
        else:
            raise tornado.web.HTTPError(400)

    def querydb(self, isbn, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ? ", (isbn,)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        epubfile = EPUB(path)
        output = epubfile.info["manifest"]
        return callback(output)

class ListFiles(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self):
        response = yield gen.Task(self.cataloguedump)
        if "text/html" in accepted_formats(self.request.headers.get("accept")):
            self.render("catalogue.html",
                        output=response, search=False)
        else:
            self.set_header("Content-Type", "application/json")
            self.set_header("Charset", "UTF-8")
            self.write(json.dumps(response, indent=4))
            self.finish()

    def cataloguedump(self, callback):
        response = listFiles()
        return callback(response)

class ShowFileToc(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier):
        if identifier:
            try:
                output = yield gen.Task(self.queryToc, identifier)
                self.set_header("Content-Type", "application/json")
                self.write(json.dumps(output, indent=4))
                self.finish()
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(400)

    def queryToc(self, identifier, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ?", (identifier, )).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        output = EPUB(path).contents
        return callback(output)

class GetFilePart(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier, part, section=False):
        if identifier and part and not section:
            try:
                output = yield gen.Task(self.perform, identifier, part, section=False)
            except IOError:
                raise tornado.web.HTTPError(404)
        elif identifier and part and section:
            try:
                output = yield gen.Task(self.perform, identifier, part, section)
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(405)

        self.set_header("Content-Type", "text/html")
        self.set_header("Charset", "UTF-8")
        self.write(output)
        self.finish()

    def perform(self, identifier, part, section, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = ?", (identifier, )).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        try:
            epub = EPUB(path)
            part_path = ""
            for i in epub.contents:
                if part == i.get("id"):
                    part_path = i.get("src")
            output = epub.read(re.sub(r"#(.*)", "", part_path))  # strip fragment id.

            output = re.sub(r'(href|src)="(\.\./)?(.*?)"', '\g<1>="/getpath/{0}/\g<3>"'.format(identifier), output)
            output = re.sub(r"(href|src)='(\.\./)?(.*?)'", '\g<1>="/getpath/{0}/\g<3>"'.format(identifier), output)

            if section:
                try:
                    from htmlentitydefs import entitydefs
                    parser = ET.XMLParser()
                    parser.parser.UseForeignDTD(True)
                    parser.entity.update(entitydefs)
                    source = StringIO(output)
                    root = ET.parse(source, parser)
                    section = int(section) - 1
                    name = root.find(".//{http://www.w3.org/1999/xhtml}body")[section]
                    output = " ".join([t for t in list(name.itertext())])
                except Exception, e:
                    print e
                    raise tornado.web.HTTPError(404)

        except KeyError:
            raise tornado.web.HTTPError(404)

        return callback(output)

class GetFilePath(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier, part):
        if identifier and part:
            try:
                output = yield gen.Task(self.perform, identifier, part)
                self.set_header("Content-Type", "text/html")
                self.write(output)
                self.finish()
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(400)

    def perform(self, identifier, part, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = '{0}'".format(identifier)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        filepath = ""
        try:
            epub = EPUB(path)
            for i in epub.namelist():
                if i.endswith(part):
                    filepath = i
            output = epub.read(filepath)

        except KeyError:
            output = "Nope."
            pass
        return callback(output)

class GetResource(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self, identifier, manifest_id):
        if identifier and manifest_id:
            try:
                output, mimetype = yield gen.Task(self.perform, identifier, manifest_id)
                self.set_header("Content-Type", mimetype)
                self.write(output)
                self.finish()
            except IOError:
                raise tornado.web.HTTPError(404)
        else:
            raise tornado.web.HTTPError(400)

    def perform(self, identifier, toc_id, callback):
        database, conn = opendb()
        try:
            path = database.execute("SELECT path FROM books WHERE isbn = '{0}'".format(identifier)).fetchone()["path"]
        except TypeError:
            raise tornado.web.HTTPError(404)
        finally:
            conn.close()
        filepath = ""
        mimetype = ""
        try:
            epub = EPUB(path)
            for i in epub.info["manifest"]:
                if i["id"] == toc_id:
                    filepath = i["href"]
                    mimetype = i["mimetype"]
            if not mimetype:
                mimetype = "text/html"
            output = epub.read(os.path.join(epub.root_folder, filepath)), mimetype

        except KeyError:
            output = "KEY ERROR"
            pass
        return callback(output)

class DownloadPublication(tornado.web.RequestHandler):

    def get(self, filename):
        if filename:
            database, conn = opendb()
            try:
                path = database.execute("SELECT path FROM books WHERE isbn = '{0}' ".format(filename)).fetchone()["path"]
            except TypeError:
                raise tornado.web.HTTPError(404)
            finally:
                conn.close()
            output = open(path, "r")
            self.set_header('Content-Type', 'application/zip')
            self.set_header('Content-Disposition', 'attachment; filename='+os.path.basename(path)+'')
            self.write(output.read())
        else:
            raise tornado.web.HTTPError(404)

class OPDSCatalogue(tornado.web.RequestHandler):

    @tornado.web.asynchronous
    @gen.engine
    def get(self):
        if os.path.exists("feed.xml") and (time.time() - os.path.getmtime("feed.xml")) < 200:
            self.set_header("Content-Type", "application/atom+xml")
            with open("feed.xml", "r") as f:
                self.write(f.read())
                self.finish()
        else:
            catalogue = yield gen.Task(self.perform, )
            self.set_header("Content-Type", "application/atom+xml")
            self.write(catalogue)
            self.finish()

    def perform(self, callback):
        catalogue = opds.generateCatalogRoot()
        # Raw cache. TODO: better cache handling
        with open("feed.xml", "w") as f:
            f.write(catalogue)
        return callback(catalogue)

class MainQuery(tornado.web.RequestHandler):

    def get(self):
        query = parse_querystring(self.request.query)
        if not query:
            self.redirect("/catalogue")
        connessione = DatabaseConnection()
        result = connessione.query(query.keys()[0], query.values()[0][0])
        meta = {}
        for entry in result:
            meta[entry["isbn"]] = [entry["title"], os.path.basename(entry["path"]), entry["author"]]
        connessione.exit()
        if "text/html" in accepted_formats(self.request.headers.get("accept")):
            self.render("catalogue.html", output=meta, search=query.values()[0][0])
        else:
            self.write(meta)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# Software License Agreement (BSD License)
#
# Copyright (c) 2008, Willow Garage, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of Willow Garage, Inc. nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Revision $Id$

"""
Python utilities for manipulating ROS Stacks.
See: U{http://ros.org/wiki/Stacks}

Warning: this API is still fairly experimental and incomplete.
"""

import os
import sys
import re

import roslib.exceptions
import roslib.packages
import roslib.stack_manifest
from roslib.rosenv import ROS_ROOT, ROS_PACKAGE_PATH

STACK_FILE = 'stack.xml'
ROS_STACK = 'ros'

class ROSStackException(roslib.exceptions.ROSLibException): pass
class InvalidROSStackException(ROSStackException): pass

def stack_of(pkg, env=None):
    """
    @param env: override environment variables
    @type  env: {str: str}
    @return: name of stack that pkg is in, or None if pkg is not part of a stack
    @rtype: str
    @raise roslib.packages.InvalidROSPkgException: if pkg cannot be located
    """
    if env is None:
        env = os.environ
    pkg_dir = roslib.packages.get_pkg_dir(pkg, ros_root=env[ROS_ROOT], ros_package_path=env.get(ROS_PACKAGE_PATH, None))
    d = pkg_dir
    while d and os.path.dirname(d) != d:
        stack_file = os.path.join(d, STACK_FILE)
        if os.path.exists(stack_file):
            #TODO: need to resolve issues regarding whether the
            #stack.xml or the directory defines the stack name
            return os.path.basename(d)
        d = os.path.dirname(d)

def packages_of(stack, env=None):
    """
    @param env: override environment variables
    @type  env: {str: str}
    @return: name of packages that are part of stack
    @rtype: [str]
    @raise InvalidROSStackException: if stack cannot be located
    @raise ValueError: if stack name is invalid
    """
    # record settings for error messages
    if env is None:
        env = os.environ
    ros_root = env[ROS_ROOT]
    ros_package_path = env.get(ROS_PACKAGE_PATH, '')

    if not stack:
        raise ValueError("stack name not specified")
    stack_dir = get_stack_dir(stack, env=env)
    if stack_dir is None:
        raise InvalidROSStackException("Cannot locate installation of stack %s. ROS_ROOT[%s] ROS_PACKAGE_PATH[%s]"%(stack, ros_root,ros_package_path))
    # unary stack
    if roslib.packages.is_pkg_dir(stack_dir):
        return [stack]
    packages = []
    l = [os.path.join(stack_dir, d) for d in os.listdir(stack_dir)]
    # kwc: this really is just a 1-directory reimplementation of
    # list_pkgs(). Should merge implementations, though have to deal
    # with issues of cache, etc...
    while l:
        d = l.pop()
        if os.path.isdir(d):
            if roslib.packages.is_pkg_dir(d):
                p = os.path.basename(d)
                # this is sometimes true if we've descended into a build directory
                if not p in packages:
                    packages.append(p)
            elif os.path.exists(os.path.join(d, 'rospack_nosubdirs')):
                # don't descend
                pass
            elif os.path.basename(d) not in ['build', '.svn', '.git']: #recurse
                l.extend([os.path.join(d, e) for e in os.listdir(d)])
    return packages

def get_stack_dir(stack, env=None):
    """
    Get the directory of a ROS stack. This will initialize an internal
    cache and return cached results if possible.

    This routine is not thread-safe to os.environ changes.

    @param env: override environment variables
    @type  env: {str: str}
    @param stack: name of ROS stack to locate on disk
    @type  stack: str
    @return: directory of stack.
    @rtype: str
    @raise InvalidROSStackException: if stack cannot be located.
    """

    # it's possible to get incorrect results from this cache
    # implementation by manipulating the environment and calling this
    # from multiple threads.  as that is an unusual use case and would
    # require a slower implmentation, it's not supported. the
    # interpretation of this routine is get_stack_dir for the
    # environment this process was launched in.
    global _dir_cache_marker

    if env is None:
        env = os.environ
    if stack in _dir_cache:
        ros_root = env[ROS_ROOT]
        ros_package_path = env.get(ROS_PACKAGE_PATH, '')

        # we don't attempt to be thread-safe to environment changes,
        # however we do need to be threadsafe to cache invalidation.
        try:
            if _dir_cache_marker == (ros_root, ros_package_path):
                d = _dir_cache[stack]
                if os.path.isfile(os.path.join(d, STACK_FILE)):
                    return d
                else:
                    # invalidate the cache
                    _dir_cache_marker = None
                    _dir_cache.clear()
        except KeyError:
            pass
    _update_stack_cache(env=env) #update cache
    val = _dir_cache.get(stack, None)
    if val is None:
        raise InvalidROSStackException("Cannot location installation of stack %s. ROS_ROOT[%s] ROS_PACKAGE_PATH[%s]"%(stack, env[ROS_ROOT], env.get(ROS_PACKAGE_PATH, '')))
    return val

# rosstack directory cache
_dir_cache = {}
# stores ROS_ROOT, ROS_PACKAGE_PATH of _dir_cache
_dir_cache_marker = None

def _update_stack_cache(force=False, env=None):
    """
    Update _dir_cache if environment has changed since last cache build.

    @param env: override environment variables
    @type  env: {str: str}
    @param force: force cache rebuild regardless of environment variables
    @type  force: bool
    """
    global _dir_cache_marker
    if env is None:
        env = os.environ
    ros_root = env[ROS_ROOT]
    ros_package_path = env.get(ROS_PACKAGE_PATH, '')

    if _dir_cache_marker == (ros_root, ros_package_path):
        return
    _dir_cache.clear()
    _dir_cache_marker = ros_root, ros_package_path

    pkg_dirs = roslib.packages.get_package_paths(env=env)
    # ros is assumed to be at ROS_ROOT
    if os.path.exists(os.path.join(ros_root, 'stack.xml')):
        _dir_cache['ros'] = ros_root
        pkg_dirs.remove(ros_root)

    # pass in accumulated stacks list to each call. This ensures
    # precedence (i.e. that stacks first on pkg_dirs path win).
    stacks = []
    for pkg_root in pkg_dirs:
        # list_stacks_by_path will append list into stacks, so that
        # each call accumulates in it.
        list_stacks_by_path(pkg_root, stacks, cache=_dir_cache)

def list_stacks(env=None):
    """
    Get list of all ROS stacks. This uses an internal cache.

    This routine is not thread-safe to os.environ changes.

    @param env: override environment variables
    @type  env: {str: str}
    @return: complete list of stacks names in ROS environment
    @rtype: [str]
    """
    _update_stack_cache(env=env)
    return _dir_cache.keys()

def list_stacks_by_path(path, stacks=None, cache=None):
    """
    List ROS stacks within the specified path.

    Optionally, a cache dictionary can be provided, which will be
    updated with the stack->path mappings. list_stacks_by_path() does
    NOT returned cached results -- it only updates the cache.

    @param path: path to list stacks in
    @type  path: str
    @param stacks: list of stacks to append to. If stack is
      already present in stacks, it will be ignored.
    @type  stacks: [str]
    @param cache: (optional) stack path cache to update. Maps stack name to directory path.
    @type  cache: {str: str}
    @return: complete list of stack names in ROS environment. Same as stacks parameter.
    @rtype: [str]
    """
    if stacks is None:
        stacks = []
    MANIFEST_FILE = roslib.packages.MANIFEST_FILE
    basename = os.path.basename
    for d, dirs, files in os.walk(path, topdown=True):
        if STACK_FILE in files:
            stack = basename(d)
            if stack not in stacks:
                stacks.append(stack)
                if cache is not None:
                    cache[stack] = d
            del dirs[:]
            continue #leaf
        elif MANIFEST_FILE in files:
            del dirs[:]
            continue #leaf
        elif 'rospack_nosubdirs' in files:
            del dirs[:]
            continue  #leaf
        # remove hidden dirs (esp. .svn/.git)
        [dirs.remove(di) for di in dirs if di[0] == '.']
        for sub_d in dirs:
            # followlinks=True only available in Python 2.6, so we
            # have to implement manually
            sub_p = os.path.join(d, sub_d)
            if os.path.islink(sub_p):
                stacks.extend(list_stacks_by_path(sub_p, cache=cache))
    return stacks

# #2022
def expand_to_packages(names, env=None):
    """
    Expand names into a list of packages. Names can either be of packages or stacks.

    @param names: names of stacks or packages
    @type  names: [str]
    @return: ([packages], [not_found]). expand_packages() returns two
    lists. The first is of packages names. The second is a list of
    names for which no matching stack or package was found. Lists may have duplicates.
    @rtype: ([str], [str])
    """
    if type(names) not in [tuple, list]:
        raise ValueError("names must be a list of strings")

    # do full package list first. This forces an entire tree
    # crawl. This is less efficient for a small list of names, but
    # much more efficient for many names.
    package_list = roslib.packages.list_pkgs(env=env)
    valid = []
    invalid = []
    for n in names:
        if not n in package_list:
            try:
                valid.extend(roslib.stacks.packages_of(n, env=env))
            except roslib.stacks.InvalidROSStackException, e:
                invalid.append(n)
        else:
            valid.append(n)
    return valid, invalid

def get_stack_version(stack, env=None):
    """
    @param env: override environment variables
    @type  env: {str: str}

    @return: version number of stack, or None if stack is unversioned.
    @rtype: str
    """
    return get_stack_version_by_dir(get_stack_dir(stack, env=env))

def get_stack_version_by_dir(stack_dir):
    """
    Get stack version where stack_dir points to root directory of stack.

    @param env: override environment variables
    @type  env: {str: str}

    @return: version number of stack, or None if stack is unversioned.
    @rtype: str
    """
    # REP 109: check for <version> tag first, then CMakeLists.txt
    manifest_filename = os.path.join(stack_dir, STACK_FILE)
    if os.path.isfile(manifest_filename):
        m = roslib.stack_manifest.parse_file(manifest_filename)
        if m.version:
            return m.version

    cmake_filename = os.path.join(stack_dir, 'CMakeLists.txt')
    if os.path.isfile(cmake_filename):
        with open(cmake_filename) as f:
            return _get_cmake_version(f.read())
    else:
        return None

def _get_cmake_version(text):
    for l in text.split('\n'):
        if l.strip().startswith('rosbuild_make_distribution'):
            x_re = re.compile(r'[()]')
            lsplit = x_re.split(l.strip())
            if len(lsplit) < 2:
                raise ReleaseException("couldn't find version number in CMakeLists.txt:\n\n%s"%l)
            return lsplit[1]
<commit_msg>#3576: roslib.stacks py3k support<commit_after>#! /usr/bin/env python
# Software License Agreement (BSD License)
#
# Copyright (c) 2008, Willow Garage, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of Willow Garage, Inc. nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Revision $Id$

"""
Python utilities for manipulating ROS Stacks.
See: U{http://ros.org/wiki/Stacks}

Warning: this API is still fairly experimental and incomplete.
"""

import os
import sys
import re

import roslib.exceptions
import roslib.packages
import roslib.stack_manifest
from roslib.rosenv import ROS_ROOT, ROS_PACKAGE_PATH

STACK_FILE = 'stack.xml'
ROS_STACK = 'ros'

class ROSStackException(roslib.exceptions.ROSLibException): pass
class InvalidROSStackException(ROSStackException): pass

def stack_of(pkg, env=None):
    """
    @param env: override environment variables
    @type  env: {str: str}
    @return: name of stack that pkg is in, or None if pkg is not part of a stack
    @rtype: str
    @raise roslib.packages.InvalidROSPkgException: if pkg cannot be located
    """
    if env is None:
        env = os.environ
    pkg_dir = roslib.packages.get_pkg_dir(pkg, ros_root=env[ROS_ROOT], ros_package_path=env.get(ROS_PACKAGE_PATH, None))
    d = pkg_dir
    while d and os.path.dirname(d) != d:
        stack_file = os.path.join(d, STACK_FILE)
        if os.path.exists(stack_file):
            #TODO: need to resolve issues regarding whether the
            #stack.xml or the directory defines the stack name
            return os.path.basename(d)
        d = os.path.dirname(d)

def packages_of(stack, env=None):
    """
    @param env: override environment variables
    @type  env: {str: str}
    @return: name of packages that are part of stack
    @rtype: [str]
    @raise InvalidROSStackException: if stack cannot be located
    @raise ValueError: if stack name is invalid
    """
    # record settings for error messages
    if env is None:
        env = os.environ
    ros_root = env[ROS_ROOT]
    ros_package_path = env.get(ROS_PACKAGE_PATH, '')

    if not stack:
        raise ValueError("stack name not specified")
    stack_dir = get_stack_dir(stack, env=env)
    if stack_dir is None:
        raise InvalidROSStackException("Cannot locate installation of stack %s. ROS_ROOT[%s] ROS_PACKAGE_PATH[%s]"%(stack, ros_root,ros_package_path))
    # unary stack
    if roslib.packages.is_pkg_dir(stack_dir):
        return [stack]
    packages = []
    l = [os.path.join(stack_dir, d) for d in os.listdir(stack_dir)]
    # kwc: this really is just a 1-directory reimplementation of
    # list_pkgs(). Should merge implementations, though have to deal
    # with issues of cache, etc...
    while l:
        d = l.pop()
        if os.path.isdir(d):
            if roslib.packages.is_pkg_dir(d):
                p = os.path.basename(d)
                # this is sometimes true if we've descended into a build directory
                if not p in packages:
                    packages.append(p)
            elif os.path.exists(os.path.join(d, 'rospack_nosubdirs')):
                # don't descend
                pass
            elif os.path.basename(d) not in ['build', '.svn', '.git']: #recurse
                l.extend([os.path.join(d, e) for e in os.listdir(d)])
    return packages

def get_stack_dir(stack, env=None):
    """
    Get the directory of a ROS stack. This will initialize an internal
    cache and return cached results if possible.

    This routine is not thread-safe to os.environ changes.

    @param env: override environment variables
    @type  env: {str: str}
    @param stack: name of ROS stack to locate on disk
    @type  stack: str
    @return: directory of stack.
    @rtype: str
    @raise InvalidROSStackException: if stack cannot be located.
    """

    # it's possible to get incorrect results from this cache
    # implementation by manipulating the environment and calling this
    # from multiple threads.  as that is an unusual use case and would
    # require a slower implmentation, it's not supported. the
    # interpretation of this routine is get_stack_dir for the
    # environment this process was launched in.
    global _dir_cache_marker

    if env is None:
        env = os.environ
    if stack in _dir_cache:
        ros_root = env[ROS_ROOT]
        ros_package_path = env.get(ROS_PACKAGE_PATH, '')

        # we don't attempt to be thread-safe to environment changes,
        # however we do need to be threadsafe to cache invalidation.
        try:
            if _dir_cache_marker == (ros_root, ros_package_path):
                d = _dir_cache[stack]
                if os.path.isfile(os.path.join(d, STACK_FILE)):
                    return d
                else:
                    # invalidate the cache
                    _dir_cache_marker = None
                    _dir_cache.clear()
        except KeyError:
            pass
    _update_stack_cache(env=env) #update cache
    try:
        val = _dir_cache[stack]
    except KeyError:
        raise InvalidROSStackException("Cannot location installation of stack %s. ROS_ROOT[%s] ROS_PACKAGE_PATH[%s]"%(stack, env[ROS_ROOT], env.get(ROS_PACKAGE_PATH, '')))
    return val

# rosstack directory cache
_dir_cache = {}
# stores ROS_ROOT, ROS_PACKAGE_PATH of _dir_cache
_dir_cache_marker = None

def _update_stack_cache(force=False, env=None):
    """
    Update _dir_cache if environment has changed since last cache build.

    @param env: override environment variables
    @type  env: {str: str}
    @param force: force cache rebuild regardless of environment variables
    @type  force: bool
    """
    global _dir_cache_marker
    if env is None:
        env = os.environ
    ros_root = env[ROS_ROOT]
    ros_package_path = env.get(ROS_PACKAGE_PATH, '')

    if _dir_cache_marker == (ros_root, ros_package_path):
        return
    _dir_cache.clear()
    _dir_cache_marker = ros_root, ros_package_path

    pkg_dirs = roslib.packages.get_package_paths(env=env)
    # ros is assumed to be at ROS_ROOT
    if os.path.exists(os.path.join(ros_root, 'stack.xml')):
        _dir_cache['ros'] = ros_root
        pkg_dirs.remove(ros_root)

    # pass in accumulated stacks list to each call. This ensures
    # precedence (i.e. that stacks first on pkg_dirs path win).
    stacks = []
    for pkg_root in pkg_dirs:
        # list_stacks_by_path will append list into stacks, so that
        # each call accumulates in it.
        list_stacks_by_path(pkg_root, stacks, cache=_dir_cache)

def list_stacks(env=None):
    """
    Get list of all ROS stacks. This uses an internal cache.

    This routine is not thread-safe to os.environ changes.

    @param env: override environment variables
    @type  env: {str: str}
    @return: complete list of stacks names in ROS environment
    @rtype: [str]
    """
    _update_stack_cache(env=env)
    return list(_dir_cache.keys()) #py3k

def list_stacks_by_path(path, stacks=None, cache=None):
    """
    List ROS stacks within the specified path.

    Optionally, a cache dictionary can be provided, which will be
    updated with the stack->path mappings. list_stacks_by_path() does
    NOT returned cached results -- it only updates the cache.

    @param path: path to list stacks in
    @type  path: str
    @param stacks: list of stacks to append to. If stack is
      already present in stacks, it will be ignored.
    @type  stacks: [str]
    @param cache: (optional) stack path cache to update. Maps stack name to directory path.
    @type  cache: {str: str}
    @return: complete list of stack names in ROS environment. Same as stacks parameter.
    @rtype: [str]
    """
    if stacks is None:
        stacks = []
    MANIFEST_FILE = roslib.packages.MANIFEST_FILE
    basename = os.path.basename
    for d, dirs, files in os.walk(path, topdown=True):
        if STACK_FILE in files:
            stack = basename(d)
            if stack not in stacks:
                stacks.append(stack)
                if cache is not None:
                    cache[stack] = d
            del dirs[:]
            continue #leaf
        elif MANIFEST_FILE in files:
            del dirs[:]
            continue #leaf
        elif 'rospack_nosubdirs' in files:
            del dirs[:]
            continue  #leaf
        # remove hidden dirs (esp. .svn/.git)
        [dirs.remove(di) for di in dirs if di[0] == '.']
        for sub_d in dirs:
            # followlinks=True only available in Python 2.6, so we
            # have to implement manually
            sub_p = os.path.join(d, sub_d)
            if os.path.islink(sub_p):
                stacks.extend(list_stacks_by_path(sub_p, cache=cache))
    return stacks

# #2022
def expand_to_packages(names, env=None):
    """
    Expand names into a list of packages. Names can either be of packages or stacks.

    @param names: names of stacks or packages
    @type  names: [str]
    @return: ([packages], [not_found]). expand_packages() returns two
    lists. The first is of packages names. The second is a list of
    names for which no matching stack or package was found. Lists may have duplicates.
    @rtype: ([str], [str])
    """
    if type(names) not in [tuple, list]:
        raise ValueError("names must be a list of strings")

    # do full package list first. This forces an entire tree
    # crawl. This is less efficient for a small list of names, but
    # much more efficient for many names.
    package_list = roslib.packages.list_pkgs(env=env)
    valid = []
    invalid = []
    for n in names:
        if not n in package_list:
            try:
                valid.extend(roslib.stacks.packages_of(n, env=env))
            except roslib.stacks.InvalidROSStackException as e:
                invalid.append(n)
        else:
            valid.append(n)
    return valid, invalid

def get_stack_version(stack, env=None):
    """
    @param env: override environment variables
    @type  env: {str: str}

    @return: version number of stack, or None if stack is unversioned.
    @rtype: str
    """
    return get_stack_version_by_dir(get_stack_dir(stack, env=env))

def get_stack_version_by_dir(stack_dir):
    """
    Get stack version where stack_dir points to root directory of stack.

    @param env: override environment variables
    @type  env: {str: str}

    @return: version number of stack, or None if stack is unversioned.
    @rtype: str
    """
    # REP 109: check for <version> tag first, then CMakeLists.txt
    manifest_filename = os.path.join(stack_dir, STACK_FILE)
    if os.path.isfile(manifest_filename):
        m = roslib.stack_manifest.parse_file(manifest_filename)
        if m.version:
            return m.version

    cmake_filename = os.path.join(stack_dir, 'CMakeLists.txt')
    if os.path.isfile(cmake_filename):
        with open(cmake_filename) as f:
            return _get_cmake_version(f.read())
    else:
        return None

def _get_cmake_version(text):
    for l in text.split('\n'):
        if l.strip().startswith('rosbuild_make_distribution'):
            x_re = re.compile(r'[()]')
            lsplit = x_re.split(l.strip())
            if len(lsplit) < 2:
                raise ReleaseException("couldn't find version number in CMakeLists.txt:\n\n%s"%l)
            return lsplit[1]
<|endoftext|>