<commit_before>import fnmatch
import os
from os import path
import sys
import re
import lxml.html
import lxml.etree

# Get all missing images on FS from nav's htmls

# =================================
# just some fancy colors for output

HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = "\033[1m"

def disable():
    HEADER = ''
    OKBLUE = ''
    OKGREEN = ''
    WARNING = ''
    FAIL = ''
    ENDC = ''

def infog(msg):
    print OKGREEN + msg + ENDC

def info(msg):
    print OKBLUE + msg + ENDC

def warn(msg):
    print WARNING + msg + ENDC

def err(msg):
    print FAIL + 'ERROR: ' + msg + ENDC

startpath = '.'
regex_html_file = '^.*\.x?html?$' # TODO: Lower-/Uppercase
regex_misc_file = '^.*\.x?html?$' # negate! # TODO: Lower-/Uppercase

# =================================

def convert_to_relpaths(filelist):
    for i, filename in enumerate(filelist):
        filelist[i] = path.relpath(filename)
    return filelist

# =================================

def getcontainer_opfs():
    container_file = path.join(startpath, 'META-INF', 'container.xml')
    if not(path.isfile(container_file)):
        err('ERROR: META-INF/container.xml was not found! Not valid EPUB!')
        sys.exit(1)
    meta_opfs = lxml.etree.parse(container_file).xpath('//o:rootfile/@full-path', namespaces={'o': 'urn:oasis:names:tc:opendocument:xmlns:container'})
    if (len(meta_opfs) < 1):
        err('ERROR: META-INF/container.xml has no OPFs inside. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(meta_opfs)

# =================================

def getfs_opfs():
    fs_opfs = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*.opf'):
          fs_opfs.append(os.path.join(root, filename))
    if (len(fs_opfs) < 1):
        err('ERROR: No OPFs found in filesystem. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(fs_opfs)

# =================================

def getfs_navs():
    fs_navs = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*nav*html'):
          fs_navs.append(os.path.join(root, filename))
    if (len(fs_navs) < 1):
        err('ERROR: No NAVs found in filesystem. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(fs_navs)

# =================================

def getfs_htmls():
    fs_htmls = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*html'):
          fs_htmls.append(os.path.join(root, filename))
    if (len(fs_navs) < 1):
        err('ERROR: No HTMLs found in filesystem. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(fs_navs)

# =================================

def getmapped_navs():
    nav_htmls = []
    for opf in getcontainer_opfs():
        found_navs = lxml.etree.parse(opf).xpath("//o:item[@properties = 'nav']/@href", namespaces={'o': 'http://www.idpf.org/2007/opf'})
        if len(found_navs) < 1:
            err('ERROR: No nav found in OPF: ' + opf)
            sys.exit(1)
        elif len(found_navs) > 1:
            err('ERROR: More than one nav found in OPF: ' + opf)
            sys.exit(1)
        nav_htmls.append(found_navs[0])
    return convert_to_relpaths(nav_htmls)

# =================================

# gets all OPFs from container and returns all [href, OPF_filename] which matches a regex pattern
def _getmapped_opf_regex_files(regex, negate=False):
    opf_files = []
    for opf in getcontainer_opfs():
        opf = path.relpath(opf)
        found_files = lxml.etree.parse(opf).xpath("//o:item[not(@properties = 'nav')]/@href", namespaces={'o': 'http://www.idpf.org/2007/opf'})
        if len(found_files) < 1:
            err('ERROR: No referenced files found in: ' + opf)
            sys.exit(1)
        for found_file in found_files:
            if ((re.match(regex, found_file) and not(negate)) \
            or (not(re.match(regex, found_file)) and negate)):
                found_file = path.relpath(path.join(path.dirname(opf), found_file))
                opf_files.append([found_file, opf])
    return opf_files # convert_to_relpaths(opf_files) # TODO for two arrays

# =================================

def getmapped_opf_htmls():
    return _getmapped_opf_regex_files(regex_html_file)

# =================================

def getmapped_opf_misc():
    return _getmapped_opf_regex_files(regex_misc_file, negate=True)

# =================================

def main():
    # print getmapped_opf_images()
    print getmapped_opf_htmls()
    print '==================='
    print getmapped_opf_misc()

# =================================

def oldmain():
    # nav_matches contains all found *nav*html files. Useful later for checking against opf files
    nav_matches = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*nav*html'):
          nav_matches.append(os.path.join(root, filename))

    # Check if all files in nav files exist and check if images exist
    for nav_file in nav_matches:
        print '====== Analysing navigation HTML: ' + nav_file
        nav_htmls = lxml.html.parse(nav_file).xpath("//a/@href")
        for nav_html in nav_htmls:
            nav_html = os.path.join(os.path.dirname(nav_file), nav_html)
            if not(os.path.isfile(nav_html)):
                err('HTML file {0} in {1} does not exist!'.format(nav_html, os.path.basename(nav_file)))
                continue
            #print '----- Analysing: ' + nav_html
            img_links = lxml.html.parse(nav_html).xpath("//img/@src")
            for img_link in img_links:
                img_file = os.path.join(os.path.dirname(nav_html), img_link)
                if not(os.path.isfile(img_file)):
                    err('Image file {0} in {1} does not exist!'.format(img_file, nav_html))
                    continue

# =================================

if  __name__ == '__main__':
    main()
<commit_msg>file extension html is not necessary an indicator for a valid entry in nav<commit_after>import fnmatch
import os
from os import path
import sys
import re
import lxml.html
import lxml.etree

# Get all missing images on FS from nav's htmls

# =================================
# just some fancy colors for output

HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = "\033[1m"

def disable():
    HEADER = ''
    OKBLUE = ''
    OKGREEN = ''
    WARNING = ''
    FAIL = ''
    ENDC = ''

def infog(msg):
    print OKGREEN + msg + ENDC

def info(msg):
    print OKBLUE + msg + ENDC

def warn(msg):
    print WARNING + msg + ENDC

def err(msg):
    print FAIL + 'ERROR: ' + msg + ENDC

startpath = '.'
regex_html_file = '^.*\.x?html?$' # TODO: Lower-/Uppercase
regex_misc_file = '^.*\.x?html?$' # negate! # TODO: Lower-/Uppercase

# =================================

def convert_to_relpaths(filelist):
    for i, filename in enumerate(filelist):
        filelist[i] = path.relpath(filename)
    return filelist

# =================================

def getcontainer_opfs():
    container_file = path.join(startpath, 'META-INF', 'container.xml')
    if not(path.isfile(container_file)):
        err('ERROR: META-INF/container.xml was not found! Not valid EPUB!')
        sys.exit(1)
    meta_opfs = lxml.etree.parse(container_file).xpath('//o:rootfile/@full-path', namespaces={'o': 'urn:oasis:names:tc:opendocument:xmlns:container'})
    if (len(meta_opfs) < 1):
        err('ERROR: META-INF/container.xml has no OPFs inside. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(meta_opfs)

# =================================

def getfs_opfs():
    fs_opfs = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*.opf'):
          fs_opfs.append(os.path.join(root, filename))
    if (len(fs_opfs) < 1):
        err('ERROR: No OPFs found in filesystem. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(fs_opfs)

# =================================

def getfs_navs():
    fs_navs = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*nav*html'):
          fs_navs.append(os.path.join(root, filename))
    if (len(fs_navs) < 1):
        err('ERROR: No NAVs found in filesystem. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(fs_navs)

# =================================

def getfs_htmls():
    fs_htmls = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*html'):
          fs_htmls.append(os.path.join(root, filename))
    if (len(fs_navs) < 1):
        err('ERROR: No HTMLs found in filesystem. This should not happen!')
        sys.exit(1)
    return convert_to_relpaths(fs_navs)

# =================================

def getmapped_navs():
    nav_htmls = []
    for opf in getcontainer_opfs():
        found_navs = lxml.etree.parse(opf).xpath("//o:item[@properties = 'nav']/@href", namespaces={'o': 'http://www.idpf.org/2007/opf'})
        if len(found_navs) < 1:
            err('ERROR: No nav found in OPF: ' + opf)
            sys.exit(1)
        elif len(found_navs) > 1:
            err('ERROR: More than one nav found in OPF: ' + opf)
            sys.exit(1)
        nav_htmls.append(path.join(path.dirname(opf), found_navs[0]))
    return convert_to_relpaths(nav_htmls)

# =================================

def get_nav_htmls():
    nav_files = []
    for nav in getmapped_navs():
        found_files = lxml.html.parse(nav).xpath('//a/@href')
        if (len(found_files) < 1):
            err('ERROR: No referenced files found in: ' + nav)
            sys.exit(1)
        for found_file in found_files:
            found_file = path.relpath(path.join(path.dirname(nav), found_file))
            nav_files.append([found_file, nav])
    return nav_files # TODO: NAV and OPF file in third array place and convert_to_relpaths

# =================================

# gets all OPFs from container and returns all [href, OPF_filename] which matches a regex pattern
def _getmapped_opf_regex_files(regex, negate=False):
    opf_files = []
    for opf in getcontainer_opfs():
        opf = path.relpath(opf)
        found_files = lxml.etree.parse(opf).xpath("//o:item[not(@properties = 'nav')]/@href", namespaces={'o': 'http://www.idpf.org/2007/opf'})
        if len(found_files) < 1:
            err('ERROR: No referenced files found in: ' + opf)
            sys.exit(1)
        for found_file in found_files:
            if ((re.match(regex, found_file) and not(negate)) \
            or (not(re.match(regex, found_file)) and negate)):
                found_file = path.relpath(path.join(path.dirname(opf), found_file))
                opf_files.append([found_file, opf])
    return opf_files # convert_to_relpaths(opf_files) # TODO for two arrays

# =================================

def getmapped_opf_htmls():
    return _getmapped_opf_regex_files(regex_html_file)

# =================================

def getmapped_opf_misc():
    return _getmapped_opf_regex_files(regex_misc_file, negate=True)

# =================================

# gets all NAVs from opfs and returns all [href, nav_filename, opf_filename] which matches a regex pattern.
# regex is not really needed because it seems only HTMLs are in the nav inside.
def _getmapped_nav_regex_files(regex, negate=False):
    nav_files = []
    for nav in getmapped_navs():
        found_files = lxml.html.parse(nav).xpath('//a/@href')
        if len(found_files) < 1:
            err('ERROR: No referenced files found in: ' + nav)
            sys.exit(1)
        for found_file in found_files:
            if ((re.match(regex, found_file) and not(negate)) \
            or (not(re.match(regex, found_file)) and negate)):
                found_file = path.relpath(path.join(path.dirname(nav), found_file))
                nav_files.append([found_file, nav])
    return nav_files # TODO: OPF file in third array place and convert_to_relpaths

def getmapped_nav_htmls():
    return _getmapped_nav_regex_files(regex_html_file)

def main():
    #print getmapped_nav_htmls()
    print get_nav_htmls()

# =================================

def oldmain():
    # nav_matches contains all found *nav*html files. Useful later for checking against opf files
    nav_matches = []
    for root, dirnames, filenames in os.walk('.'):
      for filename in fnmatch.filter(filenames, '*nav*html'):
          nav_matches.append(os.path.join(root, filename))

    # Check if all files in nav files exist and check if images exist
    for nav_file in nav_matches:
        print '====== Analysing navigation HTML: ' + nav_file
        nav_htmls = lxml.html.parse(nav_file).xpath("//a/@href")
        for nav_html in nav_htmls:
            nav_html = os.path.join(os.path.dirname(nav_file), nav_html)
            if not(os.path.isfile(nav_html)):
                err('HTML file {0} in {1} does not exist!'.format(nav_html, os.path.basename(nav_file)))
                continue
            #print '----- Analysing: ' + nav_html
            img_links = lxml.html.parse(nav_html).xpath("//img/@src")
            for img_link in img_links:
                img_file = os.path.join(os.path.dirname(nav_html), img_link)
                if not(os.path.isfile(img_file)):
                    err('Image file {0} in {1} does not exist!'.format(img_file, nav_html))
                    continue

# =================================

if  __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Two-dimensional pattern generators drawing from various random distributions.

At present, supports uniform random distributions, but could support
normal distributions or 1/f noise.

$Id$
"""
__version__='$Revision$'

from topo.base.parameter import Number,Parameter
from topo.base.patterngenerator import PatternGenerator
from topo.base.sheet import bounds2slice
import RandomArray

class UniformRandom(PatternGenerator):
    """2D uniform random noise pattern generator."""

    # The standard x, y, and orientation variables are currently ignored,
    # so they aren't shown in auto-generated lists of parameters (e.g. in the GUI)
    x       = Number(hidden = True)
    y       = Number(hidden = True)
    orientation   = Number(hidden = True)

    # Optimization: We use a simpler __call__ method here to skip the
    # coordinate transformations (which would have no effect anyway)
    def __call__(self,**params):
        bounds = params.get('bounds',self.bounds)
        density = params.get('density',self.density)

        # CEBHACKALERT: temporary, density will become one again soon...
        if type(density)!=tuple:
            xdensity=density
            ydensity=density
        else:
            xdensity,ydensity = density

        r1,r2,c1,c2 = bounds2slice(bounds,bounds,xdensity,ydensity)
        shape = (r2-r1,c2-c1)

        offset_=params.get('offset',self.offset)
        scale_=params.get('scale',self.scale)

        return RandomArray.uniform( offset_, offset_+scale_, shape)
<commit_msg>Made RandomGenerator base class to make it simpler to add other Random classes.  Added GaussianRandom<commit_after>"""
Two-dimensional pattern generators drawing from various random distributions.

At present, supports uniform random distributions, but could support
normal distributions or 1/f noise.

$Id$
"""
__version__='$Revision$'

from topo.base.parameter import Number,Parameter
from topo.base.patterngenerator import PatternGenerator
from topo.base.sheet import bounds2slice
import RandomArray

class RandomGenerator(PatternGenerator):
    """2D random noise pattern generator abstract class."""

    # The standard x, y, and orientation variables are currently ignored,
    # so they aren't shown in auto-generated lists of parameters (e.g. in the GUI)
    x       = Number(hidden = True)
    y       = Number(hidden = True)
    orientation   = Number(hidden = True)

    def _distrib(self,shape,**params):
        """Method for subclasses to override with a particular random distribution."""
        raise NotImplementedError

    # Optimization: We use a simpler __call__ method here to skip the
    # coordinate transformations (which would have no effect anyway)
    def __call__(self,**params):
        bounds = params.get('bounds',self.bounds)
        density = params.get('density',self.density)

        # CEBHACKALERT: temporary, density will become one again soon...
        if type(density)!=tuple:
            xdensity=density
            ydensity=density
        else:
            xdensity,ydensity = density

        r1,r2,c1,c2 = bounds2slice(bounds,bounds,xdensity,ydensity)
        shape = (r2-r1,c2-c1)

        return self._distrib(shape)

class UniformRandom(RandomGenerator):
    """2D uniform random noise pattern generator."""

    def _distrib(self,shape,**params):
        offset_=params.get('offset',self.offset)
        scale_=params.get('scale',self.scale)
        return RandomArray.uniform( offset_, offset_+scale_, shape)

class GaussianRandom(RandomGenerator):
    """
    2D Gaussian random noise pattern generator.

    Each pixel is chosen independently from a Gaussian distribution
    of zero mean and unit variance, then multiplied by the given
    scale and adjusted by the given offset.
    """

    scale  = Number(default=0.25,softbounds=(0.0,2.0))
    offset = Number(default=0.50,softbounds=(-2.0,2.0))

    def _distrib(self,shape,**params):
        offset_=params.get('offset',self.offset)
        scale_ =params.get('scale', self.scale)
        return offset_+scale_*RandomArray.standard_normal(shape)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.

__project__ = "semetric-python"
__author__  = "Matt Jeffery <matt@clan.se>"
__status__  = "alpha"
__version__ = "0.1.0"

import sys
import warnings
import httplib2

# Special imports for Python 3
if sys.version_info >= (3,):
    from urllib.parse import urlparse, urlunparse, urlencode, parse_qs
else:
    from urllib import urlencode
    from urlparse import urlparse, urlunparse, parse_qs

try:
    import json
except ImportError: # pragma: no cover
    import simplejson as json

class APIError(Exception):
    pass

class APIClient(object):
    """
        API Client
    """

    API_BASE_URL = "http://api.semetric.com"
    HTTP_REQUEST = ('GET', 'POST') # these kinds of requests won't be set in _method
    ANY_GET_REQUEST = ('GET', 'PUT')
    USER_AGENT = "{0}/{1}".format(__project__, __version__)
    USER_AGENT_HEADER = {"User-Agent": USER_AGENT}

    def __init__(self, apikey, baseurl=API_BASE_URL):
        self.apikey = apikey
        self.baseurl = baseurl.rstrip('/')
        self.http = httplib2.Http()

    def _request(self, path, method='GET', **params):
        """
            Make an API request
        """
        base = "{0}/{1}".format(self.baseurl, path)
        # normalise the method argument
        method = method.upper().strip()

        # parse the existing url
        urlparts = urlparse(base)
        qstr = parse_qs(urlparts.query)

        # add the token to the query string
        qstr['token'] = self.apikey
        if method not in APIClient.HTTP_REQUEST:
            qstr['_method'] = method
        else:
            try:
                del qstr['_method']
            except KeyError:
                pass

        if method in APIClient.ANY_GET_REQUEST:
            # if it's a get request then update the query string with
            # the params
            qstr.update(params)
            # all of the params go in the query string
            query_string = urlencode(qstr)
            # reconstruct the url
            url = urlunparse((urlparts.scheme,
                                   urlparts.netloc,
                                   urlparts.path,
                                   urlparts.params,
                                   query_string,
                                   "")) # empty fragment
            resp, content = self.http.request(url, "GET", headers=self.USER_AGENT_HEADER)
        else:
            # all of the params go in the query string
            query_string = urlencode(qstr)
            # reconstruct the url
            url = urlunparse((urlparts.scheme,
                                   urlparts.netloc,
                                   urlparts.path,
                                   urlparts.params,
                                   query_string,
                                   "")) # empty fragment
            resp, content = self.http.request(url, "POST", urlencode(params), headers=self.USER_AGENT_HEADER)

        status = int(resp['status'])

        if status != 200:
            raise APIError("An unknown error occurred")

        return resp, json.loads(content)

    def __call__(self, path):
        pass

class Entity(object):
    """
        Base class for an entity in the Semetric API
    """
    __apiclass__ = "entity"

    @staticmethod
    def subclass_mapping():
        """
            Generate a mapping from Semetric API classes to Python
            Classes.
        """
        subclasses= dict((subclass.__apiclass__, subclass)
                         for subclass
                         in Entity.__subclasses__())
        subclasses['entity'] = Entity
        return subclasses

    @staticmethod
    def entity_factory(entity_dict):
        subclasses = Entity.subclass_mapping()

        apiclass = entity_dict.get('class', 'entity')
        try:
            entity_class = subclasses[apiclass]
        except KeyError:
            entity_class = Entity
            warnings.warn("Could not map api class `{0}' to a python class, using Entity".format(apiclass), stacklevel=2)

        return entity_class(**entity_dict)

    def __init__(self, **kwargs):
        """
            Set up some basics for an Entity
        """
        pass

class Artist(Entity):
    __apiclass__ = "artist"

    def __init__(self, id, name, **kwargs):
        self.id = id
        self.name = name
        self.extras = kwargs

<commit_msg>don't cover test python version specific imports<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.

__project__ = "semetric-python"
__author__  = "Matt Jeffery <matt@clan.se>"
__status__  = "alpha"
__version__ = "0.1.0"

import sys
import warnings
import httplib2

# Special imports for Python 3
if sys.version_info >= (3,): # pragma: no cover
    from urllib.parse import urlparse, urlunparse, urlencode, parse_qs
else:
    from urllib import urlencode
    from urlparse import urlparse, urlunparse, parse_qs

try:
    import json
except ImportError: # pragma: no cover
    import simplejson as json

class APIError(Exception):
    pass

class APIClient(object):
    """
        API Client
    """

    API_BASE_URL = "http://api.semetric.com"
    HTTP_REQUEST = ('GET', 'POST') # these kinds of requests won't be set in _method
    ANY_GET_REQUEST = ('GET', 'PUT')
    USER_AGENT = "{0}/{1}".format(__project__, __version__)
    USER_AGENT_HEADER = {"User-Agent": USER_AGENT}

    def __init__(self, apikey, baseurl=API_BASE_URL):
        self.apikey = apikey
        self.baseurl = baseurl.rstrip('/')
        self.http = httplib2.Http()

    def _request(self, path, method='GET', **params):
        """
            Make an API request
        """
        base = "{0}/{1}".format(self.baseurl, path)
        # normalise the method argument
        method = method.upper().strip()

        # parse the existing url
        urlparts = urlparse(base)
        qstr = parse_qs(urlparts.query)

        # add the token to the query string
        qstr['token'] = self.apikey
        if method not in APIClient.HTTP_REQUEST:
            qstr['_method'] = method
        else:
            try:
                del qstr['_method']
            except KeyError:
                pass

        if method in APIClient.ANY_GET_REQUEST:
            # if it's a get request then update the query string with
            # the params
            qstr.update(params)
            # all of the params go in the query string
            query_string = urlencode(qstr)
            # reconstruct the url
            url = urlunparse((urlparts.scheme,
                                   urlparts.netloc,
                                   urlparts.path,
                                   urlparts.params,
                                   query_string,
                                   "")) # empty fragment
            resp, content = self.http.request(url, "GET", headers=self.USER_AGENT_HEADER)
        else:
            # all of the params go in the query string
            query_string = urlencode(qstr)
            # reconstruct the url
            url = urlunparse((urlparts.scheme,
                                   urlparts.netloc,
                                   urlparts.path,
                                   urlparts.params,
                                   query_string,
                                   "")) # empty fragment
            resp, content = self.http.request(url, "POST", urlencode(params), headers=self.USER_AGENT_HEADER)

        status = int(resp['status'])

        if status != 200:
            raise APIError("An unknown error occurred")

        return resp, json.loads(content)

    def __call__(self, path):
        pass

class Entity(object):
    """
        Base class for an entity in the Semetric API
    """
    __apiclass__ = "entity"

    @staticmethod
    def subclass_mapping():
        """
            Generate a mapping from Semetric API classes to Python
            Classes.
        """
        subclasses= dict((subclass.__apiclass__, subclass)
                         for subclass
                         in Entity.__subclasses__())
        subclasses['entity'] = Entity
        return subclasses

    @staticmethod
    def entity_factory(entity_dict):
        subclasses = Entity.subclass_mapping()

        apiclass = entity_dict.get('class', 'entity')
        try:
            entity_class = subclasses[apiclass]
        except KeyError:
            entity_class = Entity
            warnings.warn("Could not map api class `{0}' to a python class, using Entity".format(apiclass), stacklevel=2)

        return entity_class(**entity_dict)

    def __init__(self, **kwargs):
        """
            Set up some basics for an Entity
        """
        pass

class Artist(Entity):
    __apiclass__ = "artist"

    def __init__(self, id, name, **kwargs):
        self.id = id
        self.name = name
        self.extras = kwargs

<|endoftext|>


<commit_before>"""
Utilities used within urbansim that don't yet have a better home.

"""
from __future__ import print_function

import os

import numpy as np
import pandas as pd
import toolz

def _mkifnotexists(folder):
    d = os.path.join(os.getenv('DATA_HOME', "."), folder)
    if not os.path.exists(d):
        os.makedirs(d)
    return d

def data_dir():
    """
    Return the directory for the input data.
    """
    return _mkifnotexists("data")

def configs_dir():
    """
    Return the directory for the model configuration files.
    """
    return _mkifnotexists("configs")

def runs_dir():
    """
    Return the directory for the run output.
    """
    return _mkifnotexists("runs")

def models_dir():
    """
    Return the directory for the model configuration files (used by the
    website).
    """
    return _mkifnotexists("configs")

def charts_dir():
    """
    Return the directory for the chart configuration files (used by the
    website).
    """
    return _mkifnotexists("web/charts")

def maps_dir():
    """
    Return the directory for the map configuration files (used by the
    website).
    """
    return _mkifnotexists("web/maps")

def simulations_dir():
    """
    Return the directory for the simulation configuration files (used by the
    website).
    """
    return _mkifnotexists("web/simulations")

def reports_dir():
    """
    Return the directory for the report configuration files (used by the
    website).
    """
    return _mkifnotexists("web/reports")

def edits_dir():
    """
    Return the directory for the editable files (used by the
    website).
    """
    return _mkifnotexists("")

def config(fname):
    """
    Return the config path for the file with the given filename.
    """
    return os.path.join(configs_dir(), fname)

def get_run_number():
    """
    Get a run number for this execution of the model system, for
    identifying the output hdf5 files).

    Returns
    -------
    The integer number for this run of the model system.
    """
    try:
        f = open(os.path.join(os.getenv('DATA_HOME', "."), 'RUNNUM'), 'r')
        num = int(f.read())
        f.close()
    except:
        num = 1
    f = open(os.path.join(os.getenv('DATA_HOME', "."), 'RUNNUM'), 'w')
    f.write(str(num + 1))
    f.close()
    return num

def compute_range(travel_data, attr, travel_time_attr, dist, agg=np.sum):
    """
    Compute a zone-based accessibility query using the urbansim format
    travel data dataframe.

    Parameters
    ----------
    travel_data : dataframe
        The dataframe of urbansim format travel data.  Has from_zone_id as
        first index, to_zone_id as second index, and different impedances
        between zones as columns.
    attr : series
        The attr to aggregate.  Should be indexed by zone_id and the values
        will be aggregated.
    travel_time_attr : string
        The column name in travel_data to use as the impedance.
    dist : float
        The max distance to aggregate up to
    agg : function, optional, np.sum by default
        The numpy function to use for aggregation
    """
    travel_data = travel_data.reset_index(level=1)
    travel_data = travel_data[travel_data[travel_time_attr] < dist]
    travel_data["attr"] = attr[travel_data.to_zone_id].values
    return travel_data.groupby(level=0).attr.apply(agg)

def reindex(series1, series2):
    """
    This reindexes the first series by the second series.  This is an extremely
    common operation that does not appear to  be in Pandas at this time.
    If anyone knows of an easier way to do this in Pandas, please inform the
    UrbanSim developers.

    The canonical example would be a parcel series which has an index which is
    parcel_ids and a value which you want to fetch, let's say it's land_area.
    Another dataset, let's say of buildings has a series which indicate the
    parcel_ids that the buildings are located on, but which does not have
    land_area.  If you pass parcels.land_area as the first series and
    buildings.parcel_id as the second series, this function returns a series
    which is indexed by buildings and has land_area as values and can be
    added to the buildings dataset.

    In short, this is a join on to a different table using a foreign key
    stored in the current table, but with only one attribute rather than
    for a full dataset.

    This is very similar to the pandas "loc" function or "reindex" function,
    but neither of those functions return the series indexed on the current
    table.  In both of those cases, the series would be indexed on the foreign
    table and would require a second step to change the index.
    """
    return pd.Series(series1.loc[series2.values].values, index=series2.index)

def signif(val):
    """
    Convert a statistical significance to its ascii representation - this
    should be the same representation created in R.
    """
    val = abs(val)
    if val > 3.1:
        return '***'
    elif val > 2.33:
        return '**'
    elif val > 1.64:
        return '*'
    elif val > 1.28:
        return '.'
    return ''

naics_d = {
    11: 'Agriculture',
    21: 'Mining',
    22: 'Utilities',
    23: 'Construction',
    31: 'Manufacturing1',
    32: 'Manufacturing2',
    33: 'Manufacturing3',
    42: 'Wholesale',
    44: 'Retail1',
    45: 'Retail2',
    48: 'Transportation',
    49: 'Warehousing',
    51: 'Information',
    52: 'Finance and Insurance',
    53: 'Real Estate',
    54: 'Professional',
    55: 'Management',
    56: 'Administrative',
    61: 'Educational',
    62: 'Health Care',
    71: 'Arts',
    72: 'Accomodation and Food',
    81: 'Other',
    92: 'Public',
    99: 'Unknown'
}

def naicsname(val):
    """
    This function maps NAICS (job codes) from number to name.
    """
    return naics_d[val]

def numpymat2df(mat):
    """
    Sometimes (though not very often) it is useful to convert a numpy matrix
    which has no column names to a Pandas dataframe for use of the Pandas
    functions.  This method converts a 2D numpy matrix to Pandas dataframe
    with default column headers.

    Parameters
    ----------
    mat : The numpy matrix

    Returns
    -------
    A pandas dataframe with the same data as the input matrix but with columns
    named x0,  x1, ... x[n-1] for the number of columns.
    """
    return pd.DataFrame(
        dict(('x%d' % i, mat[:, i]) for i in range(mat.shape[1])))

def df64bitto32bit(tbl):
    """
    Convert a Pandas dataframe from 64 bit types to 32 bit types to save
    memory or disk space.

    Parameters
    ----------
    tbl : The dataframe to convert

    Returns
    -------
    The converted dataframe
    """
    newtbl = pd.DataFrame(index=tbl.index)
    for colname in tbl.columns:
        newtbl[colname] = series64bitto32bit(tbl[colname])
    return newtbl

def series64bitto32bit(s):
    """
    Convert a Pandas series from 64 bit types to 32 bit types to save
    memory or disk space.

    Parameters
    ----------
    s : The series to convert

    Returns
    -------
    The converted series
    """
    if s.dtype == np.float64:
        return s.astype('float32')
    elif s.dtype == np.int64:
        return s.astype('int32')
    return s

def _pandassummarytojson(v, ndigits=3):
    return {i: round(float(v.ix[i]), ndigits) for i in v.index}

def pandasdfsummarytojson(df, ndigits=3):
    """
    Convert the result of a

    Parameters
    ----------
    df : The result of a Pandas describe operation.
    ndigits : int, optional - The number of significant digits to round to.

    Returns
    -------
    A json object which captures the describe.  Keys are field names and
    values are dictionaries with all of the indexes returned by the Pandas
    describe.
    """
    df = df.transpose()
    return {k: _pandassummarytojson(v, ndigits) for k, v in df.iterrows()}

def column_map(tables, columns):
    """
    Take a list of tables and a list of column names and resolve which
    columns come from which table.

    Parameters
    ----------
    tables : sequence of _DataFrameWrapper or _TableFuncWrapper
        Could also be sequence of modified pandas.DataFrames, the important
        thing is that they have ``.name`` and ``.columns`` attributes.
    columns : sequence of str
        The column names of interest.

    Returns
    -------
    col_map : dict
        Maps table names to lists of column names.

    """
    if not columns:
        return {t.name: None for t in tables}

    columns = set(columns)
    colmap = {t.name: list(set(t.columns).intersection(columns)) for t in tables}
    foundcols = toolz.reduce(lambda x, y: x.union(y), (set(v) for v in colmap.values()))
    if foundcols != columns:
        raise RuntimeError('Not all required columns were found. '
                           'Missing: {}'.format(list(columns - foundcols)))
    return colmap

def column_list(tables, columns):
    """
    Take a list of tables and a list of column names and return the columns
    that are present in the tables.

    Parameters
    ----------
    tables : sequence of _DataFrameWrapper or _TableFuncWrapper
        Could also be sequence of modified pandas.DataFrames, the important
        thing is that they have ``.name`` and ``.columns`` attributes.
    columns : sequence of str
        The column names of interest.

    Returns
    -------
    cols : list
        Lists of column names available in the tables.

    """
    columns = set(columns)
    foundcols = toolz.reduce(lambda x, y: x.union(y), (set(t.columns) for t in tables))
    return list(columns.intersection(foundcols))
<commit_msg>reindex needs to use merge not loc<commit_after>"""
Utilities used within urbansim that don't yet have a better home.

"""
from __future__ import print_function

import os

import numpy as np
import pandas as pd
import toolz

def _mkifnotexists(folder):
    d = os.path.join(os.getenv('DATA_HOME', "."), folder)
    if not os.path.exists(d):
        os.makedirs(d)
    return d

def data_dir():
    """
    Return the directory for the input data.
    """
    return _mkifnotexists("data")

def configs_dir():
    """
    Return the directory for the model configuration files.
    """
    return _mkifnotexists("configs")

def runs_dir():
    """
    Return the directory for the run output.
    """
    return _mkifnotexists("runs")

def models_dir():
    """
    Return the directory for the model configuration files (used by the
    website).
    """
    return _mkifnotexists("configs")

def charts_dir():
    """
    Return the directory for the chart configuration files (used by the
    website).
    """
    return _mkifnotexists("web/charts")

def maps_dir():
    """
    Return the directory for the map configuration files (used by the
    website).
    """
    return _mkifnotexists("web/maps")

def simulations_dir():
    """
    Return the directory for the simulation configuration files (used by the
    website).
    """
    return _mkifnotexists("web/simulations")

def reports_dir():
    """
    Return the directory for the report configuration files (used by the
    website).
    """
    return _mkifnotexists("web/reports")

def edits_dir():
    """
    Return the directory for the editable files (used by the
    website).
    """
    return _mkifnotexists("")

def config(fname):
    """
    Return the config path for the file with the given filename.
    """
    return os.path.join(configs_dir(), fname)

def get_run_number():
    """
    Get a run number for this execution of the model system, for
    identifying the output hdf5 files).

    Returns
    -------
    The integer number for this run of the model system.
    """
    try:
        f = open(os.path.join(os.getenv('DATA_HOME', "."), 'RUNNUM'), 'r')
        num = int(f.read())
        f.close()
    except:
        num = 1
    f = open(os.path.join(os.getenv('DATA_HOME', "."), 'RUNNUM'), 'w')
    f.write(str(num + 1))
    f.close()
    return num

def compute_range(travel_data, attr, travel_time_attr, dist, agg=np.sum):
    """
    Compute a zone-based accessibility query using the urbansim format
    travel data dataframe.

    Parameters
    ----------
    travel_data : dataframe
        The dataframe of urbansim format travel data.  Has from_zone_id as
        first index, to_zone_id as second index, and different impedances
        between zones as columns.
    attr : series
        The attr to aggregate.  Should be indexed by zone_id and the values
        will be aggregated.
    travel_time_attr : string
        The column name in travel_data to use as the impedance.
    dist : float
        The max distance to aggregate up to
    agg : function, optional, np.sum by default
        The numpy function to use for aggregation
    """
    travel_data = travel_data.reset_index(level=1)
    travel_data = travel_data[travel_data[travel_time_attr] < dist]
    travel_data["attr"] = attr[travel_data.to_zone_id].values
    return travel_data.groupby(level=0).attr.apply(agg)

def reindex(series1, series2):
    """
    This reindexes the first series by the second series.  This is an extremely
    common operation that does not appear to  be in Pandas at this time.
    If anyone knows of an easier way to do this in Pandas, please inform the
    UrbanSim developers.

    The canonical example would be a parcel series which has an index which is
    parcel_ids and a value which you want to fetch, let's say it's land_area.
    Another dataset, let's say of buildings has a series which indicate the
    parcel_ids that the buildings are located on, but which does not have
    land_area.  If you pass parcels.land_area as the first series and
    buildings.parcel_id as the second series, this function returns a series
    which is indexed by buildings and has land_area as values and can be
    added to the buildings dataset.

    In short, this is a join on to a different table using a foreign key
    stored in the current table, but with only one attribute rather than
    for a full dataset.

    This is very similar to the pandas "loc" function or "reindex" function,
    but neither of those functions return the series indexed on the current
    table.  In both of those cases, the series would be indexed on the foreign
    table and would require a second step to change the index.
    """

    # turns out the merge is much faster than the .loc below
    df = pd.merge(pd.DataFrame({"left": series2}),
                  pd.DataFrame({"right": series1}),
                  left_on="left",
                  right_index=True,
                  how="left")
    return df.right

    #return pd.Series(series1.loc[series2.values].values, index=series2.index)

def signif(val):
    """
    Convert a statistical significance to its ascii representation - this
    should be the same representation created in R.
    """
    val = abs(val)
    if val > 3.1:
        return '***'
    elif val > 2.33:
        return '**'
    elif val > 1.64:
        return '*'
    elif val > 1.28:
        return '.'
    return ''

naics_d = {
    11: 'Agriculture',
    21: 'Mining',
    22: 'Utilities',
    23: 'Construction',
    31: 'Manufacturing1',
    32: 'Manufacturing2',
    33: 'Manufacturing3',
    42: 'Wholesale',
    44: 'Retail1',
    45: 'Retail2',
    48: 'Transportation',
    49: 'Warehousing',
    51: 'Information',
    52: 'Finance and Insurance',
    53: 'Real Estate',
    54: 'Professional',
    55: 'Management',
    56: 'Administrative',
    61: 'Educational',
    62: 'Health Care',
    71: 'Arts',
    72: 'Accomodation and Food',
    81: 'Other',
    92: 'Public',
    99: 'Unknown'
}

def naicsname(val):
    """
    This function maps NAICS (job codes) from number to name.
    """
    return naics_d[val]

def numpymat2df(mat):
    """
    Sometimes (though not very often) it is useful to convert a numpy matrix
    which has no column names to a Pandas dataframe for use of the Pandas
    functions.  This method converts a 2D numpy matrix to Pandas dataframe
    with default column headers.

    Parameters
    ----------
    mat : The numpy matrix

    Returns
    -------
    A pandas dataframe with the same data as the input matrix but with columns
    named x0,  x1, ... x[n-1] for the number of columns.
    """
    return pd.DataFrame(
        dict(('x%d' % i, mat[:, i]) for i in range(mat.shape[1])))

def df64bitto32bit(tbl):
    """
    Convert a Pandas dataframe from 64 bit types to 32 bit types to save
    memory or disk space.

    Parameters
    ----------
    tbl : The dataframe to convert

    Returns
    -------
    The converted dataframe
    """
    newtbl = pd.DataFrame(index=tbl.index)
    for colname in tbl.columns:
        newtbl[colname] = series64bitto32bit(tbl[colname])
    return newtbl

def series64bitto32bit(s):
    """
    Convert a Pandas series from 64 bit types to 32 bit types to save
    memory or disk space.

    Parameters
    ----------
    s : The series to convert

    Returns
    -------
    The converted series
    """
    if s.dtype == np.float64:
        return s.astype('float32')
    elif s.dtype == np.int64:
        return s.astype('int32')
    return s

def _pandassummarytojson(v, ndigits=3):
    return {i: round(float(v.ix[i]), ndigits) for i in v.index}

def pandasdfsummarytojson(df, ndigits=3):
    """
    Convert the result of a

    Parameters
    ----------
    df : The result of a Pandas describe operation.
    ndigits : int, optional - The number of significant digits to round to.

    Returns
    -------
    A json object which captures the describe.  Keys are field names and
    values are dictionaries with all of the indexes returned by the Pandas
    describe.
    """
    df = df.transpose()
    return {k: _pandassummarytojson(v, ndigits) for k, v in df.iterrows()}

def column_map(tables, columns):
    """
    Take a list of tables and a list of column names and resolve which
    columns come from which table.

    Parameters
    ----------
    tables : sequence of _DataFrameWrapper or _TableFuncWrapper
        Could also be sequence of modified pandas.DataFrames, the important
        thing is that they have ``.name`` and ``.columns`` attributes.
    columns : sequence of str
        The column names of interest.

    Returns
    -------
    col_map : dict
        Maps table names to lists of column names.

    """
    if not columns:
        return {t.name: None for t in tables}

    columns = set(columns)
    colmap = {t.name: list(set(t.columns).intersection(columns)) for t in tables}
    foundcols = toolz.reduce(lambda x, y: x.union(y), (set(v) for v in colmap.values()))
    if foundcols != columns:
        raise RuntimeError('Not all required columns were found. '
                           'Missing: {}'.format(list(columns - foundcols)))
    return colmap

def column_list(tables, columns):
    """
    Take a list of tables and a list of column names and return the columns
    that are present in the tables.

    Parameters
    ----------
    tables : sequence of _DataFrameWrapper or _TableFuncWrapper
        Could also be sequence of modified pandas.DataFrames, the important
        thing is that they have ``.name`` and ``.columns`` attributes.
    columns : sequence of str
        The column names of interest.

    Returns
    -------
    cols : list
        Lists of column names available in the tables.

    """
    columns = set(columns)
    foundcols = toolz.reduce(lambda x, y: x.union(y), (set(t.columns) for t in tables))
    return list(columns.intersection(foundcols))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# This script pulls the list of Mozilla trusted certificate authorities
# from the web at the "mozurl" below, parses the file to grab the PEM
# for each cert, and then generates DER files in a new ./data directory
# Upload these to an on-chip filesystem and use the CertManager to parse
# and use them for your outgoing SSL connections.
#
# Script by Earle F. Philhower, III.  Released to the public domain.
from __future__ import print_function
import csv
import os
import sys
from subprocess import Popen, PIPE, call
try:
    from urllib.request import urlopen
except Exception:
    from urllib2 import urlopen
try:
    from StringIO import StringIO
except Exception:
    from io import StringIO

# Mozilla's URL for the CSV file with included PEM certs
mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV"

# Load the manes[] and pems[] array from the URL
names = []
pems = []
response = urlopen(mozurl)
csvData = response.read()
if sys.version_info[0] > 2:
    csvData = csvData.decode('utf-8')
csvFile = StringIO(csvData)
csvReader = csv.reader(csvFile)
for row in csvReader:
    names.append(row[0]+":"+row[1]+":"+row[2])
    pems.append(row[30])
del names[0] # Remove headers
del pems[0] # Remove headers

# Try and make ./data, skip if present
try:
    os.mkdir("data")
except Exception:
    pass

derFiles = []
idx = 0
# Process the text PEM using openssl into DER files
for i in range(0, len(pems)):
    certName = "data/ca_%03d.der" % (idx);
    thisPem = pems[i].replace("'", "")
    print(names[i] + " -> " + certName)
    ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE)
    pipe = ssl.stdin
    pipe.write(thisPem.encode('utf-8'))
    pipe.close()
    ssl.wait()
    if os.path.exists(certName):
        derFiles.append(certName)
        idx = idx + 1

if os.path.exists("data/certs.ar"):
    os.unlink("data/certs.ar");

arCmd = ['ar', 'q', 'data/certs.ar'] + derFiles;
call( arCmd )

for der in derFiles:
    os.unlink(der)
<commit_msg>Update certs-from-mozilla.py (#7573)<commit_after>#!/usr/bin/env python3

# This script pulls the list of Mozilla trusted certificate authorities
# from the web at the "mozurl" below, parses the file to grab the PEM
# for each cert, and then generates DER files in a new ./data directory
# Upload these to an on-chip filesystem and use the CertManager to parse
# and use them for your outgoing SSL connections.
#
# Script by Earle F. Philhower, III.  Released to the public domain.
from __future__ import print_function
import csv
import os
import sys
from subprocess import Popen, PIPE, call
try:
    from urllib.request import urlopen
except Exception:
    from urllib2 import urlopen
try:
    from StringIO import StringIO
except Exception:
    from io import StringIO

# Mozilla's URL for the CSV file with included PEM certs
mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV"

# Load the names[] and pems[] array from the URL
names = []
pems = []
response = urlopen(mozurl)
csvData = response.read()
if sys.version_info[0] > 2:
    csvData = csvData.decode('utf-8')
csvFile = StringIO(csvData)
csvReader = csv.reader(csvFile)
for row in csvReader:
    names.append(row[0]+":"+row[1]+":"+row[2])
    pems.append(row[32])
del names[0] # Remove headers
del pems[0] # Remove headers

# Try and make ./data, skip if present
try:
    os.mkdir("data")
except Exception:
    pass

derFiles = []
idx = 0
# Process the text PEM using openssl into DER files
for i in range(0, len(pems)):
    certName = "data/ca_%03d.der" % (idx);
    thisPem = pems[i].replace("'", "")
    print(names[i] + " -> " + certName)
    ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE)
    pipe = ssl.stdin
    pipe.write(thisPem.encode('utf-8'))
    pipe.close()
    ssl.wait()
    if os.path.exists(certName):
        derFiles.append(certName)
        idx = idx + 1

if os.path.exists("data/certs.ar"):
    os.unlink("data/certs.ar");

arCmd = ['ar', 'q', 'data/certs.ar'] + derFiles;
call( arCmd )

for der in derFiles:
    os.unlink(der)
<|endoftext|>


<commit_before>import os.path

import boto3
from boto3.session import Session as boto3_session
from uritemplate import URITemplate

from mapbox.compat import text_types, urlparse
from mapbox.errors import InvalidFileError
from mapbox.services.base import Service

class Uploader(Service):
    """Access to the Upload API.

    Example usage:

        from mapbox import Uploader

        u = Uploader()
        url = u.stage(open('test.tif'))
        job = u.create(url, 'test1').json()

        assert job in u.list().json()

        # ... wait unti finished ...
        finished = u.status(job).json()['complete']

        u.delete(job)
        assert job not in u.list().json()
    """

    @property
    def baseuri(self):
        return 'https://{0}/uploads/v1'.format(self.host)

    def _get_credentials(self):
        """Gets temporary S3 credentials to stage user-uploaded files
        """
        uri = URITemplate(self.baseuri + '/{username}/credentials').expand(
            username=self.username)
        resp = self.session.get(uri)
        self.handle_http_error(
            resp,
            custom_messages={
                401: "Token is not authorized",
                404: "Token does not have upload scope",
                429: "Too many requests"})
        return resp

    def stage(self, fileobj_or_url, creds=None, callback=None):
        """Stages data in a Mapbox-owned S3 bucket

        If creds are not provided, temporary credentials will be
        generated using the Mapbox API.

        Note that when a s3:// URL is passed, identifying an object
        in a bucket owned by the caller, the caller must ensure that
        AWS credentials for access to that object are properly
        configured.

        Parameters
        ----------
        fileobj_or_url : file object or str
            A filename, s3:// URL, or a Python file object opened in
            binary mode.
        creds : dict
            AWS credentials allowing uploads to the destination bucket.
        callback : func
            A function that takes a number of bytes processed as its
            sole argument.

        Returns
        -------
        str
            The URL of the staged data
        """
        if (not isinstance(fileobj_or_url, text_types) and
                not hasattr(fileobj_or_url, 'read')):
            raise InvalidFileError(
                "a filename, URL, or file-like object is required")

        if not creds:
            res = self._get_credentials()
            creds = res.json()

        session = boto3_session(
            aws_access_key_id=creds['accessKeyId'],
            aws_secret_access_key=creds['secretAccessKey'],
            aws_session_token=creds['sessionToken'],
            region_name='us-east-1')
        s3 = session.resource('s3')
        bucket = s3.Bucket(creds['bucket'])
        key = creds['key']

        if isinstance(fileobj_or_url, text_types):
            # s3:// URLs are a special case. We can copy these directly
            # between AWS buckets.
            if fileobj_or_url.startswith('s3://'):
                parse_results = urlparse(fileobj_or_url)
                assert parse_results.scheme == 's3'
                source_bucket = parse_results.netloc
                source_key = parse_results.path.lstrip('/')

                copy_source = {'Bucket': source_bucket, 'Key': source_key}

                # The source client credentials and region must be
                # configured by the caller.
                source_session = boto3_session()
                source_client = source_session.client('s3')
                bucket.copy(copy_source, key, SourceClient=source_client,
                            Callback=callback)

            else:
                bucket.upload_file(fileobj_or_url, key, Callback=callback)
        else:
            bucket.upload_fileobj(fileobj_or_url, key, Callback=callback)

        return creds['url']

    def create(self, stage_url, tileset, name=None, patch=False):
        """Create a tileset

        Note: this step is refered to as "upload" in the API docs;
        This class's upload() method is a high-level function
        which acts like the Studio upload form.

        Returns a response object where the json() contents are
        an upload dict. Completion of the tileset may take several
        seconds or minutes depending on size of the data. The status()
        method of this class may be used to poll the API endpoint for
        tileset creation status.

        Parameters
        ----------
        stage_url : str
            URL to resource on S3, typically provided in the response
            of this class's stage() method. Does not work on arbitrary
            URLs (TODO).
        tileset : str
            The id of the tileset set to be created. Username will be
            prefixed if not present. For example, 'my-tileset' becomes
            '{username}.my-tileset'.
        name : str
            A short name for the tileset that will appear in Mapbox
            studio.
        patch : bool
            Optional patch mode which requires a flag on the owner's
            account.

        Returns
        -------
        requests.Response
        """
        if not tileset.startswith(self.username + "."):
            tileset = "{0}.{1}".format(self.username, tileset)

        msg = {'tileset': tileset,
               'url': stage_url}

        if patch:
            msg['patch'] = patch

        if name is not None:
            msg['name'] = name

        uri = URITemplate(self.baseuri + '/{username}').expand(
            username=self.username)

        resp = self.session.post(uri, json=msg)
        self.handle_http_error(resp)

        return resp

    def list(self):
        """List uploads

        Returns a Response object, the json() method of which returns
        a list of uploads

        Returns
        -------
        requests.Response
        """
        uri = URITemplate(self.baseuri + '/{username}').expand(
            username=self.username)
        resp = self.session.get(uri)
        self.handle_http_error(resp)

        return resp

    def delete(self, upload):
        """Delete an upload

        Returns a Response object. A response status code of 200
        indicates success.

        Parameters
        ----------
        upload : str
            The id of the upload or a dict with key 'id'.

        Returns
        -------
        requests.Response
        """
        if isinstance(upload, dict):
            upload_id = upload['id']
        else:
            upload_id = upload

        uri = URITemplate(self.baseuri + '/{username}/{upload_id}').expand(
            username=self.username, upload_id=upload_id)
        resp = self.session.delete(uri)
        self.handle_http_error(resp)

        return resp

    def status(self, upload):
        """Status of an upload

        Returns a Response object, the json() method of which returns
        and updated upload dict.

        Parameters
        ----------
        upload : str
            The id of the upload or a dict with key 'id'.

        Returns
        -------
        requests.Response
        """
        if isinstance(upload, dict):
            upload_id = upload['id']
        else:
            upload_id = upload

        uri = URITemplate(self.baseuri + '/{username}/{upload_id}').expand(
            username=self.username, upload_id=upload_id)
        resp = self.session.get(uri)
        self.handle_http_error(resp)

        return resp

    def upload(self, fileobj_or_url, tileset, name=None, patch=False,
               callback=None, region=None):
        """Upload data and create a Mapbox tileset

        Effectively replicates the Studio upload feature. Returns a
        Response object, the json() of which returns a dict with upload
        metadata.

        Note that when a s3:// URL is passed, identifying an object
        in a bucket owned by the caller, the caller must ensure that
        AWS credentials for access to that object are properly
        configured.

        Parameters
        ----------
        fileobj_or_url : file object or str
            A filename, s3:// URL, or a Python file object opened in
            binary mode.
        tileset : str
            A tileset identifier such as '{owner}.my-tileset'.
        name : str
            A short name for the tileset that will appear in Mapbox
            studio.
        patch : bool
            Optional patch mode which requires a flag on the owner's
            account.
        callback : func
            A function that takes a number of bytes processed as its
            sole argument.
        region : str
            Upload data is staged to a bucket in 'us-east-1'. If source
            data is in a different region it must be specified by name.

        Returns
        -------
        requests.Response
        """
        url = self.stage(fileobj_or_url, callback=callback)
        return self.create(url, tileset, name=name, patch=patch)
<commit_msg>Limited s3 copy support<commit_after>import os.path

import boto3
from boto3.session import Session as boto3_session
from uritemplate import URITemplate

from mapbox.compat import text_types, urlparse
from mapbox.errors import InvalidFileError
from mapbox.services.base import Service

class Uploader(Service):
    """Access to the Upload API.

    Example usage:

        from mapbox import Uploader

        u = Uploader()
        url = u.stage(open('test.tif'))
        job = u.create(url, 'test1').json()

        assert job in u.list().json()

        # ... wait unti finished ...
        finished = u.status(job).json()['complete']

        u.delete(job)
        assert job not in u.list().json()
    """

    @property
    def baseuri(self):
        return 'https://{0}/uploads/v1'.format(self.host)

    def _get_credentials(self):
        """Gets temporary S3 credentials to stage user-uploaded files
        """
        uri = URITemplate(self.baseuri + '/{username}/credentials').expand(
            username=self.username)
        resp = self.session.get(uri)
        self.handle_http_error(
            resp,
            custom_messages={
                401: "Token is not authorized",
                404: "Token does not have upload scope",
                429: "Too many requests"})
        return resp

    def stage(self, fileobj_or_url, creds=None, callback=None,
              region_name=None):
        """Stages data in a Mapbox-owned S3 bucket

        If creds are not provided, temporary credentials will be
        generated using the Mapbox API.

        Note that when a s3:// URL is passed, identifying an object
        in a bucket owned by the caller, the caller must ensure that
        AWS credentials for access to that object are properly
        configured.

        Parameters
        ----------
        fileobj_or_url : file object or str
            A filename, s3:// URL, or a Python file object opened in
            binary mode.
        creds : dict
            AWS credentials allowing uploads to the destination bucket.
        callback : func
            A function that takes a number of bytes processed as its
            sole argument.

        Returns
        -------
        str
            The URL of the staged data
        """
        if (not isinstance(fileobj_or_url, text_types) and
                not hasattr(fileobj_or_url, 'read')):
            raise InvalidFileError(
                "a filename, URL, or file-like object is required")

        if not creds:
            res = self._get_credentials()
            creds = res.json()

        session = boto3_session(
            aws_access_key_id=creds['accessKeyId'],
            aws_secret_access_key=creds['secretAccessKey'],
            aws_session_token=creds['sessionToken'],
            region_name='us-east-1')
        s3 = session.resource('s3')
        bucket = s3.Bucket(creds['bucket'])
        key = creds['key']

        if isinstance(fileobj_or_url, text_types):
            # s3:// URLs are a special case. We copy these directly
            # between AWS buckets if we can. This requires s3:GetObject
            # to be granted to our federated user.
            if fileobj_or_url.startswith('s3://'):
                parse_results = urlparse(fileobj_or_url)
                assert parse_results.scheme == 's3'
                source_bucket = parse_results.netloc
                source_key = parse_results.path.lstrip('/')

                copy_source = {'Bucket': source_bucket, 'Key': source_key}

                # The source client credentials and region must be
                # configured by the caller.
                source_client = boto3.client('s3', region_name)
                bucket.copy(copy_source, key, SourceClient=source_client,
                            Callback=callback)

            else:
                bucket.upload_file(fileobj_or_url, key, Callback=callback)
        else:
            bucket.upload_fileobj(fileobj_or_url, key, Callback=callback)

        return creds['url']

    def create(self, stage_url, tileset, name=None, patch=False):
        """Create a tileset

        Note: this step is refered to as "upload" in the API docs;
        This class's upload() method is a high-level function
        which acts like the Studio upload form.

        Returns a response object where the json() contents are
        an upload dict. Completion of the tileset may take several
        seconds or minutes depending on size of the data. The status()
        method of this class may be used to poll the API endpoint for
        tileset creation status.

        Parameters
        ----------
        stage_url : str
            URL to resource on S3, typically provided in the response
            of this class's stage() method. Does not work on arbitrary
            URLs (TODO).
        tileset : str
            The id of the tileset set to be created. Username will be
            prefixed if not present. For example, 'my-tileset' becomes
            '{username}.my-tileset'.
        name : str
            A short name for the tileset that will appear in Mapbox
            studio.
        patch : bool
            Optional patch mode which requires a flag on the owner's
            account.

        Returns
        -------
        requests.Response
        """
        if not tileset.startswith(self.username + "."):
            tileset = "{0}.{1}".format(self.username, tileset)

        msg = {'tileset': tileset,
               'url': stage_url}

        if patch:
            msg['patch'] = patch

        if name is not None:
            msg['name'] = name

        uri = URITemplate(self.baseuri + '/{username}').expand(
            username=self.username)

        resp = self.session.post(uri, json=msg)
        self.handle_http_error(resp)

        return resp

    def list(self):
        """List uploads

        Returns a Response object, the json() method of which returns
        a list of uploads

        Returns
        -------
        requests.Response
        """
        uri = URITemplate(self.baseuri + '/{username}').expand(
            username=self.username)
        resp = self.session.get(uri)
        self.handle_http_error(resp)

        return resp

    def delete(self, upload):
        """Delete an upload

        Returns a Response object. A response status code of 200
        indicates success.

        Parameters
        ----------
        upload : str
            The id of the upload or a dict with key 'id'.

        Returns
        -------
        requests.Response
        """
        if isinstance(upload, dict):
            upload_id = upload['id']
        else:
            upload_id = upload

        uri = URITemplate(self.baseuri + '/{username}/{upload_id}').expand(
            username=self.username, upload_id=upload_id)
        resp = self.session.delete(uri)
        self.handle_http_error(resp)

        return resp

    def status(self, upload):
        """Status of an upload

        Returns a Response object, the json() method of which returns
        and updated upload dict.

        Parameters
        ----------
        upload : str
            The id of the upload or a dict with key 'id'.

        Returns
        -------
        requests.Response
        """
        if isinstance(upload, dict):
            upload_id = upload['id']
        else:
            upload_id = upload

        uri = URITemplate(self.baseuri + '/{username}/{upload_id}').expand(
            username=self.username, upload_id=upload_id)
        resp = self.session.get(uri)
        self.handle_http_error(resp)

        return resp

    def upload(self, fileobj_or_url, tileset, name=None, patch=False,
               callback=None, region=None):
        """Upload data and create a Mapbox tileset

        Effectively replicates the Studio upload feature. Returns a
        Response object, the json() of which returns a dict with upload
        metadata.

        Note that when a s3:// URL is passed, identifying an object
        in a bucket owned by the caller, the caller must ensure that
        AWS credentials for access to that object are properly
        configured.

        Parameters
        ----------
        fileobj_or_url : file object or str
            A filename, s3:// URL, or a Python file object opened in
            binary mode.
        tileset : str
            A tileset identifier such as '{owner}.my-tileset'.
        name : str
            A short name for the tileset that will appear in Mapbox
            studio.
        patch : bool
            Optional patch mode which requires a flag on the owner's
            account.
        callback : func
            A function that takes a number of bytes processed as its
            sole argument.
        region : str
            Upload data is staged to a bucket in 'us-east-1'. If source
            data is in a different region it must be specified by name.

        Returns
        -------
        requests.Response
        """
        url = self.stage(fileobj_or_url, callback=callback)
        return self.create(url, tileset, name=name, patch=patch)
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
# This module implements the base NDData class.

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections

import numpy as np

from .nddatabase import NDDataBase
from ..units import Unit, Quantity
from .. import log
from ..utils.compat.odict import OrderedDict

from ..io import registry as io_registry
from ..config import ConfigAlias

__all__ = ['NDData']

__doctest_skip__ = ['NDData']

WARN_UNSUPPORTED_CORRELATED = ConfigAlias(
    '0.4', 'WARN_UNSUPPORTED_CORRELATED', 'warn_unsupported_correlated',
    'astropy.nddata.nddata', 'astropy.nddata')

class NDData(NDDataBase):
    """
    A basic class for array-based data.

    The key distinction from raw numpy arrays is the presence of
    additional metadata such as uncertainties, a mask, units,
    and/or a coordinate system.

    Parameters
    -----------
    data : `~numpy.ndarray`, `~numpy.ndarray`-like, or `NDData`
        The actual data contained in this `NDData` object. Note that this
        will always be copies by *reference* if `data` is a numpy array or
        `NDData`, so you should make copy the ``data`` before passing it in
        if that's the  desired behavior.

    uncertainty : any type, optional
        Uncertainty on the data. The uncertainty *must* have a string attribute
        named ``uncertainty_type``, but there is otherwise no restriction.

    mask : `~numpy.ndarray`-like, optional
        Mask for the data. The values must be ``False`` where
        the data is *valid* and ``True`` when it is not (like Numpy
        masked arrays). If ``data`` is a numpy masked array, providing
        ``mask`` here will causes the mask from the masked array to be
        ignored.

    wcs : undefined, optional
        WCS-object containing the world coordinate system for the data.

    meta : `dict`-like object, optional
        Metadata for this object. Must be dict-like but no further restriction
        is placed on meta.

    unit : `~astropy.units.UnitBase` instance or str, optional
        The units of the data.

    Raises
    ------
    ValueError :
        If the `uncertainty` or `mask` inputs cannot be broadcast (e.g., match
        shape) onto ``data``.

    Notes
    -----
    The data in a `NDData` object can should be accessed through the data
    attribute.

    For example::

        >>> from astropy.nddata import NDData
        >>> import numpy as np
        >>> x = NDData([1,2,3])
        >>> np.asarray(x.data)
        array([1, 2, 3])
    """

    def __init__(self, data, uncertainty=None, mask=None, wcs=None,
                 meta=None, unit=None):

        if isinstance(data, self.__class__):
            self._data = np.array(data.data, subok=True, copy=False)
            self.uncertainty = data.uncertainty
            self._mask = data.mask
            self._wcs = data.wcs
            self._meta = data.meta
            self._unit = data.unit

            if uncertainty is not None:
                self._uncertainty = uncertainty
                log.info("Overwriting NDData's current uncertainty being"
                         " overwritten with specified uncertainty")

            if mask is not None:
                self._mask = mask
                log.info("Overwriting NDData's current "
                         "mask with specified mask")

            if wcs is not None:
                self._wcs = wcs
                log.info("Overwriting NDData's current wcs with specified wcs")

            if meta is not None:
                self._meta = meta
                log.info("Overwriting NDData's current meta "
                         "with specified meta")

            if unit is not None:
                raise ValueError('To convert to different unit please use .to')
        else:
            if hasattr(data, 'mask'):
                self._data = np.array(data.data, subok=True, copy=False)

                if mask is not None:
                    self._mask = mask
                    log.info("NDData was created with a masked array, and a "
                             "mask was explictly provided to NDData. The  "
                             "explicitly passed-in mask will be used and the "
                             "masked array's mask will be ignored.")
                else:
                    self._mask = data.mask
            elif isinstance(data, Quantity):
                self._data = np.array(data.value, subok=True, copy=False)
                self._mask = mask
            elif (not hasattr(data, 'shape') or
                  not hasattr(data, '__getitem__') or
                  not hasattr(data, '__array__')):
                # Data doesn't look like a numpy array, try converting it to
                # one.
                self._data = np.array(data, subok=True, copy=False)
                # Quick check to see if what we got out looks like an array
                # rather than an object (since numpy will convert a
                # non-numerical input to an array of objects).
                if self._data.dtype == 'O':
                    raise TypeError("Could not convert data to numpy array.")
                self._mask = mask
            else:
                self._data = data  # np.array(data, subok=True, copy=False)
                self._mask = mask

            self._wcs = wcs

            if meta is None:
                self._meta = OrderedDict()
            elif not isinstance(meta, collections.Mapping):
                raise TypeError("meta attribute must be dict-like")
            else:
                self._meta = meta

            if isinstance(data, Quantity):
                if unit is not None:
                    raise ValueError("Cannot use the unit argument when data "
                                     "is a Quantity")
                else:
                    self._unit = data.unit
            else:
                if unit is not None:
                    self._unit = Unit(unit)
                else:
                    self._unit = None
            # This must come after self's unit has been set so that the unit
            # of the uncertainty, if any, can be converted to the unit of the
            # unit of self.
            self.uncertainty = uncertainty

    def __str__(self):
        return str(self.data)

    def __repr__(self):
        prefix = self.__class__.__name__ + '('
        body = np.array2string(self.data, separator=', ', prefix=prefix)
        return ''.join([prefix, body, ')'])

    @property
    def data(self):
        return self._data

    @property
    def mask(self):
        return self._mask

    @mask.setter
    def mask(self, value):
        self._mask = value

    @property
    def unit(self):
        return self._unit

    @property
    def wcs(self):
        return self._wcs

    @property
    def meta(self):
        return self._meta
<commit_msg>Several docstring fixes<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
# This module implements the base NDData class.

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import collections

import numpy as np

from .nddatabase import NDDataBase
from ..units import Unit, Quantity
from .. import log
from ..utils.compat.odict import OrderedDict

from ..io import registry as io_registry
from ..config import ConfigAlias

__all__ = ['NDData']

__doctest_skip__ = ['NDData']

WARN_UNSUPPORTED_CORRELATED = ConfigAlias(
    '0.4', 'WARN_UNSUPPORTED_CORRELATED', 'warn_unsupported_correlated',
    'astropy.nddata.nddata', 'astropy.nddata')

class NDData(NDDataBase):
    """
    A basic class for array-based data.

    The key distinction from raw numpy arrays is the presence of
    additional metadata such as uncertainties, a mask, units,
    and/or a coordinate system.

    Parameters
    -----------
    data : `~numpy.ndarray`, `~numpy.ndarray`-like, or `NDData`
        The actual data contained in this `NDData` object. If possible, data
        will not be copied`data`, so you should make copy the ``data`` before
        passing it in if that's the desired behavior.

    uncertainty : any type, optional
        Uncertainty on the data. The uncertainty *must* have a string attribute
        named ``uncertainty_type``, but there is otherwise no restriction.

    mask : `~numpy.ndarray`-like, optional
        Mask for the data. The values must be ``False`` where
        the data is *valid* and ``True`` when it is not (like Numpy
        masked arrays). If ``data`` is a numpy masked array, providing
        ``mask`` here will causes the mask from the masked array to be
        ignored.

    wcs : undefined, optional
        WCS-object containing the world coordinate system for the data.

    meta : `dict`-like object, optional
        Metadata for this object. Must be dict-like but no further restriction
        is placed on meta.

    unit : `~astropy.units.UnitBase` instance or str, optional
        The units of the data. If data is an `~astropy.units.Quantity` then
        ``unit`` is set to the unit of the data.

    Notes
    -----
    The data in a `NDData` object should be accessed through the data
    attribute.

    For example::

        >>> from astropy.nddata import NDData
        >>> x = NDData([1,2,3])
        >>> x.data
        array([1, 2, 3])
    """

    def __init__(self, data, uncertainty=None, mask=None, wcs=None,
                 meta=None, unit=None):

        if isinstance(data, self.__class__):
            self._data = np.array(data.data, subok=True, copy=False)
            self.uncertainty = data.uncertainty
            self._mask = data.mask
            self._wcs = data.wcs
            self._meta = data.meta
            self._unit = data.unit

            if uncertainty is not None:
                self._uncertainty = uncertainty
                log.info("Overwriting NDData's current uncertainty being"
                         " overwritten with specified uncertainty")

            if mask is not None:
                self._mask = mask
                log.info("Overwriting NDData's current "
                         "mask with specified mask")

            if wcs is not None:
                self._wcs = wcs
                log.info("Overwriting NDData's current wcs with specified wcs")

            if meta is not None:
                self._meta = meta
                log.info("Overwriting NDData's current meta "
                         "with specified meta")

            if unit is not None and unit is not data.unit:
                raise ValueError('Unit provided in initializer does not '
                                 'match data unit.')
        else:
            if hasattr(data, 'mask'):
                self._data = np.array(data.data, subok=True, copy=False)

                if mask is not None:
                    self._mask = mask
                    log.info("NDData was created with a masked array, and a "
                             "mask was explicitly provided to NDData. The  "
                             "explicitly passed-in mask will be used and the "
                             "masked array's mask will be ignored.")
                else:
                    self._mask = data.mask
            elif isinstance(data, Quantity):
                self._data = np.array(data.value, subok=True, copy=False)
                self._mask = mask
            elif (not hasattr(data, 'shape') or
                  not hasattr(data, '__getitem__') or
                  not hasattr(data, '__array__')):
                # Data doesn't look like a numpy array, try converting it to
                # one.
                self._data = np.array(data, subok=True, copy=False)
                # Quick check to see if what we got out looks like an array
                # rather than an object (since numpy will convert a
                # non-numerical input to an array of objects).
                if self._data.dtype == 'O':
                    raise TypeError("Could not convert data to numpy array.")
                self._mask = mask
            else:
                self._data = data  # np.array(data, subok=True, copy=False)
                self._mask = mask

            self._wcs = wcs

            if meta is None:
                self._meta = OrderedDict()
            elif not isinstance(meta, collections.Mapping):
                raise TypeError("meta attribute must be dict-like")
            else:
                self._meta = meta

            if isinstance(data, Quantity):
                if unit is not None:
                    raise ValueError("Cannot use the unit argument when data "
                                     "is a Quantity")
                else:
                    self._unit = data.unit
            else:
                if unit is not None:
                    self._unit = Unit(unit)
                else:
                    self._unit = None
            # This must come after self's unit has been set so that the unit
            # of the uncertainty, if any, can be converted to the unit of the
            # unit of self.
            self.uncertainty = uncertainty

    def __str__(self):
        return str(self.data)

    def __repr__(self):
        prefix = self.__class__.__name__ + '('
        body = np.array2string(self.data, separator=', ', prefix=prefix)
        return ''.join([prefix, body, ')'])

    @property
    def data(self):
        return self._data

    @property
    def mask(self):
        return self._mask

    @mask.setter
    def mask(self, value):
        self._mask = value

    @property
    def unit(self):
        return self._unit

    @property
    def wcs(self):
        return self._wcs

    @property
    def meta(self):
        return self._meta
<|endoftext|>


<commit_before>import codecs, cStringIO as StringIO

class HTMLInputStream(object):
    """For reading data from an input stream

    This deals with character encoding issues automatically.

    This keeps track of the current line and column number in the file
    automatically, as you consume and unconsume characters.
    """

    def __init__(self, file, encoding = None):
        """ Initialise the HTMLInputReader.

        The file parameter must be a File object.

        The optional encoding parameter must be a string that indicates
        the encoding.  If specified, that encoding will be used,
        regardless of any BOM or later declaration (such as in a meta
        element)
        """

        self.line = 1 # Current line number
        self.col = 0  # Current column number
        self.lineBreaks = [0]

        # Keep a reference to the unencoded file object so that a new
        # EncodedFile can be created later if the encoding is declared
        # in a meta element
        if hasattr(file, 'tell'):
            self.file = file
        else:
            self.file = StringIO.StringIO(file)

        skipBOM = False
        self.charEncoding = self.detectBOM(self.file)
        if self.charEncoding:
            # The encoding is known from the BOM, don't allow later
            # declarations from the meta element to override this.
            skipBOM = True
            self.allowEncodingOverride = False
        else:
            # Using the default encoding, don't allow later
            # declarations from the meta element to override this.
            self.allowEncodingOverride = True
            self.charEncoding = "cp1252" # default to Windows-1252

        self.encodedFile = codecs.EncodedFile(self.file, self.charEncoding)
        if skipBOM:
            self.encodedFile.read(1)

    def detectBOM(self, fp):
        """ Attempts to detect the character encoding of the html file
        given by a file object fp. fp must not be a codec wrapped file
        object!

        The return value can be:
            - if detection of the BOM succeeds, the codec name of the
              corresponding unicode charset is returned

            - if BOM detection fails, None is returned.
        """
        # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/363841

        ### detection using BOM

        ## the BOMs we know, by their pattern
        bomDict = { # bytepattern : name
                   (0x00, 0x00, 0xFE, 0xFF) : "utf_32_be",
                   (0xFF, 0xFE, 0x00, 0x00) : "utf_32_le",
                   (0xFE, 0xFF, None, None) : "utf_16_be",
                   (0xFF, 0xFE, None, None) : "utf_16_le",
                   (0xEF, 0xBB, 0xBF, None) : "utf_8",
                  }

        ## go to beginning of file and get the first 4 bytes
        oldFP = fp.tell()
        fp.seek(0)
        (byte1, byte2, byte3, byte4) = tuple(map(ord, fp.read(4)))

        ## try bom detection using 4 bytes, 3 bytes, or 2 bytes
        bomDetection = bomDict.get((byte1, byte2, byte3, byte4))
        if not bomDetection :
            bomDetection = bomDict.get((byte1, byte2, byte3, None))
            if not bomDetection :
                bomDetection = bomDict.get((byte1, byte2, None, None))

        ## if BOM detected, we're done :-)
        fp.seek(0) # No BOM, return to the beginning of the file
        if bomDetection :
            return bomDetection
        return None

    def consumeChar(self):
        char = unicode(self.encodedFile.read(1), self.charEncoding)
        if char == "\n":
            # Move to next line and reset column count
            self.line += 1
            self.col = 0
            self.lineBreaks.append(self.encodedFile.tell())
        else:
            # Just increment the column counter
            self.col += 1
        return char or None

    def unconsumeChar(self):
        """Unconsume the previous character by seeking backwards thorough
        the file.
        """
        self.encodedFile.seek(-1, 1)
        if self.encodedFile.tell()+1 == self.lineBreaks[-1]:
            self.line -= 1
            self.lineBreaks.pop()
            self.col = self.encodedFile.tell()-self.lineBreaks[-1]
        else:
            self.col -= 1

    def declareEncoding(self, encoding):
        """Report the encoding declared by the meta element

        If the encoding is currently only guessed, then this
        will read subsequent characters in that encoding.

        If the encoding is not compatible with the guessed encoding
        and non-US-ASCII characters have been seen, parsing will
        have to begin again.
        """
        pass

if __name__ == "__main__":
    try:
        # Hard coded file name for now, this will need to be fixed later
        htmlFile = open("tests/utf-8-bom.html", "rU")
        stream = HTMLInputStream(htmlFile)

        char = stream.consumeChar()
        while char:
            line = stream.line
            col = stream.col
            if char == "\n":
                print "LF (%d, %d)" % (line, col)
            else:
                print "%s (%d, %d)" % (char, line, col)
            char = stream.consumeChar()
        print "EOF"
        htmlFile.close()
    except IOError:
        print "The file does not exist."
<commit_msg>HTMLInputStream now accepts either a file object, local filename, url or a string as input<commit_after>import codecs

def openSource(source):
    """ Opens source first trying to open a local file, if that fails
    try to open as a URL and finally treating source as a string.

    Returns a file-like object.
    """
    # Already a file-like object?
    if hasattr(source, 'tell'):
        return source

    # Try opening source normally
    try:
        return open(source)
    except: pass

    # Try opening source as a URL and storing the bytes returned so
    # they can be turned into a file-like object below
    try:
        import urllib
        source = urllib.urlopen(source).read(-1)
    except: pass

    # Treat source as a string and make it into a file-like object
    import cStringIO as StringIO
    return StringIO.StringIO(str(source))

class HTMLInputStream(object):
    """For reading data from an input stream

    This deals with character encoding issues automatically.

    This keeps track of the current line and column number in the file
    automatically, as you consume and unconsume characters.
    """

    def __init__(self, source, encoding = None):
        """ Initialise the HTMLInputReader.

        The file parameter must be a File object.

        The optional encoding parameter must be a string that indicates
        the encoding.  If specified, that encoding will be used,
        regardless of any BOM or later declaration (such as in a meta
        element)
        """

        self.line = 1 # Current line number
        self.col = 0  # Current column number
        self.lineBreaks = [0]

        # Keep a reference to the unencoded file object so that a new
        # EncodedFile can be created later if the encoding is declared
        # in a meta element
        self.file = openSource(source)

        skipBOM = False
        self.charEncoding = self.detectBOM(self.file)
        if self.charEncoding:
            # The encoding is known from the BOM, don't allow later
            # declarations from the meta element to override this.
            skipBOM = True
            self.allowEncodingOverride = False
        else:
            # Using the default encoding, don't allow later
            # declarations from the meta element to override this.
            self.allowEncodingOverride = True
            self.charEncoding = "cp1252" # default to Windows-1252

        self.encodedFile = codecs.EncodedFile(self.file, self.charEncoding)
        if skipBOM:
            self.encodedFile.read(1)

    def detectBOM(self, fp):
        """ Attempts to detect the character encoding of the html file
        given by a file object fp. fp must not be a codec wrapped file
        object!

        The return value can be:
            - if detection of the BOM succeeds, the codec name of the
              corresponding unicode charset is returned

            - if BOM detection fails, None is returned.
        """
        # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/363841

        ### detection using BOM

        ## the BOMs we know, by their pattern
        bomDict = { # bytepattern : name
                   (0x00, 0x00, 0xFE, 0xFF) : "utf_32_be",
                   (0xFF, 0xFE, 0x00, 0x00) : "utf_32_le",
                   (0xFE, 0xFF, None, None) : "utf_16_be",
                   (0xFF, 0xFE, None, None) : "utf_16_le",
                   (0xEF, 0xBB, 0xBF, None) : "utf_8",
                  }

        ## go to beginning of file and get the first 4 bytes
        oldFP = fp.tell()
        fp.seek(0)
        (byte1, byte2, byte3, byte4) = tuple(map(ord, fp.read(4)))

        ## try bom detection using 4 bytes, 3 bytes, or 2 bytes
        bomDetection = bomDict.get((byte1, byte2, byte3, byte4))
        if not bomDetection :
            bomDetection = bomDict.get((byte1, byte2, byte3, None))
            if not bomDetection :
                bomDetection = bomDict.get((byte1, byte2, None, None))

        ## if BOM detected, we're done :-)
        fp.seek(0) # No BOM, return to the beginning of the file
        if bomDetection :
            return bomDetection
        return None

    def consumeChar(self):
        char = unicode(self.encodedFile.read(1), self.charEncoding)
        if char == "\n":
            # Move to next line and reset column count
            self.line += 1
            self.col = 0
            self.lineBreaks.append(self.encodedFile.tell())
        else:
            # Just increment the column counter
            self.col += 1
        return char or None

    def unconsumeChar(self):
        """Unconsume the previous character by seeking backwards thorough
        the file.
        """
        self.encodedFile.seek(-1, 1)
        if self.encodedFile.tell()+1 == self.lineBreaks[-1]:
            self.line -= 1
            self.lineBreaks.pop()
            self.col = self.encodedFile.tell()-self.lineBreaks[-1]
        else:
            self.col -= 1

    def declareEncoding(self, encoding):
        """Report the encoding declared by the meta element

        If the encoding is currently only guessed, then this
        will read subsequent characters in that encoding.

        If the encoding is not compatible with the guessed encoding
        and non-US-ASCII characters have been seen, parsing will
        have to begin again.
        """
        pass

if __name__ == "__main__":
    try:
        # Hard coded file name for now, this will need to be fixed later
        htmlFile = open("tests/utf-8-bom.html", "rU")
        stream = HTMLInputStream(htmlFile)

        char = stream.consumeChar()
        while char:
            line = stream.line
            col = stream.col
            if char == "\n":
                print "LF (%d, %d)" % (line, col)
            else:
                print "%s (%d, %d)" % (char, line, col)
            char = stream.consumeChar()
        print "EOF"
        htmlFile.close()
    except IOError:
        print "The file does not exist."
<|endoftext|>