<commit_before>import astropy.io.fits as pyfits
import numpy as np
import warnings
from threeML.io.file_utils import file_existing_and_readable, sanitize_filename

class Response(object):

    def __init__(self, rsp_file, arf_file=None):

        # Now make sure that the response file exist

        rsp_file = sanitize_filename(rsp_file)

        assert file_existing_and_readable(rsp_file.split("{")[0]), "Response file %s not existing or not " \
                                                                   "readable" % rsp_file

        # Check if we are dealing with a .rsp2 file (containing more than
        # one response). This is checked by looking for the syntax
        # [responseFile]{[responseNumber]}

        if '{' in rsp_file:

            tokens = rsp_file.split("{")
            rsp_file = tokens[0]
            rsp_number = int(tokens[-1].split('}')[0].replace(" ", ""))

        else:

            rsp_number = 1

        # Read the response
        with pyfits.open(rsp_file) as f:

            try:

                # This is usually when the response file contains only the energy dispersion

                data = f['MATRIX', rsp_number].data
                header = f['MATRIX', rsp_number].header

                if arf_file is None:

                    warnings.warn("The response is in an extension called MATRIX, which usually means you also "
                                  "need an ancillary file (ARF) which you didn't provide. You should refer to the "
                                  "documentation  of the instrument and make sure you don't need an ARF.")

            except:

                # Other detectors might use the SPECRESP MATRIX name instead, usually when the response has been
                # already convoluted with the effective area

                # Note that here we are not catching any exception, because
                # we have to fail if we cannot read the matrix

                data = f['SPECRESP MATRIX', rsp_number].data
                header = f['SPECRESP MATRIX', rsp_number].header

            # Sometimes .rsp files contains a weird format featuring variable-length
            # arrays. Historically those confuse pyfits quite a lot, so we ensure
            # to transform them into standard numpy matrices to avoid issues

            self._matrix = self._read_matrix(data, header)

            self._ebounds = np.vstack([f['EBOUNDS'].data.field("E_MIN"),
                                       f['EBOUNDS'].data.field("E_MAX")]).T

            self._mc_channels = np.vstack([data.field("ENERG_LO"),
                                           data.field("ENERG_HI")]).T

            # Now let's see if we have a ARF, if yes, read it

            if arf_file is not None:

                arf_file = sanitize_filename(arf_file)

                assert file_existing_and_readable(arf_file.split("{")[0]), "Ancillary file %s not existing or not " \
                                                                           "readable" % arf_file

                with pyfits.open(arf_file) as f:

                    data = f['SPECRESP'].data

                arf = data.field('SPECRESP')

                # Check that arf and rmf have same dimensions

                if arf.shape[0] != self._matrix.shape[1]:

                    raise IOError("The ARF and the RMF file does not have the same number of channels")

                # Check that the ENERG_LO and ENERG_HI for the RMF and the ARF
                # are the same

                arf_mc_channels = np.vstack([data.field("ENERG_LO"),
                                             data.field("ENERG_HI")]).T

                # Declare the mc channels different if they differ by more than
                # 1%

                idx = (self._mc_channels > 0)

                diff = (self._mc_channels[idx] - arf_mc_channels[idx]) / self._mc_channels[idx]

                if diff.max() > 0.01:
                    raise IOError("The ARF and the RMF have one or more MC channels which differ by more than 1%")

                # Multiply ARF and RMF

                self._matrix = self._matrix * arf

        # Init everything else to none
        self._differential_function = None
        self._integral_function = None

        # Store the name of the file
        self._rsp_file = rsp_file
        self._arf_file = arf_file

    @property
    def rsp_filename(self):
        """
        Returns the name of the RSP/RMF file from which the response has been loaded
        """

        return self._rsp_file

    @property
    def arf_filename(self):
        """
        Returns the name of the ARF file (or None if there is none)
        """

        return self._arf_file

    @property
    def first_channel(self):
        """
        The first channel of the channel array. Correpsonds to
        TLMIN keyword in FITS files

        :return: first channel
        """
        return int(self._first_channel)

    def _read_matrix(self, data, header, column_name='MATRIX'):

        n_channels = header.get("DETCHANS")

        assert n_channels is not None, "Matrix is improperly formatted. No DETCHANS keyword."

        # The header contains a keyword which tells us the first legal channel. It is TLMIN of the F_CHAN column
        # NOTE: TLMIN keywords start at 1, so TLMIN1 is the minimum legal value for the first column. So we need
        # to add a +1 because of course the numbering of lists (data.columns.names) starts at 0

        f_chan_column_pos = data.columns.names.index("F_CHAN") + 1

        try:
            tlmin_fchan = header["TLMIN%i" % f_chan_column_pos]

        except(KeyError):
            warnings.warn('No TLMIN keyword found. This DRM does not follow OGIP standards. Assuming TLMIN=1')
            tlmin_fchan = 1

        # Store the first channel as a property
        self._first_channel = tlmin_fchan

        rsp = np.zeros([data.shape[0], n_channels], float)

        n_grp = data.field("N_GRP")

        # The numbering of channels could start at 0, or at some other number (usually 1). Of course the indexing
        # of arrays starts at 0. So let's offset the F_CHAN column to account for that

        f_chan = data.field("F_CHAN") - tlmin_fchan
        n_chan = data.field("N_CHAN")

        # In certain matrices where compression has not been used, n_grp, f_chan and n_chan are not array columns,
        # but simple scalars. Expand then their dimensions so that we don't need to customize the code below

        if n_grp.ndim == 1:

            n_grp = np.expand_dims(n_grp, 1)

        if f_chan.ndim == 1:

            f_chan = np.expand_dims(f_chan, 1)

        if n_chan.ndim == 1:

            n_chan = np.expand_dims(n_chan, 1)

        matrix = data.field(column_name)

        for i, row in enumerate(data):

            m_start = 0

            for j in range(n_grp[i]):

                rsp[i, f_chan[i][j]: f_chan[i][j] + n_chan[i][j]] = matrix[i][m_start:m_start + n_chan[i][j]]

                m_start += n_chan[i][j]

        return rsp.T

    @property
    def ebounds(self):
        return self._ebounds

    @ebounds.setter
    def ebounds(self, value):
        raise PrivateMember('ebounds should not be altered manually, silly rabbit!')

    @ebounds.getter
    def ebounds(self):
        return self._ebounds

    def set_function(self, differentialFunction, integralFunction=None):
        """
        Set the function to be used for the convolution
        """

        self._differential_function = differentialFunction

        self._integral_function = integralFunction

    def convolve(self):

        true_fluxes = self._integral_function(self._mc_channels[:, 0],
                                              self._mc_channels[:, 1])

        # Sometimes some channels have 0 lenths, or maybe they start at 0, where
        # many functions (like a power law) are not defined. In the response these
        # channels have usually a 0, but unfortunately for a computer
        # inf * zero != zero. Thus, let's force this. We avoid checking this situation
        # in details because this would have a HUGE hit on performances

        idx = np.isfinite(true_fluxes)
        true_fluxes[~idx] = 0

        folded_counts = np.dot(true_fluxes, self._matrix.T)

        return folded_counts

    def energy_to_channel(self, energy):

        '''Finds the channel containing the provided energy.
        NOTE: returns the channel index (starting at zero),
        not the channel number (likely starting from 1)'''

        # Get the index of the first ebounds upper bound larger than energy

        try:

            idx = next(idx for idx,
                               value in enumerate(self._ebounds[:, 1])
                       if value >= energy)

        except StopIteration:

            # No values above the given energy, return the last channel
            return self._ebounds[:, 1].shape[0]

        return idx
<commit_msg>make ebounds and mc_energies default as floats to avoid overflow in astromodels<commit_after>import astropy.io.fits as pyfits
import numpy as np
import warnings
from threeML.io.file_utils import file_existing_and_readable, sanitize_filename

class Response(object):

    def __init__(self, rsp_file, arf_file=None):

        # Now make sure that the response file exist

        rsp_file = sanitize_filename(rsp_file)

        assert file_existing_and_readable(rsp_file.split("{")[0]), "Response file %s not existing or not " \
                                                                   "readable" % rsp_file

        # Check if we are dealing with a .rsp2 file (containing more than
        # one response). This is checked by looking for the syntax
        # [responseFile]{[responseNumber]}

        if '{' in rsp_file:

            tokens = rsp_file.split("{")
            rsp_file = tokens[0]
            rsp_number = int(tokens[-1].split('}')[0].replace(" ", ""))

        else:

            rsp_number = 1

        # Read the response
        with pyfits.open(rsp_file) as f:

            try:

                # This is usually when the response file contains only the energy dispersion

                data = f['MATRIX', rsp_number].data
                header = f['MATRIX', rsp_number].header

                if arf_file is None:

                    warnings.warn("The response is in an extension called MATRIX, which usually means you also "
                                  "need an ancillary file (ARF) which you didn't provide. You should refer to the "
                                  "documentation  of the instrument and make sure you don't need an ARF.")

            except:

                # Other detectors might use the SPECRESP MATRIX name instead, usually when the response has been
                # already convoluted with the effective area

                # Note that here we are not catching any exception, because
                # we have to fail if we cannot read the matrix

                data = f['SPECRESP MATRIX', rsp_number].data
                header = f['SPECRESP MATRIX', rsp_number].header

            # Sometimes .rsp files contains a weird format featuring variable-length
            # arrays. Historically those confuse pyfits quite a lot, so we ensure
            # to transform them into standard numpy matrices to avoid issues

            self._matrix = self._read_matrix(data, header)

            self._ebounds = np.vstack([f['EBOUNDS'].data.field("E_MIN"),
                                       f['EBOUNDS'].data.field("E_MAX")]).T

            self._ebounds = self._ebounds.astype(float)

            self._mc_channels = np.vstack([data.field("ENERG_LO"),
                                           data.field("ENERG_HI")]).T

            self._mc_channels = self._mc_channels.astype(float)

            # Now let's see if we have a ARF, if yes, read it

            if arf_file is not None:

                arf_file = sanitize_filename(arf_file)

                assert file_existing_and_readable(arf_file.split("{")[0]), "Ancillary file %s not existing or not " \
                                                                           "readable" % arf_file

                with pyfits.open(arf_file) as f:

                    data = f['SPECRESP'].data

                arf = data.field('SPECRESP')

                # Check that arf and rmf have same dimensions

                if arf.shape[0] != self._matrix.shape[1]:

                    raise IOError("The ARF and the RMF file does not have the same number of channels")

                # Check that the ENERG_LO and ENERG_HI for the RMF and the ARF
                # are the same

                arf_mc_channels = np.vstack([data.field("ENERG_LO"),
                                             data.field("ENERG_HI")]).T

                # Declare the mc channels different if they differ by more than
                # 1%

                idx = (self._mc_channels > 0)

                diff = (self._mc_channels[idx] - arf_mc_channels[idx]) / self._mc_channels[idx]

                if diff.max() > 0.01:
                    raise IOError("The ARF and the RMF have one or more MC channels which differ by more than 1%")

                # Multiply ARF and RMF

                self._matrix = self._matrix * arf

        # Init everything else to none
        self._differential_function = None
        self._integral_function = None

        # Store the name of the file
        self._rsp_file = rsp_file
        self._arf_file = arf_file

    @property
    def rsp_filename(self):
        """
        Returns the name of the RSP/RMF file from which the response has been loaded
        """

        return self._rsp_file

    @property
    def arf_filename(self):
        """
        Returns the name of the ARF file (or None if there is none)
        """

        return self._arf_file

    @property
    def first_channel(self):
        """
        The first channel of the channel array. Correpsonds to
        TLMIN keyword in FITS files

        :return: first channel
        """
        return int(self._first_channel)

    def _read_matrix(self, data, header, column_name='MATRIX'):

        n_channels = header.get("DETCHANS")

        assert n_channels is not None, "Matrix is improperly formatted. No DETCHANS keyword."

        # The header contains a keyword which tells us the first legal channel. It is TLMIN of the F_CHAN column
        # NOTE: TLMIN keywords start at 1, so TLMIN1 is the minimum legal value for the first column. So we need
        # to add a +1 because of course the numbering of lists (data.columns.names) starts at 0

        f_chan_column_pos = data.columns.names.index("F_CHAN") + 1

        try:
            tlmin_fchan = header["TLMIN%i" % f_chan_column_pos]

        except(KeyError):
            warnings.warn('No TLMIN keyword found. This DRM does not follow OGIP standards. Assuming TLMIN=1')
            tlmin_fchan = 1

        # Store the first channel as a property
        self._first_channel = tlmin_fchan

        rsp = np.zeros([data.shape[0], n_channels], float)

        n_grp = data.field("N_GRP")

        # The numbering of channels could start at 0, or at some other number (usually 1). Of course the indexing
        # of arrays starts at 0. So let's offset the F_CHAN column to account for that

        f_chan = data.field("F_CHAN") - tlmin_fchan
        n_chan = data.field("N_CHAN")

        # In certain matrices where compression has not been used, n_grp, f_chan and n_chan are not array columns,
        # but simple scalars. Expand then their dimensions so that we don't need to customize the code below

        if n_grp.ndim == 1:

            n_grp = np.expand_dims(n_grp, 1)

        if f_chan.ndim == 1:

            f_chan = np.expand_dims(f_chan, 1)

        if n_chan.ndim == 1:

            n_chan = np.expand_dims(n_chan, 1)

        matrix = data.field(column_name)

        for i, row in enumerate(data):

            m_start = 0

            for j in range(n_grp[i]):

                rsp[i, f_chan[i][j]: f_chan[i][j] + n_chan[i][j]] = matrix[i][m_start:m_start + n_chan[i][j]]

                m_start += n_chan[i][j]

        return rsp.T

    @property
    def ebounds(self):
        """

        Returns the ebounds of the RSP.

        :return:
        """
        return self._ebounds

    def set_function(self, differentialFunction, integralFunction=None):
        """
        Set the function to be used for the convolution
        """

        self._differential_function = differentialFunction

        self._integral_function = integralFunction

    def convolve(self):

        true_fluxes = self._integral_function(self._mc_channels[:, 0],
                                              self._mc_channels[:, 1])

        # Sometimes some channels have 0 lenths, or maybe they start at 0, where
        # many functions (like a power law) are not defined. In the response these
        # channels have usually a 0, but unfortunately for a computer
        # inf * zero != zero. Thus, let's force this. We avoid checking this situation
        # in details because this would have a HUGE hit on performances

        idx = np.isfinite(true_fluxes)
        true_fluxes[~idx] = 0

        folded_counts = np.dot(true_fluxes, self._matrix.T)

        return folded_counts

    def energy_to_channel(self, energy):

        '''Finds the channel containing the provided energy.
        NOTE: returns the channel index (starting at zero),
        not the channel number (likely starting from 1)'''

        # Get the index of the first ebounds upper bound larger than energy

        try:

            idx = next(idx for idx,
                               value in enumerate(self._ebounds[:, 1])
                       if value >= energy)

        except StopIteration:

            # No values above the given energy, return the last channel
            return self._ebounds[:, 1].shape[0]

        return idx
<|endoftext|>


<commit_before>""" Setup script for PyPI """
import os
from setuptools import setup
from ConfigParser import SafeConfigParser

settings = SafeConfigParser()
settings.read(os.path.realpath('cumulus_bundle_handler/settings.conf'))

setup(
    name='cumulus-bundle-handler',
    version=settings.get('general', 'version'),
    license='Proprietary',
    description='Cumulus AWS deployment tools',
    author='Sebastian Dahlgren',
    author_email='sebastian.dahlgren@skymill.se',
    url='http://www.skymillsolutions.com',
    keywords="cumulus cloudformation amazon web services",
    platforms=['Any'],
    packages=['cumulus_bundle_handler'],
    scripts=['cumulus-bundle-handler'],
    include_package_data=True,
    zip_safe=False,
    install_requires=[
        'boto >= 2.12.0'
    ],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: OS Independent',
        'Programming Language :: Python'
    ]
)
<commit_msg>Fixed license information<commit_after>""" Setup script for PyPI """
import os
from setuptools import setup
from ConfigParser import SafeConfigParser

settings = SafeConfigParser()
settings.read(os.path.realpath('cumulus_bundle_handler/settings.conf'))

setup(
    name='cumulus-bundle-handler',
    version=settings.get('general', 'version'),
    license='Apache Software License',
    description='Cumulus AWS deployment tools',
    author='Sebastian Dahlgren',
    author_email='sebastian.dahlgren@skymill.se',
    url='http://www.skymillsolutions.com',
    keywords="cumulus cloudformation amazon web services",
    platforms=['Any'],
    packages=['cumulus_bundle_handler'],
    scripts=['cumulus-bundle-handler'],
    include_package_data=True,
    zip_safe=False,
    install_requires=[
        'boto >= 2.12.0'
    ],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: OS Independent',
        'Programming Language :: Python'
    ]
)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
convert gpx into epp dist/height profile
"""

from __future__ import print_function
from xml.dom import minidom

import math
import sys

import eppread as epp

EARTH_RADIUS = 6378137.0

class Point:
    lat = None
    lon = None
    ele = None
    dist = None
    total = None
    def __init__(self, lat=None, lon=None, ele=None, dist=None, total=None):
        self.lat = lat
        self.lon = lon
        self.ele = ele
        self.dist = dist
        self.total = total

class Profile:
    dist = None
    ele = None
    def __init__(self, dist=None, ele=None):
        self.dist = dist
        self.ele = ele

# see https://en.wikipedia.org/wiki/Great-circle_distance
# for calculating the distance between two geographical coordinates
def great_circle_distance(p1, p2):
    sdlat = math.sin((p1.lat - p2.lat) * math.pi / 180.0) / 2.0
    sdlon = math.sin((p1.lon - p2.lon) * math.pi / 180.0) / 2.0
    central_angle = 2.0 * math.asin(math.sqrt(sdlat * sdlat
                                              + math.cos(p1.lat * math.pi / 180.0)
                                              * math.cos(p2.lat * math.pi / 180.0)
                                              * sdlon * sdlon))
    return central_angle * EARTH_RADIUS

def read_gpx(filename):
    xml = minidom.parse(filename)
    pts = []
    for trk in xml.getElementsByTagName('trk')[0:1]:
        trkpts = trk.getElementsByTagName('trkpt')
        prev = None
        for trkpt in trkpts:
            p = Point(float(trkpt.getAttribute('lat')),
                      float(trkpt.getAttribute('lon')),
                      float(trkpt.getElementsByTagName('ele')[0].firstChild.nodeValue),
                      0.0, 0.0)
            if prev:
                p.dist = great_circle_distance(prev, p)
                p.total = prev.total + p.dist
            pts.append(p)
            prev = p
    return pts

class State:
    profile = None
    prev = None
    target = None
    raster = None
    def __init__(self, raster, profile=[], target=0.0, prev=None):
        self.raster = raster
        self.profile = profile
        self.prev = prev
        self.target = target

def transform(s, x):
    if len(s.profile) == 0:
        s.target += s.raster
        s.profile.append(Profile(0.0, x.ele))
    elif x.total < s.target:
        next
    else:
        while s.target < x.total:
            if x.total != s.prev.total:
                section = (s.target - s.prev.total) / (x.total - s.prev.total)
                climb = section * (x.ele - s.prev.ele)
                ele = s.prev.ele + climb
            else:
                ele = (x.ele - s.prev.ele) / 2 + s.prev.ele
            s.profile.append(Profile(s.target, ele))
            s.target += s.raster
    s.prev = x
    return s

def calculate_profile(points, raster):
    state = State(raster=raster)
    reduce(transform, points, state)
    return state.profile

def build_epp(profile, raster):
    header = dict(title='test',
                  description='test',
                  type='DIST_HEIGHT',
                  third='NONE',
                  length=len(profile),
                  graphmin=0,
                  graphmax=500,
                  raster=raster,
                  blr=dict(run=1, lyps=1, bike=1),
                  startheight=0,
                  maxwatt=0,
                  maxpulse=0,
                  maxspeed=0)
    data = map(lambda x : dict(val1=int(x.dist), val2=x.ele, val3=0), profile)
    eppinfo = dict(version='VERSION_7',
                   header=header,
                   data=data)
    eppdata = epp.epp_file.build(eppinfo)
    return eppdata

if __name__ == "__main__":
    if (len(sys.argv) > 1):
        raster = 500.0
        points = read_gpx(sys.argv[1])
        profile = calculate_profile(points, raster)
        eppdata = build_epp(profile, raster)
        print(eppdata)
    else:
        print("usage: gpx2epp <file>")
        sys.exit(1);
<commit_msg>Simplify height profile calculation<commit_after>#!/usr/bin/env python

"""
convert gpx into epp dist/height profile
"""

from __future__ import print_function
from xml.dom import minidom

import math
import sys

import eppread as epp

EARTH_RADIUS = 6378137.0

class Point:
    lat = None
    lon = None
    ele = None
    dist = None
    total = None
    def __init__(self, lat=None, lon=None, ele=None, dist=None, total=None):
        self.lat = lat
        self.lon = lon
        self.ele = ele
        self.dist = dist
        self.total = total

class Profile:
    dist = None
    ele = None
    def __init__(self, dist=None, ele=None):
        self.dist = dist
        self.ele = ele

# see https://en.wikipedia.org/wiki/Great-circle_distance
# for calculating the distance between two geographical coordinates
def great_circle_distance(p1, p2):
    sdlat = math.sin((p1.lat - p2.lat) * math.pi / 180.0) / 2.0
    sdlon = math.sin((p1.lon - p2.lon) * math.pi / 180.0) / 2.0
    central_angle = 2.0 * math.asin(math.sqrt(sdlat * sdlat
                                              + math.cos(p1.lat * math.pi / 180.0)
                                              * math.cos(p2.lat * math.pi / 180.0)
                                              * sdlon * sdlon))
    return central_angle * EARTH_RADIUS

def read_gpx(filename):
    xml = minidom.parse(filename)
    pts = []
    for trk in xml.getElementsByTagName('trk')[0:1]:
        trkpts = trk.getElementsByTagName('trkpt')
        prev = None
        for trkpt in trkpts:
            p = Point(float(trkpt.getAttribute('lat')),
                      float(trkpt.getAttribute('lon')),
                      float(trkpt.getElementsByTagName('ele')[0].firstChild.nodeValue),
                      0.0, 0.0)
            if prev:
                p.dist = great_circle_distance(prev, p)
                p.total = prev.total + p.dist
            pts.append(p)
            prev = p
    return pts

class State:
    profile = None
    prev = None
    target = None
    raster = None
    def __init__(self, raster, profile=[], target=0.0, prev=None):
        self.raster = raster
        self.profile = profile
        self.prev = prev
        self.target = target

def transform(s, x):
    while x.total > s.target:
        if x.total != s.prev.total:
            section = (s.target - s.prev.total) / (x.total - s.prev.total)
            climb = section * (x.ele - s.prev.ele)
            ele = s.prev.ele + climb
        else:
            ele = (x.ele - s.prev.ele) / 2 + s.prev.ele
        s.profile.append(Profile(s.target, ele))
        s.target += s.raster
    s.prev = x
    return s

def calculate_profile(points, raster):
    state = State(raster=raster)
    reduce(transform, points, state)
    return state.profile

def build_epp(profile, raster):
    header = dict(title='test',
                  description='test',
                  type='DIST_HEIGHT',
                  third='NONE',
                  length=len(profile),
                  graphmin=0,
                  graphmax=500,
                  raster=raster,
                  blr=dict(run=1, lyps=1, bike=1),
                  startheight=0,
                  maxwatt=0,
                  maxpulse=0,
                  maxspeed=0)
    data = map(lambda x : dict(val1=int(x.dist), val2=x.ele, val3=0), profile)
    eppinfo = dict(version='VERSION_7',
                   header=header,
                   data=data)
    eppdata = epp.epp_file.build(eppinfo)
    return eppdata

if __name__ == "__main__":
    if (len(sys.argv) > 1):
        raster = 500.0
        points = read_gpx(sys.argv[1])
        profile = calculate_profile(points, raster)
        eppdata = build_epp(profile, raster)
        print(eppdata)
    else:
        print("usage: gpx2epp <file>")
        sys.exit(1);
<|endoftext|>


<commit_before>{
    "name": "Avalara Avatax Connector",
    "version": "13.0.3.1.0",
    "author": "Fabrice Henrion, Odoo Community Association (OCA)",
    "summary": "Automatic Tax application using the Avalara Avatax Service",
    "license": "AGPL-3",
    "category": "Accounting",
    "depends": ["account", "sale_stock", "base_geolocalize"],
    "data": [
        "security/avalara_salestax_security.xml",
        "security/ir.model.access.csv",
        "data/avalara_salestax_data.xml",
        "data/avalara_salestax_exemptions.xml",
        "wizard/avalara_salestax_ping_view.xml",
        "wizard/avalara_salestax_address_validate_view.xml",
        "views/avalara_salestax_view.xml",
        "views/partner_view.xml",
        "views/product_view.xml",
        "views/account_move_action.xml",
        "views/account_move_view.xml",
        "views/account_tax_view.xml",
        "views/account_fiscal_position_view.xml",
    ],
    "image": ["static/description/avatax_icon.png"],
    "installable": True,
    "application": True,
    "external_dependencies": {"python": ["Avalara"]},
    "development_status": "Beta",
    "maintainers": ["dreispt"],
}
<commit_msg>account_avatax 13.0.3.1.1<commit_after>{
    "name": "Avalara Avatax Connector",
    "version": "13.0.3.1.1",
    "author": "Fabrice Henrion, Odoo Community Association (OCA)",
    "summary": "Automatic Tax application using the Avalara Avatax Service",
    "license": "AGPL-3",
    "category": "Accounting",
    "depends": ["account", "sale_stock", "base_geolocalize"],
    "data": [
        "security/avalara_salestax_security.xml",
        "security/ir.model.access.csv",
        "data/avalara_salestax_data.xml",
        "data/avalara_salestax_exemptions.xml",
        "wizard/avalara_salestax_ping_view.xml",
        "wizard/avalara_salestax_address_validate_view.xml",
        "views/avalara_salestax_view.xml",
        "views/partner_view.xml",
        "views/product_view.xml",
        "views/account_move_action.xml",
        "views/account_move_view.xml",
        "views/account_tax_view.xml",
        "views/account_fiscal_position_view.xml",
    ],
    "image": ["static/description/avatax_icon.png"],
    "installable": True,
    "application": True,
    "external_dependencies": {"python": ["Avalara"]},
    "development_status": "Beta",
    "maintainers": ["dreispt"],
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function, division

import hostlist as modulehostlist
from utility import *
from error import LSFError

import sys
import re
from time import strftime, strptime, mktime
from subprocess import Popen, check_output, PIPE

class Job():
    """representation of a single LSF batch job"""
    strregexps = {
        "Job Name": "Job Name <(.*?)>, ?User",
        "User": "User <(.*?)>[,;]",
        "Project": "Project <(.*?)>[,;]",
        "User Group": "User Group <(.*?)>[,;]",
        "Mail": "Mail <(.*?)>[,;]",
        "Status": "Status <(.*?)>[,;]",
        "Queue": "Queue <(.*?)>[,;]",
        "Command": "Command <(.*?)>, ?(?:Job Description|Share group charged)",
        "Submitted from host": "Submitted from host <(.*?)>[,;]",
        "CWD": "CWD <(.*?)>[,;]",
        "Output File": "Output File <(.*?)>[,;]",
        "Error File": "Error File <(.*?)>[,;]",
        "Requested Resources": "Requested Resources <(.*?) ?>[,;]",
        "Dependency Condition": "Dependency Condition <(.*?)>[,;]",
        "Share group charged": "Share group charged <(.*?)>[,;]",
        "Job Description": "Job Description <(.*?)>",
        "Specified Hosts": "Specified Hosts <(.*?)>[,;]",
        "Execution Home": "Execution Home <(.*?)>[,;]",
        "Execution CWD": "Execution CWD <(.*?)>[,;]",
        "Processors": "(?:Hosts/Processors|\[\d+\] started on) <(.*?)>[,;]",
        "Reserved": "Reserved <\d+> job slots? on host(?:\(s\))? <(.*?)>[,;]",
        "Started on": "Started on <(.*?)>[,;]",
        "Complete": "Completed <(.*?)>",
        "PENDING REASONS": "PENDING REASONS:\n(.*?)\n\n",
        "RUNLIMIT": "RUNLIMIT\s*\n (.*?) min of",
        "MEMLIMIT": "MEMLIMIT\s*\n .*? (\d+ [BKMGT])\s+\n",
        "CORELIMIT": "CORELIMIT\s*\n (\d+ [BKMGT])",
    }
    numregexps = {
        "Job Priority": "Job Priority <(\d+)>,",
        "Processors Requested": ", (\d+) Processors Requested,",
        "nReserved": "Reserved <(\d+)> job slots? on host",
        "CPU time": "The CPU time used is (.*?) seconds.",
        "ptile": "span\[ptile=(\d+)\]",
    }
    tfregexps = {
        "Notify when job ends": "Notify when job (begins/)?ends",
        "Notify when job begins": "Notify when job begins",
        "Exclusive Execution": "Exclusive Execution",
    }
    timeregexp = "([A-Z][a-z]{2} +\d+ \d+:\d+:\d+)"
    timeregexps = {
        "submittime": timeregexp + ": Submitted",
        "starttime": timeregexp + ": (?:\[\d+\] )?[sS]tarted",
        "resourcetime": timeregexp + ": Resource",
        "endtime": timeregexp + ": Done",
        "exittime": timeregexp + ": Exited",
        "completetime": timeregexp + ": Completed",
    }

    def __init__(self, init):
        """Init job from LSF or data"""
        self.initialized = False
        self.initializing = False
        if type(init) in (int, str):
            self.data = {"Job": str(init)}
        elif type(init) == dict:
            if not "Job" in init:
                raise KeyError("No 'Job' in Job init data")
            self.data = init
        else:
            raise TypeError("Job init argument must be int, str or dict")

    def init(self):
        """Read the full job information from LSF"""
        if self.initializing or self.initialized:
            return True
        self.initializing = True
        if self.read():
            self.initialized = True
        self.initializing = False
        return True

    def kill(self):
        """Kill the job"""
        p = Popen(["bkill", str(self["Job"])], stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        out = out.decode()
        err = err.decode()
        if "User permission denied" in err:
            self.initialized = False
            return False
        else:
            return True

    def read(self):
        """read information on the job from LSF.
        Return true on successful reading of the job.
        """
        if not self["Job"]:
            return False
        self.data = {"Job": self["Job"]}
        p = Popen(["bjobs", "-l", self["Job"]], stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        out = str(out.decode(errors="ignore"))
        err = str(err.decode(errors="ignore"))
        if err and not out:
            return False
        if 78 * "-" in out:
            return False
        outjoin = out.replace("\n                     ", "")
        #data retrieval
        for key, regexp in Job.strregexps.iteritems():
            match = re.search(regexp, outjoin, re.DOTALL)
            if match:
                self[key] = match.groups()[0]
        for key, regexp in Job.numregexps.iteritems():
            match = re.search(regexp, outjoin)
            if match:
                try:
                    self[key] = int(match.groups()[0])
                except:
                    self[key] = float(match.groups()[0])
        for key, regexp in Job.tfregexps.iteritems():
            self[key] = not (re.search(regexp, outjoin) is None)
        for key, regexp in Job.timeregexps.iteritems():
            match = re.search(regexp, outjoin)
            if match:
                tstr = strftime("%Y ") + match.groups()[0]
                t = strptime(tstr, "%Y %b %d %H:%M:%S")
                self[key] = mktime(t)
        #processing
        if "User" in self:
            if "Mail" in self:
                self["Userstr"] = "{User} <{Mail}>".format(**self)
            else:
                self["Userstr"] = "{User}".format(**self)
        if "Command" in self:
            for _ in xrange(3):
                self["Command"] = self["Command"].replace("; ", ";;")
            self["Command"] = self["Command"].replace(";;;; ", ";    ")
            self["Command"] = self["Command"].replace(";", "\n")
            self["Command"] = re.sub("for \(\((.*?)\n\n(.*?)\n\n(.*?)\)\)",
                                     "for ((\\1; \\2; \\3))", self["Command"])
        if "Started on" in self:
            self["Processors"] = self["Started on"]
            del self["Started on"]
        if "Processors" in self and isinstance(self['Processors'], str):
            procs = {}
            self["Processors"] = self["Processors"].replace("><", "> <")
            for proc in self["Processors"].split("> <"):
                proc = proc.split("*")
                if len(proc) == 1:
                    procs[proc[0]] = 1
                else:
                    procs[proc[1]] = int(proc[0])
            self["Processors"] = procs
            strs = (str(c).rjust(3) + "*" + p for p, c in procs.iteritems())
            self["Processorsstr"] = " ".join(strs)
            self["Hosts"] = modulehostlist.Hostlist([p for p in procs])
            hgs = {}
            for host in self["Hosts"]:
                hg = host["Hostgroup"]
                if not hg in hgs:
                    hgs[hg] = 0
                hgs[hg] += procs[host["HOST"]]
            self["Hostgroups"] = hgs
            strs = (str(c).rjust(3) + "*" + p + "*"
                    for p, c in hgs.iteritems())
            self["Hostgroupsstr"] = " ".join(strs)
        if not "Processors Requested" in self:
            self["Processors Requested"] = 1
        self["Nodes Requested"] = self["Processors Requested"]
        if "ptile" in self:
            self["Nodes Requested"] //= self["ptile"]
        if "nReserved" in self:
            if self["nReserved"] == 1:
                self["Reserved"] = {self["Reserved"]: 1}
            else:
                procs = {}
                for proc in self["Reserved"].split("> <"):
                    proc = proc.split("*")
                    procs[proc[1]] = int(proc[0])
                self["Reserved"] = procs
        if "PENDING REASONS" in self:
            reasons = self["PENDING REASONS"]
            match = re.findall(" (.*?): (\d+) hosts?;", str(reasons))
            d = {r: int(n) for r, n in match}
            match = re.findall("^ ([^:]*?);", str(reasons))
            d.update((m, True) for m in match)
            self["PENDING REASONS"] = d
        if "RUNLIMIT" in self:
            self["RUNLIMIT"] = int(60 * float(self["RUNLIMIT"]))
        if "MEMLIMIT" in self:
            groups = re.search("(.*) ([BKMGT])", self["MEMLIMIT"]).groups()
            units = {"B": 0, "K": 1, "M": 2, "G": 3, "T": 4}
            self["MEMLIMIT"] = int(groups[0]) * 1024 ** units[groups[1]]
        if "CORELIMIT" in self:
            groups = re.search("(.*) ([BKMGT])", self["CORELIMIT"]).groups()
            units = {"B": 0, "K": 1, "M": 2, "G": 3, "T": 4}
            self["CORELIMIT"] = int(groups[0]) * 1024 ** units[groups[1]]
        return True

    def __str__(self):
        """long format job details"""
        self.init()
        h, w = map(int, check_output(("stty", "size")).split())
        wk = 25
        wv = w - wk
        data = {}
        for k in self.data.keys():
            val = self[k]
            if k in Job.timeregexps:
                val = format_time(val)
            elif k in ("RUNLIMIT", "CPU time"):
                val = format_duration(val)
            elif k == "MEMLIMIT":
                val = format_mem(val)
            elif type(val) is dict:
                val = "\n".join("{}\t{}".format(dv, dk)
                                for dk, dv in val.iteritems())
            elif isinstance(val, list):
                val = "\n".join(str(v) for v in val)
            elif k in ("Processorsstr", "Userstr"):
                continue
            strs = (s[i:i + wv]
                    for s in str(val).splitlines()
                    for i in range(0, len(s) + 1, wv))
            s = ("\n" + wk * " ").join(strs)
            data[k] = k.ljust(wk) + s + "\n"
        result = ""
        for k in ("Job", "Job Name", "User", "Status", "Command", "submittime",
                  "starttime", "endtime", "Pending Reasons", "RUNLIMIT",
                  "MEMLIMIT", "Processors Requested", "Processors", "ptile",
                  "Exclusive Execution", "Requested Resources", "Reserved",
                  "Job Description"):
            if k in data:
                result += data[k]
                del data[k]
        for v in data.itervalues():
            result += v
        return result

    def __getitem__(self, key):
        """Access job attributes"""
        if key in self.data:
            return self.data[key]
        self.init()
        return self.data.__getitem__(key)

    def __setitem__(self, key, value):
        """Access job attributes"""
        return self.data.__setitem__(key, value)

    def __contains__(self, key):
        """Access job attributes"""
        if key in self.data:
            return True
        self.init()
        return self.data.__contains__(key)

    def keys(self):
        """Access job attributes"""
        self.init()
        return self.data.keys()

    def __delitem__(self, key):
        """Access job attributes"""
        return self.data.__delitem__(key)

def submit(data, shell=False):
    """Submit a job to LSF"""
    if not "Command" in data:
        print("no command given", file=sys.stderr)
        return False
    if "Job Name" in data:
        data["-J"] = data["Job Name"]
    if "Output File" in data:
        data["-o"] = data["Output File"]
    if not "-o" in data and "-J" in data:
        data["-o"] = data["-J"] + ".%J.out"
    if "Error File" in data:
        data["-e"] = data["Error File"]
    if "Mail" in data:
        data["-u"] = data["Mail"]
    if "Notify when job ends" in data and data["Notify when job ends"]:
        data["-N"] = True
    if "Notify when job begins" in data and data["Notify when job begins"]:
        data["-B"] = True
    if "Exclusive Execution" in data and data["Exclusive Execution"]:
        data["-x"] = True
    if "Processors Requested" in data:
        data["-n"] = str(int(data["Processors Requested"]))
    if "RUNLIMIT" in data:
        data["-W"] = str(data["RUNLIMIT"] // 60)
    if "MEMLIMIT" in data:
        data["-M"] = str(data["MEMLIMIT"] // 1024)
    if "CORELIMIT" in data:
        data["-C"] = str(data["CORELIMIT"] // 1024)
    if "Project" in data:
        data["-P"] = data["Project"]
    if "Resource Request" in data:
        if "-R" in data:
            data["-R"] = [data["-R"]]
        else:
            data["-R"] = []
        data["-R"].append(data["Resource Request"])
    if "Dependency Condition" in data:
        data["-w"] = data["Dependency Condition"]
    cmd = ["bsub"]
    for key, value in data.iteritems():
        if key[0] != "-":
            continue
        val = data[key]
        if type(val) is bool:
            cmd += [key]
        elif type(val) is str:
            cmd += [key, val]
        elif type(val) is list:
            for v in val:
                cmd += [key, str(v)]
    if shell:
        script = '#!/bin/bash -l\n'
    else:
        script = ''
    script += data["Command"]
    p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
    out, err = p.communicate(script)
    out = out.decode()
    err = err.decode()
    match = re.search("Job <(.*?)> is submitted", out)
    if match:
        return Job(str(match.groups()[0]))
    else:
        match = re.search("Error: (.*)\n", err)
        if match:
            raise LSFError(1, match.groups()[0])
        else:
            raise LSFError(1, err)
<commit_msg>correct splitting of nodelists<commit_after>#!/usr/bin/env python
from __future__ import print_function, division

import hostlist as modulehostlist
from utility import *
from error import LSFError

import sys
import re
from time import strftime, strptime, mktime
from subprocess import Popen, check_output, PIPE

class Job():
    """representation of a single LSF batch job"""
    strregexps = {
        "Job Name": "Job Name <(.*?)>, ?User",
        "User": "User <(.*?)>[,;]",
        "Project": "Project <(.*?)>[,;]",
        "User Group": "User Group <(.*?)>[,;]",
        "Mail": "Mail <(.*?)>[,;]",
        "Status": "Status <(.*?)>[,;]",
        "Queue": "Queue <(.*?)>[,;]",
        "Command": "Command <(.*?)>, ?(?:Job Description|Share group charged)",
        "Submitted from host": "Submitted from host <(.*?)>[,;]",
        "CWD": "CWD <(.*?)>[,;]",
        "Output File": "Output File <(.*?)>[,;]",
        "Error File": "Error File <(.*?)>[,;]",
        "Requested Resources": "Requested Resources <(.*?) ?>[,;]",
        "Dependency Condition": "Dependency Condition <(.*?)>[,;]",
        "Share group charged": "Share group charged <(.*?)>[,;]",
        "Job Description": "Job Description <(.*?)>",
        "Specified Hosts": "Specified Hosts <(.*?)>[,;]",
        "Execution Home": "Execution Home <(.*?)>[,;]",
        "Execution CWD": "Execution CWD <(.*?)>[,;]",
        "Processors": "(?:Hosts/Processors|\[\d+\] started on) <(.*?)>[,;]",
        "Reserved": "Reserved <\d+> job slots? on host(?:\(s\))? <(.*?)>[,;]",
        "Started on": "Started on <(.*?)>[,;]",
        "Complete": "Completed <(.*?)>",
        "PENDING REASONS": "PENDING REASONS:\n(.*?)\n\n",
        "RUNLIMIT": "RUNLIMIT\s*\n (.*?) min of",
        "MEMLIMIT": "MEMLIMIT\s*\n .*? (\d+ [BKMGT])\s+\n",
        "CORELIMIT": "CORELIMIT\s*\n (\d+ [BKMGT])",
    }
    numregexps = {
        "Job Priority": "Job Priority <(\d+)>,",
        "Processors Requested": ", (\d+) Processors Requested,",
        "nReserved": "Reserved <(\d+)> job slots? on host",
        "CPU time": "The CPU time used is (.*?) seconds.",
        "ptile": "span\[ptile=(\d+)\]",
    }
    tfregexps = {
        "Notify when job ends": "Notify when job (begins/)?ends",
        "Notify when job begins": "Notify when job begins",
        "Exclusive Execution": "Exclusive Execution",
    }
    timeregexp = "([A-Z][a-z]{2} +\d+ \d+:\d+:\d+)"
    timeregexps = {
        "submittime": timeregexp + ": Submitted",
        "starttime": timeregexp + ": (?:\[\d+\] )?[sS]tarted",
        "resourcetime": timeregexp + ": Resource",
        "endtime": timeregexp + ": Done",
        "exittime": timeregexp + ": Exited",
        "completetime": timeregexp + ": Completed",
    }

    def __init__(self, init):
        """Init job from LSF or data"""
        self.initialized = False
        self.initializing = False
        if type(init) in (int, str):
            self.data = {"Job": str(init)}
        elif type(init) == dict:
            if not "Job" in init:
                raise KeyError("No 'Job' in Job init data")
            self.data = init
        else:
            raise TypeError("Job init argument must be int, str or dict")

    def init(self):
        """Read the full job information from LSF"""
        if self.initializing or self.initialized:
            return True
        self.initializing = True
        if self.read():
            self.initialized = True
        self.initializing = False
        return True

    def kill(self):
        """Kill the job"""
        p = Popen(["bkill", str(self["Job"])], stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        out = out.decode()
        err = err.decode()
        if "User permission denied" in err:
            self.initialized = False
            return False
        else:
            return True

    def read(self):
        """read information on the job from LSF.
        Return true on successful reading of the job.
        """
        if not self["Job"]:
            return False
        self.data = {"Job": self["Job"]}
        p = Popen(["bjobs", "-l", self["Job"]], stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        out = str(out.decode(errors="ignore"))
        err = str(err.decode(errors="ignore"))
        if err and not out:
            return False
        if 78 * "-" in out:
            return False
        outjoin = out.replace("\n                     ", "")
        #data retrieval
        for key, regexp in Job.strregexps.iteritems():
            match = re.search(regexp, outjoin, re.DOTALL)
            if match:
                self[key] = match.groups()[0]
        for key, regexp in Job.numregexps.iteritems():
            match = re.search(regexp, outjoin)
            if match:
                try:
                    self[key] = int(match.groups()[0])
                except:
                    self[key] = float(match.groups()[0])
        for key, regexp in Job.tfregexps.iteritems():
            self[key] = not (re.search(regexp, outjoin) is None)
        for key, regexp in Job.timeregexps.iteritems():
            match = re.search(regexp, outjoin)
            if match:
                tstr = strftime("%Y ") + match.groups()[0]
                t = strptime(tstr, "%Y %b %d %H:%M:%S")
                self[key] = mktime(t)
        #processing
        if "User" in self:
            if "Mail" in self:
                self["Userstr"] = "{User} <{Mail}>".format(**self)
            else:
                self["Userstr"] = "{User}".format(**self)
        if "Command" in self:
            for _ in xrange(3):
                self["Command"] = self["Command"].replace("; ", ";;")
            self["Command"] = self["Command"].replace(";;;; ", ";    ")
            self["Command"] = self["Command"].replace(";", "\n")
            self["Command"] = re.sub("for \(\((.*?)\n\n(.*?)\n\n(.*?)\)\)",
                                     "for ((\\1; \\2; \\3))", self["Command"])
        if "Started on" in self:
            self["Processors"] = self["Started on"]
            del self["Started on"]
        if "Processors" in self and isinstance(self['Processors'], str):
            procs = {}
            self["Processors"] = self["Processors"].replace("><", "> <")
            for proc in re.split("> ?<", self["Processors"]):
                proc = proc.split("*")
                if len(proc) == 1:
                    procs[proc[0]] = 1
                else:
                    procs[proc[1]] = int(proc[0])
            self["Processors"] = procs
            strs = (str(c).rjust(3) + "*" + p for p, c in procs.iteritems())
            self["Processorsstr"] = " ".join(strs)
            self["Hosts"] = modulehostlist.Hostlist([p for p in procs])
            hgs = {}
            for host in self["Hosts"]:
                hg = host["Hostgroup"]
                if not hg in hgs:
                    hgs[hg] = 0
                hgs[hg] += procs[host["HOST"]]
            self["Hostgroups"] = hgs
            strs = (str(c).rjust(3) + "*" + p + "*"
                    for p, c in hgs.iteritems())
            self["Hostgroupsstr"] = " ".join(strs)
        if not "Processors Requested" in self:
            self["Processors Requested"] = 1
        self["Nodes Requested"] = self["Processors Requested"]
        if "ptile" in self:
            self["Nodes Requested"] //= self["ptile"]
        if "nReserved" in self:
            if self["nReserved"] == 1:
                self["Reserved"] = {self["Reserved"]: 1}
            else:
                procs = {}
                for proc in re.split("> ?<", self["Reserved"]):
                    proc = proc.split("*")
                    procs[proc[1]] = int(proc[0])
                self["Reserved"] = procs
        if "PENDING REASONS" in self:
            reasons = self["PENDING REASONS"]
            match = re.findall(" (.*?): (\d+) hosts?;", str(reasons))
            d = {r: int(n) for r, n in match}
            match = re.findall("^ ([^:]*?);", str(reasons))
            d.update((m, True) for m in match)
            self["PENDING REASONS"] = d
        if "RUNLIMIT" in self:
            self["RUNLIMIT"] = int(60 * float(self["RUNLIMIT"]))
        if "MEMLIMIT" in self:
            groups = re.search("(.*) ([BKMGT])", self["MEMLIMIT"]).groups()
            units = {"B": 0, "K": 1, "M": 2, "G": 3, "T": 4}
            self["MEMLIMIT"] = int(groups[0]) * 1024 ** units[groups[1]]
        if "CORELIMIT" in self:
            groups = re.search("(.*) ([BKMGT])", self["CORELIMIT"]).groups()
            units = {"B": 0, "K": 1, "M": 2, "G": 3, "T": 4}
            self["CORELIMIT"] = int(groups[0]) * 1024 ** units[groups[1]]
        return True

    def __str__(self):
        """long format job details"""
        self.init()
        h, w = map(int, check_output(("stty", "size")).split())
        wk = 25
        wv = w - wk
        data = {}
        for k in self.data.keys():
            val = self[k]
            if k in Job.timeregexps:
                val = format_time(val)
            elif k in ("RUNLIMIT", "CPU time"):
                val = format_duration(val)
            elif k == "MEMLIMIT":
                val = format_mem(val)
            elif type(val) is dict:
                val = "\n".join("{}\t{}".format(dv, dk)
                                for dk, dv in val.iteritems())
            elif isinstance(val, list):
                val = "\n".join(str(v) for v in val)
            elif k in ("Processorsstr", "Userstr"):
                continue
            strs = (s[i:i + wv]
                    for s in str(val).splitlines()
                    for i in range(0, len(s) + 1, wv))
            s = ("\n" + wk * " ").join(strs)
            data[k] = k.ljust(wk) + s + "\n"
        result = ""
        for k in ("Job", "Job Name", "User", "Status", "Command", "submittime",
                  "starttime", "endtime", "Pending Reasons", "RUNLIMIT",
                  "MEMLIMIT", "Processors Requested", "Processors", "ptile",
                  "Exclusive Execution", "Requested Resources", "Reserved",
                  "Job Description"):
            if k in data:
                result += data[k]
                del data[k]
        for v in data.itervalues():
            result += v
        return result

    def __getitem__(self, key):
        """Access job attributes"""
        if key in self.data:
            return self.data[key]
        self.init()
        return self.data.__getitem__(key)

    def __setitem__(self, key, value):
        """Access job attributes"""
        return self.data.__setitem__(key, value)

    def __contains__(self, key):
        """Access job attributes"""
        if key in self.data:
            return True
        self.init()
        return self.data.__contains__(key)

    def keys(self):
        """Access job attributes"""
        self.init()
        return self.data.keys()

    def __delitem__(self, key):
        """Access job attributes"""
        return self.data.__delitem__(key)

def submit(data, shell=False):
    """Submit a job to LSF"""
    if not "Command" in data:
        print("no command given", file=sys.stderr)
        return False
    if "Job Name" in data:
        data["-J"] = data["Job Name"]
    if "Output File" in data:
        data["-o"] = data["Output File"]
    if not "-o" in data and "-J" in data:
        data["-o"] = data["-J"] + ".%J.out"
    if "Error File" in data:
        data["-e"] = data["Error File"]
    if "Mail" in data:
        data["-u"] = data["Mail"]
    if "Notify when job ends" in data and data["Notify when job ends"]:
        data["-N"] = True
    if "Notify when job begins" in data and data["Notify when job begins"]:
        data["-B"] = True
    if "Exclusive Execution" in data and data["Exclusive Execution"]:
        data["-x"] = True
    if "Processors Requested" in data:
        data["-n"] = str(int(data["Processors Requested"]))
    if "RUNLIMIT" in data:
        data["-W"] = str(data["RUNLIMIT"] // 60)
    if "MEMLIMIT" in data:
        data["-M"] = str(data["MEMLIMIT"] // 1024)
    if "CORELIMIT" in data:
        data["-C"] = str(data["CORELIMIT"] // 1024)
    if "Project" in data:
        data["-P"] = data["Project"]
    if "Resource Request" in data:
        if "-R" in data:
            data["-R"] = [data["-R"]]
        else:
            data["-R"] = []
        data["-R"].append(data["Resource Request"])
    if "Dependency Condition" in data:
        data["-w"] = data["Dependency Condition"]
    cmd = ["bsub"]
    for key, value in data.iteritems():
        if key[0] != "-":
            continue
        val = data[key]
        if type(val) is bool:
            cmd += [key]
        elif type(val) is str:
            cmd += [key, val]
        elif type(val) is list:
            for v in val:
                cmd += [key, str(v)]
    if shell:
        script = '#!/bin/bash -l\n'
    else:
        script = ''
    script += data["Command"]
    p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
    out, err = p.communicate(script)
    out = out.decode()
    err = err.decode()
    match = re.search("Job <(.*?)> is submitted", out)
    if match:
        return Job(str(match.groups()[0]))
    else:
        match = re.search("Error: (.*)\n", err)
        if match:
            raise LSFError(1, match.groups()[0])
        else:
            raise LSFError(1, err)
<|endoftext|>


<commit_before>""" greeker.py
    scrambles nouns in an XML document to produce a specimine for layout testing
"""
import sys
import nltk
from lxml import etree
import re
import inflect
p = inflect.engine()
import random
from string import maketrans

def main(argv=None):
    if argv is None:
        argv = sys.argv
    # TODO: make this real the argv
    # TODO: allow different scrambler options via sys.argv
    scrambler = consonant_vowel_sensitive_random_word
    greekize_file("sample.xml", "out.xml", scrambler)

def greekize_file(infile, outfile, scrambler):
    """greekize the infile to outfile"""
    file = etree.parse(infile)
    text_nodes = file.xpath("//text()")
    # pull sample text from the text nodes
    text = ''.join(text_nodes)
    greek_text = greekize_text(text, scrambler).split()
    # pass an array because we are recursivly .pop(0) the new words from it
    update_xml(file.getroot(), greek_text)
    file.write(outfile, pretty_print=True)

def greekize_text(text, scrambler):
    """takes a string of text as input; changes nouns to pig latin"""

    # array of all the sentences (is this necessary?)
    sentences = nltk.sent_tokenize(text)
    tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]

    # part of speech tagging
    tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]

    # pig latinize all nouns in the text
    for sentence in tagged_sentences:
        for tagged_word in sentence:
            # skip "(", not sure what other characters nltk will put in the parse tree
            if tagged_word[0] in ["(", ")", "[", "]"]:
                continue
            # replace plural nouns with pig latin
            if tagged_word[1] == 'NNS':
                singular = p.singular_noun(tagged_word[0]);
                scrambled = p.plural_noun(scrambler(singular))
                text = re.sub(tagged_word[0]+"(\W)",scrambled+"\\1",text)
            # replace proper nouns and nouns
            if tagged_word[1] in ['NN', 'NNP']:
                text = re.sub(tagged_word[0]+"(\W)",scrambler(tagged_word[0])+"\\1",text)
    return text

def update_xml(node, greek_text):
    """update the xml document with the new words"""
    # TODO; construction of the new_ texts need to retain whitespace
    # pop some new words off the greek_text array
    if node.text:
        node.text = update_text(node.text, greek_text)
    # this gets child elements... not all DOM nodes
    for desc in node.getchildren():
        # recursive call
        update_xml(desc, greek_text)
    # ElementTree supports mixed content via .tail...
    if node.tail:
        node.tail = update_text(node.tail, greek_text)

def update_text(text_from_node,greek_text):
    """create new string for element .text or .tail"""
    # if I don't have any words; just copy the whitespace
    if not(re.search("\w", text_from_node)):
        return text_from_node
    # otherwise; pop some words off the stack
    new_text = ''
    for word in text_from_node.split():
        new_text += greek_text.pop(0)
        new_text += " "
    return new_text

def consonant_vowel_sensitive_random_word(word):
    """scramble word, keeping vowles in the same place"""
    random.seed(word)
    vowles = "aeiouy"
    consonants = "bcdfghjklmnpqrstvwxz"
    new_vowles = list(vowles)
    new_consonants = list(consonants)
    # shuffles in place
    random.shuffle(new_vowles)
    random.shuffle(new_consonants)
    vowles = vowles + vowles.upper()
    consonants = consonants + consonants.upper()
    trans_to = ''.join(new_vowles) + ''.join(new_vowles).upper()
    trans_to += ''.join(new_consonants) + ''.join(new_consonants).upper()
    randomize = maketrans(vowles + consonants, trans_to)
    return word.translate(randomize)

def pig_latinize(noun):
    """ convert one word into pig latin """
    # http://pythonicprose.blogspot.com/2009/09/python-pig-latin-generator.html
    word = noun.lower()
    m = len(word)
    vowels = "a", "e", "i", "o", "u", "y"
    # short words are not converted
    if m<3 or word=="the":
        return word
    else:
        # m==0 when the word starts with a vowel
        for i in vowels:
            if word.find(i) < m and word.find(i) != -1:
                m = word.find(i)
        if m==0:
            piglatin = word+"way"
        else:
            piglatin = word[m:]+word[:m]+"ay"
        # give the return the same title case as the input
        if noun.istitle():
            piglatin = piglatin.lower().title()
        return piglatin

# main() idiom for importing into REPL for debugging
if __name__ == "__main__":
    sys.exit(main())
<commit_msg>added command line arguments with argparse<commit_after>#!/usr/bin/env python
""" greeker.py
    scrambles nouns in an XML document to produce a specimine for layout testing
"""
import sys
import argparse
import nltk
from lxml import etree
import re
import inflect
p = inflect.engine()
import random
from string import maketrans
import argparse

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(description='Create greeked text for XML testing.')

    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
                     help='input XML (or standard input)',
                     default=sys.stdin)
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
                     help='output greeked XML (or standard out)',
                     default=sys.stdout)
    parser.add_argument('--piglatin', action='store_const',
                   const='pig',
                   help='replace using pig latin rather than more random "words"')

    args = parser.parse_args()

    if args.piglatin:
        scrambler = pig_latinize
    else:
        scrambler = consonant_vowel_sensitive_random_word

    greekize_file(args.infile, args.outfile, scrambler)

def greekize_file(infile, outfile, scrambler):
    """greekize the infile to outfile"""
    file = etree.parse(infile)
    text_nodes = file.xpath("//text()")
    # pull sample text from the text nodes
    text = ''.join(text_nodes)
    greek_text = greekize_text(text, scrambler).split()
    # pass an array because we are recursivly .pop(0) the new words from it
    update_xml(file.getroot(), greek_text)
    file.write(outfile, pretty_print=True)

def greekize_text(text, scrambler):
    """takes a string of text as input; changes nouns to pig latin"""

    # array of all the sentences (is this necessary?)
    sentences = nltk.sent_tokenize(text)
    tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]

    # part of speech tagging
    tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]

    # pig latinize all nouns in the text
    for sentence in tagged_sentences:
        for tagged_word in sentence:
            # skip "(", not sure what other characters nltk will put in the parse tree
            if tagged_word[0] in ["(", ")", "[", "]"]:
                continue
            # replace plural nouns with pig latin
            if tagged_word[1] == 'NNS':
                singular = p.singular_noun(tagged_word[0]);
                scrambled = p.plural_noun(scrambler(singular))
                text = re.sub(tagged_word[0]+"(\W)",scrambled+"\\1",text)
            # replace proper nouns and nouns
            if tagged_word[1] in ['NN', 'NNP']:
                text = re.sub(tagged_word[0]+"(\W)",scrambler(tagged_word[0])+"\\1",text)
    return text

def update_xml(node, greek_text):
    """update the xml document with the new words"""
    # TODO; construction of the new_ texts need to retain whitespace
    # pop some new words off the greek_text array
    if node.text:
        node.text = update_text(node.text, greek_text)
    # this gets child elements... not all DOM nodes
    for desc in node.getchildren():
        # recursive call
        update_xml(desc, greek_text)
    # ElementTree supports mixed content via .tail...
    if node.tail:
        node.tail = update_text(node.tail, greek_text)

def update_text(text_from_node,greek_text):
    """create new string for element .text or .tail"""
    # if I don't have any words; just copy the whitespace
    if not(re.search("\w", text_from_node)):
        return text_from_node
    # otherwise; pop some words off the stack
    new_text = ''
    for word in text_from_node.split():
        new_text += greek_text.pop(0)
        new_text += " "
    return new_text

def consonant_vowel_sensitive_random_word(word):
    """scramble word, keeping vowles in the same place"""
    random.seed(word)
    vowles = "aeiouy"
    consonants = "bcdfghjklmnpqrstvwxz"
    new_vowles = list(vowles)
    new_consonants = list(consonants)
    # shuffles in place
    random.shuffle(new_vowles)
    random.shuffle(new_consonants)
    vowles = vowles + vowles.upper()
    consonants = consonants + consonants.upper()
    trans_to = ''.join(new_vowles) + ''.join(new_vowles).upper()
    trans_to += ''.join(new_consonants) + ''.join(new_consonants).upper()
    randomize = maketrans(vowles + consonants, trans_to)
    return word.translate(randomize)

def pig_latinize(noun):
    """ convert one word into pig latin """
    # http://pythonicprose.blogspot.com/2009/09/python-pig-latin-generator.html
    word = noun.lower()
    m = len(word)
    vowels = "a", "e", "i", "o", "u", "y"
    # short words are not converted
    if m<3 or word=="the":
        return word
    else:
        # m==0 when the word starts with a vowel
        for i in vowels:
            if word.find(i) < m and word.find(i) != -1:
                m = word.find(i)
        if m==0:
            piglatin = word+"way"
        else:
            piglatin = word[m:]+word[:m]+"ay"
        # give the return the same title case as the input
        if noun.istitle():
            piglatin = piglatin.lower().title()
        return piglatin

# main() idiom for importing into REPL for debugging
if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
    xhtml2odt - XHTML to ODT XML transformation.
    Copyright (C) 2009-2010 Aurelien Bompard

This script can convert a wiki page to the OpenDocument Text (ODT) format,
standardized as ISO/IEC 26300:2006, and the native format of office suites such
as OpenOffice.org, KOffice, and others.

It uses a template ODT file which will be filled with the converted content of
the exported Wiki page.

Based on the work on docbook2odt, by Roman Fordinal
http://open.comsultia.com/docbook2odf/

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
"""

import tempfile
import shutil
import re
import os
import sys
import zipfile
import urllib2
import urlparse
from glob import glob
from StringIO import StringIO
from optparse import OptionParser

import tidy
from lxml import etree
from PIL import Image

#pylint#: disable-msg=C0301,C0111

INSTALL_PATH = "."

INCH_TO_CM = 2.54
CHARSET = "utf-8"

class ODTExportError(Exception): pass

class HTMLFile(object):

    def __init__(self, options):
        self.options = options
        self.html = ""

    def read(self):
        in_file = open(self.options.input)
        self.html = in_file.read()
        in_file.close()
        self.cleanup()
        if self.options.htmlid:
            self.select_id()

    def cleanup(self):
        tidy_options = dict(output_xhtml=1, add_xml_decl=1, indent=1,
                            tidy_mark=0, #input_encoding=str(self.charset),
                            output_encoding='utf8', doctype='auto',
                            wrap=0, char_encoding='utf8')
        self.html = str(tidy.parseString(self.html, **tidy_options))
        # Replace nbsp with entity
        # http://www.mail-archive.com/analog-help@lists.meer.net/msg03670.html
        self.html = self.html.replace(" ", " ") \
                             .replace("\xa0", " ")
        # Tidy creates newlines after <pre> (by indenting)
        self.html = re.sub('<pre([^>]*)>\n', '<pre\\1>', self.html)

    def select_id(self):
        html_tree = etree.fromstring(self.html)
        selected = html_tree.xpath("//*[@id='%s']" % self.options.htmlid)
        self.html = etree.tostring(selected[0], method="html")

class ODTFile(object):
    """Handles the conversion and production of an ODT file"""

    def __init__(self, options):
        self.options = options
        self.template_dirs = []
        if options.tpldir:
            self.template_dirs.append(options.tpldir)
        self.template_dirs.append(
            os.path.join(INSTALL_PATH, "styles")
        )
        self.xml = {
            "content": "",
            "styles": "",
        }
        self.tmpdir = tempfile.mkdtemp(prefix="xhtml2odt-")

    def open(self):
        self.zfile = zipfile.ZipFile(self.options.template, "r")
        for name in self.zfile.namelist():
            fname = os.path.join(self.tmpdir, name)
            if not os.path.exists(os.path.dirname(fname)):
                os.makedirs(os.path.dirname(fname))
            if name[-1] == "/":
                if not os.path.exists(fname):
                    os.mkdir(fname)
                continue
            fname_h = open(fname, "w")
            fname_h.write(self.zfile.read(name))
            fname_h.close()
        for xmlfile in self.xml:
            self.xml[xmlfile] = self.zfile.read("%s.xml" % xmlfile)

    def import_xhtml(self, xhtml):
        odt = self.xhtml_to_odt(xhtml)
        self.insert_content(odt)
        self.add_styles()

    def xhtml_to_odt(self, xhtml):
        xsl_dir = os.path.join(INSTALL_PATH, 'xsl')
        xslt_doc = etree.parse(os.path.join(xsl_dir, "xhtml2odt.xsl"))
        transform = etree.XSLT(xslt_doc)
        xhtml = self.handle_images(xhtml)
        xhtml = self.handle_links(xhtml)
        xhtml = etree.fromstring(xhtml) # must be valid xml at this point
        params = {
            "url": "/",
            "heading_minus_level": str(self.options.top_header_level - 1),
        }
        if self.options.verbose:
            params["debug"] = "1"
        if self.options.img_width:
            params["img_default_width"] = etree.XSLT.strparam(
                                            self.options.img_width)
        if self.options.img_height:
            params["img_default_height"] = etree.XSLT.strparam(
                                            self.options.img_height)
        odt = transform(xhtml, **params)
        return str(odt).replace('<?xml version="1.0" encoding="utf-8"?>','')

    def handle_images(self, xhtml):
        # Handle local images
        xhtml = re.sub('<img [^>]*src="([^"]+)"',
                      self.handle_local_img, xhtml)
        # Handle remote images
        if self.options.with_network:
            xhtml = re.sub('<img [^>]*src="(https?://[^"]+)"',
                          self.handle_remote_img, xhtml)
        return xhtml

    def handle_local_img(self, img_mo):
        log("handling local image: %s" % img_mo.group(1), self.options.verbose)
        src = img_mo.group(1)
        if src.count("://") and not src.startswith("file://"):
            # This is an absolute link, don't touch it
            return img_mo.group()
        if src.startswith("file://"):
            filename = src[7:]
        elif src.startswith("/"):
            filename = src
        else: # relative link
            filename = os.path.join(os.path.dirname(self.options.input), src)
        if os.path.exists(filename):
            return self.handle_img(img_mo.group(), src, filename)
        if src.startswith("file://") or not self.options.url:
            # There's nothing we can do here
            return img_mo.group()
        newsrc = urlparse.urljoin(self.options.url, os.path.normpath(src))
        if not self.options.with_network:
            # Don't download it, just update the URL
            return img_mo.group().replace(src, newsrc)
        try:
            tmpfile = self.download_img(newsrc)
        except (urllib2.HTTPError, urllib2.URLError):
            log("Failed getting %s" % newsrc, self.options.verbose)
            return img_mo.group()
        ret = self.handle_img(img_mo.group(), src, tmpfile)
        os.remove(tmpfile)
        return ret

    def handle_remote_img(self, img_mo):
        log('handling remote image: %s' % img_mo.group(), self.options.verbose)
        src = img_mo.group(1)
        try:
            tmpfile = self.download_img(src)
        except (urllib2.HTTPError, urllib2.URLError):
            return img_mo.group()
        ret = self.handle_img(img_mo.group(), src, tmpfile)
        os.remove(tmpfile)
        return ret

    def download_img(self, src):
        """
        Download the image to a temporary location
        """
        log('Downloading image: %s' % src, self.options.verbose)
        # TODO: proxy support
        remoteimg = urllib2.urlopen(src)
        tmpimg_fd, tmpfile = tempfile.mkstemp()
        tmpimg = os.fdopen(tmpimg_fd, 'w')
        tmpimg.write(remoteimg.read())
        tmpimg.close()
        remoteimg.close()
        return tmpfile

    def handle_img(self, full_tag, src, filename):
        log('Importing image: %s' % filename, self.options.verbose)
        if not os.path.exists(filename):
            raise ODTExportError('Image "%s" is not readable or does not exist'
                                 % filename)
        # TODO: generate a filename (with tempfile.mkstemp) to avoid weird
        # filenames. Maybe use img.format for the extension
        if not os.path.exists(os.path.join(self.tmpdir, "Pictures")):
            os.mkdir(os.path.join(self.tmpdir, "Pictures"))
        shutil.copy(filename, os.path.join(self.tmpdir, "Pictures",
                                           os.path.basename(filename)))
        newsrc = "Pictures/%s" % os.path.basename(filename)
        try:
            img = Image.open(filename)
        except IOError:
            log('Failed to identify image: %s' % filename,
                self.options.verbose)
        else:
            width, height = img.size
            log('Detected size: %spx x %spx' % (width, height),
                self.options.verbose)
            width = width / float(self.options.img_dpi) * INCH_TO_CM
            height = height / float(self.options.img_dpi) * INCH_TO_CM
            newsrc += '" width="%scm" height="%scm' % (width, height)
        return full_tag.replace(src, newsrc)

    def handle_links(self, xhtml):
        """Turn relative links into absolute links"""
        # Handle local images
        xhtml = re.sub('<a [^>]*href="([^"]+)"',
                      self.handle_relative_links, xhtml)
        return xhtml

    def handle_relative_links(self, link_mo):
        log("handling relative link: %s" % link_mo.group(1),
            self.options.verbose)
        href = link_mo.group(1)
        if href.startswith("file://") or not self.options.url:
            # There's nothing we can do here
            return link_mo.group()
        if href.count("://"):
            # This is an absolute link, don't touch it
            return link_mo.group()
        newhref = urlparse.urljoin(self.options.url, os.path.normpath(href))
        return link_mo.group().replace(href, newhref)

    def insert_content(self, content):
        if self.options.replace_keyword and \
            self.xml["content"].count(self.options.replace_keyword) > 0:
            # TODO: this creates an empty line before and after the
            # replace_keyword. It's not optimal, I should use a regexp to
            # remove the previous opening <text:p> tag and the corresponding
            # closing tag.
            self.xml["content"] = self.xml["content"].replace(
                self.options.replace_keyword,
                '</text:p>%s<text:p text:style-name="Text_20_body">' % content)
        else:
            self.xml["content"] = self.xml["content"].replace(
                '</office:text>',
                content + '</office:text>')

    def add_styles(self):
        xsl_dir = os.path.join(INSTALL_PATH, 'xsl')
        xslt_doc = etree.parse(os.path.join(xsl_dir, "styles.xsl"))
        transform = etree.XSLT(xslt_doc)
        contentxml = etree.fromstring(self.xml["content"])
        stylesxml = etree.fromstring(self.xml["styles"])
        params = {}
        if self.options.verbose:
            params["debug"] = "1"
        self.xml["content"] = str(transform(contentxml, **params))
        self.xml["styles"] = str(transform(stylesxml, **params))

    def compile(self):
        # Store the new content
        for xmlfile in self.xml:
            xmlf = open(os.path.join(self.tmpdir, "%s.xml" % xmlfile), "w")
            xmlf.write(self.xml[xmlfile])
            xmlf.close()

    def _build_zip(self, document):
        newzf = zipfile.ZipFile(document, "w", zipfile.ZIP_DEFLATED)
        for root, dirs, files in os.walk(self.tmpdir):
            for file in files:
                realpath = os.path.join(root, file)
                internalpath = os.path.join(root.replace(self.tmpdir, ""), file)
                newzf.write(realpath, internalpath)
        newzf.close()

    def save(self, output=None):
        self.compile()
        if output:
            document = output
        else:
            document = StringIO()
        self._build_zip(document)
        shutil.rmtree(self.tmpdir)
        if not output:
            return document.getvalue()

def log(msg, verbose=False):
    if verbose:
        sys.stderr.write(msg+"\n")

def get_options():
    usage = "usage: %prog [options] -i input -o output -t template.odt"
    parser = OptionParser(usage=usage)
    parser.add_option("-i", "--input", dest="input", metavar="FILE",
                      help="Read the html from this file")
    parser.add_option("-o", "--output", dest="output", metavar="FILE",
                      help="Location of the output ODT file")
    parser.add_option("-t", "--template", dest="template", metavar="FILE",
                      help="Location of the template ODT file")
    parser.add_option("-u", "--url", dest="url",
                      help="Use this URL for relative links")
    parser.add_option("-v", "--verbose", dest="verbose",
                      action="store_true", default=False,
                      help="Show what's going on")
    parser.add_option("--html-id", dest="htmlid", metavar="ID",
                      help="Only export from the element with this ID")
    parser.add_option("--replace", dest="replace_keyword",
                      default="ODT-INSERT", metavar="KEYWORD",
                      help="Keyword to replace in the ODT template "
                      "(default is %default)")
    parser.add_option("--top-header-level", dest="top_header_level",
                      type="int", default="1", metavar="LEVEL",
                      help="Level of highest header in the HTML "
                      "(default is %default)")
    parser.add_option("--img-default-width", dest="img_width",
                      metavar="WIDTH", default="8cm",
                      help="Default image width (default is %default)")
    parser.add_option("--img-default-height", dest="img_height",
                      metavar="HEIGHT", default="6cm",
                      help="Default image height (default is %default)")
    parser.add_option("--dpi", dest="img_dpi", type="int",
                      default=96, metavar="DPI", help="Screen resolution "
                      "in Dots Per Inch (default is %default)")
    parser.add_option("--no-network", dest="with_network",
                      action="store_false", default=True,
                      help="Do not download remote images")
    parser.add_option("--tpldir", dest="tpldir",
                      help="Override templates directory")
    options, args = parser.parse_args()
    if len(args) > 0:
        parser.error("illegal arguments: %s"% ", ".join(args))
    if not options.input:
        parser.error("No input provided")
    if not options.output:
        parser.error("No output provided")
    if not options.template:
        parser.error("No ODT template provided")
    if not os.path.exists(options.input):
        parser.error("Can't find input file: %s" % options.input)
    if not os.path.exists(options.template):
        parser.error("Can't find template file: %s" % options.template)
    return options, args

def main():
    options, args = get_options()
    htmlfile = HTMLFile(options)
    htmlfile.read()
    odtfile = ODTFile(options)
    odtfile.open()
    odtfile.import_xhtml(htmlfile.html)
    odtfile.save(options.output)

if __name__ == '__main__':
    main()

<commit_msg>Remove abusive escaping<commit_after>#!/usr/bin/env python

"""
    xhtml2odt - XHTML to ODT XML transformation.
    Copyright (C) 2009-2010 Aurelien Bompard

This script can convert a wiki page to the OpenDocument Text (ODT) format,
standardized as ISO/IEC 26300:2006, and the native format of office suites such
as OpenOffice.org, KOffice, and others.

It uses a template ODT file which will be filled with the converted content of
the exported Wiki page.

Based on the work on docbook2odt, by Roman Fordinal
http://open.comsultia.com/docbook2odf/

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
"""

import tempfile
import shutil
import re
import os
import sys
import zipfile
import urllib2
import urlparse
from glob import glob
from StringIO import StringIO
from optparse import OptionParser

import tidy
from lxml import etree
from PIL import Image

#pylint#: disable-msg=C0301,C0111

INSTALL_PATH = "."

INCH_TO_CM = 2.54
CHARSET = "utf-8"

class ODTExportError(Exception): pass

class HTMLFile(object):

    def __init__(self, options):
        self.options = options
        self.html = ""

    def read(self):
        in_file = open(self.options.input)
        self.html = in_file.read()
        in_file.close()
        self.cleanup()
        if self.options.htmlid:
            self.select_id()

    def cleanup(self):
        tidy_options = dict(output_xhtml=1, add_xml_decl=1, indent=1,
                            tidy_mark=0, #input_encoding=str(self.charset),
                            output_encoding='utf8', doctype='auto',
                            wrap=0, char_encoding='utf8')
        self.html = str(tidy.parseString(self.html, **tidy_options))
        # Replace nbsp with entity
        # http://www.mail-archive.com/analog-help@lists.meer.net/msg03670.html
        self.html = self.html.replace(" ", " ")
        # Tidy creates newlines after <pre> (by indenting)
        self.html = re.sub('<pre([^>]*)>\n', '<pre\\1>', self.html)

    def select_id(self):
        html_tree = etree.fromstring(self.html)
        selected = html_tree.xpath("//*[@id='%s']" % self.options.htmlid)
        self.html = etree.tostring(selected[0], method="html")

class ODTFile(object):
    """Handles the conversion and production of an ODT file"""

    def __init__(self, options):
        self.options = options
        self.template_dirs = []
        if options.tpldir:
            self.template_dirs.append(options.tpldir)
        self.template_dirs.append(
            os.path.join(INSTALL_PATH, "styles")
        )
        self.xml = {
            "content": "",
            "styles": "",
        }
        self.tmpdir = tempfile.mkdtemp(prefix="xhtml2odt-")

    def open(self):
        self.zfile = zipfile.ZipFile(self.options.template, "r")
        for name in self.zfile.namelist():
            fname = os.path.join(self.tmpdir, name)
            if not os.path.exists(os.path.dirname(fname)):
                os.makedirs(os.path.dirname(fname))
            if name[-1] == "/":
                if not os.path.exists(fname):
                    os.mkdir(fname)
                continue
            fname_h = open(fname, "w")
            fname_h.write(self.zfile.read(name))
            fname_h.close()
        for xmlfile in self.xml:
            self.xml[xmlfile] = self.zfile.read("%s.xml" % xmlfile)

    def import_xhtml(self, xhtml):
        odt = self.xhtml_to_odt(xhtml)
        self.insert_content(odt)
        self.add_styles()

    def xhtml_to_odt(self, xhtml):
        xsl_dir = os.path.join(INSTALL_PATH, 'xsl')
        xslt_doc = etree.parse(os.path.join(xsl_dir, "xhtml2odt.xsl"))
        transform = etree.XSLT(xslt_doc)
        xhtml = self.handle_images(xhtml)
        xhtml = self.handle_links(xhtml)
        xhtml = etree.fromstring(xhtml) # must be valid xml at this point
        params = {
            "url": "/",
            "heading_minus_level": str(self.options.top_header_level - 1),
        }
        if self.options.verbose:
            params["debug"] = "1"
        if self.options.img_width:
            params["img_default_width"] = etree.XSLT.strparam(
                                            self.options.img_width)
        if self.options.img_height:
            params["img_default_height"] = etree.XSLT.strparam(
                                            self.options.img_height)
        odt = transform(xhtml, **params)
        return str(odt).replace('<?xml version="1.0" encoding="utf-8"?>','')

    def handle_images(self, xhtml):
        # Handle local images
        xhtml = re.sub('<img [^>]*src="([^"]+)"',
                      self.handle_local_img, xhtml)
        # Handle remote images
        if self.options.with_network:
            xhtml = re.sub('<img [^>]*src="(https?://[^"]+)"',
                          self.handle_remote_img, xhtml)
        return xhtml

    def handle_local_img(self, img_mo):
        log("handling local image: %s" % img_mo.group(1), self.options.verbose)
        src = img_mo.group(1)
        if src.count("://") and not src.startswith("file://"):
            # This is an absolute link, don't touch it
            return img_mo.group()
        if src.startswith("file://"):
            filename = src[7:]
        elif src.startswith("/"):
            filename = src
        else: # relative link
            filename = os.path.join(os.path.dirname(self.options.input), src)
        if os.path.exists(filename):
            return self.handle_img(img_mo.group(), src, filename)
        if src.startswith("file://") or not self.options.url:
            # There's nothing we can do here
            return img_mo.group()
        newsrc = urlparse.urljoin(self.options.url, os.path.normpath(src))
        if not self.options.with_network:
            # Don't download it, just update the URL
            return img_mo.group().replace(src, newsrc)
        try:
            tmpfile = self.download_img(newsrc)
        except (urllib2.HTTPError, urllib2.URLError):
            log("Failed getting %s" % newsrc, self.options.verbose)
            return img_mo.group()
        ret = self.handle_img(img_mo.group(), src, tmpfile)
        os.remove(tmpfile)
        return ret

    def handle_remote_img(self, img_mo):
        log('handling remote image: %s' % img_mo.group(), self.options.verbose)
        src = img_mo.group(1)
        try:
            tmpfile = self.download_img(src)
        except (urllib2.HTTPError, urllib2.URLError):
            return img_mo.group()
        ret = self.handle_img(img_mo.group(), src, tmpfile)
        os.remove(tmpfile)
        return ret

    def download_img(self, src):
        """
        Download the image to a temporary location
        """
        log('Downloading image: %s' % src, self.options.verbose)
        # TODO: proxy support
        remoteimg = urllib2.urlopen(src)
        tmpimg_fd, tmpfile = tempfile.mkstemp()
        tmpimg = os.fdopen(tmpimg_fd, 'w')
        tmpimg.write(remoteimg.read())
        tmpimg.close()
        remoteimg.close()
        return tmpfile

    def handle_img(self, full_tag, src, filename):
        log('Importing image: %s' % filename, self.options.verbose)
        if not os.path.exists(filename):
            raise ODTExportError('Image "%s" is not readable or does not exist'
                                 % filename)
        # TODO: generate a filename (with tempfile.mkstemp) to avoid weird
        # filenames. Maybe use img.format for the extension
        if not os.path.exists(os.path.join(self.tmpdir, "Pictures")):
            os.mkdir(os.path.join(self.tmpdir, "Pictures"))
        shutil.copy(filename, os.path.join(self.tmpdir, "Pictures",
                                           os.path.basename(filename)))
        newsrc = "Pictures/%s" % os.path.basename(filename)
        try:
            img = Image.open(filename)
        except IOError:
            log('Failed to identify image: %s' % filename,
                self.options.verbose)
        else:
            width, height = img.size
            log('Detected size: %spx x %spx' % (width, height),
                self.options.verbose)
            width = width / float(self.options.img_dpi) * INCH_TO_CM
            height = height / float(self.options.img_dpi) * INCH_TO_CM
            newsrc += '" width="%scm" height="%scm' % (width, height)
        return full_tag.replace(src, newsrc)

    def handle_links(self, xhtml):
        """Turn relative links into absolute links"""
        # Handle local images
        xhtml = re.sub('<a [^>]*href="([^"]+)"',
                      self.handle_relative_links, xhtml)
        return xhtml

    def handle_relative_links(self, link_mo):
        log("handling relative link: %s" % link_mo.group(1),
            self.options.verbose)
        href = link_mo.group(1)
        if href.startswith("file://") or not self.options.url:
            # There's nothing we can do here
            return link_mo.group()
        if href.count("://"):
            # This is an absolute link, don't touch it
            return link_mo.group()
        newhref = urlparse.urljoin(self.options.url, os.path.normpath(href))
        return link_mo.group().replace(href, newhref)

    def insert_content(self, content):
        if self.options.replace_keyword and \
            self.xml["content"].count(self.options.replace_keyword) > 0:
            # TODO: this creates an empty line before and after the
            # replace_keyword. It's not optimal, I should use a regexp to
            # remove the previous opening <text:p> tag and the corresponding
            # closing tag.
            self.xml["content"] = self.xml["content"].replace(
                self.options.replace_keyword,
                '</text:p>%s<text:p text:style-name="Text_20_body">' % content)
        else:
            self.xml["content"] = self.xml["content"].replace(
                '</office:text>',
                content + '</office:text>')

    def add_styles(self):
        xsl_dir = os.path.join(INSTALL_PATH, 'xsl')
        xslt_doc = etree.parse(os.path.join(xsl_dir, "styles.xsl"))
        transform = etree.XSLT(xslt_doc)
        contentxml = etree.fromstring(self.xml["content"])
        stylesxml = etree.fromstring(self.xml["styles"])
        params = {}
        if self.options.verbose:
            params["debug"] = "1"
        self.xml["content"] = str(transform(contentxml, **params))
        self.xml["styles"] = str(transform(stylesxml, **params))

    def compile(self):
        # Store the new content
        for xmlfile in self.xml:
            xmlf = open(os.path.join(self.tmpdir, "%s.xml" % xmlfile), "w")
            xmlf.write(self.xml[xmlfile])
            xmlf.close()

    def _build_zip(self, document):
        newzf = zipfile.ZipFile(document, "w", zipfile.ZIP_DEFLATED)
        for root, dirs, files in os.walk(self.tmpdir):
            for file in files:
                realpath = os.path.join(root, file)
                internalpath = os.path.join(root.replace(self.tmpdir, ""), file)
                newzf.write(realpath, internalpath)
        newzf.close()

    def save(self, output=None):
        self.compile()
        if output:
            document = output
        else:
            document = StringIO()
        self._build_zip(document)
        shutil.rmtree(self.tmpdir)
        if not output:
            return document.getvalue()

def log(msg, verbose=False):
    if verbose:
        sys.stderr.write(msg+"\n")

def get_options():
    usage = "usage: %prog [options] -i input -o output -t template.odt"
    parser = OptionParser(usage=usage)
    parser.add_option("-i", "--input", dest="input", metavar="FILE",
                      help="Read the html from this file")
    parser.add_option("-o", "--output", dest="output", metavar="FILE",
                      help="Location of the output ODT file")
    parser.add_option("-t", "--template", dest="template", metavar="FILE",
                      help="Location of the template ODT file")
    parser.add_option("-u", "--url", dest="url",
                      help="Use this URL for relative links")
    parser.add_option("-v", "--verbose", dest="verbose",
                      action="store_true", default=False,
                      help="Show what's going on")
    parser.add_option("--html-id", dest="htmlid", metavar="ID",
                      help="Only export from the element with this ID")
    parser.add_option("--replace", dest="replace_keyword",
                      default="ODT-INSERT", metavar="KEYWORD",
                      help="Keyword to replace in the ODT template "
                      "(default is %default)")
    parser.add_option("--top-header-level", dest="top_header_level",
                      type="int", default="1", metavar="LEVEL",
                      help="Level of highest header in the HTML "
                      "(default is %default)")
    parser.add_option("--img-default-width", dest="img_width",
                      metavar="WIDTH", default="8cm",
                      help="Default image width (default is %default)")
    parser.add_option("--img-default-height", dest="img_height",
                      metavar="HEIGHT", default="6cm",
                      help="Default image height (default is %default)")
    parser.add_option("--dpi", dest="img_dpi", type="int",
                      default=96, metavar="DPI", help="Screen resolution "
                      "in Dots Per Inch (default is %default)")
    parser.add_option("--no-network", dest="with_network",
                      action="store_false", default=True,
                      help="Do not download remote images")
    parser.add_option("--tpldir", dest="tpldir",
                      help="Override templates directory")
    options, args = parser.parse_args()
    if len(args) > 0:
        parser.error("illegal arguments: %s"% ", ".join(args))
    if not options.input:
        parser.error("No input provided")
    if not options.output:
        parser.error("No output provided")
    if not options.template:
        parser.error("No ODT template provided")
    if not os.path.exists(options.input):
        parser.error("Can't find input file: %s" % options.input)
    if not os.path.exists(options.template):
        parser.error("Can't find template file: %s" % options.template)
    return options, args

def main():
    options, args = get_options()
    htmlfile = HTMLFile(options)
    htmlfile.read()
    odtfile = ODTFile(options)
    odtfile.open()
    odtfile.import_xhtml(htmlfile.html)
    odtfile.save(options.output)

if __name__ == '__main__':
    main()

<|endoftext|>