<commit_before>#! /usr/bin/env python

"""
Module with helping functions.
"""

from __future__ import division
from __future__ import print_function

__author__ = 'C. Gomez @ ULg'
__all__ = ['matrix_scaling',
           'prepare_matrix',
           'reshape_matrix',
           'pca_annulus',
           'scale_cube_for_pca']

import numpy as np
from sklearn.preprocessing import scale
from ..var import mask_circle, get_annulus, get_square_robust, frame_center
from ..preproc import cube_derotate, cube_collapse, cube_rescaling

def scale_cube_for_pca(cube,scal_list, full_output=True, inverse=False, y_in=1,
                       x_in=1):
    """
    Wrapper to scale or descale a cube by factors given in scal_list, without
    any loss of information (zero-padding if scaling > 1).
    Important: in case of ifs data, the scaling factors in var_list should be
    >= 1 (ie. provide the scaling factors as for scaling to the longest
    wavelength channel).

    Parameters:
    -----------
    cube: 3D-array
       Datacube that whose frames have to be rescaled.
    scal_list: 1D-array
       Vector of same dimension as the first dimension of datacube, containing
       the scaling factor for each frame.
    full_output: bool, {True,False}, optional
       Whether to output just the rescaled cube (False) or also its median,
       the new y and x shapes of the cube, and the new centers cy and cx of the
       frames (True).
    inverse: bool, {True,False}, optional
       Whether to inverse the scaling factors in scal_list before applying them
       or not; i.e. True is to descale the cube (typically after a first scaling
       has already been done)
    y_in, x-in:
       Initial y and x sizes.
       In case the cube is descaled, these values will be used to crop back the
       cubes/frames to their original size.

    Returns:
    --------
    frame: 2D-array
        The median of the rescaled cube.
    If full_output is set to True, the function returns:
    cube,frame,y,x,cy,cx: 3D-array,2D-array,int,int,int,int
        The rescaled cube, its median, the new y and x shapes of the cube, and
        the new centers cy and cx of the frames
    """
    #First pad the cube with zeros appropriately to not loose info when scaling
    # the cube.
    # TBD next: pad with random gaussian noise instead of zeros. Padding with
    # only zeros can make the svd not converge in a pca per zone.

    n, y, x = cube.shape

    max_sc = np.amax(scal_list)

    if not inverse and max_sc > 1:
        new_y = int(np.ceil(max_sc*y))
        new_x = int(np.ceil(max_sc*x))
        if (new_y - y)%2 != 0: new_y = new_y+1
        if (new_x - x)%2 != 0: new_x = new_x+1
        pad_len_y = (new_y - y)//2
        pad_len_x = (new_x - x)//2
        big_cube = np.pad(cube, ((0,0), (pad_len_y, pad_len_y),
                                 (pad_len_x, pad_len_x)), 'constant',
                          constant_values=(0,))
    else:
        big_cube = cube.copy()

    n, y, x = big_cube.shape
    cy,cx = frame_center(big_cube[0])
    var_list = scal_list

    if inverse:
        var_list = 1./scal_list[:]
        cy,cx = frame_center(cube[0])

    # (de)scale the cube, so that a planet would now move radially
    cube,frame = cube_rescaling(big_cube,var_list,ref_y=cy, ref_x=cx)

    if inverse:
        if max_sc > 1:
            frame = get_square_robust(frame,max(y_in,x_in), cy,cx,strict=False)
            if full_output:
                n_z = cube.shape[0]
                array_old = cube.copy()
                cube = np.zeros([n_z,max(y_in,x_in),max(y_in,x_in)])
                for zz in range(n_z):
                    cube[zz]=get_square_robust(array_old[zz],max(y_in,x_in),
                                               cy,cx,strict=False)

    if full_output:
        return cube,frame,y,x,cy,cx
    else:
        return frame

def pca_annulus(cube, angs, ncomp, annulus_width, r_guess, cube_ref=None,
                svd_mode='lapack', scaling=None, collapse='median'):
    """
    PCA process the cube only for an annulus of a given width and at a given
    radial distance to the frame center. It returns a PCA processed frame with
    only non-zero values at the positions of the annulus.

    Parameters
    ----------
    cube : array_like
        The cube of fits images expressed as a numpy.array.
    angs : array_like
        The parallactic angles expressed as a numpy.array.
    ncomp : int
        The number of principal component.
    annulus_width : float
        The annulus width in pixel on which the PCA is performed.
    r_guess : float
        Radius of the annulus in pixels.
    cube_ref : array_like, 3d, optional
        Reference library cube. For Reference Star Differential Imaging.
    svd_mode : {'lapack', 'randsvd', 'eigen', 'arpack'}, str optional
        Switch for different ways of computing the SVD and selected PCs.
    scaling : {None, 'temp-mean', 'spat-mean', 'temp-standard', 'spat-standard'}
        With None, no scaling is performed on the input data before SVD. With
        "temp-mean" then temporal px-wise mean subtraction is done, with
        "spat-mean" then the spatial mean is subtracted, with "temp-standard"
        temporal mean centering plus scaling to unit variance is done and with
        "spat-standard" spatial mean centering plus scaling to unit variance is
        performed.
    collapse : {'median', 'mean', 'sum', 'trimmean', None}, str or None, optional
        Sets the way of collapsing the frames for producing a final image. If
        None then the cube of residuals is returned.

    Returns
    -------
    Depending on ``collapse`` parameter a final collapsed frame or the cube of
    residuals is returned.
    """
    data, ind = prepare_matrix(cube, scaling, mode='annular', annulus_radius=r_guess,
                               annulus_width=annulus_width, verbose=False)
    yy, xx = ind

    if cube_ref is not None:
        data_svd, _  = prepare_matrix(cube_ref, scaling, mode='annular',
                                      annulus_radius=r_guess,
                                      annulus_width=annulus_width, verbose=False)
    else:
        data_svd = data

    V = svd_wrapper(data_svd, svd_mode, ncomp, debug=False, verbose=False)

    transformed = np.dot(data, V.T)
    reconstructed = np.dot(transformed, V)
    residuals = data - reconstructed
    cube_zeros = np.zeros_like(cube)
    cube_zeros[:, yy, xx] = residuals
    cube_res_der = cube_derotate(cube_zeros, angs)
    if collapse is not None:
        pca_frame = cube_collapse(cube_res_der, mode=collapse)
        return pca_frame
    else:
        return cube_res_der

def matrix_scaling(matrix, scaling):
    """ Scales a matrix using sklearn.preprocessing.scale function.

    scaling : {None, 'temp-mean', 'spat-mean', 'temp-standard', 'spat-standard'}
    With None, no scaling is performed on the input data before SVD. With
    "temp-mean" then temporal px-wise mean subtraction is done, with
    "spat-mean" then the spatial mean is subtracted, with "temp-standard"
    temporal mean centering plus scaling to unit variance is done and with
    "spat-standard" spatial mean centering plus scaling to unit variance is
    performed.
    """
    if scaling is None:
        pass
    elif scaling=='temp-mean':
        matrix = scale(matrix, with_mean=True, with_std=False)
    elif scaling=='spat-mean':
        matrix = scale(matrix, with_mean=True, with_std=False, axis=1)
    elif scaling=='temp-standard':
        matrix = scale(matrix, with_mean=True, with_std=True)
    elif scaling=='spat-standard':
        matrix = scale(matrix, with_mean=True, with_std=True, axis=1)
    else:
        raise ValueError('Scaling mode not recognized')

    return matrix

def prepare_matrix(array, scaling=None, mask_center_px=None, mode='fullfr',
                   annulus_radius=None, annulus_width=None, verbose=True):
    """ Builds the matrix for the SVD/PCA and other matrix decompositions,
    centers the data and masks the frames central area if needed.

    Parameters
    ----------
    array : array_like
        Input cube, 3d array.
    scaling : {None, 'temp-mean', 'spat-mean', 'temp-standard', 'spat-standard'}
        With None, no scaling is performed on the input data before SVD. With
        "temp-mean" then temporal px-wise mean subtraction is done, with
        "spat-mean" then the spatial mean is subtracted, with "temp-standard"
        temporal mean centering plus scaling to unit variance is done and with
        "spat-standard" spatial mean centering plus scaling to unit variance is
        performed.
    mask_center_px : None or Int, optional
        Whether to mask the center of the frames or not.
    mode : {'fullfr', 'annular'}
        Whether to use the whole frames or a single annulus.
    annulus_radius : float
        Distance in pixels from the center of the frame to the center of the
        annulus.
    annulus_width : float
        Width of the annulus in pixels.
    verbose : {True, False}, bool optional
        If True prints intermediate info and timing.

    Returns
    -------
    If mode is `annular` then the indices of the annulus (yy, xx) are returned
    along with the matrix.

    matrix : array_like
        Out matrix whose rows are vectorized frames from the input cube.

    """
    if mode == 'annular':
        if annulus_radius is None or annulus_width is None:
            msgerr = 'Annulus_radius and/or annulus_width can be None in annular '
            msgerr += 'mode'
            raise ValueError(msgerr)

        ind = get_annulus(array[0], annulus_radius - annulus_width / 2.,
                          annulus_width, output_indices=True)
        yy, xx = ind
        matrix = array[:, yy, xx]

        matrix = matrix_scaling(matrix, scaling)

        if verbose:
            msg = 'Done vectorizing the cube annulus. Matrix shape [{:},{:}]'
            print(msg.format(matrix.shape[0], matrix.shape[1]))
        return matrix, ind

    elif mode == 'fullfr':
        if mask_center_px:
            array = mask_circle(array, mask_center_px)

        nfr = array.shape[0]
        matrix = np.reshape(array, (nfr, -1))  # == for i: array[i].flatten()

        matrix = matrix_scaling(matrix, scaling)

        if verbose:
            msg = 'Done vectorizing the frames. Matrix shape [{:},{:}]'
            print(msg.format(matrix.shape[0], matrix.shape[1]))
        return matrix

def reshape_matrix(array, y, x):
    """ Converts a matrix whose rows are vectorized frames to a cube with
    reshaped frames.
    """
    return array.reshape(array.shape[0], y, x)
<commit_msg>Bug fix: svd_wrapper needs to be imported for the pca_annulus function.<commit_after>#! /usr/bin/env python

"""
Module with helping functions.
"""

from __future__ import division
from __future__ import print_function

__author__ = 'C. Gomez @ ULg'
__all__ = ['matrix_scaling',
           'prepare_matrix',
           'reshape_matrix',
           'pca_annulus',
           'scale_cube_for_pca']

import numpy as np
from sklearn.preprocessing import scale
from ..var import mask_circle, get_annulus, get_square_robust, frame_center
from ..preproc import cube_derotate, cube_collapse, cube_rescaling
from .svd import svd_wrapper

def scale_cube_for_pca(cube,scal_list, full_output=True, inverse=False, y_in=1,
                       x_in=1):
    """
    Wrapper to scale or descale a cube by factors given in scal_list, without
    any loss of information (zero-padding if scaling > 1).
    Important: in case of ifs data, the scaling factors in var_list should be
    >= 1 (ie. provide the scaling factors as for scaling to the longest
    wavelength channel).

    Parameters:
    -----------
    cube: 3D-array
       Datacube that whose frames have to be rescaled.
    scal_list: 1D-array
       Vector of same dimension as the first dimension of datacube, containing
       the scaling factor for each frame.
    full_output: bool, {True,False}, optional
       Whether to output just the rescaled cube (False) or also its median,
       the new y and x shapes of the cube, and the new centers cy and cx of the
       frames (True).
    inverse: bool, {True,False}, optional
       Whether to inverse the scaling factors in scal_list before applying them
       or not; i.e. True is to descale the cube (typically after a first scaling
       has already been done)
    y_in, x-in:
       Initial y and x sizes.
       In case the cube is descaled, these values will be used to crop back the
       cubes/frames to their original size.

    Returns:
    --------
    frame: 2D-array
        The median of the rescaled cube.
    If full_output is set to True, the function returns:
    cube,frame,y,x,cy,cx: 3D-array,2D-array,int,int,int,int
        The rescaled cube, its median, the new y and x shapes of the cube, and
        the new centers cy and cx of the frames
    """
    #First pad the cube with zeros appropriately to not loose info when scaling
    # the cube.
    # TBD next: pad with random gaussian noise instead of zeros. Padding with
    # only zeros can make the svd not converge in a pca per zone.

    n, y, x = cube.shape

    max_sc = np.amax(scal_list)

    if not inverse and max_sc > 1:
        new_y = int(np.ceil(max_sc*y))
        new_x = int(np.ceil(max_sc*x))
        if (new_y - y)%2 != 0: new_y = new_y+1
        if (new_x - x)%2 != 0: new_x = new_x+1
        pad_len_y = (new_y - y)//2
        pad_len_x = (new_x - x)//2
        big_cube = np.pad(cube, ((0,0), (pad_len_y, pad_len_y),
                                 (pad_len_x, pad_len_x)), 'constant',
                          constant_values=(0,))
    else:
        big_cube = cube.copy()

    n, y, x = big_cube.shape
    cy,cx = frame_center(big_cube[0])
    var_list = scal_list

    if inverse:
        var_list = 1./scal_list[:]
        cy,cx = frame_center(cube[0])

    # (de)scale the cube, so that a planet would now move radially
    cube,frame = cube_rescaling(big_cube,var_list,ref_y=cy, ref_x=cx)

    if inverse:
        if max_sc > 1:
            frame = get_square_robust(frame,max(y_in,x_in), cy,cx,strict=False)
            if full_output:
                n_z = cube.shape[0]
                array_old = cube.copy()
                cube = np.zeros([n_z,max(y_in,x_in),max(y_in,x_in)])
                for zz in range(n_z):
                    cube[zz]=get_square_robust(array_old[zz],max(y_in,x_in),
                                               cy,cx,strict=False)

    if full_output:
        return cube,frame,y,x,cy,cx
    else:
        return frame

def pca_annulus(cube, angs, ncomp, annulus_width, r_guess, cube_ref=None,
                svd_mode='lapack', scaling=None, collapse='median'):
    """
    PCA process the cube only for an annulus of a given width and at a given
    radial distance to the frame center. It returns a PCA processed frame with
    only non-zero values at the positions of the annulus.

    Parameters
    ----------
    cube : array_like
        The cube of fits images expressed as a numpy.array.
    angs : array_like
        The parallactic angles expressed as a numpy.array.
    ncomp : int
        The number of principal component.
    annulus_width : float
        The annulus width in pixel on which the PCA is performed.
    r_guess : float
        Radius of the annulus in pixels.
    cube_ref : array_like, 3d, optional
        Reference library cube. For Reference Star Differential Imaging.
    svd_mode : {'lapack', 'randsvd', 'eigen', 'arpack'}, str optional
        Switch for different ways of computing the SVD and selected PCs.
    scaling : {None, 'temp-mean', 'spat-mean', 'temp-standard', 'spat-standard'}
        With None, no scaling is performed on the input data before SVD. With
        "temp-mean" then temporal px-wise mean subtraction is done, with
        "spat-mean" then the spatial mean is subtracted, with "temp-standard"
        temporal mean centering plus scaling to unit variance is done and with
        "spat-standard" spatial mean centering plus scaling to unit variance is
        performed.
    collapse : {'median', 'mean', 'sum', 'trimmean', None}, str or None, optional
        Sets the way of collapsing the frames for producing a final image. If
        None then the cube of residuals is returned.

    Returns
    -------
    Depending on ``collapse`` parameter a final collapsed frame or the cube of
    residuals is returned.
    """
    data, ind = prepare_matrix(cube, scaling, mode='annular', annulus_radius=r_guess,
                               annulus_width=annulus_width, verbose=False)
    yy, xx = ind

    if cube_ref is not None:
        data_svd, _  = prepare_matrix(cube_ref, scaling, mode='annular',
                                      annulus_radius=r_guess,
                                      annulus_width=annulus_width, verbose=False)
    else:
        data_svd = data

    V = svd_wrapper(data_svd, svd_mode, ncomp, debug=False, verbose=False)

    transformed = np.dot(data, V.T)
    reconstructed = np.dot(transformed, V)
    residuals = data - reconstructed
    cube_zeros = np.zeros_like(cube)
    cube_zeros[:, yy, xx] = residuals
    cube_res_der = cube_derotate(cube_zeros, angs)
    if collapse is not None:
        pca_frame = cube_collapse(cube_res_der, mode=collapse)
        return pca_frame
    else:
        return cube_res_der

def matrix_scaling(matrix, scaling):
    """ Scales a matrix using sklearn.preprocessing.scale function.

    scaling : {None, 'temp-mean', 'spat-mean', 'temp-standard', 'spat-standard'}
    With None, no scaling is performed on the input data before SVD. With
    "temp-mean" then temporal px-wise mean subtraction is done, with
    "spat-mean" then the spatial mean is subtracted, with "temp-standard"
    temporal mean centering plus scaling to unit variance is done and with
    "spat-standard" spatial mean centering plus scaling to unit variance is
    performed.
    """
    if scaling is None:
        pass
    elif scaling=='temp-mean':
        matrix = scale(matrix, with_mean=True, with_std=False)
    elif scaling=='spat-mean':
        matrix = scale(matrix, with_mean=True, with_std=False, axis=1)
    elif scaling=='temp-standard':
        matrix = scale(matrix, with_mean=True, with_std=True)
    elif scaling=='spat-standard':
        matrix = scale(matrix, with_mean=True, with_std=True, axis=1)
    else:
        raise ValueError('Scaling mode not recognized')

    return matrix

def prepare_matrix(array, scaling=None, mask_center_px=None, mode='fullfr',
                   annulus_radius=None, annulus_width=None, verbose=True):
    """ Builds the matrix for the SVD/PCA and other matrix decompositions,
    centers the data and masks the frames central area if needed.

    Parameters
    ----------
    array : array_like
        Input cube, 3d array.
    scaling : {None, 'temp-mean', 'spat-mean', 'temp-standard', 'spat-standard'}
        With None, no scaling is performed on the input data before SVD. With
        "temp-mean" then temporal px-wise mean subtraction is done, with
        "spat-mean" then the spatial mean is subtracted, with "temp-standard"
        temporal mean centering plus scaling to unit variance is done and with
        "spat-standard" spatial mean centering plus scaling to unit variance is
        performed.
    mask_center_px : None or Int, optional
        Whether to mask the center of the frames or not.
    mode : {'fullfr', 'annular'}
        Whether to use the whole frames or a single annulus.
    annulus_radius : float
        Distance in pixels from the center of the frame to the center of the
        annulus.
    annulus_width : float
        Width of the annulus in pixels.
    verbose : {True, False}, bool optional
        If True prints intermediate info and timing.

    Returns
    -------
    If mode is `annular` then the indices of the annulus (yy, xx) are returned
    along with the matrix.

    matrix : array_like
        Out matrix whose rows are vectorized frames from the input cube.

    """
    if mode == 'annular':
        if annulus_radius is None or annulus_width is None:
            msgerr = 'Annulus_radius and/or annulus_width can be None in annular '
            msgerr += 'mode'
            raise ValueError(msgerr)

        ind = get_annulus(array[0], annulus_radius - annulus_width / 2.,
                          annulus_width, output_indices=True)
        yy, xx = ind
        matrix = array[:, yy, xx]

        matrix = matrix_scaling(matrix, scaling)

        if verbose:
            msg = 'Done vectorizing the cube annulus. Matrix shape [{:},{:}]'
            print(msg.format(matrix.shape[0], matrix.shape[1]))
        return matrix, ind

    elif mode == 'fullfr':
        if mask_center_px:
            array = mask_circle(array, mask_center_px)

        nfr = array.shape[0]
        matrix = np.reshape(array, (nfr, -1))  # == for i: array[i].flatten()

        matrix = matrix_scaling(matrix, scaling)

        if verbose:
            msg = 'Done vectorizing the frames. Matrix shape [{:},{:}]'
            print(msg.format(matrix.shape[0], matrix.shape[1]))
        return matrix

def reshape_matrix(array, y, x):
    """ Converts a matrix whose rows are vectorized frames to a cube with
    reshaped frames.
    """
    return array.reshape(array.shape[0], y, x)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2010 Mats Ekberg
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import jsonrpc
import base64
import re
import sys
import tempfile

import subprocess
from front import Front
from blobrepo import repository
from common import *
from boar_exceptions import *

BOAR_URL_PATTERN = "boar\+([a-z]+)://.*"

def is_boar_url(string):
    return re.match(BOAR_URL_PATTERN, string) != None

def localize(repourl):
    m = re.match(BOAR_URL_PATTERN, repourl)
    if not m:
        repourl="boar+local://" + os.path.abspath(repourl)
    return repourl

def connect(repourl):
    #repourl = localize(repourl)

    m = re.match(BOAR_URL_PATTERN, repourl)
    if not m:
        return Front(user_friendly_open_local_repository(repourl))

    transporter = m.group(1)
    if transporter == "ssh":
        front = connect_ssh(repourl)
    elif transporter == "nc":
        front = connect_nc(repourl)
    elif transporter == "local":
        front = connect_local(repourl)
    else:
        raise UserError("No such transporter: '%s'" % transporter)
    assert front
    return front

def user_friendly_open_local_repository(path):
    # This won't catch invalid/nonexisting repos. Let the repo constructor do that
    path = os.path.abspath(path)
    if repository.looks_like_repo(path) and repository.has_pending_operations(path):
        notice("The repository at %s has pending operations. Resuming..." % os.path.basename(path))
        repo = repository.Repo(path)
        notice("Pending operations completed.")
    else:
        repo = repository.Repo(path)
    return repo

def _connect_cmd(cmd):
    errorlog = tempfile.TemporaryFile()
    #errorlog = open("/tmp/errors.txt", "w")
    p = subprocess.Popen(cmd,
                         shell = True,
                         stdout = subprocess.PIPE,
                         stdin = subprocess.PIPE,
                         stderr = errorlog)
    if p.poll():
        errorlog.seek(0)
        errorstr = errorlog.read().strip()
        raise UserError("Transport command failed with error code %s: %s" % (p.returncode, errorstr))
    server = jsonrpc.ServerProxy(jsonrpc.JsonRpc20(),
                                 jsonrpc.TransportStream(p.stdout, p.stdin))
    try:
        assert server.ping() == "pong"
    except:
        print "*** Transport command stderr output:"
        errorlog.seek(0)
        print errorlog.read()
        print "*** Local stack trace:"
        raise
    server.initialize()
    return server.front

def connect_ssh(url):
    url_match = re.match("boar\+ssh://(.*)@([^/]+)(/.*)", url)
    assert url_match, "Not a valid ssh Boar URL:" + url
    user, host, path = url_match.groups()
    cmd = "ssh '%s'@'%s' boarserve.py '%s'" % (user, host, path)
    return _connect_cmd(cmd)

def connect_nc(url):
    url_match = re.match("boar\+nc://(.*):(\d+)/?", url)
    assert url_match, "Not a valid netcat Boar URL:" + url
    host, port = url_match.groups()
    cmd = "nc %s %s" % (host, port)
    return _connect_cmd(cmd)

def connect_local(url):
    url_match = re.match("boar\+local://(.*)/?", url)
    assert url_match, "Not a valid local Boar URL:" + url
    repopath, = url_match.groups()
    boarhome = os.path.dirname(os.path.abspath(__file__))
    cmd = "'%s/boarserve.py' '%s'" % (boarhome, repopath)
    return _connect_cmd(cmd)

<commit_msg>Simpler error handling by just piping transport command stderr to application stderr<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2010 Mats Ekberg
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import jsonrpc
import base64
import re
import sys
import tempfile

import subprocess
from front import Front
from blobrepo import repository
from common import *
from boar_exceptions import *

BOAR_URL_PATTERN = "boar\+([a-z]+)://.*"

def is_boar_url(string):
    return re.match(BOAR_URL_PATTERN, string) != None

def localize(repourl):
    m = re.match(BOAR_URL_PATTERN, repourl)
    if not m:
        repourl="boar+local://" + os.path.abspath(repourl)
    return repourl

def connect(repourl):
    #repourl = localize(repourl)

    m = re.match(BOAR_URL_PATTERN, repourl)
    if not m:
        return Front(user_friendly_open_local_repository(repourl))

    transporter = m.group(1)
    if transporter == "ssh":
        front = connect_ssh(repourl)
    elif transporter == "nc":
        front = connect_nc(repourl)
    elif transporter == "local":
        front = connect_local(repourl)
    else:
        raise UserError("No such transporter: '%s'" % transporter)
    assert front
    return front

def user_friendly_open_local_repository(path):
    # This won't catch invalid/nonexisting repos. Let the repo constructor do that
    path = os.path.abspath(path)
    if repository.looks_like_repo(path) and repository.has_pending_operations(path):
        notice("The repository at %s has pending operations. Resuming..." % os.path.basename(path))
        repo = repository.Repo(path)
        notice("Pending operations completed.")
    else:
        repo = repository.Repo(path)
    return repo

def _connect_cmd(cmd):
    p = subprocess.Popen(cmd,
                         shell = True,
                         stdout = subprocess.PIPE,
                         stdin = subprocess.PIPE,
                         stderr = None)
    if p.poll():
        raise UserError("Transport command failed with error code %s" % (p.returncode))
    server = jsonrpc.ServerProxy(jsonrpc.JsonRpc20(),
                                 jsonrpc.TransportStream(p.stdout, p.stdin))
    try:
        assert server.ping() == "pong"
    except:
        raise
    server.initialize()
    return server.front

def connect_ssh(url):
    url_match = re.match("boar\+ssh://(.*)@([^/]+)(/.*)", url)
    assert url_match, "Not a valid ssh Boar URL:" + url
    user, host, path = url_match.groups()
    cmd = "ssh '%s'@'%s' boarserve.py '%s'" % (user, host, path)
    return _connect_cmd(cmd)

def connect_nc(url):
    url_match = re.match("boar\+nc://(.*):(\d+)/?", url)
    assert url_match, "Not a valid netcat Boar URL:" + url
    host, port = url_match.groups()
    cmd = "nc %s %s" % (host, port)
    return _connect_cmd(cmd)

def connect_local(url):
    url_match = re.match("boar\+local://(.*)/?", url)
    assert url_match, "Not a valid local Boar URL:" + url
    repopath, = url_match.groups()
    boarhome = os.path.dirname(os.path.abspath(__file__))
    cmd = "'%s/boarserve.py' '%s'" % (boarhome, repopath)
    return _connect_cmd(cmd)

<|endoftext|>


<commit_before>from .ast import AST, ast_to_string
from .grammar import GRAMMAR, NoParse, SingleParse, MultipleParse
<commit_msg>Export Parse superclass in viper.parser<commit_after>from .ast import AST, ast_to_string
from .grammar import GRAMMAR, Parse, NoParse, SingleParse, MultipleParse
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
``software`` module contains functionality to parse out
latest software versions from respective vendors' web sites.
'''

STABLE_SENDMAIL_URL = "http://www.sendmail.com/sm/open_source/download/"
STABLE_SENDMAIL_PATTERN = \
    r'Sendmail.*(.*) is ' \
    r'available from'

STABLE_BIND_URL = \
    "https://www.isc.org/downloads/"
STABLE_BIND_PATTERN = \
    r'
\n(\d+.*)	\n<a href'

STABLE_DHCP_URL = \
    "https://www.isc.org/downloads/"
STABLE_DHCP_PATTERN = \
    r'
\n(\d+.*)	\n<td class="c_sw"'

STABLE_CYRUS_IMAP_URL = "http://www.cyrusimap.org/"
STABLE_CYRUS_IMAP_PATTERN = \
    r'
Version: (.*?)
'

STABLE_APACHE_URL = "http://httpd.apache.org/download.cgi"
STABLE_APACHE_PATTERN = \
    r'(.*?) \(released'

STABLE_SQUID_URL = "http://www.squid-cache.org/Versions/"
STABLE_SQUID_PATTERN = \
    r'
.*?	.*?	(.*?)
'

STABLE_CANIT_URL = "http://roaringpenguin.com/news"
STABLE_CANIT_PATTERN = \
    r'>Roaring Penguin Software releases CanIt (.*?)<'

STABLE_PSQL_URL = "http://www.postgresql.org/versions.rss"
STABLE_PSQL_PATTERN = \
    r'<item><title>(.*?)(\<\/|$)'

STABLE_SQUIRRELMAIL_URL = "http://squirrelmail.org/download.php"
STABLE_SQUIRRELMAIL_PATTERN = \
    r'Stable version.*?Version: (.*?)
'

STABLE_OPENLDAP_URL = "http://www.openldap.org/software/download/"
STABLE_OPENLDAP_PATTERN = \
    r'Our latest release.*?of OpenLDAP Software for general use' \
    r'.*?OpenLDAP-(.*?) is currently available'

STABLE_PLA_URL = "http://phpldapadmin.sourceforge.net/wiki/index.php/Release"
STABLE_PLA_PATTERN = \
    r'' \
    r'phpLDAPadmin (.*?)
'

STABLE_POSTFIX_URL = "http://de.postfix.org/ftpmirror/"
STABLE_POSTFIX_PATTERN = \
    r'Source code'

STABLE_NGINX_URL = "http://nginx.org/en/download.html"
STABLE_NGINX_PATTERN = \
    r'Stable version
.*?'

STABLE_MYSQL_URL = "http://www.mysql.com/downloads/mysql/"
STABLE_MYSQL_PATTERN = \
    r'
MySQL Community Server (.*?)\s
'

STABLE_PROFTPD_URL = "http://www.proftpd.org/"
STABLE_PROFTPD_PATTERN = \
    r'Stable: (.*?)'

STABLE_VSFTPD_URL = "https://security.appspot.com/vsftpd.html"
STABLE_VSFTPD_PATTERN = \
    r'
vsftpd-(.*?) is released'

STABLE_SENDMAILANALYZER_URL = \
    "http://sourceforge.net/projects/sa-report/files/"
STABLE_SENDMAILANALYZER_PATTERN = \
    r'Download sendmailanalyzer-(.*?).tar.gz '

STABLE_DJANGO_URL = "https://www.djangoproject.com/download/"
STABLE_DJANGO_PATTERN = \
    r'
The latest official version is (.*?).
'

STABLE_STUNNEL_URL = "https://www.stunnel.org/downloads.html"
STABLE_STUNNEL_PATTERN = \
    r'stunnel-(.*?).tar.gz	'

STABLE_POUND_URL = "http://www.apsis.ch/pound/"
STABLE_POUND_PATTERN = \
    r'Download the latest version Pound-(.*?).tgz'

STABLE_LINUX_URL = "http://www.kernel.org/"
STABLE_LINUX_PATTERN = \
    r'.*?' \
    r'(.*?)'

STABLE_VARNISH_URL = "https://www.varnish-cache.org/releases"
STABLE_VARNISH_PATTERN = \
    r'
Current supported releases
.*?' \
    r'Varnish Cache (.*?)'

STABLE_CLAMAV_URL = "http://www.clamav.net/download.html"
STABLE_CLAMAV_PATTERN = \
    r'
The latest stable release is (.*?)
'

STABLE_OPENSSH_URL = "http://www.openssh.com/"
STABLE_OPENSSH_PATTERN = \
    r'OpenSSH (.*?) released'

STABLE_DOVECOT_URL = "http://www.dovecot.org/download.html"
STABLE_DOVECOT_PATTERN = \
    r'
Stable releases
.*?' \
    r'Download '

STABLE_MONGODB_URL = "http://www.mongodb.org/downloads"
STABLE_MONGODB_PATTERN = \
    r'
Production Release \((.*?)\)'

STABLE_SPAMASSASSIN_URL = "http://spamassassin.apache.org/downloads.cgi"
STABLE_SPAMASSASSIN_PATTERN = \
    r'
Released version, (.*?)
'

STABLE_CONSUL_URL = "http://www.consul.io/downloads.html"
STABLE_CONSUL_PATTERN = \
    r'Below are all available downloads for the latest version of Consul ' \
    r'\((.*?)\)'

STABLE_MIMEDEFANG_URL = "http://www.mimedefang.org/"
STABLE_MIMEDEFANG_PATTERN = \
    r'Release (.*?)'

STABLE_ROUNDCUBE_URL = "http://roundcube.net/download/"
STABLE_ROUNDCUBE_PATTERN = \
    r'Complete: (.*?)	'

STABLE_VAGRANT_URL = "http://www.vagrantup.com/downloads.html"
STABLE_VAGRANT_PATTERN = \
    r'
 Below are all available downloads for the latest version of ' \
    r'Vagrant \((.*?)\)'

STABLE_HAPROXY_URL = "http://www.haproxy.org/#down"
STABLE_HAPROXY_PATTERN = \
    r'.*?-stable	.*?' \
    r'(.*?)	'

STABLE_MONIT_URL = "http://mmonit.com/monit/#download"
STABLE_MONIT_PATTERN = \
    r'
.*?' \
    r'
Monit (.*?) Downloads
'

STABLE_WORDPRESS_URL = "https://wordpress.org/download/"
STABLE_WORDPRESS_PATTERN = \
    r'
The latest stable release of WordPress ' \
    r'\(Version (.*?)\)'

STABLE_BACULA_URL = "http://sourceforge.net/projects/bacula/files/bacula/"
STABLE_BACULA_PATTERN = \
    r'

Looking for the latest ' \
    r'version\? .*?' \
    r'<a href="/projects/bacula/files/latest/download\?source=files" ' \
    r'title="/bacula/.*?/bacula-(.*?).tar.gz'

STABLE_REDIS_URL = "http://redis.io/download"
STABLE_REDIS_PATTERN = \
    r'(.*?)	.*?' \
    r'Stable	'

STABLE_UNBOUND_URL = "http://unbound.net/download.html"
STABLE_UNBOUND_PATTERN = \
    r'
Unbound Downloads
.*?' \
    r'The latest version of unbound \(currently (.*?)\)'

STABLE_SOGO_URL = "http://www.sogo.nu/downloads/backend.html"
STABLE_SOGO_PATTERN = \
    r'
Source Code
.*?' \
    r'id="downbutton">.*?SOGo-(.*?).tar.gz
'

STABLE_COUCHDB_URL = "http://couchdb.apache.org/"
STABLE_COUCHDB_PATTERN = \
    r'
.*?' \
    r'
Download CouchDB (.*?)
'

STABLE_OWNCLOUD_URL = "https://owncloud.org/install/"
STABLE_OWNCLOUD_PATTERN = \
    r'
Install ownCloud Server.*?' \
    r'
Latest stable version: (.*?)\s\('

STABLE_OPENSMTPD_URL = "https://opensmtpd.org/"
STABLE_OPENSMTPD_PATTERN = \
    r'OpenSMTPD (.*?) released'

STABLE_PHP_URL = "http://php.net/"
STABLE_PHP_PATTERN = \
    r"
(.*?)" \
    r"·Release Notes
"

__all__ = [
    "STABLE_SENDMAIL_URL", "STABLE_SENDMAIL_PATTERN",
    "STABLE_BIND_URL", "STABLE_BIND_PATTERN",
    "STABLE_CYRUS_IMAP_URL", "STABLE_CYRUS_IMAP_PATTERN",
    "STABLE_APACHE_URL", "STABLE_APACHE_PATTERN",
    "STABLE_SQUID_URL", "STABLE_SQUID_PATTERN",
    "STABLE_CANIT_URL", "STABLE_CANIT_PATTERN",
    "STABLE_PSQL_URL", "STABLE_PSQL_PATTERN",
    "STABLE_SQUIRRELMAIL_URL", "STABLE_SQUIRRELMAIL_PATTERN",
    "STABLE_OPENLDAP_URL", "STABLE_OPENLDAP_PATTERN",
    "STABLE_PLA_URL", "STABLE_PLA_PATTERN",
    "STABLE_POSTFIX_URL", "STABLE_POSTFIX_PATTERN",
    "STABLE_NGINX_URL", "STABLE_NGINX_PATTERN",
    "STABLE_MYSQL_URL", "STABLE_MYSQL_PATTERN",
    "STABLE_PROFTPD_URL", "STABLE_PROFTPD_PATTERN",
    "STABLE_VSFTPD_URL", "STABLE_VSFTPD_PATTERN",
    "STABLE_SENDMAILANALYZER_URL", "STABLE_SENDMAILANALYZER_PATTERN",
    "STABLE_DJANGO_URL", "STABLE_DJANGO_PATTERN",
    "STABLE_STUNNEL_URL", "STABLE_STUNNEL_PATTERN",
    "STABLE_POUND_URL", "STABLE_POUND_PATTERN",
    "STABLE_LINUX_URL", "STABLE_LINUX_PATTERN",
    "STABLE_VARNISH_URL", "STABLE_VARNISH_PATTERN",
    "STABLE_CLAMAV_URL", "STABLE_CLAMAV_PATTERN",
    "STABLE_OPENSSH_URL", "STABLE_OPENSSH_PATTERN",
    "STABLE_DOVECOT_URL", "STABLE_DOVECOT_PATTERN",
    "STABLE_MONGODB_URL", "STABLE_MONGODB_PATTERN",
    "STABLE_SPAMASSASSIN_URL", "STABLE_SPAMASSASSIN_PATTERN",
    "STABLE_CONSUL_URL", "STABLE_CONSUL_PATTERN",
    "STABLE_MIMEDEFANG_URL", "STABLE_MIMEDEFANG_PATTERN",
    "STABLE_VAGRANT_URL", "STABLE_VAGRANT_PATTERN",
    "STABLE_HAPROXY_URL", "STABLE_HAPROXY_PATTERN",
    "STABLE_MONIT_URL", "STABLE_MONIT_PATTERN",
    "STABLE_WORDPRESS_URL", "STABLE_WORDPRESS_PATTERN",
    "STABLE_BACULA_URL", "STABLE_BACULA_PATTERN",
    "STABLE_REDIS_URL", "STABLE_REDIS_PATTERN",
    "STABLE_UNBOUND_URL", "STABLE_UNBOUND_PATTERN",
    "STABLE_SOGO_URL", "STABLE_SOGO_PATTERN",
    "STABLE_COUCHDB_URL", "STABLE_COUCHDB_PATTERN",
    "STABLE_OWNCLOUD_URL", "STABLE_OWNCLOUD_PATTERN",
    "STABLE_OPENSMTPD_URL", "STABLE_OPENSMTPD_PATTERN",
    "STABLE_PHP_URL", "STABLE_PHP_PATTERN"
]
<commit_msg>Fix Consul version parsing<commit_after>#!/usr/bin/env python

'''
``software`` module contains functionality to parse out
latest software versions from respective vendors' web sites.
'''

STABLE_SENDMAIL_URL = "http://www.sendmail.com/sm/open_source/download/"
STABLE_SENDMAIL_PATTERN = \
    r'Sendmail.*(.*) is ' \
    r'available from'

STABLE_BIND_URL = \
    "https://www.isc.org/downloads/"
STABLE_BIND_PATTERN = \
    r'
\n(\d+.*)	\n<a href'

STABLE_DHCP_URL = \
    "https://www.isc.org/downloads/"
STABLE_DHCP_PATTERN = \
    r'
\n(\d+.*)	\n<td class="c_sw"'

STABLE_CYRUS_IMAP_URL = "http://www.cyrusimap.org/"
STABLE_CYRUS_IMAP_PATTERN = \
    r'
Version: (.*?)
'

STABLE_APACHE_URL = "http://httpd.apache.org/download.cgi"
STABLE_APACHE_PATTERN = \
    r'(.*?) \(released'

STABLE_SQUID_URL = "http://www.squid-cache.org/Versions/"
STABLE_SQUID_PATTERN = \
    r'
.*?	.*?	(.*?)
'

STABLE_CANIT_URL = "http://roaringpenguin.com/news"
STABLE_CANIT_PATTERN = \
    r'>Roaring Penguin Software releases CanIt (.*?)<'

STABLE_PSQL_URL = "http://www.postgresql.org/versions.rss"
STABLE_PSQL_PATTERN = \
    r'<item><title>(.*?)(\<\/|$)'

STABLE_SQUIRRELMAIL_URL = "http://squirrelmail.org/download.php"
STABLE_SQUIRRELMAIL_PATTERN = \
    r'Stable version.*?Version: (.*?)
'

STABLE_OPENLDAP_URL = "http://www.openldap.org/software/download/"
STABLE_OPENLDAP_PATTERN = \
    r'Our latest release.*?of OpenLDAP Software for general use' \
    r'.*?OpenLDAP-(.*?) is currently available'

STABLE_PLA_URL = "http://phpldapadmin.sourceforge.net/wiki/index.php/Release"
STABLE_PLA_PATTERN = \
    r'' \
    r'phpLDAPadmin (.*?)
'

STABLE_POSTFIX_URL = "http://de.postfix.org/ftpmirror/"
STABLE_POSTFIX_PATTERN = \
    r'Source code'

STABLE_NGINX_URL = "http://nginx.org/en/download.html"
STABLE_NGINX_PATTERN = \
    r'Stable version
.*?'

STABLE_MYSQL_URL = "http://www.mysql.com/downloads/mysql/"
STABLE_MYSQL_PATTERN = \
    r'
MySQL Community Server (.*?)\s
'

STABLE_PROFTPD_URL = "http://www.proftpd.org/"
STABLE_PROFTPD_PATTERN = \
    r'Stable: (.*?)'

STABLE_VSFTPD_URL = "https://security.appspot.com/vsftpd.html"
STABLE_VSFTPD_PATTERN = \
    r'
vsftpd-(.*?) is released'

STABLE_SENDMAILANALYZER_URL = \
    "http://sourceforge.net/projects/sa-report/files/"
STABLE_SENDMAILANALYZER_PATTERN = \
    r'Download sendmailanalyzer-(.*?).tar.gz '

STABLE_DJANGO_URL = "https://www.djangoproject.com/download/"
STABLE_DJANGO_PATTERN = \
    r'
The latest official version is (.*?).
'

STABLE_STUNNEL_URL = "https://www.stunnel.org/downloads.html"
STABLE_STUNNEL_PATTERN = \
    r'stunnel-(.*?).tar.gz	'

STABLE_POUND_URL = "http://www.apsis.ch/pound/"
STABLE_POUND_PATTERN = \
    r'Download the latest version Pound-(.*?).tgz'

STABLE_LINUX_URL = "http://www.kernel.org/"
STABLE_LINUX_PATTERN = \
    r'.*?' \
    r'(.*?)'

STABLE_VARNISH_URL = "https://www.varnish-cache.org/releases"
STABLE_VARNISH_PATTERN = \
    r'
Current supported releases
.*?' \
    r'Varnish Cache (.*?)'

STABLE_CLAMAV_URL = "http://www.clamav.net/download.html"
STABLE_CLAMAV_PATTERN = \
    r'
The latest stable release is (.*?)
'

STABLE_OPENSSH_URL = "http://www.openssh.com/"
STABLE_OPENSSH_PATTERN = \
    r'OpenSSH (.*?) released'

STABLE_DOVECOT_URL = "http://www.dovecot.org/download.html"
STABLE_DOVECOT_PATTERN = \
    r'
Stable releases
.*?' \
    r'Download '

STABLE_MONGODB_URL = "http://www.mongodb.org/downloads"
STABLE_MONGODB_PATTERN = \
    r'
Production Release \((.*?)\)'

STABLE_SPAMASSASSIN_URL = "http://spamassassin.apache.org/downloads.cgi"
STABLE_SPAMASSASSIN_PATTERN = \
    r'
Released version, (.*?)
'

STABLE_CONSUL_URL = "http://www.consul.io/downloads.html"
STABLE_CONSUL_PATTERN = \
    r'
Linux
.*?' \
    r'
32-bit
'

STABLE_MIMEDEFANG_URL = "http://www.mimedefang.org/"
STABLE_MIMEDEFANG_PATTERN = \
    r'Release (.*?)'

STABLE_ROUNDCUBE_URL = "http://roundcube.net/download/"
STABLE_ROUNDCUBE_PATTERN = \
    r'Complete: (.*?)	'

STABLE_VAGRANT_URL = "http://www.vagrantup.com/downloads.html"
STABLE_VAGRANT_PATTERN = \
    r'
 Below are all available downloads for the latest version of ' \
    r'Vagrant \((.*?)\)'

STABLE_HAPROXY_URL = "http://www.haproxy.org/#down"
STABLE_HAPROXY_PATTERN = \
    r'.*?-stable	.*?' \
    r'(.*?)	'

STABLE_MONIT_URL = "http://mmonit.com/monit/#download"
STABLE_MONIT_PATTERN = \
    r'
.*?' \
    r'
Monit (.*?) Downloads
'

STABLE_WORDPRESS_URL = "https://wordpress.org/download/"
STABLE_WORDPRESS_PATTERN = \
    r'
The latest stable release of WordPress ' \
    r'\(Version (.*?)\)'

STABLE_BACULA_URL = "http://sourceforge.net/projects/bacula/files/bacula/"
STABLE_BACULA_PATTERN = \
    r'

Looking for the latest ' \
    r'version\? .*?' \
    r'<a href="/projects/bacula/files/latest/download\?source=files" ' \
    r'title="/bacula/.*?/bacula-(.*?).tar.gz'

STABLE_REDIS_URL = "http://redis.io/download"
STABLE_REDIS_PATTERN = \
    r'(.*?)	.*?' \
    r'Stable	'

STABLE_UNBOUND_URL = "http://unbound.net/download.html"
STABLE_UNBOUND_PATTERN = \
    r'
Unbound Downloads
.*?' \
    r'The latest version of unbound \(currently (.*?)\)'

STABLE_SOGO_URL = "http://www.sogo.nu/downloads/backend.html"
STABLE_SOGO_PATTERN = \
    r'
Source Code
.*?' \
    r'id="downbutton">.*?SOGo-(.*?).tar.gz
'

STABLE_COUCHDB_URL = "http://couchdb.apache.org/"
STABLE_COUCHDB_PATTERN = \
    r'
.*?' \
    r'
Download CouchDB (.*?)
'

STABLE_OWNCLOUD_URL = "https://owncloud.org/install/"
STABLE_OWNCLOUD_PATTERN = \
    r'
Install ownCloud Server.*?' \
    r'
Latest stable version: (.*?)\s\('

STABLE_OPENSMTPD_URL = "https://opensmtpd.org/"
STABLE_OPENSMTPD_PATTERN = \
    r'OpenSMTPD (.*?) released'

STABLE_PHP_URL = "http://php.net/"
STABLE_PHP_PATTERN = \
    r"
(.*?)" \
    r"·Release Notes
"

__all__ = [
    "STABLE_SENDMAIL_URL", "STABLE_SENDMAIL_PATTERN",
    "STABLE_BIND_URL", "STABLE_BIND_PATTERN",
    "STABLE_CYRUS_IMAP_URL", "STABLE_CYRUS_IMAP_PATTERN",
    "STABLE_APACHE_URL", "STABLE_APACHE_PATTERN",
    "STABLE_SQUID_URL", "STABLE_SQUID_PATTERN",
    "STABLE_CANIT_URL", "STABLE_CANIT_PATTERN",
    "STABLE_PSQL_URL", "STABLE_PSQL_PATTERN",
    "STABLE_SQUIRRELMAIL_URL", "STABLE_SQUIRRELMAIL_PATTERN",
    "STABLE_OPENLDAP_URL", "STABLE_OPENLDAP_PATTERN",
    "STABLE_PLA_URL", "STABLE_PLA_PATTERN",
    "STABLE_POSTFIX_URL", "STABLE_POSTFIX_PATTERN",
    "STABLE_NGINX_URL", "STABLE_NGINX_PATTERN",
    "STABLE_MYSQL_URL", "STABLE_MYSQL_PATTERN",
    "STABLE_PROFTPD_URL", "STABLE_PROFTPD_PATTERN",
    "STABLE_VSFTPD_URL", "STABLE_VSFTPD_PATTERN",
    "STABLE_SENDMAILANALYZER_URL", "STABLE_SENDMAILANALYZER_PATTERN",
    "STABLE_DJANGO_URL", "STABLE_DJANGO_PATTERN",
    "STABLE_STUNNEL_URL", "STABLE_STUNNEL_PATTERN",
    "STABLE_POUND_URL", "STABLE_POUND_PATTERN",
    "STABLE_LINUX_URL", "STABLE_LINUX_PATTERN",
    "STABLE_VARNISH_URL", "STABLE_VARNISH_PATTERN",
    "STABLE_CLAMAV_URL", "STABLE_CLAMAV_PATTERN",
    "STABLE_OPENSSH_URL", "STABLE_OPENSSH_PATTERN",
    "STABLE_DOVECOT_URL", "STABLE_DOVECOT_PATTERN",
    "STABLE_MONGODB_URL", "STABLE_MONGODB_PATTERN",
    "STABLE_SPAMASSASSIN_URL", "STABLE_SPAMASSASSIN_PATTERN",
    "STABLE_CONSUL_URL", "STABLE_CONSUL_PATTERN",
    "STABLE_MIMEDEFANG_URL", "STABLE_MIMEDEFANG_PATTERN",
    "STABLE_VAGRANT_URL", "STABLE_VAGRANT_PATTERN",
    "STABLE_HAPROXY_URL", "STABLE_HAPROXY_PATTERN",
    "STABLE_MONIT_URL", "STABLE_MONIT_PATTERN",
    "STABLE_WORDPRESS_URL", "STABLE_WORDPRESS_PATTERN",
    "STABLE_BACULA_URL", "STABLE_BACULA_PATTERN",
    "STABLE_REDIS_URL", "STABLE_REDIS_PATTERN",
    "STABLE_UNBOUND_URL", "STABLE_UNBOUND_PATTERN",
    "STABLE_SOGO_URL", "STABLE_SOGO_PATTERN",
    "STABLE_COUCHDB_URL", "STABLE_COUCHDB_PATTERN",
    "STABLE_OWNCLOUD_URL", "STABLE_OWNCLOUD_PATTERN",
    "STABLE_OPENSMTPD_URL", "STABLE_OPENSMTPD_PATTERN",
    "STABLE_PHP_URL", "STABLE_PHP_PATTERN"
]
<|endoftext|>


<commit_before>#!/usr/bin/python3
# vim:fileencoding=utf-8:sw=4:et

# generate-chinese-variants
#
# Copyright (c) 2013-2018 Mike FABIAN <mfabian@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from typing import Any
import re
import logging
import sys

# Unihan_Variants.txt contains the following 2 lines:
#
# U+50DE  kSimplifiedVariant      U+4F2A
# U+50DE  kTraditionalVariant     U+507D U+50DE
#
# This seems to be currently the only case in Unihan_Variants.txt where
# a character which has entries for kTraditionalVariant and
# the same character is listed again among the traditional variants
# is *not* simplified Chinese.
#
# U+50DE 僞 is traditional Chinese.
# U+507D 偽 is also traditional Chinese.
# U+4F2A 伪 is simplified Chinese
#
# This does not cause a problem with the current parsing code
# of Unihan_Variants.txt because the line
#
# U+50DE  kSimplifiedVariant      U+4F2A
#
# is read first and thus the character is already inserted in the
# “VARIANTS_TABLE_ORIG” dictionary as traditional Chinese, which is correct.
# If a character is already in the dictionary and more lines for the
# same character are read from Unihan_Variants.txt, these extra lines
# are ignored.
#
# But maybe for some corner cases more tweaking of the code is
# necessary. One can also add overrides manually to the
# initial content of “VARIANTS_TABLE_ORIG”.

VARIANTS_TABLE_ORIG = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
    #
    # overrides can be added manually here. For example the following
    # line marks the 〇 character as used in both
    # simplified and traditional Chinese:
    u'〇': 3 # simplified *and* traditional Chinese
    }

# keep the lines from Unihan_Variants.txt which were used for debugging
VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED = {}

def read_unihan_variants(unihan_variants_file) -> None:
    '''
    Read the Unihan_Variants.txt file downloaded  from Unicode.org.
    '''
    for line in unihan_variants_file:
        line = line.strip()
        if not re.match('^#', line):
            if re.search('(kTraditionalVariant|kSimplifiedVariant)', line):
                match = re.match(r'^U\+([0-9A-F]{4,5})', line)
                if match:
                    char = chr(int(match.group(1), 16))
                    category = 0 # should never  stay at this value
                    if re.match(re.escape(match.group(0))
                                + r'.*'
                                + re.escape(match.group(0)), line):
                        # is both simplified and traditional
                        category = 1 | 1 << 1
                    elif re.search('kTraditionalVariant', line):
                        category = 1 # simplified only
                    elif re.search('kSimplifiedVariant', line):
                        category = 1 << 1 # traditional only
                    logging.debug(
                        'char=%s category=%d line=%s',
                        char, category, line)
                    if not char in VARIANTS_TABLE_ORIG:
                        VARIANTS_TABLE_ORIG[char] = category
                    if (not char
                            in VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED):
                        VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED[
                            char] = line

def detect_chinese_category_old(phrase: str) -> int:
    '''
    Old function using encoding conversion to guess whether
    a text is simplified Chinese, traditional Chinese, both,
    or unknown. Does not work well, is included here for reference
    and for comparing with the results of the new, improved function
    using the data from the Unihan database.
    '''
    # this is the bitmask we will use,
    # from low to high, 1st bit is simplified Chinese,
    # 2nd bit is traditional Chinese,
    # 3rd bit means out of gbk
    category = 0
    # make sure that we got a unicode string
    tmp_phrase = ''.join(re.findall(u'['
                                    + u'\u4E00-\u9FCB'
                                    + u'\u3400-\u4DB5'
                                    + u'\uF900-\uFaFF'
                                    + u'\U00020000-\U0002A6D6'
                                    + u'\U0002A700-\U0002B734'
                                    + u'\U0002B740-\U0002B81D'
                                    + u'\U0002F800-\U0002FA1D'
                                    + u']+',
                                    phrase))
    # first whether in gb2312
    try:
        tmp_phrase.encode('gb2312')
        category |= 1
    except:
        if u'〇' in tmp_phrase:
            # we add '〇' into SC as well
            category |= 1
    # second check big5-hkscs
    try:
        tmp_phrase.encode('big5hkscs')
        category |= 1 << 1
    except:
        # then check whether in gbk,
        if category & 1:
            # already know in SC
            pass
        else:
            # need to check
            try:
                tmp_phrase.encode('gbk')
                category |= 1
            except:
                # not in gbk
                pass
    # then set for 3rd bit, if not in SC and TC
    if not category & (1 | 1 << 1):
        category |= (1 << 2)
    return category

def write_variants_script(script_file) -> None:
    '''
    Write the generated Python script.
    '''
    script_file.write('''#!/usr/bin/python
# vim:fileencoding=utf-8:sw=4:et

# auto-generated by “generate-chinese-variants.py”, do not edit here!
#
# Copyright (c) 2013 Mike FABIAN <mfabian@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
''')

    script_file.write('''
import sys
''')

    script_file.write('''
VARIANTS_TABLE = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
''')

    for phrase in sorted(VARIANTS_TABLE_ORIG):
        script_file.write(
            "    u'" + phrase + "': "
            + "%s" %VARIANTS_TABLE_ORIG[phrase] + ",\n")

    script_file.write('''    }
''')

    script_file.write('''
def detect_chinese_category(phrase):
    \'\'\'
    New function using Unihan data to guess whether a text is
    simplified Chinese, traditional Chinese, both, or something rare
    like a mixture of exclusively simplified with exclusively traditional
    characters.

    Meaning of the bits in the category value returned by this function:
    1 = 1 << 0       simplified Chinese
    2 = 1 << 1       traditional Chinese
    3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    4 = 1 << 2       mixture of simplified and traditional Chinese
    \'\'\'
    # make sure that we got a unicode string
    if phrase in VARIANTS_TABLE:
        # the complete phrase is in VARIANTS_TABLE, just return the
        # value found:
        return VARIANTS_TABLE[phrase]
    category = 0xFF
    for char in phrase:
        if char in VARIANTS_TABLE:
            category &= VARIANTS_TABLE[char]
        else:
            # If it is not listed in VARIANTS_TABLE, assume it is
            # both simplified and traditional Chinese.
            # It could be something non-Chinese as well then, but
            # if it is non-Chinese, it should also be allowed to
            # occur in any Chinese text and thus classified as
            # both simplified *and* traditional Chinese (the emoji
            # table for example uses many non-Chinese characters)
            category &= (1 | 1 << 1)
    if category == 0:
        # If category is 0 after binary & of the categories of all the
        # characters in the phrase, it means that the phrase contained
        # exclusively simplified *and* exclusively traditional
        # characters at the same time.  For example if the phrase is
        # “乌烏” then “乌” gets category 1 (simplified Chinese)
        # and “烏” gets category 2 (traditional Chinese), the result
        # of the binary & is thus 0. In that case, classify it as
        # category 4 which is for weird, excentric, rare stuff. If the
        # user selects one of the modes “all characters but
        # simplified Chinese first” or “all characters but
        # traditional Chinese first”, phrases with category 4 will be
        # shown but filtered to be shown only at the end of the
        # candidate list.
        category = 1 << 2
    return category
''')

TEST_DATA = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
    u'乌': 1,
    u'烏': 2,
    u'晞': 3,
    u'䖷': 3,
    u'乌烏': 4,
    u'a☺α乌': 1,
    u'a☺α烏': 2,
    u'台': 3,
    u'同': 3,
    # Bug in Unihan_Variants.txt? Unihan_Variants.txt says this is
    # “traditional Chinese only”:
    u'覆': 3,
    u'表': 3,
    u'杰': 3,
    u'面': 3,
    u'系': 3,
    u'乾': 3,
    u'著': 3,
    }

def test_detection(generated_script) -> None:
    '''
    Test whether the generated script does the detection correctly.
    '''
    logging.info('Testing detection ...')
    for phrase in TEST_DATA:
        if (generated_script.detect_chinese_category(phrase)
                != TEST_DATA[phrase]):
            print('phrase', phrase, repr(phrase),
                  'detected as',
                  generated_script.detect_chinese_category(phrase),
                  'should have been', TEST_DATA[phrase],
                  'Test failed. exiting...')
            exit(1)
        else:
            logging.info('phrase=%s %s detected as %d OK.',
                         phrase,
                         repr(phrase),
                         TEST_DATA[phrase])
    logging.info('All tests passed.')

def compare_old_new_detection(phrase, generated_script) -> None:
    '''
    Only for debugging.

    Compares results of the Chinese category detection using the
    old and the new function.
    '''
    if (detect_chinese_category_old(phrase)
            != generated_script.detect_chinese_category(phrase)):
        logging.debug(
            '%s %s old=%d new=%d',
            phrase.encode('utf-8'),
            repr(phrase),
            detect_chinese_category_old(phrase),
            generated_script.detect_chinese_category(phrase))
        if phrase in VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED:
            logging.debug(
                VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED[phrase])

def parse_args() -> Any:
    '''Parse the command line arguments'''
    import argparse
    parser = argparse.ArgumentParser(
        description=(
            'Generate a script containing a table and a function '
            + 'to check whether a string of Chinese characters '
            + 'is simplified or traditional'))
    parser.add_argument('-i', '--inputfilename',
                        nargs='?',
                        type=str,
                        default='./Unihan_Variants.txt',
                        help='input file, default is ./Unihan_Variants.txt')
    parser.add_argument('-o', '--outputfilename',
                        nargs='?',
                        type=str,
                        default='./chinese_variants.py',
                        help='output file, default is ./chinese_variants.py')
    parser.add_argument('-d', '--debug',
                        action='store_true',
                        help='print debugging output')
    return parser.parse_args()

def main() -> None:
    '''Main program'''
    args = parse_args()
    log_level = logging.INFO
    if args.debug:
        log_level = logging.DEBUG
    logging.basicConfig(format="%(levelname)s: %(message)s", level=log_level)
    with open(args.inputfilename, 'r') as inputfile:
        logging.info("input file=%s", inputfile)
        read_unihan_variants(inputfile)
    with open(args.outputfilename, 'w') as outputfile:
        logging.info("output file=%s", outputfile)
        write_variants_script(outputfile)

    import imp
    generated_script = imp.load_source('dummy', args.outputfilename)

    test_detection(generated_script)

    for phrase in generated_script.VARIANTS_TABLE: # type: ignore
        compare_old_new_detection(phrase, generated_script)

if __name__ == '__main__':
    main()
<commit_msg>Small improvements to the test function in generate-chinese-variants.py<commit_after>#!/usr/bin/python3
# vim:fileencoding=utf-8:sw=4:et

# generate-chinese-variants
#
# Copyright (c) 2013-2018 Mike FABIAN <mfabian@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from typing import Any
import re
import logging
import sys

# Unihan_Variants.txt contains the following 2 lines:
#
# U+50DE  kSimplifiedVariant      U+4F2A
# U+50DE  kTraditionalVariant     U+507D U+50DE
#
# This seems to be currently the only case in Unihan_Variants.txt where
# a character which has entries for kTraditionalVariant and
# the same character is listed again among the traditional variants
# is *not* simplified Chinese.
#
# U+50DE 僞 is traditional Chinese.
# U+507D 偽 is also traditional Chinese.
# U+4F2A 伪 is simplified Chinese
#
# This does not cause a problem with the current parsing code
# of Unihan_Variants.txt because the line
#
# U+50DE  kSimplifiedVariant      U+4F2A
#
# is read first and thus the character is already inserted in the
# “VARIANTS_TABLE_ORIG” dictionary as traditional Chinese, which is correct.
# If a character is already in the dictionary and more lines for the
# same character are read from Unihan_Variants.txt, these extra lines
# are ignored.
#
# But maybe for some corner cases more tweaking of the code is
# necessary. One can also add overrides manually to the
# initial content of “VARIANTS_TABLE_ORIG”.

VARIANTS_TABLE_ORIG = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
    #
    # overrides can be added manually here. For example the following
    # line marks the 〇 character as used in both
    # simplified and traditional Chinese:
    u'〇': 3 # simplified *and* traditional Chinese
    }

# keep the lines from Unihan_Variants.txt which were used for debugging
VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED = {}

def read_unihan_variants(unihan_variants_file) -> None:
    '''
    Read the Unihan_Variants.txt file downloaded  from Unicode.org.
    '''
    for line in unihan_variants_file:
        line = line.strip()
        if not re.match('^#', line):
            if re.search('(kTraditionalVariant|kSimplifiedVariant)', line):
                match = re.match(r'^U\+([0-9A-F]{4,5})', line)
                if match:
                    char = chr(int(match.group(1), 16))
                    category = 0 # should never  stay at this value
                    if re.match(re.escape(match.group(0))
                                + r'.*'
                                + re.escape(match.group(0)), line):
                        # is both simplified and traditional
                        category = 1 | 1 << 1
                    elif re.search('kTraditionalVariant', line):
                        category = 1 # simplified only
                    elif re.search('kSimplifiedVariant', line):
                        category = 1 << 1 # traditional only
                    logging.debug(
                        'char=%s category=%d line=%s',
                        char, category, line)
                    if not char in VARIANTS_TABLE_ORIG:
                        VARIANTS_TABLE_ORIG[char] = category
                    if (not char
                            in VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED):
                        VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED[
                            char] = line

def detect_chinese_category_old(phrase: str) -> int:
    '''
    Old function using encoding conversion to guess whether
    a text is simplified Chinese, traditional Chinese, both,
    or unknown. Does not work well, is included here for reference
    and for comparing with the results of the new, improved function
    using the data from the Unihan database.
    '''
    # this is the bitmask we will use,
    # from low to high, 1st bit is simplified Chinese,
    # 2nd bit is traditional Chinese,
    # 3rd bit means out of gbk
    category = 0
    # make sure that we got a unicode string
    tmp_phrase = ''.join(re.findall(u'['
                                    + u'\u4E00-\u9FCB'
                                    + u'\u3400-\u4DB5'
                                    + u'\uF900-\uFaFF'
                                    + u'\U00020000-\U0002A6D6'
                                    + u'\U0002A700-\U0002B734'
                                    + u'\U0002B740-\U0002B81D'
                                    + u'\U0002F800-\U0002FA1D'
                                    + u']+',
                                    phrase))
    # first whether in gb2312
    try:
        tmp_phrase.encode('gb2312')
        category |= 1
    except:
        if u'〇' in tmp_phrase:
            # we add '〇' into SC as well
            category |= 1
    # second check big5-hkscs
    try:
        tmp_phrase.encode('big5hkscs')
        category |= 1 << 1
    except:
        # then check whether in gbk,
        if category & 1:
            # already know in SC
            pass
        else:
            # need to check
            try:
                tmp_phrase.encode('gbk')
                category |= 1
            except:
                # not in gbk
                pass
    # then set for 3rd bit, if not in SC and TC
    if not category & (1 | 1 << 1):
        category |= (1 << 2)
    return category

def write_variants_script(script_file) -> None:
    '''
    Write the generated Python script.
    '''
    script_file.write('''#!/usr/bin/python
# vim:fileencoding=utf-8:sw=4:et

# auto-generated by “generate-chinese-variants.py”, do not edit here!
#
# Copyright (c) 2013 Mike FABIAN <mfabian@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
''')

    script_file.write('''
import sys
''')

    script_file.write('''
VARIANTS_TABLE = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
''')

    for phrase in sorted(VARIANTS_TABLE_ORIG):
        script_file.write(
            "    u'" + phrase + "': "
            + "%s" %VARIANTS_TABLE_ORIG[phrase] + ",\n")

    script_file.write('''    }
''')

    script_file.write('''
def detect_chinese_category(phrase):
    \'\'\'
    New function using Unihan data to guess whether a text is
    simplified Chinese, traditional Chinese, both, or something rare
    like a mixture of exclusively simplified with exclusively traditional
    characters.

    Meaning of the bits in the category value returned by this function:
    1 = 1 << 0       simplified Chinese
    2 = 1 << 1       traditional Chinese
    3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    4 = 1 << 2       mixture of simplified and traditional Chinese
    \'\'\'
    # make sure that we got a unicode string
    if phrase in VARIANTS_TABLE:
        # the complete phrase is in VARIANTS_TABLE, just return the
        # value found:
        return VARIANTS_TABLE[phrase]
    category = 0xFF
    for char in phrase:
        if char in VARIANTS_TABLE:
            category &= VARIANTS_TABLE[char]
        else:
            # If it is not listed in VARIANTS_TABLE, assume it is
            # both simplified and traditional Chinese.
            # It could be something non-Chinese as well then, but
            # if it is non-Chinese, it should also be allowed to
            # occur in any Chinese text and thus classified as
            # both simplified *and* traditional Chinese (the emoji
            # table for example uses many non-Chinese characters)
            category &= (1 | 1 << 1)
    if category == 0:
        # If category is 0 after binary & of the categories of all the
        # characters in the phrase, it means that the phrase contained
        # exclusively simplified *and* exclusively traditional
        # characters at the same time.  For example if the phrase is
        # “乌烏” then “乌” gets category 1 (simplified Chinese)
        # and “烏” gets category 2 (traditional Chinese), the result
        # of the binary & is thus 0. In that case, classify it as
        # category 4 which is for weird, excentric, rare stuff. If the
        # user selects one of the modes “all characters but
        # simplified Chinese first” or “all characters but
        # traditional Chinese first”, phrases with category 4 will be
        # shown but filtered to be shown only at the end of the
        # candidate list.
        category = 1 << 2
    return category
''')

TEST_DATA = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
    u'乌': 1,
    u'烏': 2,
    u'晞': 3,
    u'䖷': 3,
    u'乌烏': 4,
    u'a☺α乌': 1,
    u'a☺α烏': 2,
    u'台': 3,
    u'同': 3,
    # Characters below this comments probably have buggy entries
    # in Unihan_Variants.txt:
    u'覆': 3, # U+8986
    u'表': 3, # U+8868
    u'杰': 3, # U+6770
    u'面': 3, # U+9762
    u'系': 3, # U+7CFB
    u'乾': 3, # U+4E7E
    u'著': 3, # U+8457 Patch by Heiher <r@hev.cc>
    }

def test_detection(generated_script) -> int:
    '''
    Test whether the generated script does the detection correctly.

    Returns the number of errors found.
    '''
    logging.info('Testing detection ...')
    error_count = 0
    for phrase in TEST_DATA:
        if (generated_script.detect_chinese_category(phrase)
                != TEST_DATA[phrase]):
            print('phrase', phrase, repr(phrase),
                  'detected as',
                  generated_script.detect_chinese_category(phrase),
                  'should have been', TEST_DATA[phrase],
                  'FAIL.')
            error_count += 1
        else:
            logging.info('phrase=%s %s detected as %d PASS.',
                         phrase,
                         repr(phrase),
                         TEST_DATA[phrase])
    return error_count

def compare_old_new_detection(phrase, generated_script) -> None:
    '''
    Only for debugging.

    Compares results of the Chinese category detection using the
    old and the new function.
    '''
    if (detect_chinese_category_old(phrase)
            != generated_script.detect_chinese_category(phrase)):
        logging.debug(
            '%s %s old=%d new=%d',
            phrase.encode('utf-8'),
            repr(phrase),
            detect_chinese_category_old(phrase),
            generated_script.detect_chinese_category(phrase))
        if phrase in VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED:
            logging.debug(
                VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED[phrase])

def parse_args() -> Any:
    '''Parse the command line arguments'''
    import argparse
    parser = argparse.ArgumentParser(
        description=(
            'Generate a script containing a table and a function '
            + 'to check whether a string of Chinese characters '
            + 'is simplified or traditional'))
    parser.add_argument('-i', '--inputfilename',
                        nargs='?',
                        type=str,
                        default='./Unihan_Variants.txt',
                        help='input file, default is ./Unihan_Variants.txt')
    parser.add_argument('-o', '--outputfilename',
                        nargs='?',
                        type=str,
                        default='./chinese_variants.py',
                        help='output file, default is ./chinese_variants.py')
    parser.add_argument('-d', '--debug',
                        action='store_true',
                        help='print debugging output')
    return parser.parse_args()

def main() -> None:
    '''Main program'''
    args = parse_args()
    log_level = logging.INFO
    if args.debug:
        log_level = logging.DEBUG
    logging.basicConfig(format="%(levelname)s: %(message)s", level=log_level)
    with open(args.inputfilename, 'r') as inputfile:
        logging.info("input file=%s", inputfile)
        read_unihan_variants(inputfile)
    with open(args.outputfilename, 'w') as outputfile:
        logging.info("output file=%s", outputfile)
        write_variants_script(outputfile)

    import imp
    generated_script = imp.load_source('dummy', args.outputfilename)

    logging.info('Testing detection ...')
    error_count = test_detection(generated_script)
    if error_count:
        logging.info('FAIL: %s tests failed, exiting ...', error_count)
        exit(1)
    else:
        logging.info('PASS: All tests passed.')

    for phrase in generated_script.VARIANTS_TABLE: # type: ignore
        compare_old_new_detection(phrase, generated_script)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2014, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

"""
Implements the global singleton app object.

"""

from __future__ import division

import sys

from. import backends
from .backends import BACKENDS, BACKENDMAP, ATTEMPTED_BACKENDS
from .. import config
from .base import BaseApplicationBackend as ApplicationBackend  # noqa
from ..util import logger

class Application(object):

    """ Representation of the vispy application. This wraps a native
    GUI application instance. Vispy has a default instance of this class
    at vispy.app.default_app.

    There are multiple stages for an Application object:
        * Backend-less - the state when it is just initialized
        * Backend selected - use() has been successfully called. Note that
          the Canvas calls use() without arguments reight before creating
          its backend widget.
        * Native application is created - the Canvas probes the
          Application,native property to ensure that there is a native
          application right before a native widget is created.

    """

    def __init__(self):
        self._backend_module = None
        self._backend = None

    def __repr__(self):
        name = self.backend_name
        if not name:
            return '<Vispy app with no backend>'
        else:
            return '<Vispy app, wrapping the %s GUI toolkit>' % name

    @property
    def backend_name(self):
        """ The name of the GUI backend that this app wraps.
        """
        if self._backend is not None:
            return self._backend._vispy_get_backend_name()
        else:
            return ''

    @property
    def backend_module(self):
        """ The module object that defines the backend.
        """
        return self._backend_module

    def process_events(self):
        """ Process all pending GUI events. If the mainloop is not
        running, this should be done regularly to keep the visualization
        interactive and to keep the event system going.
        """
        return self._backend._vispy_process_events()

    def create(self):
        """ Create the native application.
        """
        self.native

    def run(self):
        """ Enter the native GUI event loop.
        """
        return self._backend._vispy_run()

    def quit(self):
        """ Quit the native GUI event loop.
        """
        return self._backend._vispy_quit()

    @property
    def native(self):
        """ The native GUI application instance.
        """
        return self._backend._vispy_get_native_app()

    def use(self, backend_name=None):
        """ Select a backend by name. If the backend name is omitted,
        will chose a suitable backend automatically. It is an error to
        try to select a particular backend if one is already selected.
        Available backends: 'PySide', 'PyQt4', 'Glut', 'Pyglet', 'qt'.
        The latter will use PySide or PyQt4, whichever works.

        If a backend name is provided, and that backend could not be
        loaded, an error is raised.

        If no backend name is provided, this function will first check
        if the GUI toolkit corresponding to each backend is already
        imported, and try that backend first. If this is unsuccessful,
        it will try the 'default_backend' provided in the vispy config.
        If still not succesful, it will try each backend in a
        predetermined order.

        """
        if backend_name is not None:
            if backend_name.lower() not in BACKENDMAP:
                raise ValueError('backend_name must be one of %s or None, not '
                                 '%s' % (BACKENDMAP, backend_name))

        # Should we try and load any backend, or just this specific one?
        try_others = backend_name is None

        # Check if already selected
        if self._backend is not None:
            if backend_name and \
                    backend_name.lower() != self.backend_name.lower():
                raise RuntimeError('Can only select a backend once.')
            return

        # Get backends to try ...
        backends_to_try = []
        if not try_others:
            # Test if given name is ok
            if backend_name.lower() not in BACKENDMAP.keys():
                raise ValueError('Backend name not known: "%s"' % backend_name)
            # Add it
            backends_to_try.append(backend_name.lower())
        else:
            # See if a backend is loaded
            for name, module_name, native_module_name in BACKENDS:
                if native_module_name and native_module_name in sys.modules:
                    backends_to_try.append(name.lower())
            # See if a default is given
            default_backend = config['default_backend'].lower()
            if default_backend.lower() in BACKENDMAP.keys():
                if default_backend not in backends_to_try:
                    backends_to_try.append(default_backend)
            # After this, try each one
            for name, module_name, native_module_name in BACKENDS:
                name = name.lower()
                if name not in backends_to_try:
                    backends_to_try.append(name)

        # Now try each one
        for key in backends_to_try:
            # Get info for this backend
            try:
                name, module_name, native_module_name = BACKENDMAP[key]
            except KeyError:
                logger.warn('This should not happen, unknown backend: "".'
                            % key)
                continue
            # Try to import it ...

            try:
                ATTEMPTED_BACKENDS.append(name)
                mod_name = 'backends.' + module_name
                __import__(mod_name, globals(), level=1)
            except ImportError as err:
                msg = 'Could not import backend "%s":\n%s' % (name, str(err))
                if not try_others:
                    raise RuntimeError(msg)
            except Exception as err:
                msg = ('Error while importing backend "%s":\n%s'
                       % (name, str(err)))
                if not try_others:
                    raise RuntimeError(msg)
                else:
                    logger.info(msg)
            else:
                # Success!
                self._backend_module = getattr(backends, module_name)
                break
        else:
            raise RuntimeError('Could not import any of the backends.')

        # Store classes for app backend and canvas backend
        self._backend = self.backend_module.ApplicationBackend()
<commit_msg>Add info message on backend creation.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2014, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

"""
Implements the global singleton app object.

"""

from __future__ import division

import sys

from. import backends
from .backends import BACKENDS, BACKENDMAP, ATTEMPTED_BACKENDS
from .. import config
from .base import BaseApplicationBackend as ApplicationBackend  # noqa
from ..util import logger

class Application(object):

    """ Representation of the vispy application. This wraps a native
    GUI application instance. Vispy has a default instance of this class
    at vispy.app.default_app.

    There are multiple stages for an Application object:
        * Backend-less - the state when it is just initialized
        * Backend selected - use() has been successfully called. Note that
          the Canvas calls use() without arguments reight before creating
          its backend widget.
        * Native application is created - the Canvas probes the
          Application,native property to ensure that there is a native
          application right before a native widget is created.

    """

    def __init__(self):
        self._backend_module = None
        self._backend = None

    def __repr__(self):
        name = self.backend_name
        if not name:
            return '<Vispy app with no backend>'
        else:
            return '<Vispy app, wrapping the %s GUI toolkit>' % name

    @property
    def backend_name(self):
        """ The name of the GUI backend that this app wraps.
        """
        if self._backend is not None:
            return self._backend._vispy_get_backend_name()
        else:
            return ''

    @property
    def backend_module(self):
        """ The module object that defines the backend.
        """
        return self._backend_module

    def process_events(self):
        """ Process all pending GUI events. If the mainloop is not
        running, this should be done regularly to keep the visualization
        interactive and to keep the event system going.
        """
        return self._backend._vispy_process_events()

    def create(self):
        """ Create the native application.
        """
        self.native

    def run(self):
        """ Enter the native GUI event loop.
        """
        return self._backend._vispy_run()

    def quit(self):
        """ Quit the native GUI event loop.
        """
        return self._backend._vispy_quit()

    @property
    def native(self):
        """ The native GUI application instance.
        """
        return self._backend._vispy_get_native_app()

    def use(self, backend_name=None):
        """ Select a backend by name. If the backend name is omitted,
        will chose a suitable backend automatically. It is an error to
        try to select a particular backend if one is already selected.
        Available backends: 'PySide', 'PyQt4', 'Glut', 'Pyglet', 'qt'.
        The latter will use PySide or PyQt4, whichever works.

        If a backend name is provided, and that backend could not be
        loaded, an error is raised.

        If no backend name is provided, this function will first check
        if the GUI toolkit corresponding to each backend is already
        imported, and try that backend first. If this is unsuccessful,
        it will try the 'default_backend' provided in the vispy config.
        If still not succesful, it will try each backend in a
        predetermined order.

        """
        if backend_name is not None:
            if backend_name.lower() not in BACKENDMAP:
                raise ValueError('backend_name must be one of %s or None, not '
                                 '%s' % (BACKENDMAP, backend_name))

        # Should we try and load any backend, or just this specific one?
        try_others = backend_name is None

        # Check if already selected
        if self._backend is not None:
            if backend_name and \
                    backend_name.lower() != self.backend_name.lower():
                raise RuntimeError('Can only select a backend once.')
            return

        # Get backends to try ...
        backends_to_try = []
        if not try_others:
            # Test if given name is ok
            if backend_name.lower() not in BACKENDMAP.keys():
                raise ValueError('Backend name not known: "%s"' % backend_name)
            # Add it
            backends_to_try.append(backend_name.lower())
        else:
            # See if a backend is loaded
            for name, module_name, native_module_name in BACKENDS:
                if native_module_name and native_module_name in sys.modules:
                    backends_to_try.append(name.lower())
            # See if a default is given
            default_backend = config['default_backend'].lower()
            if default_backend.lower() in BACKENDMAP.keys():
                if default_backend not in backends_to_try:
                    backends_to_try.append(default_backend)
            # After this, try each one
            for name, module_name, native_module_name in BACKENDS:
                name = name.lower()
                if name not in backends_to_try:
                    backends_to_try.append(name)

        # Now try each one
        for key in backends_to_try:
            # Get info for this backend
            try:
                name, module_name, native_module_name = BACKENDMAP[key]
            except KeyError:
                logger.warn('This should not happen, unknown backend: "".'
                            % key)
                continue
            # Try to import it ...

            try:
                ATTEMPTED_BACKENDS.append(name)
                mod_name = 'backends.' + module_name
                __import__(mod_name, globals(), level=1)
            except ImportError as err:
                msg = 'Could not import backend "%s":\n%s' % (name, str(err))
                if not try_others:
                    raise RuntimeError(msg)
            except Exception as err:
                msg = ('Error while importing backend "%s":\n%s'
                       % (name, str(err)))
                if not try_others:
                    raise RuntimeError(msg)
                else:
                    logger.info(msg)
            else:
                # Success!
                self._backend_module = getattr(backends, module_name)
                logger.info('Selected backend %s' % module_name)
                break
        else:
            raise RuntimeError('Could not import any of the backends.')

        # Store classes for app backend and canvas backend
        self._backend = self.backend_module.ApplicationBackend()
<|endoftext|>