<commit_before>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Prepare the Application Configurations."""
import collections
import json
import logging
import os
from base64 import b64decode

import gitlab

from ..consts import GIT_URL, GITLAB_TOKEN

ENVS = ('build', 'dev', 'stage', 'prod', 'prodp', 'stagepci', 'prods',
        'stagesox')
LOG = logging.getLogger(__name__)
JSON_ERROR_MSG = '"{0}" appears to be invalid json. Please validate it with http://jsonlint.com.'

def process_git_configs(git_short=''):
    """Retrieve _application.json_ files from GitLab.

    Args:
        git_short (str): Short Git representation of repository, e.g.
            forrest/core.

    Returns:
        collections.defaultdict: Configurations stored for each environment
        found.
    """
    LOG.info('Processing application.json files from GitLab.')

    server = gitlab.Gitlab(GIT_URL, token=GITLAB_TOKEN)

    project_id = server.getproject(git_short)['id']

    app_configs = collections.defaultdict(dict)
    for env in ENVS:
        app_json = 'runway/application-master-{env}.json'.format(env=env)
        file_blob = server.getfile(
            project_id,
            app_json,
            'master')
        LOG.debug('GitLab file response:\n%s', file_blob)

        if not file_blob:
            LOG.debug('Application configuration not available for %s.', env)
            # TODO: Use default configs anyways?
            continue
        else:
            file_contents = b64decode(file_blob['content'])
            try:
                app_configs[env] = json.loads(file_contents.decode())
            except ValueError:
                msg = JSON_ERROR_MSG.format(app_json)
                raise SystemExit(msg)

    LOG.info('Processing pipeline.json from GitLab.')
    pipeline_json = 'runway/pipeline.json'
    pipeline_blob = server.getfile(project_id,
                                   pipeline_json,
                                   'master', )

    if not pipeline_blob:
        LOG.info('Pipeline configuration not available, using defaults.')
        app_configs['pipeline'] = {'env': ['stage', 'prod']}
    else:
        LOG.info('Pipeline configuration found.')
        pipeline_contents = b64decode(pipeline_blob['content'])
        try:
            LOG.info(pipeline_contents.decode())
            app_configs['pipeline'] = json.loads(pipeline_contents.decode())
        except ValueError:
            msg = JSON_ERROR_MSG.format(pipeline_json)
            raise SystemExit(msg)

    config_commit = server.getbranch(project_id, 'master')['commit']['id']
    LOG.info('Commit ID used: %s', config_commit)
    app_configs['pipeline']['config_commit'] = config_commit

    LOG.debug('Application configs:\n%s', app_configs)
    return app_configs

def process_runway_configs(runway_dir=''):
    """Read the _application.json_ files.

    Args:
        runway_dir (str): Name of runway directory with app.json files.

    Returns:
        collections.defaultdict: Configurations stored for each environment
        found.
    """
    LOG.info('Processing application.json files from local directory.')

    app_configs = collections.defaultdict(dict)
    for env in ENVS:
        file_json = 'application-master-{env}.json'.format(
            env=env)
        file_name = os.path.join(runway_dir,
                                 file_json)
        LOG.debug('File to read: %s', file_name)

        try:
            with open(file_name, 'rt') as json_file:
                LOG.info('Processing %s.', file_name)
                app_configs[env] = json.load(json_file)
        except FileNotFoundError:
            continue
        except ValueError:
            msg = JSON_ERROR_MSG.format(file_json)
            raise SystemExit(msg)

    LOG.info('Processing pipeline.json from local directory')
    try:
        pipeline_file = os.path.join(runway_dir, 'pipeline.json')
        LOG.debug('Reading pipeline.json from %s', pipeline_file)
        with open(pipeline_file) as pipeline:
            app_configs['pipeline'] = json.load(pipeline)
            LOG.info(app_configs['pipeline'])
    except FileNotFoundError:
        LOG.warning('Unable to process pipeline.json. Using defaults.')
        app_configs['pipeline'] = {'env': ['stage', 'prod']}
    except ValueError:
        msg = JSON_ERROR_MSG.format(pipeline_file)
        raise SystemExit(msg)

    LOG.debug('Application configs:\n%s', app_configs)
    return app_configs
<commit_msg>feat: Include short Git name in INFO<commit_after>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Prepare the Application Configurations."""
import collections
import json
import logging
import os
from base64 import b64decode

import gitlab

from ..consts import GIT_URL, GITLAB_TOKEN

ENVS = ('build', 'dev', 'stage', 'prod', 'prodp', 'stagepci', 'prods',
        'stagesox')
LOG = logging.getLogger(__name__)
JSON_ERROR_MSG = '"{0}" appears to be invalid json. Please validate it with http://jsonlint.com.'

def process_git_configs(git_short=''):
    """Retrieve _application.json_ files from GitLab.

    Args:
        git_short (str): Short Git representation of repository, e.g.
            forrest/core.

    Returns:
        collections.defaultdict: Configurations stored for each environment
        found.
    """
    LOG.info('Processing application.json files from GitLab "%s".', git_short)

    server = gitlab.Gitlab(GIT_URL, token=GITLAB_TOKEN)

    project_id = server.getproject(git_short)['id']

    app_configs = collections.defaultdict(dict)
    for env in ENVS:
        app_json = 'runway/application-master-{env}.json'.format(env=env)
        file_blob = server.getfile(
            project_id,
            app_json,
            'master')
        LOG.debug('GitLab file response:\n%s', file_blob)

        if not file_blob:
            LOG.debug('Application configuration not available for %s.', env)
            # TODO: Use default configs anyways?
            continue
        else:
            file_contents = b64decode(file_blob['content'])
            try:
                app_configs[env] = json.loads(file_contents.decode())
            except ValueError:
                msg = JSON_ERROR_MSG.format(app_json)
                raise SystemExit(msg)

    LOG.info('Processing pipeline.json from GitLab.')
    pipeline_json = 'runway/pipeline.json'
    pipeline_blob = server.getfile(project_id,
                                   pipeline_json,
                                   'master', )

    if not pipeline_blob:
        LOG.info('Pipeline configuration not available, using defaults.')
        app_configs['pipeline'] = {'env': ['stage', 'prod']}
    else:
        LOG.info('Pipeline configuration found.')
        pipeline_contents = b64decode(pipeline_blob['content'])
        try:
            LOG.info(pipeline_contents.decode())
            app_configs['pipeline'] = json.loads(pipeline_contents.decode())
        except ValueError:
            msg = JSON_ERROR_MSG.format(pipeline_json)
            raise SystemExit(msg)

    config_commit = server.getbranch(project_id, 'master')['commit']['id']
    LOG.info('Commit ID used: %s', config_commit)
    app_configs['pipeline']['config_commit'] = config_commit

    LOG.debug('Application configs:\n%s', app_configs)
    return app_configs

def process_runway_configs(runway_dir=''):
    """Read the _application.json_ files.

    Args:
        runway_dir (str): Name of runway directory with app.json files.

    Returns:
        collections.defaultdict: Configurations stored for each environment
        found.
    """
    LOG.info('Processing application.json files from local directory.')

    app_configs = collections.defaultdict(dict)
    for env in ENVS:
        file_json = 'application-master-{env}.json'.format(
            env=env)
        file_name = os.path.join(runway_dir,
                                 file_json)
        LOG.debug('File to read: %s', file_name)

        try:
            with open(file_name, 'rt') as json_file:
                LOG.info('Processing %s.', file_name)
                app_configs[env] = json.load(json_file)
        except FileNotFoundError:
            continue
        except ValueError:
            msg = JSON_ERROR_MSG.format(file_json)
            raise SystemExit(msg)

    LOG.info('Processing pipeline.json from local directory')
    try:
        pipeline_file = os.path.join(runway_dir, 'pipeline.json')
        LOG.debug('Reading pipeline.json from %s', pipeline_file)
        with open(pipeline_file) as pipeline:
            app_configs['pipeline'] = json.load(pipeline)
            LOG.info(app_configs['pipeline'])
    except FileNotFoundError:
        LOG.warning('Unable to process pipeline.json. Using defaults.')
        app_configs['pipeline'] = {'env': ['stage', 'prod']}
    except ValueError:
        msg = JSON_ERROR_MSG.format(pipeline_file)
        raise SystemExit(msg)

    LOG.debug('Application configs:\n%s', app_configs)
    return app_configs
<|endoftext|>


<commit_before>
import pkg_resources
import os
import numpy as np
import unittest

from openmdao.main.api import Assembly

from fusedwind.turbine.configurations import configure_bladesurface
from fusedwind.turbine.geometry import read_blade_planform

PATH = pkg_resources.resource_filename('fusedwind', 'test')

def configure_blade():

    top = Assembly()

    configure_bladesurface(top, planform_nC=6)

    # load the planform file
    top.pf_splines.pfIn = read_blade_planform('data/DTU_10MW_RWT_blade_axis_prebend.dat')
    top.blade_length = 86.366
    top.span_ni = 5

    b = top.blade_surface

    # distribute 200 points evenly along the airfoil sections
    b.chord_ni = 40

    # load the airfoil shapes defining the blade
    for f in ['data/ffaw3241.dat',
              'data/ffaw3301.dat',
              'data/ffaw3360.dat',
              'data/ffaw3480.dat' ,
              'data/cylinder.dat']:

        b.base_airfoils.append(np.loadtxt(f))

    b.blend_var = np.array([0.241, 0.301, 0.36, 0.48, 1.])

    return top

class LoftedBladeSurfaceTest(unittest.TestCase):

    def test_build(self):
        top = configure_blade()
        top.run()
        d = np.loadtxt(os.path.join(PATH, 'data/blade_test_data.dat')).reshape(40, 5, 3)
        self.assertEqual(np.testing.assert_array_almost_equal(top.blade_surface.surfout.surface, d, decimal=6), None)

if __name__ == '__main__':

    unittest.main()
<commit_msg>fixed path issue in a test<commit_after>
import pkg_resources
import os
import numpy as np
import unittest

from openmdao.main.api import Assembly

from fusedwind.turbine.configurations import configure_bladesurface
from fusedwind.turbine.geometry import read_blade_planform

PATH = pkg_resources.resource_filename('fusedwind', 'test')

def configure_blade():

    top = Assembly()

    configure_bladesurface(top, planform_nC=6)

    # load the planform file
    top.pf_splines.pfIn = read_blade_planform(os.path.join(PATH, 'data/DTU_10MW_RWT_blade_axis_prebend.dat'))
    top.blade_length = 86.366
    top.span_ni = 5

    b = top.blade_surface

    # distribute 200 points evenly along the airfoil sections
    b.chord_ni = 40

    # load the airfoil shapes defining the blade
    for f in [os.path.join(PATH, 'data/ffaw3241.dat'),
              os.path.join(PATH, 'data/ffaw3301.dat'),
              os.path.join(PATH, 'data/ffaw3360.dat'),
              os.path.join(PATH, 'data/ffaw3480.dat') ,
              os.path.join(PATH, 'data/cylinder.dat')]:

        b.base_airfoils.append(np.loadtxt(f))

    b.blend_var = np.array([0.241, 0.301, 0.36, 0.48, 1.])

    return top

class LoftedBladeSurfaceTest(unittest.TestCase):

    def test_build(self):
        top = configure_blade()
        top.run()
        d = np.loadtxt(os.path.join(PATH, 'data/blade_test_data.dat')).reshape(40, 5, 3)
        self.assertEqual(np.testing.assert_array_almost_equal(top.blade_surface.surfout.surface, d, decimal=6), None)

if __name__ == '__main__':

    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Feature manipulation utilities"""

import numpy as np
import scipy.signal

from .. import cache
from ..util.exceptions import ParameterError
from ..util.deprecation import Deprecated
__all__ = ['delta', 'stack_memory']

@cache(level=40)
def delta(data, width=9, order=1, axis=-1, trim=Deprecated()):
    r'''Compute delta features: local estimate of the derivative
    of the input data along the selected axis.

    Parameters
    ----------
    data      : np.ndarray
        the input data matrix (eg, spectrogram)

    width     : int >= 3, odd [scalar]
        Number of frames over which to compute the delta feature

    order     : int > 0 [scalar]
        the order of the difference operator.
        1 for first derivative, 2 for second, etc.

    axis      : int [scalar]
        the axis along which to compute deltas.
        Default is -1 (columns).

    trim      : bool [DEPRECATED]
        This parameter is deprecated in 0.6.0 and will be removed
        in 0.7.0.

    Returns
    -------
    delta_data   : np.ndarray [shape=(d, t)]
        delta matrix of `data` at specified order

    Notes
    -----
    This function caches at level 40.

    Examples
    --------
    Compute MFCC deltas, delta-deltas

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> mfcc = librosa.feature.mfcc(y=y, sr=sr)
    >>> mfcc_delta = librosa.feature.delta(mfcc)
    >>> mfcc_delta
    array([[ 29.278,  30.896, ...,  -0.   ,  -0.   ],
           [ 22.27 ,  25.54 , ...,   0.   ,   0.   ],
           ...,
           [ -1.198,  -0.613, ...,   0.   ,   0.   ],
           [ -0.529,  -0.204, ...,   0.   ,   0.   ]])

    >>> mfcc_delta2 = librosa.feature.delta(mfcc, order=2)
    >>> mfcc_delta2
    array([[-0.992, -2.336, ..., -0.   , -0.   ],
           [-0.024, -0.986, ...,  0.   ,  0.   ],
           ...,
           [ 0.258,  0.341, ...,  0.   ,  0.   ],
           [ 0.158,  0.198, ...,  0.   ,  0.   ]])

    >>> import matplotlib.pyplot as plt
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(mfcc)
    >>> plt.title('MFCC')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 2)
    >>> librosa.display.specshow(mfcc_delta)
    >>> plt.title(r'MFCC-$\Delta$')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 3)
    >>> librosa.display.specshow(mfcc_delta2, x_axis='time')
    >>> plt.title(r'MFCC-$\Delta^2$')
    >>> plt.colorbar()
    >>> plt.tight_layout()

    '''

    data = np.atleast_1d(data)

    if width < 3 or np.mod(width, 2) != 1:
        raise ParameterError('width must be an odd integer >= 3')

    if order <= 0 or not isinstance(order, int):
        raise ParameterError('order must be a positive integer')

    half_length = 1 + int(width // 2)
    window = np.arange(half_length - 1., -half_length, -1.)

    # Normalize the window so we're scale-invariant
    window /= np.sum(np.abs(window)**2)

    delta_x = data
    for _ in range(order):
        delta_x = scipy.ndimage.convolve1d(delta_x,
                                           window,
                                           axis=axis,
                                           mode='nearest')

    return delta_x

@cache(level=40)
def stack_memory(data, n_steps=2, delay=1, **kwargs):
    """Short-term history embedding: vertically concatenate a data
    vector or matrix with delayed copies of itself.

    Each column `data[:, i]` is mapped to::

        data[:, i] ->  [data[:, i],
                        data[:, i - delay],
                        ...
                        data[:, i - (n_steps-1)*delay]]

    For columns `i < (n_steps - 1) * delay` , the data will be padded.
    By default, the data is padded with zeros, but this behavior can be
    overridden by supplying additional keyword arguments which are passed
    to `np.pad()`.

    Parameters
    ----------
    data : np.ndarray [shape=(t,) or (d, t)]
        Input data matrix.  If `data` is a vector (`data.ndim == 1`),
        it will be interpreted as a row matrix and reshaped to `(1, t)`.

    n_steps : int > 0 [scalar]
        embedding dimension, the number of steps back in time to stack

    delay : int != 0 [scalar]
        the number of columns to step.

        Positive values embed from the past (previous columns).

        Negative values embed from the future (subsequent columns).

    kwargs : additional keyword arguments
      Additional arguments to pass to `np.pad`.

    Returns
    -------
    data_history : np.ndarray [shape=(m * d, t)]
        data augmented with lagged copies of itself,
        where `m == n_steps - 1`.

    Notes
    -----
    This function caches at level 40.

    Examples
    --------
    Keep two steps (current and previous)

    >>> data = np.arange(-3, 3)
    >>> librosa.feature.stack_memory(data)
    array([[-3, -2, -1,  0,  1,  2],
           [ 0, -3, -2, -1,  0,  1]])

    Or three steps

    >>> librosa.feature.stack_memory(data, n_steps=3)
    array([[-3, -2, -1,  0,  1,  2],
           [ 0, -3, -2, -1,  0,  1],
           [ 0,  0, -3, -2, -1,  0]])

    Use reflection padding instead of zero-padding

    >>> librosa.feature.stack_memory(data, n_steps=3, mode='reflect')
    array([[-3, -2, -1,  0,  1,  2],
           [-2, -3, -2, -1,  0,  1],
           [-1, -2, -3, -2, -1,  0]])

    Or pad with edge-values, and delay by 2

    >>> librosa.feature.stack_memory(data, n_steps=3, delay=2, mode='edge')
    array([[-3, -2, -1,  0,  1,  2],
           [-3, -3, -3, -2, -1,  0],
           [-3, -3, -3, -3, -3, -2]])

    Stack time-lagged beat-synchronous chroma edge padding

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> chroma = librosa.feature.chroma_stft(y=y, sr=sr)
    >>> tempo, beats = librosa.beat.beat_track(y=y, sr=sr, hop_length=512)
    >>> beats = librosa.util.fix_frames(beats, x_min=0, x_max=chroma.shape[1])
    >>> chroma_sync = librosa.util.sync(chroma, beats)
    >>> chroma_lag = librosa.feature.stack_memory(chroma_sync, n_steps=3,
    ...                                           mode='edge')

    Plot the result

    >>> import matplotlib.pyplot as plt
    >>> beat_times = librosa.frames_to_time(beats, sr=sr, hop_length=512)
    >>> librosa.display.specshow(chroma_lag, y_axis='chroma', x_axis='time',
    ...                          x_coords=beat_times)
    >>> plt.yticks([0, 12, 24], ['Lag=0', 'Lag=1', 'Lag=2'])
    >>> plt.title('Time-lagged chroma')
    >>> plt.colorbar()
    >>> plt.tight_layout()
    """

    if n_steps < 1:
        raise ParameterError('n_steps must be a positive integer')

    if delay == 0:
        raise ParameterError('delay must be a non-zero integer')

    data = np.atleast_2d(data)

    t = data.shape[1]
    kwargs.setdefault('mode', 'constant')

    if kwargs['mode'] == 'constant':
        kwargs.setdefault('constant_values', [0])

    # Pad the end with zeros, which will roll to the front below
    if delay > 0:
        padding = (int((n_steps - 1) * delay), 0)
    else:
        padding = (0, int((n_steps - 1) * -delay))

    data = np.pad(data, [(0, 0), padding], **kwargs)

    history = data

    for i in range(1, n_steps):
        history = np.vstack([np.roll(data, -i * delay, axis=1), history])

    # Trim to original width
    if delay > 0:
        history = history[:, :t]
    else:
        history = history[:, -t:]

    # Make contiguous
    return np.ascontiguousarray(history.T).T
<commit_msg>accelerated delta calculation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Feature manipulation utilities"""

import numpy as np
import scipy.signal

from .. import cache
from ..util.exceptions import ParameterError
from ..util.deprecation import Deprecated
__all__ = ['delta', 'stack_memory']

@cache(level=40)
def delta(data, width=9, order=1, axis=-1, trim=Deprecated()):
    r'''Compute delta features: local estimate of the derivative
    of the input data along the selected axis.

    Parameters
    ----------
    data      : np.ndarray
        the input data matrix (eg, spectrogram)

    width     : int >= 3, odd [scalar]
        Number of frames over which to compute the delta feature

    order     : int > 0 [scalar]
        the order of the difference operator.
        1 for first derivative, 2 for second, etc.

    axis      : int [scalar]
        the axis along which to compute deltas.
        Default is -1 (columns).

    trim      : bool [DEPRECATED]
        This parameter is deprecated in 0.6.0 and will be removed
        in 0.7.0.

    Returns
    -------
    delta_data   : np.ndarray [shape=(d, t)]
        delta matrix of `data` at specified order

    Notes
    -----
    This function caches at level 40.

    Examples
    --------
    Compute MFCC deltas, delta-deltas

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> mfcc = librosa.feature.mfcc(y=y, sr=sr)
    >>> mfcc_delta = librosa.feature.delta(mfcc)
    >>> mfcc_delta
    array([[ 29.278,  30.896, ...,  -0.   ,  -0.   ],
           [ 22.27 ,  25.54 , ...,   0.   ,   0.   ],
           ...,
           [ -1.198,  -0.613, ...,   0.   ,   0.   ],
           [ -0.529,  -0.204, ...,   0.   ,   0.   ]])

    >>> mfcc_delta2 = librosa.feature.delta(mfcc, order=2)
    >>> mfcc_delta2
    array([[  3.123e+00,   7.475e-01, ...,  -3.553e-15,  -3.553e-15],
           [  2.933e+00,   1.101e+00, ...,   7.396e-32,   7.396e-32],
           ...,
           [  1.285e-01,   2.718e-01, ...,   0.000e+00,   0.000e+00],
           [  1.157e-01,   1.889e-01, ...,   0.000e+00,   0.000e+00]])

    >>> import matplotlib.pyplot as plt
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(mfcc)
    >>> plt.title('MFCC')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 2)
    >>> librosa.display.specshow(mfcc_delta)
    >>> plt.title(r'MFCC-$\Delta$')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 3)
    >>> librosa.display.specshow(mfcc_delta2, x_axis='time')
    >>> plt.title(r'MFCC-$\Delta^2$')
    >>> plt.colorbar()
    >>> plt.tight_layout()

    '''

    data = np.atleast_1d(data)

    if width < 3 or np.mod(width, 2) != 1:
        raise ParameterError('width must be an odd integer >= 3')

    if order <= 0 or not isinstance(order, int):
        raise ParameterError('order must be a positive integer')

    half_length = 1 + int(width // 2)
    window = np.arange(half_length - 1., -half_length, -1.)

    # Normalize the window so we're scale-invariant
    window /= np.sum(np.abs(window)**2)

    window_in = window

    for _ in range(order-1):
        window = scipy.signal.convolve(window, window_in)

    return scipy.ndimage.convolve1d(data, window, axis=axis, mode='nearest')

@cache(level=40)
def stack_memory(data, n_steps=2, delay=1, **kwargs):
    """Short-term history embedding: vertically concatenate a data
    vector or matrix with delayed copies of itself.

    Each column `data[:, i]` is mapped to::

        data[:, i] ->  [data[:, i],
                        data[:, i - delay],
                        ...
                        data[:, i - (n_steps-1)*delay]]

    For columns `i < (n_steps - 1) * delay` , the data will be padded.
    By default, the data is padded with zeros, but this behavior can be
    overridden by supplying additional keyword arguments which are passed
    to `np.pad()`.

    Parameters
    ----------
    data : np.ndarray [shape=(t,) or (d, t)]
        Input data matrix.  If `data` is a vector (`data.ndim == 1`),
        it will be interpreted as a row matrix and reshaped to `(1, t)`.

    n_steps : int > 0 [scalar]
        embedding dimension, the number of steps back in time to stack

    delay : int != 0 [scalar]
        the number of columns to step.

        Positive values embed from the past (previous columns).

        Negative values embed from the future (subsequent columns).

    kwargs : additional keyword arguments
      Additional arguments to pass to `np.pad`.

    Returns
    -------
    data_history : np.ndarray [shape=(m * d, t)]
        data augmented with lagged copies of itself,
        where `m == n_steps - 1`.

    Notes
    -----
    This function caches at level 40.

    Examples
    --------
    Keep two steps (current and previous)

    >>> data = np.arange(-3, 3)
    >>> librosa.feature.stack_memory(data)
    array([[-3, -2, -1,  0,  1,  2],
           [ 0, -3, -2, -1,  0,  1]])

    Or three steps

    >>> librosa.feature.stack_memory(data, n_steps=3)
    array([[-3, -2, -1,  0,  1,  2],
           [ 0, -3, -2, -1,  0,  1],
           [ 0,  0, -3, -2, -1,  0]])

    Use reflection padding instead of zero-padding

    >>> librosa.feature.stack_memory(data, n_steps=3, mode='reflect')
    array([[-3, -2, -1,  0,  1,  2],
           [-2, -3, -2, -1,  0,  1],
           [-1, -2, -3, -2, -1,  0]])

    Or pad with edge-values, and delay by 2

    >>> librosa.feature.stack_memory(data, n_steps=3, delay=2, mode='edge')
    array([[-3, -2, -1,  0,  1,  2],
           [-3, -3, -3, -2, -1,  0],
           [-3, -3, -3, -3, -3, -2]])

    Stack time-lagged beat-synchronous chroma edge padding

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> chroma = librosa.feature.chroma_stft(y=y, sr=sr)
    >>> tempo, beats = librosa.beat.beat_track(y=y, sr=sr, hop_length=512)
    >>> beats = librosa.util.fix_frames(beats, x_min=0, x_max=chroma.shape[1])
    >>> chroma_sync = librosa.util.sync(chroma, beats)
    >>> chroma_lag = librosa.feature.stack_memory(chroma_sync, n_steps=3,
    ...                                           mode='edge')

    Plot the result

    >>> import matplotlib.pyplot as plt
    >>> beat_times = librosa.frames_to_time(beats, sr=sr, hop_length=512)
    >>> librosa.display.specshow(chroma_lag, y_axis='chroma', x_axis='time',
    ...                          x_coords=beat_times)
    >>> plt.yticks([0, 12, 24], ['Lag=0', 'Lag=1', 'Lag=2'])
    >>> plt.title('Time-lagged chroma')
    >>> plt.colorbar()
    >>> plt.tight_layout()
    """

    if n_steps < 1:
        raise ParameterError('n_steps must be a positive integer')

    if delay == 0:
        raise ParameterError('delay must be a non-zero integer')

    data = np.atleast_2d(data)

    t = data.shape[1]
    kwargs.setdefault('mode', 'constant')

    if kwargs['mode'] == 'constant':
        kwargs.setdefault('constant_values', [0])

    # Pad the end with zeros, which will roll to the front below
    if delay > 0:
        padding = (int((n_steps - 1) * delay), 0)
    else:
        padding = (0, int((n_steps - 1) * -delay))

    data = np.pad(data, [(0, 0), padding], **kwargs)

    history = data

    for i in range(1, n_steps):
        history = np.vstack([np.roll(data, -i * delay, axis=1), history])

    # Trim to original width
    if delay > 0:
        history = history[:, :t]
    else:
        history = history[:, -t:]

    # Make contiguous
    return np.ascontiguousarray(history.T).T
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Utilities
=========

Array operations
----------------
.. autosummary::
    :toctree: generated/

    frame
    pad_center
    fix_length
    fix_frames
    index_to_slice
    softmask
    sync
    softmask

    axis_sort
    normalize
    roll_sparse
    sparsify_rows

    buf_to_float
    tiny

Matching
--------
.. autosummary::
    :toctree: generated/

    match_intervals
    match_events

Miscellaneous
-------------
.. autosummary::
    :toctree: generated/

    localmax
    peak_pick

Input validation
----------------
.. autosummary::
    :toctree: generated/

    valid_audio
    valid_int
    valid_intervals

File operations
---------------

.. autosummary::
    :toctree: generated/

    example_audio_file
    find_files

"""

from .utils import *  # pylint: disable=wildcard-import
from .files import *  # pylint: disable=wildcard-import
from .matching import *  # pylint: disable=wildcard-import
from .deprecation import *  # pylint: disable=wildcard-import
from . import decorators
from . import exceptions

__all__ = [_ for _ in dir() if not _.startswith('_')]
<commit_msg>remove one of the two softmask items in the doc<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Utilities
=========

Array operations
----------------
.. autosummary::
    :toctree: generated/

    frame
    pad_center
    fix_length
    fix_frames
    index_to_slice
    softmask
    sync

    axis_sort
    normalize
    roll_sparse
    sparsify_rows

    buf_to_float
    tiny

Matching
--------
.. autosummary::
    :toctree: generated/

    match_intervals
    match_events

Miscellaneous
-------------
.. autosummary::
    :toctree: generated/

    localmax
    peak_pick

Input validation
----------------
.. autosummary::
    :toctree: generated/

    valid_audio
    valid_int
    valid_intervals

File operations
---------------

.. autosummary::
    :toctree: generated/

    example_audio_file
    find_files

"""

from .utils import *  # pylint: disable=wildcard-import
from .files import *  # pylint: disable=wildcard-import
from .matching import *  # pylint: disable=wildcard-import
from .deprecation import *  # pylint: disable=wildcard-import
from . import decorators
from . import exceptions

__all__ = [_ for _ in dir() if not _.startswith('_')]
<|endoftext|>


<commit_before># Copyright 2011 Grid Dynamics
# Copyright 2011 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock
import webob

from nova.api.openstack.compute import fping as fping_v21
from nova import exception
from nova import test
from nova.tests.unit.api.openstack import fakes
import nova.utils

FAKE_UUID = fakes.FAKE_UUID

def execute(*cmd, **args):
    return "".join(["%s is alive" % ip for ip in cmd[1:]])

class FpingTestV21(test.TestCase):
    controller_cls = fping_v21.FpingController

    def setUp(self):
        super(FpingTestV21, self).setUp()
        self.flags(verbose=True, use_ipv6=False)
        return_server = fakes.fake_instance_get()
        return_servers = fakes.fake_instance_get_all_by_filters()
        self.stub_out("nova.db.instance_get_all_by_filters",
                      return_servers)
        self.stub_out("nova.db.instance_get_by_uuid",
                      return_server)
        self.stubs.Set(nova.utils, "execute",
                       execute)
        self.stubs.Set(self.controller_cls, "check_fping",
                       lambda self: None)
        self.controller = self.controller_cls()

    def _get_url(self):
        return "/v2/1234"

    def test_fping_index(self):
        req = fakes.HTTPRequest.blank(self._get_url() + "/os-fping")
        res_dict = self.controller.index(req)
        self.assertIn("servers", res_dict)
        for srv in res_dict["servers"]:
            for key in "project_id", "id", "alive":
                self.assertIn(key, srv)

    def test_fping_index_policy(self):
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "os-fping?all_tenants=1")
        self.assertRaises(exception.Forbidden, self.controller.index, req)
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "/os-fping?all_tenants=1")
        req.environ["nova.context"].is_admin = True
        res_dict = self.controller.index(req)
        self.assertIn("servers", res_dict)

    def test_fping_index_include(self):
        req = fakes.HTTPRequest.blank(self._get_url() + "/os-fping")
        res_dict = self.controller.index(req)
        ids = [srv["id"] for srv in res_dict["servers"]]
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "/os-fping?include=%s" % ids[0])
        res_dict = self.controller.index(req)
        self.assertEqual(len(res_dict["servers"]), 1)
        self.assertEqual(res_dict["servers"][0]["id"], ids[0])

    def test_fping_index_exclude(self):
        req = fakes.HTTPRequest.blank(self._get_url() + "/os-fping")
        res_dict = self.controller.index(req)
        ids = [srv["id"] for srv in res_dict["servers"]]
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "/os-fping?exclude=%s" %
                                      ",".join(ids[1:]))
        res_dict = self.controller.index(req)
        self.assertEqual(len(res_dict["servers"]), 1)
        self.assertEqual(res_dict["servers"][0]["id"], ids[0])

    def test_fping_show(self):
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "os-fping/%s" % FAKE_UUID)
        res_dict = self.controller.show(req, FAKE_UUID)
        self.assertIn("server", res_dict)
        srv = res_dict["server"]
        for key in "project_id", "id", "alive":
            self.assertIn(key, srv)

    @mock.patch('nova.db.instance_get_by_uuid')
    def test_fping_show_with_not_found(self, mock_get_instance):
        mock_get_instance.side_effect = exception.InstanceNotFound(
            instance_id='')
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "os-fping/%s" % FAKE_UUID)
        self.assertRaises(webob.exc.HTTPNotFound,
                          self.controller.show, req, FAKE_UUID)

class FpingPolicyEnforcementV21(test.NoDBTestCase):

    def setUp(self):
        super(FpingPolicyEnforcementV21, self).setUp()
        self.controller = fping_v21.FpingController()
        self.req = fakes.HTTPRequest.blank('')

    def common_policy_check(self, rule, func, *arg, **kwarg):
        self.policy.set_rules(rule)
        exc = self.assertRaises(
            exception.PolicyNotAuthorized, func, *arg, **kwarg)
        self.assertEqual(
            "Policy doesn't allow %s to be performed." %
            rule.popitem()[0], exc.format_message())

    def test_list_policy_failed(self):
        rule = {"os_compute_api:os-fping": "project:non_fake"}
        self.common_policy_check(rule, self.controller.index, self.req)

        self.req.GET.update({"all_tenants": "True"})
        rule = {"os_compute_api:os-fping:all_tenants":
                "project:non_fake"}
        self.common_policy_check(rule, self.controller.index, self.req)

    def test_show_policy_failed(self):
        rule = {"os_compute_api:os-fping": "project:non_fake"}
        self.common_policy_check(
            rule, self.controller.show, self.req, FAKE_UUID)

class FpingTestDeprecation(test.NoDBTestCase):

    def setUp(self):
        super(FpingTestDeprecation, self).setUp()
        self.controller = fping_v21.FpingController()
        self.req = fakes.HTTPRequest.blank('', version='2.36')

    def test_all_apis_return_not_found(self):
        self.assertRaises(exception.VersionNotFoundForAPIMethod,
            self.controller.show, self.req, fakes.FAKE_UUID)
        self.assertRaises(exception.VersionNotFoundForAPIMethod,
            self.controller.index, self.req)
<commit_msg>Remove mox from api/openstack/compute/test_fping.py<commit_after># Copyright 2011 Grid Dynamics
# Copyright 2011 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock
import webob

from nova.api.openstack.compute import fping as fping_v21
from nova import exception
from nova import test
from nova.tests.unit.api.openstack import fakes

FAKE_UUID = fakes.FAKE_UUID

def execute(*cmd, **args):
    return "".join(["%s is alive" % ip for ip in cmd[1:]])

class FpingTestV21(test.TestCase):
    controller_cls = fping_v21.FpingController

    def setUp(self):
        super(FpingTestV21, self).setUp()
        self.flags(verbose=True, use_ipv6=False)
        return_server = fakes.fake_instance_get()
        return_servers = fakes.fake_instance_get_all_by_filters()
        self.stub_out("nova.db.instance_get_all_by_filters",
                      return_servers)
        self.stub_out("nova.db.instance_get_by_uuid",
                      return_server)
        self.stub_out('nova.utils.execute',
                      execute)
        self.stub_out("nova.api.openstack.compute.fping.FpingController."
                      "check_fping",
                      lambda self: None)
        self.controller = self.controller_cls()

    def _get_url(self):
        return "/v2/1234"

    def test_fping_index(self):
        req = fakes.HTTPRequest.blank(self._get_url() + "/os-fping")
        res_dict = self.controller.index(req)
        self.assertIn("servers", res_dict)
        for srv in res_dict["servers"]:
            for key in "project_id", "id", "alive":
                self.assertIn(key, srv)

    def test_fping_index_policy(self):
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "os-fping?all_tenants=1")
        self.assertRaises(exception.Forbidden, self.controller.index, req)
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "/os-fping?all_tenants=1")
        req.environ["nova.context"].is_admin = True
        res_dict = self.controller.index(req)
        self.assertIn("servers", res_dict)

    def test_fping_index_include(self):
        req = fakes.HTTPRequest.blank(self._get_url() + "/os-fping")
        res_dict = self.controller.index(req)
        ids = [srv["id"] for srv in res_dict["servers"]]
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "/os-fping?include=%s" % ids[0])
        res_dict = self.controller.index(req)
        self.assertEqual(len(res_dict["servers"]), 1)
        self.assertEqual(res_dict["servers"][0]["id"], ids[0])

    def test_fping_index_exclude(self):
        req = fakes.HTTPRequest.blank(self._get_url() + "/os-fping")
        res_dict = self.controller.index(req)
        ids = [srv["id"] for srv in res_dict["servers"]]
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "/os-fping?exclude=%s" %
                                      ",".join(ids[1:]))
        res_dict = self.controller.index(req)
        self.assertEqual(len(res_dict["servers"]), 1)
        self.assertEqual(res_dict["servers"][0]["id"], ids[0])

    def test_fping_show(self):
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "os-fping/%s" % FAKE_UUID)
        res_dict = self.controller.show(req, FAKE_UUID)
        self.assertIn("server", res_dict)
        srv = res_dict["server"]
        for key in "project_id", "id", "alive":
            self.assertIn(key, srv)

    @mock.patch('nova.db.instance_get_by_uuid')
    def test_fping_show_with_not_found(self, mock_get_instance):
        mock_get_instance.side_effect = exception.InstanceNotFound(
            instance_id='')
        req = fakes.HTTPRequest.blank(self._get_url() +
                                      "os-fping/%s" % FAKE_UUID)
        self.assertRaises(webob.exc.HTTPNotFound,
                          self.controller.show, req, FAKE_UUID)

class FpingPolicyEnforcementV21(test.NoDBTestCase):

    def setUp(self):
        super(FpingPolicyEnforcementV21, self).setUp()
        self.controller = fping_v21.FpingController()
        self.req = fakes.HTTPRequest.blank('')

    def common_policy_check(self, rule, func, *arg, **kwarg):
        self.policy.set_rules(rule)
        exc = self.assertRaises(
            exception.PolicyNotAuthorized, func, *arg, **kwarg)
        self.assertEqual(
            "Policy doesn't allow %s to be performed." %
            rule.popitem()[0], exc.format_message())

    def test_list_policy_failed(self):
        rule = {"os_compute_api:os-fping": "project:non_fake"}
        self.common_policy_check(rule, self.controller.index, self.req)

        self.req.GET.update({"all_tenants": "True"})
        rule = {"os_compute_api:os-fping:all_tenants":
                "project:non_fake"}
        self.common_policy_check(rule, self.controller.index, self.req)

    def test_show_policy_failed(self):
        rule = {"os_compute_api:os-fping": "project:non_fake"}
        self.common_policy_check(
            rule, self.controller.show, self.req, FAKE_UUID)

class FpingTestDeprecation(test.NoDBTestCase):

    def setUp(self):
        super(FpingTestDeprecation, self).setUp()
        self.controller = fping_v21.FpingController()
        self.req = fakes.HTTPRequest.blank('', version='2.36')

    def test_all_apis_return_not_found(self):
        self.assertRaises(exception.VersionNotFoundForAPIMethod,
            self.controller.show, self.req, fakes.FAKE_UUID)
        self.assertRaises(exception.VersionNotFoundForAPIMethod,
            self.controller.index, self.req)
<|endoftext|>


<commit_before>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import windmill
import exceptions
import os, sys, shutil, time, signal
import killableprocess
import commands
import logging
import urlparse

logger = logging.getLogger(__name__)

os.environ['MOZ_NO_REMOTE'] = str(1)
PROXY_PORT = windmill.settings['SERVER_HTTP_PORT']

# Deal with queries in urls
url_split = windmill.settings['TEST_URL'].split('?')
if len(url_split) > 1:
    DEFAULT_TEST_URL = url_split[0]+'/windmill-serv/start.html'+'?'+url_split[1]
else:
    DEFAULT_TEST_URL = windmill.settings['TEST_URL']+'/windmill-serv/start.html'

MOZILLA_PROFILE_PATH = windmill.settings['MOZILLA_PROFILE_PATH']
MOZILLA_DEFAULT_PROFILE = windmill.settings['MOZILLA_DEFAULT_PROFILE']
MOZILLA_BINARY = windmill.settings['MOZILLA_BINARY']

def setpgid_preexec_fn():
    os.setpgid(0, 0)

def runCommand(cmd):
    if sys.platform != "win32":
        return killableprocess.Popen(cmd, preexec_fn=setpgid_preexec_fn)
    else:
        return killableprocess.Popen(cmd)

class MozillaProfile(object):

    def __init__(self, path=MOZILLA_PROFILE_PATH, default_profile=MOZILLA_DEFAULT_PROFILE,
                  proxy_port=PROXY_PORT, test_url=DEFAULT_TEST_URL):
        """Create profile dir, and populate it"""

        self.proxy_host = 'localhost'
        self.proxy_port = proxy_port
        self.test_url = test_url

        self.profile_path = path

        if windmill.settings['MOZILLA_CREATE_NEW_PROFILE']:
            if os.path.exists(self.profile_path) is True:
                shutil.rmtree(self.profile_path)

            if sys.platform == 'linux2':
                print commands.getoutput('%s -CreateProfile "windmill %s"' % (
                                         MOZILLA_BINARY, self.profile_path))
            shutil.copytree(default_profile, self.profile_path)

            self.prefs_js_filename = self.profile_path + '/prefs.js'
            self.prefs_js_f = open( self.prefs_js_filename, 'w')
            self.initial_prefs()

    def initial_prefs(self):
        """Initial prefs population, separated form __init__ for ease of subclassing"""

        # Get rid of default browser check
        self.user_pref('"browser.shell.checkDefaultBrowser", false')
        # Suppress authentication confirmations
        self.user_pref('"network.http.phishy-userpass-length", 255')
        # Disable pop-up blocking
        self.user_pref('"browser.allowpopups", true')
        self.user_pref('"dom.disable_open_during_load", false')
        # Open links in new windows (Firefox 2.0)
        self.user_pref('"browser.link.open_external", 2')
        self.user_pref('"browser.link.open_newwindow", 2')
        # Configure local proxy
        self.user_pref('"network.proxy.http", "%s"' % self.proxy_host)
        self.user_pref('"network.proxy.http_port", %s' % str(self.proxy_port))
        self.user_pref('"network.proxy.no_proxies_on", ""')
        self.user_pref('"network.proxy.type", 1')

        self.user_pref('"network.http.max-connections", 40')
        self.user_pref('"network.http.max-connections-per-server", 16')
        self.user_pref('"network.http.max-persistent-connections-per-proxy", 12')
        self.user_pref('"network.http.max-persistent-connections-per-server", 6')
        self.user_pref('"network.http.pipelining.maxrequests", 6')

        # Turn off favicon requests, no need for even more requests
        self.user_pref('"browser.chrome.favicons", false')

        self.user_pref('"startup.homepage_override_url", "' + self.test_url + '"')
        self.user_pref('"browser.startup.homepage", "' + self.test_url + '"')
        self.user_pref('"startup.homepage_welcome_url", ""')
        # Disable security warnings
        self.user_pref('"security.warn_submit_insecure", false')
        self.user_pref('"security.warn_submit_insecure.show_once", false')
        self.user_pref('"security.warn_entering_secure", false')
        self.user_pref('"security.warn_entering_secure.show_once", false')
        self.user_pref('"security.warn_entering_weak", false')
        self.user_pref('"security.warn_entering_weak.show_once", false')
        self.user_pref('"security.warn_leaving_secure", false')
        self.user_pref('"security.warn_leaving_secure.show_once", false')
        self.user_pref('"security.warn_viewing_mixed", false')
        self.user_pref('"security.warn_viewing_mixed.show_once", false')
        # Disable cache
        self.user_pref('"browser.cache.disk.enable", false')
        self.user_pref('"browser.cache.memory.enable", false')
        # Disable "do you want to remember this password?"
        self.user_pref('"signon.rememberSignons", false')

    def user_pref(self, string):
        self.prefs_js_f.write('user_pref(' + string + ');\n')
        self.prefs_js_f.flush()

    def add_js(self, string):
        self.prefs_js_f.write(string + '\n')
        self.prefs_js_f.flush()

    def clean_up(self):
        shutil.rmtree(self.profile_path)

def convertPath(linuxPath):
    sysdrive = os.environ.get('SYSTEMDRIVE')
    cygdrive = '/cygdrive/%s' % sysdrive.lower().replace(':', '')

    return linuxPath.replace(cygdrive, '%s' % sysdrive).replace('/', '\\')

class MozillaBrowser(object):
    """MozillaBrowser class, init requires MozillaProfile instance"""
    def __init__(self, profile, mozilla_bin=MOZILLA_BINARY):

        self.profile = profile
        self.mozilla_bin = mozilla_bin
        self.p_handle = None

        if sys.platform == 'cygwin':
            profile_path = convertPath(self.profile.profile_path)
        else:
            profile_path = self.profile.profile_path

        if sys.platform is not 'linux2':
            self.command = [self.mozilla_bin, '-profile', profile_path]
        else:
            self.command = [self.mozilla_bin, '-P', 'windmill']

    def start(self):

        self.p_handle = runCommand(self.command)

        logger.info(self.command)

    def is_alive(self):

        if self.p_handle.poll() is None:
            return False

        try:
            self.p_handle.kill(group=True)
            return True
        except exceptions.OSError:
            return False

    def kill(self, kill_signal):

        if sys.platform == 'darwin':
            try:
                os.kill(self.p_handle.pid+1, kill_signal)
            except:
                logger.error('Cannot kill firefox')
        else:
            try:
                self.p_handle.kill(group=True)
            except:
                logger.error('Cannot kill firefox')

        shutil.rmtree(windmill.settings['MOZILLA_PROFILE_PATH'])
        if windmill.settings['MOZILLA_REMOVE_PROFILE_ON_EXIT'] is True:
            # Windows holds on to the file handlers for prefs.js indefinitely, we leave tempfiles and let the OS handle cleaning them up at a later time
            if sys.platform != "win32":
                shutil.rmtree(windmill.settings['MOZILLA_PROFILE_PATH'])

    def stop(self):

        self.kill(signal.SIGTERM)

<commit_msg>Silly me.<commit_after>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import windmill
import exceptions
import os, sys, shutil, time, signal
import killableprocess
import commands
import logging
import urlparse

logger = logging.getLogger(__name__)

os.environ['MOZ_NO_REMOTE'] = str(1)
PROXY_PORT = windmill.settings['SERVER_HTTP_PORT']

# Deal with queries in urls
url_split = windmill.settings['TEST_URL'].split('?')
if len(url_split) > 1:
    DEFAULT_TEST_URL = url_split[0]+'/windmill-serv/start.html'+'?'+url_split[1]
else:
    DEFAULT_TEST_URL = windmill.settings['TEST_URL']+'/windmill-serv/start.html'

MOZILLA_PROFILE_PATH = windmill.settings['MOZILLA_PROFILE_PATH']
MOZILLA_DEFAULT_PROFILE = windmill.settings['MOZILLA_DEFAULT_PROFILE']
MOZILLA_BINARY = windmill.settings['MOZILLA_BINARY']

def setpgid_preexec_fn():
    os.setpgid(0, 0)

def runCommand(cmd):
    if sys.platform != "win32":
        return killableprocess.Popen(cmd, preexec_fn=setpgid_preexec_fn)
    else:
        return killableprocess.Popen(cmd)

class MozillaProfile(object):

    def __init__(self, path=MOZILLA_PROFILE_PATH, default_profile=MOZILLA_DEFAULT_PROFILE,
                  proxy_port=PROXY_PORT, test_url=DEFAULT_TEST_URL):
        """Create profile dir, and populate it"""

        self.proxy_host = 'localhost'
        self.proxy_port = proxy_port
        self.test_url = test_url

        self.profile_path = path

        if windmill.settings['MOZILLA_CREATE_NEW_PROFILE']:
            if os.path.exists(self.profile_path) is True:
                shutil.rmtree(self.profile_path)

            if sys.platform == 'linux2':
                print commands.getoutput('%s -CreateProfile "windmill %s"' % (
                                         MOZILLA_BINARY, self.profile_path))
            shutil.copytree(default_profile, self.profile_path)

            self.prefs_js_filename = self.profile_path + '/prefs.js'
            self.prefs_js_f = open( self.prefs_js_filename, 'w')
            self.initial_prefs()

    def initial_prefs(self):
        """Initial prefs population, separated form __init__ for ease of subclassing"""

        # Get rid of default browser check
        self.user_pref('"browser.shell.checkDefaultBrowser", false')
        # Suppress authentication confirmations
        self.user_pref('"network.http.phishy-userpass-length", 255')
        # Disable pop-up blocking
        self.user_pref('"browser.allowpopups", true')
        self.user_pref('"dom.disable_open_during_load", false')
        # Open links in new windows (Firefox 2.0)
        self.user_pref('"browser.link.open_external", 2')
        self.user_pref('"browser.link.open_newwindow", 2')
        # Configure local proxy
        self.user_pref('"network.proxy.http", "%s"' % self.proxy_host)
        self.user_pref('"network.proxy.http_port", %s' % str(self.proxy_port))
        self.user_pref('"network.proxy.no_proxies_on", ""')
        self.user_pref('"network.proxy.type", 1')

        self.user_pref('"network.http.max-connections", 40')
        self.user_pref('"network.http.max-connections-per-server", 16')
        self.user_pref('"network.http.max-persistent-connections-per-proxy", 12')
        self.user_pref('"network.http.max-persistent-connections-per-server", 6')
        self.user_pref('"network.http.pipelining.maxrequests", 6')

        # Turn off favicon requests, no need for even more requests
        self.user_pref('"browser.chrome.favicons", false')

        self.user_pref('"startup.homepage_override_url", "' + self.test_url + '"')
        self.user_pref('"browser.startup.homepage", "' + self.test_url + '"')
        self.user_pref('"startup.homepage_welcome_url", ""')
        # Disable security warnings
        self.user_pref('"security.warn_submit_insecure", false')
        self.user_pref('"security.warn_submit_insecure.show_once", false')
        self.user_pref('"security.warn_entering_secure", false')
        self.user_pref('"security.warn_entering_secure.show_once", false')
        self.user_pref('"security.warn_entering_weak", false')
        self.user_pref('"security.warn_entering_weak.show_once", false')
        self.user_pref('"security.warn_leaving_secure", false')
        self.user_pref('"security.warn_leaving_secure.show_once", false')
        self.user_pref('"security.warn_viewing_mixed", false')
        self.user_pref('"security.warn_viewing_mixed.show_once", false')
        # Disable cache
        self.user_pref('"browser.cache.disk.enable", false')
        self.user_pref('"browser.cache.memory.enable", false')
        # Disable "do you want to remember this password?"
        self.user_pref('"signon.rememberSignons", false')

    def user_pref(self, string):
        self.prefs_js_f.write('user_pref(' + string + ');\n')
        self.prefs_js_f.flush()

    def add_js(self, string):
        self.prefs_js_f.write(string + '\n')
        self.prefs_js_f.flush()

    def clean_up(self):
        shutil.rmtree(self.profile_path)

def convertPath(linuxPath):
    sysdrive = os.environ.get('SYSTEMDRIVE')
    cygdrive = '/cygdrive/%s' % sysdrive.lower().replace(':', '')

    return linuxPath.replace(cygdrive, '%s' % sysdrive).replace('/', '\\')

class MozillaBrowser(object):
    """MozillaBrowser class, init requires MozillaProfile instance"""
    def __init__(self, profile, mozilla_bin=MOZILLA_BINARY):

        self.profile = profile
        self.mozilla_bin = mozilla_bin
        self.p_handle = None

        if sys.platform == 'cygwin':
            profile_path = convertPath(self.profile.profile_path)
        else:
            profile_path = self.profile.profile_path

        if sys.platform != 'linux2':
            self.command = [self.mozilla_bin, '-profile', profile_path]
        else:
            self.command = [self.mozilla_bin, '-P', 'windmill']

    def start(self):

        self.p_handle = runCommand(self.command)

        logger.info(self.command)

    def is_alive(self):

        if self.p_handle.poll() is None:
            return False

        try:
            self.p_handle.kill(group=True)
            return True
        except exceptions.OSError:
            return False

    def kill(self, kill_signal):

        if sys.platform == 'darwin':
            try:
                os.kill(self.p_handle.pid+1, kill_signal)
            except:
                logger.error('Cannot kill firefox')
        else:
            try:
                self.p_handle.kill(group=True)
            except:
                logger.error('Cannot kill firefox')

        shutil.rmtree(windmill.settings['MOZILLA_PROFILE_PATH'])
        if windmill.settings['MOZILLA_REMOVE_PROFILE_ON_EXIT'] is True:
            # Windows holds on to the file handlers for prefs.js indefinitely, we leave tempfiles and let the OS handle cleaning them up at a later time
            if sys.platform != "win32":
                shutil.rmtree(windmill.settings['MOZILLA_PROFILE_PATH'])

    def stop(self):

        self.kill(signal.SIGTERM)

<|endoftext|>


<commit_before>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import webbrowser
import windmill
import exceptions
import os, sys, shutil, subprocess, time
import logging

logger = logging.getLogger(__name__)

os.environ['MOZ_NO_REMOTE'] = str(1)
PROXY_PORT = windmill.settings['SERVER_HTTP_PORT']
DEFAULT_TEST_URL = windmill.settings['TEST_URL']+'/windmill-serv/start.html'
MOZILLA_PROFILE_PATH = windmill.settings['MOZILLA_PROFILE_PATH']
MOZILLA_DEFAULT_PROFILE = windmill.settings['MOZILLA_DEFAULT_PROFILE']

class MozillaProfile(object):

    def __init__(self, path=MOZILLA_PROFILE_PATH, default_profile=MOZILLA_DEFAULT_PROFILE,
                  proxy_port=PROXY_PORT, test_url=DEFAULT_TEST_URL):
        """Create profile dir, and populate it"""

        self.proxy_host = 'localhost'
        self.proxy_port = proxy_port
        self.test_url = test_url

        self.profile_path = path

        if windmill.settings['MOZILLA_CREATE_NEW_PROFILE']:
            if os.path.exists(self.profile_path) is True:
                shutil.rmtree(self.profile_path)

            shutil.copytree(default_profile, self.profile_path)

            self.prefs_js_filename = self.profile_path + '/prefs.js'
            self.prefs_js_f = open( self.prefs_js_filename, 'w')
            self.initial_prefs()

    def initial_prefs(self):
        """Initial prefs population, separated form __init__ for ease of subclassing"""

        # Get rid of default browser check
        self.user_pref('"browser.shell.checkDefaultBrowser", false')
        # Suppress authentication confirmations
        self.user_pref('"network.http.phishy-userpass-length", 255')
        # Disable pop-up blocking
        self.user_pref('"browser.allowpopups", true')
        self.user_pref('"dom.disable_open_during_load", false')
        # Open links in new windows (Firefox 2.0)
        self.user_pref('"browser.link.open_external", 2')
        self.user_pref('"browser.link.open_newwindow", 2')
        # Configure local proxy
        self.user_pref('"network.proxy.http", "%s"' % self.proxy_host)
        self.user_pref('"network.proxy.http_port", %s' % str(self.proxy_port))
        self.user_pref('"network.proxy.no_proxies_on", ""')
        self.user_pref('"network.proxy.type", 1')

        self.user_pref('"network.http.max-connections", 40')
        self.user_pref('"network.http.max-connections-per-server", 16')
        self.user_pref('"network.http.max-persistent-connections-per-proxy", 12')
        self.user_pref('"network.http.max-persistent-connections-per-server", 6')
        self.user_pref('"network.http.pipelining.maxrequests", 6')

        # Turn off favicon requests, no need for even more requests
        self.user_pref('"browser.chrome.favicons", false')

        self.user_pref('"startup.homepage_override_url", "' + self.test_url + '"')
        self.user_pref('"browser.startup.homepage", "' + self.test_url + '"')
        self.user_pref('"startup.homepage_welcome_url", ""')
        # Disable security warnings
        self.user_pref('"security.warn_submit_insecure", false')
        self.user_pref('"security.warn_submit_insecure.show_once", false')
        self.user_pref('"security.warn_entering_secure", false')
        self.user_pref('"security.warn_entering_secure.show_once", false')
        self.user_pref('"security.warn_entering_weak", false')
        self.user_pref('"security.warn_entering_weak.show_once", false')
        self.user_pref('"security.warn_leaving_secure", false')
        self.user_pref('"security.warn_leaving_secure.show_once", false')
        self.user_pref('"security.warn_viewing_mixed", false')
        self.user_pref('"security.warn_viewing_mixed.show_once", false')
        # Disable cache
        self.user_pref('"browser.cache.disk.enable", false')
        self.user_pref('"browser.cache.memory.enable", false')
        # Disable "do you want to remember this password?"
        self.user_pref('"signon.rememberSignons", false')

    def user_pref(self, string):
        self.prefs_js_f.write('user_pref(' + string + ');\n')
        self.prefs_js_f.flush()

    def add_js(self, string):
        self.prefs_js_f.write(string + '\n')
        self.prefs_js_f.flush()

    def clean_up(self):
        shutil.rmtree(self.profile_path)

MOZILLA_BINARY = windmill.settings['MOZILLA_BINARY']

class MozillaBrowser(object):
    """MozillaBrowser class, init requires MozillaProfile instance"""
    def __init__(self, profile, mozilla_bin=MOZILLA_BINARY, start_url=None):

        self.profile = profile
        self.mozilla_bin = mozilla_bin
        self.p_id = None

        if sys.platform == 'darwin':
            self.shell_command = "%s -profile %s %s" % (self.mozilla_bin, self.profile.profile_path, self.profile.test_url)
        elif sys.platform == 'linux2':
            self.shell_command = "%s -P %s %s" % (self.mozilla_bin, self.profile.profile_path, self.profile.test_url)

    def start(self):

        self.p_id = subprocess.Popen(self.shell_command, shell=True).pid

        logger.info(self.shell_command)

    def is_alive(self):

        if self.p_id is None:
            return False

        try:
            os.kill(self.p_id, 0)
            return True
        except exceptions.OSError:
            return False

    def kill(self, signal):

        os.kill(self.p_id, signal)
        try:
            os.kill(self.p_id, 0)
            if sys.platform == 'darwin':
                os.kill(self.p_id+1, signal)
                os.kill(self.p_id, signal)
        except:
            pass

    def stop(self):
        import signal

        self.kill(signal=signal.SIGTERM)

            <commit_msg>The last fix just caused more issues. It's a firefox bug.<commit_after>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import webbrowser
import windmill
import exceptions
import os, sys, shutil, subprocess, time
import logging

logger = logging.getLogger(__name__)

os.environ['MOZ_NO_REMOTE'] = str(1)
PROXY_PORT = windmill.settings['SERVER_HTTP_PORT']
DEFAULT_TEST_URL = windmill.settings['TEST_URL']+'/windmill-serv/start.html'
MOZILLA_PROFILE_PATH = windmill.settings['MOZILLA_PROFILE_PATH']
MOZILLA_DEFAULT_PROFILE = windmill.settings['MOZILLA_DEFAULT_PROFILE']

class MozillaProfile(object):

    def __init__(self, path=MOZILLA_PROFILE_PATH, default_profile=MOZILLA_DEFAULT_PROFILE,
                  proxy_port=PROXY_PORT, test_url=DEFAULT_TEST_URL):
        """Create profile dir, and populate it"""

        self.proxy_host = 'localhost'
        self.proxy_port = proxy_port
        self.test_url = test_url

        self.profile_path = path

        if windmill.settings['MOZILLA_CREATE_NEW_PROFILE']:
            if os.path.exists(self.profile_path) is True:
                shutil.rmtree(self.profile_path)

            shutil.copytree(default_profile, self.profile_path)

            self.prefs_js_filename = self.profile_path + '/prefs.js'
            self.prefs_js_f = open( self.prefs_js_filename, 'w')
            self.initial_prefs()

    def initial_prefs(self):
        """Initial prefs population, separated form __init__ for ease of subclassing"""

        # Get rid of default browser check
        self.user_pref('"browser.shell.checkDefaultBrowser", false')
        # Suppress authentication confirmations
        self.user_pref('"network.http.phishy-userpass-length", 255')
        # Disable pop-up blocking
        self.user_pref('"browser.allowpopups", true')
        self.user_pref('"dom.disable_open_during_load", false')
        # Open links in new windows (Firefox 2.0)
        self.user_pref('"browser.link.open_external", 2')
        self.user_pref('"browser.link.open_newwindow", 2')
        # Configure local proxy
        self.user_pref('"network.proxy.http", "%s"' % self.proxy_host)
        self.user_pref('"network.proxy.http_port", %s' % str(self.proxy_port))
        self.user_pref('"network.proxy.no_proxies_on", ""')
        self.user_pref('"network.proxy.type", 1')

        self.user_pref('"network.http.max-connections", 40')
        self.user_pref('"network.http.max-connections-per-server", 16')
        self.user_pref('"network.http.max-persistent-connections-per-proxy", 12')
        self.user_pref('"network.http.max-persistent-connections-per-server", 6')
        self.user_pref('"network.http.pipelining.maxrequests", 6')

        # Turn off favicon requests, no need for even more requests
        self.user_pref('"browser.chrome.favicons", false')

        self.user_pref('"startup.homepage_override_url", "' + self.test_url + '"')
        self.user_pref('"browser.startup.homepage", "' + self.test_url + '"')
        self.user_pref('"startup.homepage_welcome_url", ""')
        # Disable security warnings
        self.user_pref('"security.warn_submit_insecure", false')
        self.user_pref('"security.warn_submit_insecure.show_once", false')
        self.user_pref('"security.warn_entering_secure", false')
        self.user_pref('"security.warn_entering_secure.show_once", false')
        self.user_pref('"security.warn_entering_weak", false')
        self.user_pref('"security.warn_entering_weak.show_once", false')
        self.user_pref('"security.warn_leaving_secure", false')
        self.user_pref('"security.warn_leaving_secure.show_once", false')
        self.user_pref('"security.warn_viewing_mixed", false')
        self.user_pref('"security.warn_viewing_mixed.show_once", false')
        # Disable cache
        self.user_pref('"browser.cache.disk.enable", false')
        self.user_pref('"browser.cache.memory.enable", false')
        # Disable "do you want to remember this password?"
        self.user_pref('"signon.rememberSignons", false')

    def user_pref(self, string):
        self.prefs_js_f.write('user_pref(' + string + ');\n')
        self.prefs_js_f.flush()

    def add_js(self, string):
        self.prefs_js_f.write(string + '\n')
        self.prefs_js_f.flush()

    def clean_up(self):
        shutil.rmtree(self.profile_path)

MOZILLA_BINARY = windmill.settings['MOZILLA_BINARY']

class MozillaBrowser(object):
    """MozillaBrowser class, init requires MozillaProfile instance"""
    def __init__(self, profile, mozilla_bin=MOZILLA_BINARY, start_url=None):

        self.profile = profile
        self.mozilla_bin = mozilla_bin
        self.p_id = None

        self.shell_command = "%s -profile %s %s" % (self.mozilla_bin, self.profile.profile_path, self.profile.test_url)

    def start(self):

        self.p_id = subprocess.Popen(self.shell_command, shell=True).pid

        logger.info(self.shell_command)

    def is_alive(self):

        if self.p_id is None:
            return False

        try:
            os.kill(self.p_id, 0)
            return True
        except exceptions.OSError:
            return False

    def kill(self, signal):

        os.kill(self.p_id, signal)
        try:
            os.kill(self.p_id, 0)
            if sys.platform == 'darwin':
                os.kill(self.p_id+1, signal)
                os.kill(self.p_id, signal)
        except:
            pass

    def stop(self):
        import signal

        self.kill(signal=signal.SIGTERM)

            <|endoftext|>


<commit_before>#
#     Copyright 2011, Kay Hayen, mailto:kayhayen@gmx.de
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     If you submit Kay Hayen patches to this software in either form, you
#     automatically grant him a copyright assignment to the code, or in the
#     alternative a BSD license to the code, should your jurisdiction prevent
#     this. Obviously it won't affect code that comes to him indirectly or
#     code you don't submit to him.
#
#     This is to reserve my ability to re-license the code at any time, e.g.
#     the PSF. With this version of Nuitka, using it for Closed Source will
#     not be allowed.
#
#     This program is free software: you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation, version 3 of the License.
#
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#     Please leave the whole of this copyright notice intact.
#
""" Optimization step that expands the tree by imported modules.

Normally imports are relatively static, but Nuitka also attempts to cover the uses of
__import__ and other import techniques, that allow dynamic values. If other optimizations
make it possible to predict these, the compiler can go deeper that what it normally could.

So this is called repeatedly mayhaps, each time a constant is added.
"""

from __future__ import print_function

from .OptimizeBase import OptimizationVisitorBase, info

from nuitka import TreeBuilding, Importing, Options, Nodes, Utils

class ModuleRecursionVisitor( OptimizationVisitorBase ):
    imported_modules = {}

    def __init__( self ):
        self.stdlib = Options.shallFollowStandardLibrary()

    def _recurseTo( self, module_filename, module_package, module_relpath ):
        if module_relpath not in self.imported_modules:
            info( "Recurse to import", module_relpath )

            imported_module = TreeBuilding.buildModuleTree(
                filename = module_filename,
                package  = module_package,
                is_main  = False
            )

            self.imported_modules[ module_relpath ] = imported_module

            self.signalChange(
                "new_code",
                imported_module.getSourceReference(),
                "Recursed to module."
            )

        return self.imported_modules[ module_relpath ]

    def _consider( self, module_filename, module_package ):
        assert module_package is None or type( module_package ) is str

        if module_filename.endswith( ".py" ) or Utils.isDir( module_filename ):
            if self.stdlib or not module_filename.startswith( "/usr/lib/" ):
                module_relpath = Utils.relpath( module_filename )

                return self._recurseTo(
                    module_filename = module_filename,
                    module_package  = module_package,
                    module_relpath  = module_relpath
                )

    def _handleModule( self, module ):
        module_filename = module.getFilename()

        if module_filename not in self.imported_modules:
            self.imported_modules[ Utils.relpath( module_filename ) ] = module

        module_package = module.getPackage()

        if module_package is not None:
            package_package, package_module_name, package_filename = Importing.findModule(
                module_name    = module_package,
                parent_package = None,
                level          = 1
            )

            self._recurseTo(
                module_filename = package_module_name.replace( ".", "/" ),
                module_package  = package_package,
                module_relpath  = Utils.relpath( package_filename )
            )

    def _handleImportExternal( self, node ):
        module_package, _module_name, module_filename = Importing.findModule(
            module_name    = node.getModuleName(),
            parent_package = node.getParentModule().getPackage(),
            level          = node.getLevel()
        )

        if module_filename is not None:
            imported_module = self._consider(
                module_filename = module_filename,
                module_package  = module_package
            )

            if imported_module is not None:
                import_cut_len = len( node.getModuleName() ) - len( node.getImportName() )

                import_name = imported_module.getFullName()
                import_name = import_name[:len(import_name) - import_cut_len ]

                new_node = Nodes.CPythonStatementImportEmbedded(
                    target      = node.getTarget(),
                    module_name = imported_module.getFullName(),
                    import_name = import_name,
                    module      = imported_module,
                    source_ref  = node.getSourceReference()
                )

                node.replaceWith( new_node )

    def _handleImportFromExternal( self, node ):
        parent_module = node.getParentModule()

        module_package, _module_name, module_filename = Importing.findModule(
            module_name    = node.getModuleName(),
            parent_package = parent_module.getPackage(),
            level          = node.getLevel()
        )

        # print (module_filename,module_package,node.level)

        if module_filename is not None:
            imported_module = self._consider(
                module_filename = module_filename,
                module_package  = module_package
            )

            if imported_module:
                sub_modules = []

                if Utils.isDir( module_filename ):
                    for imported_name in node.getImports():
                        sub_module_package, _sub_module_name, sub_module_filename = Importing.findModule(
                            module_name    = imported_name,
                            parent_package = imported_module.getFullName(),
                            level          = 1,
                            warn           = True
                        )

                        if sub_module_filename is not None:
                            sub_module = self._consider(
                                module_filename = sub_module_filename,
                                module_package  = sub_module_package,
                            )

                            if sub_module is not None:
                                sub_modules.append( sub_module )

                assert len( sub_modules ) == len( node.getImports() ) or len( sub_modules ) == 0

                if sub_modules:
                    new_node = Nodes.CPythonStatementImportFromEmbedded(
                        targets     = node.getTargets(),
                        module_name = imported_module.getFullName(),
                        sub_modules = sub_modules,
                        imports     = node.getImports(),
                        source_ref  = node.getSourceReference()
                    )

                    node.replaceWith( new_node )

    def _handleImportStarExternal( self, node ):
        module_package, _module_name, module_filename = Importing.findModule(
            module_name    = node.getModuleName(),
            parent_package = node.getParentModule().getPackage(),
            level          = node.getLevel()
        )

        if module_filename is not None:
            imported_module = self._consider(
                module_filename = module_filename,
                module_package  = module_package
            )

            new_node = Nodes.CPythonStatementImportStarEmbedded(
                module_name = imported_module.getFullName(),
                source_ref  = node.getSourceReference()
            )

            node.replaceWith( new_node )

    def __call__( self, node ):
        if node.isModule():
            self._handleModule(
                module = node
            )
        elif node.isStatementImportExternal():
            self._handleImportExternal(
                node = node
            )
        elif node.isStatementImportFromExternal():
            self._handleImportFromExternal(
                node = node
            )
        elif node.isStatementImportStarExternal():
            self._handleImportStarExternal(
                node = node
            )
        elif node.isExpressionBuiltinImport():
            self._consider(
                module_filename = node.getModuleFilename(),
                module_package  = node.getModulePackage()
            )
<commit_msg>In --deep mode, packages could be duplicated.<commit_after>#
#     Copyright 2011, Kay Hayen, mailto:kayhayen@gmx.de
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     If you submit Kay Hayen patches to this software in either form, you
#     automatically grant him a copyright assignment to the code, or in the
#     alternative a BSD license to the code, should your jurisdiction prevent
#     this. Obviously it won't affect code that comes to him indirectly or
#     code you don't submit to him.
#
#     This is to reserve my ability to re-license the code at any time, e.g.
#     the PSF. With this version of Nuitka, using it for Closed Source will
#     not be allowed.
#
#     This program is free software: you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation, version 3 of the License.
#
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#     Please leave the whole of this copyright notice intact.
#
""" Optimization step that expands the tree by imported modules.

Normally imports are relatively static, but Nuitka also attempts to cover the uses of
__import__ and other import techniques, that allow dynamic values. If other optimizations
make it possible to predict these, the compiler can go deeper that what it normally could.

So this is called repeatedly mayhaps, each time a constant is added.
"""

from .OptimizeBase import OptimizationVisitorBase, info

from nuitka import TreeBuilding, Importing, Options, Nodes, Utils

class ModuleRecursionVisitor( OptimizationVisitorBase ):
    imported_modules = {}

    def __init__( self ):
        self.stdlib = Options.shallFollowStandardLibrary()

    def _recurseTo( self, module_filename, module_package, module_relpath ):
        if module_relpath not in self.imported_modules:
            info( "Recurse to import", module_relpath )

            imported_module = TreeBuilding.buildModuleTree(
                filename = module_filename,
                package  = module_package,
                is_main  = False
            )

            assert not module_relpath.endswith( "/__init__.py" )

            self.imported_modules[ module_relpath ] = imported_module

            self.signalChange(
                "new_code",
                imported_module.getSourceReference(),
                "Recursed to module."
            )

        return self.imported_modules[ module_relpath ]

    def _consider( self, module_filename, module_package ):
        assert module_package is None or type( module_package ) is str

        if module_filename.endswith( ".py" ) or Utils.isDir( module_filename ):
            if self.stdlib or not module_filename.startswith( "/usr/lib/" ):
                module_relpath = Utils.relpath( module_filename )

                return self._recurseTo(
                    module_filename = module_filename,
                    module_package  = module_package,
                    module_relpath  = module_relpath
                )

    def _handleModule( self, module ):
        module_filename = module.getFilename()

        if module_filename not in self.imported_modules:
            if module_filename.endswith( "/__init__.py" ):
                module_relpath = Utils.relpath( module_filename[:-12] )
            else:
                module_relpath = Utils.relpath( module_filename )

            self.imported_modules[ Utils.relpath( module_relpath ) ] = module

        module_package = module.getPackage()

        if module_package is not None:
            package_package, package_module_name, package_filename = Importing.findModule(
                module_name    = module_package,
                parent_package = None,
                level          = 1
            )

            self._recurseTo(
                module_filename = package_module_name.replace( ".", "/" ),
                module_package  = package_package,
                module_relpath  = Utils.relpath( package_filename )
            )

    def _handleImportExternal( self, node ):
        module_package, _module_name, module_filename = Importing.findModule(
            module_name    = node.getModuleName(),
            parent_package = node.getParentModule().getPackage(),
            level          = node.getLevel()
        )

        if module_filename is not None:
            imported_module = self._consider(
                module_filename = module_filename,
                module_package  = module_package
            )

            if imported_module is not None:
                import_cut_len = len( node.getModuleName() ) - len( node.getImportName() )

                import_name = imported_module.getFullName()
                import_name = import_name[:len(import_name) - import_cut_len ]

                new_node = Nodes.CPythonStatementImportEmbedded(
                    target      = node.getTarget(),
                    module_name = imported_module.getFullName(),
                    import_name = import_name,
                    module      = imported_module,
                    source_ref  = node.getSourceReference()
                )

                node.replaceWith( new_node )

    def _handleImportFromExternal( self, node ):
        parent_module = node.getParentModule()

        module_package, _module_name, module_filename = Importing.findModule(
            module_name    = node.getModuleName(),
            parent_package = parent_module.getPackage(),
            level          = node.getLevel()
        )

        if module_filename is not None:
            imported_module = self._consider(
                module_filename = module_filename,
                module_package  = module_package
            )

            if imported_module:
                sub_modules = []

                if Utils.isDir( module_filename ):
                    for imported_name in node.getImports():
                        sub_module_package, _sub_module_name, sub_module_filename = Importing.findModule(
                            module_name    = imported_name,
                            parent_package = imported_module.getFullName(),
                            level          = 1,
                            warn           = True
                        )

                        if sub_module_filename is not None:
                            sub_module = self._consider(
                                module_filename = sub_module_filename,
                                module_package  = sub_module_package,
                            )

                            if sub_module is not None:
                                sub_modules.append( sub_module )

                assert len( sub_modules ) == len( node.getImports() ) or len( sub_modules ) == 0

                if sub_modules:
                    new_node = Nodes.CPythonStatementImportFromEmbedded(
                        targets     = node.getTargets(),
                        module_name = imported_module.getFullName(),
                        sub_modules = sub_modules,
                        imports     = node.getImports(),
                        source_ref  = node.getSourceReference()
                    )

                    node.replaceWith( new_node )

    def _handleImportStarExternal( self, node ):
        module_package, _module_name, module_filename = Importing.findModule(
            module_name    = node.getModuleName(),
            parent_package = node.getParentModule().getPackage(),
            level          = node.getLevel()
        )

        if module_filename is not None:
            imported_module = self._consider(
                module_filename = module_filename,
                module_package  = module_package
            )

            new_node = Nodes.CPythonStatementImportStarEmbedded(
                module_name = imported_module.getFullName(),
                source_ref  = node.getSourceReference()
            )

            node.replaceWith( new_node )

    def __call__( self, node ):
        if node.isModule():
            self._handleModule(
                module = node
            )
        elif node.isStatementImportExternal():
            self._handleImportExternal(
                node = node
            )
        elif node.isStatementImportFromExternal():
            self._handleImportFromExternal(
                node = node
            )
        elif node.isStatementImportStarExternal():
            self._handleImportStarExternal(
                node = node
            )
        elif node.isExpressionBuiltinImport():
            self._consider(
                module_filename = node.getModuleFilename(),
                module_package  = node.getModulePackage()
            )
<|endoftext|>


<commit_before>import os
import time
from nose.plugins.skip import SkipTest

from nxdrive.client import LocalClient
from nxdrive.client.common import LOCALLY_EDITED_FOLDER_NAME
from nxdrive.tests.common_unit_test import UnitTestCase

DRIVE_EDIT_XATTR_NAMES = ['ndrive', 'nxdriveedit', 'nxdriveeditdigest', 'nxdriveeditname']

class TestDriveEdit(UnitTestCase):

    locally_edited_path = ('/default-domain/UserWorkspaces/'
                           + 'nuxeoDriveTestUser-user-1/Collections/'
                           + LOCALLY_EDITED_FOLDER_NAME)

    def setUpApp(self):
        super(TestDriveEdit, self).setUpApp()
        self.drive_edit = self.manager_1.get_drive_edit()
        self.drive_edit.driveEditUploadCompleted.connect(self.app.sync_completed)
        self.drive_edit.start()

        self.remote = self.remote_document_client_1
        self.local = LocalClient(os.path.join(self.nxdrive_conf_folder_1, 'edit'))

    def tearDownApp(self):
        self.drive_edit.stop()
        super(TestDriveEdit, self).tearDownApp()

    def test_filename_encoding(self):
        filename = u'Mode op\xe9ratoire.txt'
        doc_id = self.remote.make_file('/', filename, 'Some content.')

        # Linux / Win + Chrome: quoted utf-8 encoded
        browser_filename = 'Mode%20op%C3%A9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'Win + Chrome')

        # Win + IE: unquoted utf-8 encoded
        browser_filename = 'Mode op\xc3\xa9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'Win + IE')

        # Win + FF: quoted string containing unicode
        browser_filename = 'Mode%20op\xe9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'Win + FF')

        # OSX + Chrome / OSX + FF: quoted utf-8 encoded, except for white spaces!
        browser_filename = 'Mode op%C3%A9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'OS X + Chrome or FF')

    def _office_locker(self, path):
        return os.path.join(os.path.dirname(path), "~$" + os.path.basename(path)[2:])

    def _openoffice_locker(self, path):
        return os.path.join(os.path.dirname(path), ".~lock." + os.path.basename(path)[2:])

    def test_autolock_office(self):
        self._autolock(self._office_locker)

    def test_autolock_openoffice(self):
        # LibreOffice as well
        self._autolock(self._openoffice_locker)

    def _autolock(self, locker):
        global called_open, lock_file
        called_open = False
        filename = u'Document.docx'
        doc_id = self.remote.make_file('/', filename, 'Some content.')
        def open_local_file (path):
            global called_open, lock_file
            called_open = True
            # Lock file
            lock_file = locker(path)
            with open(lock_file, 'w') as f:
                f.write("plop")
        self.manager_1.open_local_file = open_local_file
        self.manager_1.set_drive_edit_auto_lock(1)
        self.drive_edit._manager.open_local_file = open_local_file
        self.drive_edit.edit(self.nuxeo_url, doc_id, filename=filename, user=self.user_1)
        self.wait_sync(timeout=2, fail_if_timeout=False)
        self.assertTrue(called_open, "Should have called open_local_file")
        # Should be able to test lock
        self.assertTrue(self.remote_restapi_client_1.is_locked(doc_id))
        os.remove(lock_file)
        self.wait_sync(timeout=2, fail_if_timeout=False)
        # Should be unlock
        self.assertFalse(self.remote_restapi_client_1.is_locked(doc_id))
        self.manager_1.set_drive_edit_auto_lock(0)
        with open(lock_file, 'w') as f:
            f.write("plop")
        self.wait_sync(timeout=2, fail_if_timeout=False)
        self.assertFalse(self.remote_restapi_client_1.is_locked(doc_id))

    def _drive_edit_update(self, doc_id, filename, browser_filename, content):
        # Download file
        local_path = '/%s/%s' % (doc_id, filename)
        self.drive_edit._prepare_edit(self.nuxeo_url, doc_id, browser_filename)
        self.assertTrue(self.local.exists(local_path))
        self.wait_sync(timeout=2, fail_if_timeout=False)
        self.local.delete_final(local_path)

        # Update file content
        self.local.update_content(local_path, content)
        self.wait_sync()
        self.assertEquals(self.remote.get_content('/' + filename), content)
<commit_msg>NXDRIVE-452: Remove lock test file<commit_after>import os
import time
from nose.plugins.skip import SkipTest

from nxdrive.client import LocalClient
from nxdrive.client.common import LOCALLY_EDITED_FOLDER_NAME
from nxdrive.tests.common_unit_test import UnitTestCase

DRIVE_EDIT_XATTR_NAMES = ['ndrive', 'nxdriveedit', 'nxdriveeditdigest', 'nxdriveeditname']

class TestDriveEdit(UnitTestCase):

    locally_edited_path = ('/default-domain/UserWorkspaces/'
                           + 'nuxeoDriveTestUser-user-1/Collections/'
                           + LOCALLY_EDITED_FOLDER_NAME)

    def setUpApp(self):
        super(TestDriveEdit, self).setUpApp()
        self.drive_edit = self.manager_1.get_drive_edit()
        self.drive_edit.driveEditUploadCompleted.connect(self.app.sync_completed)
        self.drive_edit.start()

        self.remote = self.remote_document_client_1
        self.local = LocalClient(os.path.join(self.nxdrive_conf_folder_1, 'edit'))

    def tearDownApp(self):
        self.drive_edit.stop()
        super(TestDriveEdit, self).tearDownApp()

    def test_filename_encoding(self):
        filename = u'Mode op\xe9ratoire.txt'
        doc_id = self.remote.make_file('/', filename, 'Some content.')

        # Linux / Win + Chrome: quoted utf-8 encoded
        browser_filename = 'Mode%20op%C3%A9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'Win + Chrome')

        # Win + IE: unquoted utf-8 encoded
        browser_filename = 'Mode op\xc3\xa9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'Win + IE')

        # Win + FF: quoted string containing unicode
        browser_filename = 'Mode%20op\xe9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'Win + FF')

        # OSX + Chrome / OSX + FF: quoted utf-8 encoded, except for white spaces!
        browser_filename = 'Mode op%C3%A9ratoire.txt'
        self._drive_edit_update(doc_id, filename, browser_filename, 'OS X + Chrome or FF')

    def _office_locker(self, path):
        return os.path.join(os.path.dirname(path), "~$" + os.path.basename(path)[2:])

    def _openoffice_locker(self, path):
        return os.path.join(os.path.dirname(path), ".~lock." + os.path.basename(path)[2:])

    #def test_autolock_office(self):
    #    self._autolock(self._office_locker)

    #def test_autolock_openoffice(self):
    # LibreOffice as well
    #    self._autolock(self._openoffice_locker)

    def _autolock(self, locker):
        global called_open, lock_file
        called_open = False
        filename = u'Document.docx'
        doc_id = self.remote.make_file('/', filename, 'Some content.')
        def open_local_file (path):
            global called_open, lock_file
            called_open = True
            # Lock file
            lock_file = locker(path)
            with open(lock_file, 'w') as f:
                f.write("plop")
        self.manager_1.open_local_file = open_local_file
        self.manager_1.set_drive_edit_auto_lock(1)
        self.drive_edit._manager.open_local_file = open_local_file
        self.drive_edit.edit(self.nuxeo_url, doc_id, filename=filename, user=self.user_1)
        self.wait_sync(timeout=2, fail_if_timeout=False)
        self.assertTrue(called_open, "Should have called open_local_file")
        # Should be able to test lock
        self.assertTrue(self.remote_restapi_client_1.is_locked(doc_id))
        os.remove(lock_file)
        self.wait_sync(timeout=2, fail_if_timeout=False)
        # Should be unlock
        self.assertFalse(self.remote_restapi_client_1.is_locked(doc_id))
        self.manager_1.set_drive_edit_auto_lock(0)
        with open(lock_file, 'w') as f:
            f.write("plop")
        self.wait_sync(timeout=2, fail_if_timeout=False)
        self.assertFalse(self.remote_restapi_client_1.is_locked(doc_id))

    def _drive_edit_update(self, doc_id, filename, browser_filename, content):
        # Download file
        local_path = '/%s/%s' % (doc_id, filename)
        self.drive_edit._prepare_edit(self.nuxeo_url, doc_id, browser_filename)
        self.assertTrue(self.local.exists(local_path))
        self.wait_sync(timeout=2, fail_if_timeout=False)
        self.local.delete_final(local_path)

        # Update file content
        self.local.update_content(local_path, content)
        self.wait_sync()
        self.assertEquals(self.remote.get_content('/' + filename), content)
<|endoftext|>


<commit_before>def loadFilesService():
    execfile("filesAdmin.py")
<commit_msg>Customize scripts to work with menu<commit_after>globdict = globals()
def loadFilesService():
    global globdict
    execfile("filesAdmin.py", globdict)
<|endoftext|>