<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Oct 20 17:40:07 2015

@author: casimp
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import griddata
from pyxe.fitting_functions import cos_
from pyxe.plotting_tools import plot_complex, meshgrid_res, line_extract

class StrainPlotting(object):
    """
    Takes post-processed .nxs file from the I12 EDXD detector. File should have
    been created with the XRD_analysis tool and contain detector specific peaks
    and associated strain.
    """
    def __init__(self):
        pass

    def plot_intensity(self, detector = 0, point = (), figsize = (7, 5)):
        """
        Plots q v intensity. *Not implemented for merged files.*

        # detector:   0 based indexing - 0 (default) to 23 - detector 23 empty.
        # point:      Define point (index) from which to extract q v I plot.
                      First point in array chosen if not (default) specified.
        """
        if point == ():
            point = (0, ) * len(self.strain[..., 0, 0].shape)

        q = self.q[detector]
        I = self.I[point][detector]
        plt.figure(figsize = figsize)
        plt.plot(q, I, 'k-')
        plt.xlabel('q (rad)')
        plt.ylabel('Intensity')

    def plot_fitted(self, point = (), q_idx = 0, figsize = (7, 5)):
        """
        Plots fitted in-plane strain field for given data point.
        *Not implemented for merged files.*

        # point:      Define point (index) from which to plot fitted in-plane
                      strain field.
        # q_idx:      0 based indexing - 0 (default) to 23 - detector 23 empty.
        # figsize:    Figure dimensions
        """
        if point == ():
            point = (0, ) * len(self.strain[..., 0, 0].shape)

        plt.figure(figsize = figsize)
        p = self.strain_param[point][q_idx]
        theta = self.phi
        # Data from edi12 has extra, unused detector (filled with nan)
        try:
            plt.plot(theta, self.strain[point][..., q_idx], 'k*')
        except ValueError:
            plt.plot(theta, self.strain[point][..., q_idx][:-1], 'k*')
        theta_2 = np.linspace(self.phi[0], self.phi[-1], 1000)
        plt.plot(theta_2, cos_(theta_2, *p), 'k-')
        plt.xlabel('Azimuthal angle (rad)')
        plt.ylabel('Strain')

    def plot_mohrs(self, point = (), q_idx = 0, angle = -np.pi, figsize = (7, 5)):
        """
        Use fitted in-plane styrain tensor to plot Mohr's circle.
        *Not implemented for merged files.*

        # point:      Define point (index) from which to plot fitted in-plane
                      strain field.
        # q_idx:      0 based indexing - 0 (default) to 23 - detector 23 empty.
        # angle:      Angle to highlight on circle (inc + 90deg).
        # figsize:    Figure dimensions
        """
        if point == ():
            point = (0, ) * len(self.strain[..., 0, 0].shape)

        p = self.strain_param[point][q_idx] ### test change
        R = p[0]
        theta = p[1] + angle

        e_xx, e_yy = cos_(angle, *p), cos_(angle + np.pi/2, *p)
        e_1, e_2 = (p[2] + abs(p[0])), (p[2] - abs(p[0]))
        e_xy = -np.sin(2 * theta) * ((p[2] + p[0]) - (p[2] - p[0]))/2

        fig = plt.figure(figsize = figsize)
        plt.axis('equal')
        ax = fig.add_subplot(1, 1, 1)
        circ = plt.Circle((p[2], 0), radius=R, color='k', fill = False)

        ax.add_patch(circ)

        plt.xlim([p[2] - abs(2 * R), p[2] + abs(2 * R)])
        plt.plot([e_1, e_2], [0, 0], 'ko', markersize = 3)

        plt.plot(e_xx, e_xy, 'ko', label = r'$(\epsilon_{xx}$, $\epsilon_{xy})$')
        plt.plot(e_yy, -e_xy, 'wo', label = r'$(\epsilon_{yy}$, $-\epsilon_{xy})$')

        plt.legend(numpoints=1, frameon = False, handletextpad = 0.2)
        plt.plot([e_xx, e_yy], [e_xy, -e_xy], 'k-.')
        ax.annotate('  %s' % r'$\epsilon_{1}$', xy=(e_1, 0),
                    textcoords='offset points', xytext=(e_1, 0))
        ax.annotate('  %s' % r'$\epsilon_{2}$', xy=(e_2, 0),
                    textcoords='offset points', xytext=(e_2, 0))
        plt.xlabel('Normal strain')
        plt.ylabel('Shear strain')

    def plot_line(self, az_angles = [-np.pi, -np.pi/2], detectors = [], q_idx = 0,
                  line_angle = 0, pnt = (0, 0), npts = 100, axis = 'scalar',
                  method = 'linear', data_type = 'strain', shear = False,
                  E = 200 * 10**9, v = 0.3, G = 79 * 10 **9):
        """
        Plots a line profile through a 2D strain field - extract_line method
        must be run first. *Not yet implemented in 3D.*

        # detector:   0 based indexing - 0 (default) to 23 - detector 23 empty.
        # q0_index:   Specify lattice parameter/peak to display.
        # axis:       Plot strain against the 'd1', 'd2' or 'scalar' (default)
                      position. 'scalar' co-ordinates re-zeroed/centred against
                      point specified in the extract_line command.
        """
        error = 'Line plotting only implemented for 1D or 2D data sets'
        assert len(self.dims) <= 2, error

        if len(self.dims) == 1:
            d1 = self.co_ords[self.dims[0]]
            if detectors != []:
                for i in detectors:
                    plt.plot(d1, self.strain[..., i, q_idx], '-*')
            else:
                d1, data = self.extract_line_angle(az_angles)
                for idx, angle in enumerate(az_angles):
                    plt.plot(d1, data[:, idx], '-*')

        else:
            if detectors == []:
                line_method = self.extract_line_angle
                ext = az_angles
            else:
                line_method = self.extract_line_detector
                ext = detectors

            d1, d2, data = line_method(ext, q_idx = q_idx, line_angle = line_angle,
                            pnt = pnt, npts = npts, method = method, shear = shear,
                            data_type = data_type, E = E, v = v, G = G)

            if d1[0] > d1[-1]:
                d1, d2, data = d1[::-1], d2[::-1], data[::-1]

            zero = ((pnt[0] - d1[0])**2 + (pnt[1] - d2[0])**2)**0.5
            scalar_ext = ((d1 - d1[0])**2 + (d2 - d2[0])**2)**0.5 - zero

            x = {'d1' : d1, 'd2' : d2, 'scalar' : scalar_ext}
            plt.plot(x[axis], data, '-x')

    def plot_detector(self, detector = 0, q_idx = 0, stress = False, err = False,
                      E = 200 * 10**9, v = 0.3,
                      res = 0.1, lvls = 11, figsize = (10, 10),
                      line = False, pnt = (0,0), line_angle = 0, line_props = 'w-',
                      plotting = plot_complex, ax = False, cbar = True, **kwargs):
        """
        Plot a 2D heat map of the strain field. *Not yet implemented in 3D.*

        # detector:   0 based indexing - 0 (default) to 23 - detector 23 empty.
        # q0_index:   Specify lattice parameter/peak to display.
        # res:        Resolution in points per unit length (of raw data)
                      - only implemented for merged data
        # cmap:       The colormap (default - 'RdBu_r') to use for plotting
        # lvls:       Number of contours to overlay on map. Can also explicitly
                      define levels.
        # figsize:    Tuple containing the fig size (x, y) - default (10, 10).
                      Constrained by axis being equal.

        Additional functionality allows for the overlaying of a line on top of
        the map - to be used in conjunction with the line plotting.

        # line:       Plot line (default = False)
        # line_props: Define line properties (default = 'w-')
        # mark:       Mark properties for centre point of line (default = None)
        """
        error = 'Plotting method only compatible with 2D data sets'
        assert len(self.dims) == 2, error

        d1, d2 = [self.co_ords[x] for x in self.dims]

        if err:
            data = 1 - self.strain_err[..., detector, q_idx]
        elif stress:
            data = self.extract_stress(q_idx = q_idx, E = E, v = v, detector = detector)[0]
        else:
            data = self.strain[..., detector, q_idx]

        if data.ndim != 2:
            D1, D2 = meshgrid_res(d1, d2, res)
            Z = griddata((d1.flatten(), d2.flatten()),data.flatten(), (D1, D2))
        else:
            D1, D2, Z = d1, d2, data

        ax_ = plotting(d1, d2, D1, D2, Z, lvls = lvls, figsize = figsize, ax = ax, cbar = cbar, **kwargs)

        if line == True:
            plt.figure()
            line = line_extract(D1, D2, pnt, line_angle, 100)
            ax.plot(line[0], line[1], line_props, linewidth = 2)

        return ax_

    def plot_angle(self, angle = 0, q_idx = 0, stress = False, shear = False,
                   E = 200 * 10**9, v = 0.3,   G = 79 * 10**9,
                   res = 0.1, lvls = 11, figsize = (10, 10),
                   line = False, pnt = (0,0), line_angle = 0, line_props = 'w-',
                      plotting = plot_complex, ax = False, cbar = True, **kwargs):
        """
        Plot a 2D heat map of the strain field. *Not yet implemented in 3D.*

        # angle:      Angle in radians - default (0).
        # q0_index:   Specify lattice parameter/peak to display.
        # res:        Resolution in points per unit length (of raw data)
                      - only implemented for merged data
        # cmap:       The colormap (default - 'RdBu_r') to use for plotting
        # lvls:       Number of contours to overlay on map. Can also explicitly
                      define levels.
        # figsize:    Tuple containing the fig size (x, y) - default (10, 10).
                      Constrained by axis being equal.

        Additional functionality allows for the overlaying of a line on top of
        the map - to be used in conjunction with the line plotting.

        # line:       Plot line (default = False)
        # line_props: Define line properties (default = 'w-')
        # mark:       Mark properties for centre point of line (default = None)
        """

        error = 'Plotting method only compatible with 2D data sets'
        assert len(self.dims) == 2, error

        d1, d2 = [self.co_ords[x] for x in self.dims]

        data = np.nan * self.strain[..., 0, 0]
        for idx in np.ndindex(data.shape):
            p = self.strain_param[idx][q_idx]
            e_xx, e_yy = cos_(angle, *p), cos_(angle + np.pi/2, *p)
            e_xy = -np.sin(2 * (p[1] + angle)) * p[0]
            if not stress:
                data[idx] = e_xx if not shear else e_xy
            else:
                sigma_xx = E * ((1 -v) * e_xx + v*e_yy) / ((1 + v) * (1 - 2*v))
                data[idx] = sigma_xx if not shear else e_xy * G

        if data.ndim != 2:
            D1, D2 = meshgrid_res(d1, d2, spatial_resolution = res)
            Z = griddata((d1.flatten(),d2.flatten()), data.flatten(), (D1, D2))
        else:
            D1, D2, Z = d1, d2, data

        ax_ = plotting(d1, d2, D1, D2, Z, lvls = lvls, figsize = figsize, ax = ax, cbar = cbar, **kwargs)

        if line == True:
            plt.figure()
            line = line_extract(D1, D2, pnt, line_angle, 100)
            ax.plot(line[0], line[1], line_props, linewidth = 2)

        return ax_

<commit_msg>Updated plotting in line with extract changes.<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Oct 20 17:40:07 2015

@author: casimp
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import griddata
from pyxe.fitting_functions import cos_
from pyxe.plotting_tools import plot_complex, meshgrid_res, line_extract

class StrainPlotting(object):
    """
    Takes post-processed .nxs file from the I12 EDXD detector. File should have
    been created with the XRD_analysis tool and contain detector specific peaks
    and associated strain.
    """
    def __init__(self):
        pass

    def plot_intensity(self, detector = 0, point = (), figsize = (7, 5)):
        """
        Plots q v intensity. *Not implemented for merged files.*

        # detector:   0 based indexing - 0 (default) to 23 - detector 23 empty.
        # point:      Define point (index) from which to extract q v I plot.
                      First point in array chosen if not (default) specified.
        """
        if point == ():
            point = (0, ) * len(self.strain[..., 0, 0].shape)

        q = self.q[detector]
        I = self.I[point][detector]
        plt.figure(figsize = figsize)
        plt.plot(q, I, 'k-')
        plt.xlabel('q (rad)')
        plt.ylabel('Intensity')

    def plot_fitted(self, point = (), q_idx = 0, figsize = (7, 5)):
        """
        Plots fitted in-plane strain field for given data point.
        *Not implemented for merged files.*

        # point:      Define point (index) from which to plot fitted in-plane
                      strain field.
        # q_idx:      0 based indexing - 0 (default) to 23 - detector 23 empty.
        # figsize:    Figure dimensions
        """
        if point == ():
            point = (0, ) * len(self.strain[..., 0, 0].shape)

        plt.figure(figsize = figsize)
        p = self.strain_param[point][q_idx]
        theta = self.phi
        # Data from edi12 has extra, unused detector (filled with nan)
        try:
            plt.plot(theta, self.strain[point][..., q_idx], 'k*')
        except ValueError:
            plt.plot(theta, self.strain[point][..., q_idx][:-1], 'k*')
        theta_2 = np.linspace(self.phi[0], self.phi[-1], 1000)
        plt.plot(theta_2, cos_(theta_2, *p), 'k-')
        plt.xlabel('Azimuthal angle (rad)')
        plt.ylabel('Strain')

    def plot_mohrs(self, point = (), q_idx = 0, angle = -np.pi, figsize = (7, 5)):
        """
        Use fitted in-plane styrain tensor to plot Mohr's circle.
        *Not implemented for merged files.*

        # point:      Define point (index) from which to plot fitted in-plane
                      strain field.
        # q_idx:      0 based indexing - 0 (default) to 23 - detector 23 empty.
        # angle:      Angle to highlight on circle (inc + 90deg).
        # figsize:    Figure dimensions
        """
        if point == ():
            point = (0, ) * len(self.strain[..., 0, 0].shape)

        p = self.strain_param[point][q_idx] ### test change
        R = p[0]
        theta = p[1] + angle

        e_xx, e_yy = cos_(angle, *p), cos_(angle + np.pi/2, *p)
        e_1, e_2 = (p[2] + abs(p[0])), (p[2] - abs(p[0]))
        e_xy = -np.sin(2 * theta) * ((p[2] + p[0]) - (p[2] - p[0]))/2

        fig = plt.figure(figsize = figsize)
        plt.axis('equal')
        ax = fig.add_subplot(1, 1, 1)
        circ = plt.Circle((p[2], 0), radius=R, color='k', fill = False)

        ax.add_patch(circ)

        plt.xlim([p[2] - abs(2 * R), p[2] + abs(2 * R)])
        plt.plot([e_1, e_2], [0, 0], 'ko', markersize = 3)

        plt.plot(e_xx, e_xy, 'ko',label=r'$(\epsilon_{xx}$, $\epsilon_{xy})$')
        plt.plot(e_yy,-e_xy, 'wo',label=r'$(\epsilon_{yy}$, $-\epsilon_{xy})$')

        plt.legend(numpoints=1, frameon = False, handletextpad = 0.2)
        plt.plot([e_xx, e_yy], [e_xy, -e_xy], 'k-.')
        ax.annotate('  %s' % r'$\epsilon_{1}$', xy=(e_1, 0),
                    textcoords='offset points', xytext=(e_1, 0))
        ax.annotate('  %s' % r'$\epsilon_{2}$', xy=(e_2, 0),
                    textcoords='offset points', xytext=(e_2, 0))
        plt.xlabel('Normal strain')
        plt.ylabel('Shear strain')

    def plot_line(self, phi = 0, detector = [], q_idx = 0,
                  line_angle = 0, pnt = (0, 0), npnts = 100, axis = 'scalar',
                  method = 'linear', stress = False, shear = False):
        """
        Plots a line profile through a 1D/2D strain field - extract_line
        method must be run first. *Not yet implemented in 3D.*

        # detector:   0 based indexing - 0 (default) to 23 - detector 23 empty.
        # q0_index:   Specify lattice parameter/peak to display.
        # axis:       Plot strain against the 'd1', 'd2' or 'scalar' (default)
                      position. 'scalar' co-ordinates re-zeroed/centred against
                      point specified in the extract_line command.
        """
        error = 'Line plotting only implemented for 1D or 2D data sets'
        assert len(self.dims) <= 2, error

        if len(self.dims) == 1:
            d1, data = self.extract_line(phi, detector, q_idx, pnt,
                                    line_angle, npnts, method, stress, shear)
            plt.plot(d1, data, '-*')

        else:
            d1, d2, data = self.extract_line(phi, detector, q_idx, pnt,
                                    line_angle, npnts, method, stress, shear)

            if d1[0] > d1[-1]:
                d1, d2, data = d1[::-1], d2[::-1], data[::-1]

            zero = ((pnt[0] - d1[0])**2 + (pnt[1] - d2[0])**2)**0.5
            scalar_ext = ((d1 - d1[0])**2 + (d2 - d2[0])**2)**0.5 - zero

            x = {'d1' : d1, 'd2' : d2, 'scalar' : scalar_ext}
            plt.plot(x[axis], data, '-x')

    def plot_detector(self, detector = 0, q_idx = 0, stress = False, err = False,
                      res = 0.1, lvls = 11, figsize = (10, 10), line = False,
                      pnt = (0,0), line_angle = 0, line_props = 'w-',
                      plotting = plot_complex, ax=False, cbar = True, **kwargs):
        """
        Plot a 2D heat map of the strain field. *Not yet implemented in 3D.*

        # detector:   0 based indexing - 0 (default) to 23 - detector 23 empty.
        # q0_index:   Specify lattice parameter/peak to display.
        # res:        Resolution in points per unit length (of raw data)
                      - only implemented for merged data
        # cmap:       The colormap (default - 'RdBu_r') to use for plotting
        # lvls:       Number of contours to overlay on map. Can also explicitly
                      define levels.
        # figsize:    Tuple containing the fig size (x, y) - default (10, 10).
                      Constrained by axis being equal.

        Additional functionality allows for the overlaying of a line on top of
        the map - to be used in conjunction with the line plotting.

        # line:       Plot line (default = False)
        # line_props: Define line properties (default = 'w-')
        # mark:       Mark properties for centre point of line (default = None)
        """
        error = 'Plotting method only compatible with 2D data sets'
        assert len(self.dims) == 2, error

        d1, d2 = [self.co_ords[x] for x in self.dims]

        if err:
            data = 1 - self.strain_err[..., detector, q_idx]
        else:
            data = self.extract_slice(detector = detector, q_idx = q_idx,
                                      stress = stress)

        if data.ndim != 2:
            D1, D2 = meshgrid_res(d1, d2, res)
            Z = griddata((d1.flatten(), d2.flatten()),data.flatten(), (D1, D2))
        else:
            D1, D2, Z = d1, d2, data

        ax_ = plotting(d1, d2, D1, D2, Z, lvls = lvls, figsize = figsize,
                       ax = ax, cbar = cbar, **kwargs)

        if line == True:
            plt.figure()
            line = line_extract(D1, D2, pnt, line_angle, 100)
            ax.plot(line[0], line[1], line_props, linewidth = 2)

        return ax_

    def plot_angle(self, phi = 0, q_idx = 0, stress = False, shear = False,
                   res = 0.1, lvls = 11, figsize = (10, 10),
                   line = False, pnt = (0,0), line_angle=0, line_props = 'w-',
                   plotting = plot_complex, ax = False, cbar = True, **kwargs):
        """
        Plot a 2D heat map of the strain field. *Not yet implemented in 3D.*

        # angle:      Angle in radians - default (0).
        # q0_index:   Specify lattice parameter/peak to display.
        # res:        Resolution in points per unit length (of raw data)
                      - only implemented for merged data
        # cmap:       The colormap (default - 'RdBu_r') to use for plotting
        # lvls:       Number of contours to overlay on map. Can also explicitly
                      define levels.
        # figsize:    Tuple containing the fig size (x, y) - default (10, 10).
                      Constrained by axis being equal.

        Additional functionality allows for the overlaying of a line on top of
        the map - to be used in conjunction with the line plotting.

        # line:       Plot line (default = False)
        # line_props: Define line properties (default = 'w-')
        # mark:       Mark properties for centre point of line (default = None)
        """

        error = 'Plotting method only compatible with 2D data sets'
        assert len(self.dims) == 2, error

        d1, d2 = [self.co_ords[x] for x in self.dims]
        data = self.extract_slice(phi, q_idx = q_idx, stress = stress, shear = shear)

        if data.ndim != 2:
            D1, D2 = meshgrid_res(d1, d2, spatial_resolution = res)
            Z = griddata((d1.flatten(),d2.flatten()), data.flatten(), (D1, D2))
        else:
            D1, D2, Z = d1, d2, data

        ax_ = plotting(d1, d2, D1, D2, Z, lvls = lvls, figsize = figsize,
                       ax = ax, cbar = cbar, **kwargs)

        if line == True:
            plt.figure()
            line = line_extract(D1, D2, pnt, line_angle, 100)
            ax.plot(line[0], line[1], line_props, linewidth = 2)

        return ax_

<|endoftext|>


<commit_before>"""Module for bucket object."""

# Copyright 2017 Qarnot computing
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function

import hashlib
import sys
import os
import posixpath
import shutil
import itertools

from boto3.s3.transfer import TransferConfig
from itertools import groupby
from operator import attrgetter

from . import _util
from .exceptions import BucketStorageUnavailableException
from .storage import Storage

# Max size in bytes before uploading in parts.

AWS_UPLOAD_MAX_SIZE = 8 * 1024 * 1024
# Size of parts when uploading in parts
AWS_UPLOAD_PART_SIZE = 8 * 1024 * 1024

s3_multipart_config = TransferConfig(
    multipart_threshold=AWS_UPLOAD_MAX_SIZE,
    multipart_chunksize=AWS_UPLOAD_PART_SIZE,
    max_concurrency=10,
    num_download_attempts=10,
)

class Bucket(Storage):
    """Represents a resource/result bucket.

    This class is the interface to manage resources or results from a
    :class:`qarnot.bucket.Bucket`.

    .. note::
       A :class:`Bucket` must be created with
       :meth:`qarnot.connection.Connection.create_bucket`
       or retrieved with :meth:`qarnot.connection.Connection.buckets`, :meth:`qarnot.connection.Connection.retrieve_bucket`,
       or :meth:`qarnot.connection.Connection.retrieve_or_create_bucket`.

    .. note::
       Paths given as 'remote' arguments,
       (or as path arguments for :func:`Bucket.directory`)
       **must** be valid unix-like paths.
    """

    def __init__(self, connection, name, create=True):
        if connection.s3client is None:
            raise BucketStorageUnavailableException()

        self._connection = connection
        self._uuid = name

        if create:
            self._connection.s3client.create_bucket(Bucket=name)

    @classmethod
    def _retrieve(cls, connection, bucket_uuid):
        """Retrieve information of a bucket on a cluster.

        :param :class:`qarnot.connection.Connection` connection: the cluster
            to get the bucket from
        :param str bucket_uuid: the UUID of the bucket to retrieve

        :rtype: :class:`qarnot.bucket.Bucket`
        :returns: The retrieved bucket.

        :raises qarnot.exceptions.BucketStorageUnavailableException: the bucket storage engine is not available
        """
        return connection.retrieve_bucket(uuid=bucket_uuid)

    def delete(self):
        """ Delete the bucket represented by this :class:`Bucket`."""
        bucket = self._connection.s3resource.Bucket(self._uuid)
        objectlist = list(bucket.objects.all())
        n = 1000  # delete object count max request

        if sys.version_info >= (3, 0):
            listofobjectlist = [[{'Key': x.key} for x in objectlist[i:i + n]] for i in range(0, len(objectlist), n)]
        else:
            # noinspection PyUnresolvedReferences
            listofobjectlist = [[{'Key': x.key} for x in objectlist[i:i + n]] for i in xrange(0, len(objectlist), n)]  # noqa
        for item in listofobjectlist:
            bucket.delete_objects(
                Delete={
                    'Objects': item
                }
            )
        self._connection.s3client.delete_bucket(Bucket=self._uuid)

    def list_files(self):
        """List files in the bucket

        :rtype: list(:class:`S3.ObjectSummary`)
        :returns: A list of ObjectSummary resources

        """
        bucket = self._connection.s3resource.Bucket(self._uuid)
        return bucket.objects.all()

    def directory(self, directory=''):
        """List files in a directory of the disk according to prefix.

        :rtype: list(:class:`S3.ObjectSummary`)
        :returns: A list of ObjectSummary resources
        """
        bucket = self._connection.s3resource.Bucket(self._uuid)
        return bucket.objects.filter(Prefix=directory)

    def sync_directory(self, directory, verbose=False, remote=None):
        """Synchronize a local directory with the remote disks.

        :param str directory: The local directory to use for synchronization
        :param bool verbose: Print information about synchronization operations
        :param str remote: path of the directory on remote node (defaults to *local*)

        .. warning::
           Local changes are reflected on the server, a file present on the
           disk but not in the local directory will be deleted from the disk.

           A file present in the directory but not in the disk will be uploaded.

        .. note::
           The following parameters are used to determine whether
           synchronization is required :

              * name
              * size
              * sha1sum
        """
        if not directory.endswith('/'):
            directory += '/'

        filesdict = {}
        for root, _, files in os.walk(directory):
            root = _util.decode(root)
            files = map(_util.decode, files)

            for file_ in files:
                filepath = os.path.join(root, file_)
                name = filepath[len(directory):]
                filesdict[name] = filepath

        self.sync_files(filesdict, verbose, remote)

    def sync_files(self, files, verbose=False, remote=None):
        """Synchronize files  with the remote disks.

        :param dict files: Dictionary of synchronized files
        :param bool verbose: Print information about synchronization operations
        :param str remote: path of the directory on remote node (defaults to *local*)

        Dictionary key is the remote file path while value is the local file
        path.

        .. warning::
           Local changes are reflected on the server, a file present on the
           disk but
           not in the local directory will be deleted from the disk.

           A file present in the directory but not in the disk will be uploaded.

        .. note::
           The following parameters are used to determine whether
           synchronization is required :

              * name
              * size
              * sha1sum
        """

        class Comparable(object):
            def __init__(self, name_, e_tag, filepath_):
                self.name = name_
                self.e_tag = e_tag
                self.filepath = filepath_

            def __repr__(self):
                return "Name {0}, ETag {1}".format(self.name, self.e_tag)

            def __eq__(self, other):
                return self.name == other.name and self.e_tag == other.e_tag

            def __hash__(self):
                return hash(self.name) ^ hash(self.e_tag)

        def aws_md5sum(sourcepath):
            if os.stat(sourcepath).st_size < AWS_UPLOAD_MAX_SIZE:
                hash_md5 = hashlib.md5()
                with open(sourcepath, "rb") as f:
                    for chunk in iter(lambda: f.read(4096), b""):
                        hash_md5.update(chunk)
                return "\"{0}\"".format(hash_md5.hexdigest())
            else:
                md5s = []
                with open(sourcepath, 'rb') as fp:
                    while True:

                        data = fp.read(AWS_UPLOAD_PART_SIZE)

                        if not data:
                            break
                        md5s.append(hashlib.md5(data))

                digests = b"".join(m.digest() for m in md5s)

                new_md5 = hashlib.md5(digests)
                return "\"{0}-{1}\"".format(new_md5.hexdigest(), len(md5s))

        def localtocomparable(name_, filepath_, remote):
            if remote is not None:
                name_ = os.path.join(remote, name_.lstrip('/'))
            return Comparable(name_, aws_md5sum(filepath_), filepath_)

        def objectsummarytocomparable(object_):
            return Comparable(object_.key, object_.e_tag, None)

        localfiles = set()
        for name, filepath in files.items():
            localfiles.add(localtocomparable(name.replace(os.path.sep, '/'), filepath, remote))

        remotefiles = set(map(objectsummarytocomparable, self.list_files()))

        adds = localfiles - remotefiles
        removes = remotefiles - localfiles

        for file_ in removes:
            if remote is not None and not file_.name.startswith(remote):
                continue
            renames = (x for x in adds if x.e_tag == file_.e_tag and all(rem.name != x.name for rem in remotefiles))
            for dup in renames:
                if verbose:
                    print("Copy", file_.name, "to", dup.name)
                self.copy_file(file_.name, dup.name)
            if verbose:
                print("Remove:", file_.name)
            self.delete_file(file_.name)

        remotefiles = set(map(objectsummarytocomparable, self.list_files()))

        sadds = sorted(adds, key=lambda x: x.e_tag)
        groupedadds = (list(g) for _, g in itertools.groupby(sadds, lambda x: x.e_tag))

        for entry in groupedadds:
            try:
                rem = next(x for x in remotefiles if x.e_tag == entry[0].e_tag)
                if rem.name == entry[0].name:
                    continue
                if verbose:
                    print("Copy", rem.name, "to", entry[0].name)
                self.copy_file(rem.name, entry[0].name)
            except StopIteration:
                if verbose:
                    print("Upload:", entry[0].name)
                self.add_file(entry[0].filepath, entry[0].name)
            for link in entry[1:]:  # duplicate files
                if verbose:
                    print("Copy", entry[0].name, "to", link.name)
                self.copy_file(entry[0].name, link.name)

    @_util.copy_docs(Storage.add_file)
    def add_file(self, local_or_file, remote=None):
        tobeclosed = False
        if _util.is_string(local_or_file):
            file_ = open(local_or_file, 'rb')
            tobeclosed = True
        else:
            file_ = local_or_file
        dest = remote or os.path.basename(file_.name)

        self._connection.s3client.upload_fileobj(file_, self._uuid, dest, Config=s3_multipart_config)
        if tobeclosed:
            file_.close()

    @_util.copy_docs(Storage.get_all_files)
    def get_all_files(self, output_dir, progress=None):
        for _, dupes in groupby(sorted(self.list_files(), key=attrgetter('e_tag')), attrgetter('e_tag')):
            file_info = next(dupes)
            first_file = os.path.join(output_dir, os.path.normpath(file_info.key.lstrip('/')))
            self.get_file(file_info.get()['Body'], local=first_file)  # avoids making a useless HEAD request

            for dupe in dupes:
                local = os.path.join(output_dir, os.path.normpath(dupe.key.lstrip('/')))
                directory = os.path.dirname(local)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                shutil.copy(first_file, local)

    @_util.copy_docs(Storage.get_file)
    def get_file(self, remote, local=None, progress=None):
        return super(Bucket, self).get_file(remote, local, progress)

    @_util.copy_docs(Storage.add_directory)
    def add_directory(self, local, remote=""):
        if not os.path.isdir(local):
            raise IOError("Not a valid directory")
        if not remote.endswith('/'):
            remote += '/'
        for dirpath, _, files in os.walk(local):
            dirpath = _util.decode(dirpath)
            files = map(_util.decode, files)

            remote_loc = dirpath.replace(local, remote, 1)
            for filename in files:
                self.add_file(os.path.join(dirpath, filename),
                              posixpath.join(remote_loc, filename))

    @_util.copy_docs(Storage.copy_file)
    def copy_file(self, source, dest):
        copy_source = {
            'Bucket': self._uuid,
            'Key': source
        }
        return self._connection.s3client.copy_object(CopySource=copy_source, Bucket=self._uuid, Key=dest)

    @_util.copy_docs(Storage.flush)
    def flush(self):
        pass

    @_util.copy_docs(Storage.update)
    def update(self, flush=False):
        pass

    def _download_file(self, remote, local, progress=None):
        with open(local, 'wb') as data:
            if hasattr(remote, 'read'):
                shutil.copyfileobj(remote, data)
            else:
                self._connection.s3client.download_fileobj(self._uuid, remote, data)
        return local

    @_util.copy_docs(Storage.delete_file)
    def delete_file(self, remote):
        self._connection.s3client.delete_object(Bucket=self._uuid, Key=remote)

    @property
    def uuid(self):
        """ Bucket identifier"""
        return self._uuid

    @property
    def description(self):
        """ Bucket identifier"""
        return self._uuid
<commit_msg>Fix issue with empty directories and empty files for bucket download.<commit_after>"""Module for bucket object."""

# Copyright 2017 Qarnot computing
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function

import hashlib
import sys
import os
import posixpath
import shutil
import itertools

from boto3.s3.transfer import TransferConfig
from itertools import groupby
from operator import attrgetter

from . import _util
from .exceptions import BucketStorageUnavailableException
from .storage import Storage

# Max size in bytes before uploading in parts.

AWS_UPLOAD_MAX_SIZE = 8 * 1024 * 1024
# Size of parts when uploading in parts
AWS_UPLOAD_PART_SIZE = 8 * 1024 * 1024

s3_multipart_config = TransferConfig(
    multipart_threshold=AWS_UPLOAD_MAX_SIZE,
    multipart_chunksize=AWS_UPLOAD_PART_SIZE,
    max_concurrency=10,
    num_download_attempts=10,
)

class Bucket(Storage):
    """Represents a resource/result bucket.

    This class is the interface to manage resources or results from a
    :class:`qarnot.bucket.Bucket`.

    .. note::
       A :class:`Bucket` must be created with
       :meth:`qarnot.connection.Connection.create_bucket`
       or retrieved with :meth:`qarnot.connection.Connection.buckets`, :meth:`qarnot.connection.Connection.retrieve_bucket`,
       or :meth:`qarnot.connection.Connection.retrieve_or_create_bucket`.

    .. note::
       Paths given as 'remote' arguments,
       (or as path arguments for :func:`Bucket.directory`)
       **must** be valid unix-like paths.
    """

    def __init__(self, connection, name, create=True):
        if connection.s3client is None:
            raise BucketStorageUnavailableException()

        self._connection = connection
        self._uuid = name

        if create:
            self._connection.s3client.create_bucket(Bucket=name)

    @classmethod
    def _retrieve(cls, connection, bucket_uuid):
        """Retrieve information of a bucket on a cluster.

        :param :class:`qarnot.connection.Connection` connection: the cluster
            to get the bucket from
        :param str bucket_uuid: the UUID of the bucket to retrieve

        :rtype: :class:`qarnot.bucket.Bucket`
        :returns: The retrieved bucket.

        :raises qarnot.exceptions.BucketStorageUnavailableException: the bucket storage engine is not available
        """
        return connection.retrieve_bucket(uuid=bucket_uuid)

    def delete(self):
        """ Delete the bucket represented by this :class:`Bucket`."""
        bucket = self._connection.s3resource.Bucket(self._uuid)
        objectlist = list(bucket.objects.all())
        n = 1000  # delete object count max request

        if sys.version_info >= (3, 0):
            listofobjectlist = [[{'Key': x.key} for x in objectlist[i:i + n]] for i in range(0, len(objectlist), n)]
        else:
            # noinspection PyUnresolvedReferences
            listofobjectlist = [[{'Key': x.key} for x in objectlist[i:i + n]] for i in xrange(0, len(objectlist), n)]  # noqa
        for item in listofobjectlist:
            bucket.delete_objects(
                Delete={
                    'Objects': item
                }
            )
        self._connection.s3client.delete_bucket(Bucket=self._uuid)

    def list_files(self):
        """List files in the bucket

        :rtype: list(:class:`S3.ObjectSummary`)
        :returns: A list of ObjectSummary resources

        """
        bucket = self._connection.s3resource.Bucket(self._uuid)
        return bucket.objects.all()

    def directory(self, directory=''):
        """List files in a directory of the disk according to prefix.

        :rtype: list(:class:`S3.ObjectSummary`)
        :returns: A list of ObjectSummary resources
        """
        bucket = self._connection.s3resource.Bucket(self._uuid)
        return bucket.objects.filter(Prefix=directory)

    def sync_directory(self, directory, verbose=False, remote=None):
        """Synchronize a local directory with the remote disks.

        :param str directory: The local directory to use for synchronization
        :param bool verbose: Print information about synchronization operations
        :param str remote: path of the directory on remote node (defaults to *local*)

        .. warning::
           Local changes are reflected on the server, a file present on the
           disk but not in the local directory will be deleted from the disk.

           A file present in the directory but not in the disk will be uploaded.

        .. note::
           The following parameters are used to determine whether
           synchronization is required :

              * name
              * size
              * sha1sum
        """
        if not directory.endswith('/'):
            directory += '/'

        filesdict = {}
        for root, _, files in os.walk(directory):
            root = _util.decode(root)
            files = map(_util.decode, files)

            for file_ in files:
                filepath = os.path.join(root, file_)
                name = filepath[len(directory):]
                filesdict[name] = filepath

        self.sync_files(filesdict, verbose, remote)

    def sync_files(self, files, verbose=False, remote=None):
        """Synchronize files  with the remote disks.

        :param dict files: Dictionary of synchronized files
        :param bool verbose: Print information about synchronization operations
        :param str remote: path of the directory on remote node (defaults to *local*)

        Dictionary key is the remote file path while value is the local file
        path.

        .. warning::
           Local changes are reflected on the server, a file present on the
           disk but
           not in the local directory will be deleted from the disk.

           A file present in the directory but not in the disk will be uploaded.

        .. note::
           The following parameters are used to determine whether
           synchronization is required :

              * name
              * size
              * sha1sum
        """

        class Comparable(object):
            def __init__(self, name_, e_tag, filepath_):
                self.name = name_
                self.e_tag = e_tag
                self.filepath = filepath_

            def __repr__(self):
                return "Name {0}, ETag {1}".format(self.name, self.e_tag)

            def __eq__(self, other):
                return self.name == other.name and self.e_tag == other.e_tag

            def __hash__(self):
                return hash(self.name) ^ hash(self.e_tag)

        def aws_md5sum(sourcepath):
            if os.stat(sourcepath).st_size < AWS_UPLOAD_MAX_SIZE:
                hash_md5 = hashlib.md5()
                with open(sourcepath, "rb") as f:
                    for chunk in iter(lambda: f.read(4096), b""):
                        hash_md5.update(chunk)
                return "\"{0}\"".format(hash_md5.hexdigest())
            else:
                md5s = []
                with open(sourcepath, 'rb') as fp:
                    while True:

                        data = fp.read(AWS_UPLOAD_PART_SIZE)

                        if not data:
                            break
                        md5s.append(hashlib.md5(data))

                digests = b"".join(m.digest() for m in md5s)

                new_md5 = hashlib.md5(digests)
                return "\"{0}-{1}\"".format(new_md5.hexdigest(), len(md5s))

        def localtocomparable(name_, filepath_, remote):
            if remote is not None:
                name_ = os.path.join(remote, name_.lstrip('/'))
            return Comparable(name_, aws_md5sum(filepath_), filepath_)

        def objectsummarytocomparable(object_):
            return Comparable(object_.key, object_.e_tag, None)

        localfiles = set()
        for name, filepath in files.items():
            localfiles.add(localtocomparable(name.replace(os.path.sep, '/'), filepath, remote))

        remotefiles = set(map(objectsummarytocomparable, self.list_files()))

        adds = localfiles - remotefiles
        removes = remotefiles - localfiles

        for file_ in removes:
            if remote is not None and not file_.name.startswith(remote):
                continue
            renames = (x for x in adds if x.e_tag == file_.e_tag and all(rem.name != x.name for rem in remotefiles))
            for dup in renames:
                if verbose:
                    print("Copy", file_.name, "to", dup.name)
                self.copy_file(file_.name, dup.name)
            if verbose:
                print("Remove:", file_.name)
            self.delete_file(file_.name)

        remotefiles = set(map(objectsummarytocomparable, self.list_files()))

        sadds = sorted(adds, key=lambda x: x.e_tag)
        groupedadds = (list(g) for _, g in itertools.groupby(sadds, lambda x: x.e_tag))

        for entry in groupedadds:
            try:
                rem = next(x for x in remotefiles if x.e_tag == entry[0].e_tag)
                if rem.name == entry[0].name:
                    continue
                if verbose:
                    print("Copy", rem.name, "to", entry[0].name)
                self.copy_file(rem.name, entry[0].name)
            except StopIteration:
                if verbose:
                    print("Upload:", entry[0].name)
                self.add_file(entry[0].filepath, entry[0].name)
            for link in entry[1:]:  # duplicate files
                if verbose:
                    print("Copy", entry[0].name, "to", link.name)
                self.copy_file(entry[0].name, link.name)

    @_util.copy_docs(Storage.add_file)
    def add_file(self, local_or_file, remote=None):
        tobeclosed = False
        if _util.is_string(local_or_file):
            file_ = open(local_or_file, 'rb')
            tobeclosed = True
        else:
            file_ = local_or_file
        dest = remote or os.path.basename(file_.name)

        self._connection.s3client.upload_fileobj(file_, self._uuid, dest, Config=s3_multipart_config)
        if tobeclosed:
            file_.close()

    @_util.copy_docs(Storage.get_all_files)
    def get_all_files(self, output_dir, progress=None):
        list_files_only = [x for x in self.list_files() if not x.key.endswith('/')]
        list_directories_only = [x for x in self.list_files() if x.key.endswith('/')]

        for dir in list_directories_only:
            if not os.path.isdir(os.path.join(output_dir, dir.key.lstrip('/'))):
                os.makedirs(os.path.join(output_dir, dir.key.lstrip('/')))

        for _, dupes in groupby(sorted(list_files_only, key=attrgetter('e_tag')), attrgetter('e_tag')):
            file_info = next(dupes)
            first_file = os.path.join(output_dir, file_info.key.lstrip('/'))
            self.get_file(file_info.get()['Body'], local=first_file)  # avoids making a useless HEAD request

            for dupe in dupes:
                local = os.path.join(output_dir, dupe.key.lstrip('/'))
                directory = os.path.dirname(local)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                shutil.copy(first_file, local)

    @_util.copy_docs(Storage.get_file)
    def get_file(self, remote, local=None, progress=None):
        return super(Bucket, self).get_file(remote, local, progress)

    @_util.copy_docs(Storage.add_directory)
    def add_directory(self, local, remote=""):
        if not os.path.isdir(local):
            raise IOError("Not a valid directory")
        if not remote.endswith('/'):
            remote += '/'
        for dirpath, _, files in os.walk(local):
            dirpath = _util.decode(dirpath)
            files = map(_util.decode, files)

            remote_loc = dirpath.replace(local, remote, 1)
            for filename in files:
                self.add_file(os.path.join(dirpath, filename),
                              posixpath.join(remote_loc, filename))

    @_util.copy_docs(Storage.copy_file)
    def copy_file(self, source, dest):
        copy_source = {
            'Bucket': self._uuid,
            'Key': source
        }
        return self._connection.s3client.copy_object(CopySource=copy_source, Bucket=self._uuid, Key=dest)

    @_util.copy_docs(Storage.flush)
    def flush(self):
        pass

    @_util.copy_docs(Storage.update)
    def update(self, flush=False):
        pass

    def _download_file(self, remote, local, progress=None):
        with open(local, 'wb') as data:
            if hasattr(remote, 'read'):
                shutil.copyfileobj(remote, data)
            else:
                self._connection.s3client.download_fileobj(self._uuid, remote, data)
        return local

    @_util.copy_docs(Storage.delete_file)
    def delete_file(self, remote):
        self._connection.s3client.delete_object(Bucket=self._uuid, Key=remote)

    @property
    def uuid(self):
        """ Bucket identifier"""
        return self._uuid

    @property
    def description(self):
        """ Bucket identifier"""
        return self._uuid
<|endoftext|>


<commit_before># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: jernej@reciprocitylabs.com
# Maintained By: jernej@reciprocitylabs.com

"""Utility function for selenium"""

import time
import logging

# pylint: disable=import-error
from selenium.webdriver.support import expected_conditions as EC
# pylint: disable=import-error
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common import action_chains
from selenium.common import exceptions

from lib import exception
from lib import constants

logger = logging.getLogger(__name__)

def hover_over_element(driver, element):
  """Moves the mouse pointer to the element and hovers"""
  action_chains.ActionChains(driver).move_to_element(element).perform()

def wait_until_stops_moving(element):
  """Waits until the element stops moving

  Args:
      selenium.webdriver.remote.webelement.WebElement
  """

  prev_location = None
  timer_begin = time.time()

  while prev_location != element.location:
    prev_location = element.location
    time.sleep(0.1)

    if time.time() - timer_begin > constants.ux.ELEMENT_MOVING_TIMEOUT:
      raise exception.ElementMovingTimeout

def get_when_visible(driver, locator):
  """
  Args:
    driver (base.CustomDriver)
    locator (tuple)

  Returns:
      selenium.webdriver.remote.webelement.WebElement
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_USER_WAIT_SECONDS) \
      .until(EC.visibility_of_element_located(locator))

def get_when_clickable(driver, locator):
  """
  Args:
    driver (base.CustomDriver)
    locator (tuple)

  Returns:
      selenium.webdriver.remote.webelement.WebElement
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_USER_WAIT_SECONDS) \
      .until(EC.element_to_be_clickable(locator))

def get_when_invisible(driver, locator):
  """
  Args:
    driver (base.CustomDriver)
    locator (tuple)

  Returns:
      selenium.webdriver.remote.webelement.WebElement
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_USER_WAIT_SECONDS) \
      .until(EC.invisibility_of_element_located(locator))

def scroll_to_page_bottom(driver):
  """Scrolls to te page bottom using JS

  Args:
      driver (base.CustomDriver)
  """
  driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

def is_value_in_attr(element, attr="class", value="active"):
  """Checks if the attribute value is present for given attribute

  Args:
    element (selenium.webdriver.remote.webelement.WebElement)
    attr (basestring): attribute name e.g. "class"
    value (basestring): value in the class attribute that
      indicates the element is now active/opened

  Returns:
      bool
  """
  attributes = element.get_attribute(attr)
  return value in attributes.split()

def wait_until_alert_is_present(driver):
  """
  Waits until an alert is present
  Args:
    driver (base.CustomDriver)

  Returns:
      selenium.webdriver.common.alert.Alert
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_ALERT_WAIT) \
      .until(EC.alert_is_present())

def handle_alert(driver, accept=False):
  """Wait until an alert is present and make a decision to accept or dismiss
  it"""
  try:
    wait_until_alert_is_present(driver)
    alert = driver.switch_to.alert

    if accept:
      alert.accept()
    else:
      alert.dismiss()
  except (exceptions.NoAlertPresentException, exceptions.TimeoutException):
    pass

def click_on_staleable_element(driver, el_locator):
  """Gets elements that are modified by JS for some time. When the
  element is not manipulated any more, we return it."""
  time_start = time.time()

  while time.time() - time_start < constants.ux.MAX_USER_WAIT_SECONDS:
    try:
      driver.find_element(*el_locator).click()
      break
    except exceptions.StaleElementReferenceException as e:
      logger.error(e)
      time.sleep(0.1)
  else:
    raise exception.ElementNotFound(el_locator)

def scroll_into_view(driver, element):
  driver.execute_script("return arguments[0].scrollIntoView();", element)

  # compensate for the header
  driver.execute_script(
      "window.scrollBy(0, -{});".format(constants.element.SIZE_HEADER))
  return element
<commit_msg>Fix docstring for selenium utils<commit_after># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: jernej@reciprocitylabs.com
# Maintained By: jernej@reciprocitylabs.com

"""Utility function for selenium"""

import time
import logging

# pylint: disable=import-error
from selenium.webdriver.support import expected_conditions as EC
# pylint: disable=import-error
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common import action_chains
from selenium.common import exceptions

from lib import exception
from lib import constants

logger = logging.getLogger(__name__)

def hover_over_element(driver, element):
  """Moves the mouse pointer to the element and hovers"""
  action_chains.ActionChains(driver).move_to_element(element).perform()

def wait_until_stops_moving(element):
  """Waits until the element stops moving

  Args:
      selenium.webdriver.remote.webelement.WebElement
  """

  prev_location = None
  timer_begin = time.time()

  while prev_location != element.location:
    prev_location = element.location
    time.sleep(0.1)

    if time.time() - timer_begin > constants.ux.ELEMENT_MOVING_TIMEOUT:
      raise exception.ElementMovingTimeout

def get_when_visible(driver, locator):
  """
  Args:
    driver (base.CustomDriver)
    locator (tuple)

  Returns:
      selenium.webdriver.remote.webelement.WebElement
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_USER_WAIT_SECONDS) \
      .until(EC.visibility_of_element_located(locator))

def get_when_clickable(driver, locator):
  """
  Args:
    driver (base.CustomDriver)
    locator (tuple)

  Returns:
      selenium.webdriver.remote.webelement.WebElement
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_USER_WAIT_SECONDS) \
      .until(EC.element_to_be_clickable(locator))

def get_when_invisible(driver, locator):
  """
  Args:
    driver (base.CustomDriver)
    locator (tuple)

  Returns:
      selenium.webdriver.remote.webelement.WebElement
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_USER_WAIT_SECONDS) \
      .until(EC.invisibility_of_element_located(locator))

def scroll_to_page_bottom(driver):
  """Scrolls to te page bottom using JS

  Args:
      driver (base.CustomDriver)
  """
  driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

def is_value_in_attr(element, attr="class", value="active"):
  """Checks if the attribute value is present for given attribute

  Args:
    element (selenium.webdriver.remote.webelement.WebElement)
    attr (basestring): attribute name e.g. "class"
    value (basestring): value in the class attribute that
      indicates the element is now active/opened

  Returns:
      bool
  """
  attributes = element.get_attribute(attr)
  return value in attributes.split()

def wait_until_alert_is_present(driver):
  """
  Waits until an alert is present
  Args:
    driver (base.CustomDriver)

  Returns:
      selenium.webdriver.common.alert.Alert
  """
  return WebDriverWait(
      driver,
      constants.ux.MAX_ALERT_WAIT) \
      .until(EC.alert_is_present())

def handle_alert(driver, accept=False):
  """Wait until an alert is present and make a decision to accept or dismiss
  it"""
  try:
    wait_until_alert_is_present(driver)
    alert = driver.switch_to.alert

    if accept:
      alert.accept()
    else:
      alert.dismiss()
  except (exceptions.NoAlertPresentException, exceptions.TimeoutException):
    pass

def click_on_staleable_element(driver, el_locator):
  """Clicks an element that can be modified between the time we find it
  and when we click on it."""
  time_start = time.time()

  while time.time() - time_start < constants.ux.MAX_USER_WAIT_SECONDS:
    try:
      driver.find_element(*el_locator).click()
      break
    except exceptions.StaleElementReferenceException as e:
      logger.error(e)
      time.sleep(0.1)
  else:
    raise exception.ElementNotFound(el_locator)

def scroll_into_view(driver, element):
  driver.execute_script("return arguments[0].scrollIntoView();", element)

  # compensate for the header
  driver.execute_script(
      "window.scrollBy(0, -{});".format(constants.element.SIZE_HEADER))
  return element
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Fri Jan 31 12:44:25 2014

@author: eleve
"""
import math
from pylab import *
import tkinter as tk
import os
from PIL import Image,ImageTk

x=0
y=0
theta=0
a=5
b=3
r=(1+sqrt(5))/2
x_list=list()
y_list=list()
xscale=900
yscale=700

for n in range(int(1*10**6)):
    x_list.append(x)
    y_list.append(y)
    if math.floor(r*(a*n+b))%2 ==0 :
        theta += pi/3
    else :
        theta += -2*pi/3
    x+=cos(theta)
    y+=sin(theta)

plot(x_list,y_list,',')
try :
    os.mkdir('temp')
except:
    pass
fen=tk.Tk()
savefig('temp/sauvegarde.png')
close()
image =Image.open('temp/sauvegarde.png')
largeur_image_i=image.size[0]
hauteur_image_i=image.size[1]
ratio=min([xscale/largeur_image_i,yscale/hauteur_image_i])
largeur = int(largeur_image_i*ratio)
hauteur = int(hauteur_image_i*ratio)
imagebis=image.resize((largeur,hauteur),Image.BILINEAR)
imagebis.save('temp/affichage.png')
photo = ImageTk.PhotoImage(imagebis)
figure=tk.Canvas(fen,width=xscale,height=yscale)
figure.grid(row=1,column=1,rowspan=4)
item=figure.create_image(0,0,anchor=tk.NW,image=photo)
tk.mainloop()

<commit_msg>Update figuration_de_nombre.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Fri Jan 31 12:44:25 2014

@author: eleve
"""
import math
from pylab import *
import tkinter as tk
import os
from PIL import Image,ImageTk

x=0
y=0
theta=0
x_list=list()
y_list=list()
xscale=900
yscale=700
def render():
    a=fen.a.get()
    b=fen.b.get()
    r=fen.r.get()
    for n in range(int(1*10**6)):
        x_list.append(x)
        y_list.append(y)
        if math.floor(r*(a*n+b))%2 ==0 :
            theta += pi/3
        else :
            theta += -2*pi/3
        x+=cos(theta)
        y+=sin(theta)

    plot(x_list,y_list,',')
    try :
        os.mkdir('temp')
    except:
        pass
    savefig('temp/sauvegarde.png')
    close()
    tk.mainloop()

fen=tk.Tk()
fen.a=tk.IntVar()
fen.b=tk.IntVar()
fen.r=tk.DoubleVar()
fen.entry_a=tk.Entry(fen,textvariable=fen.a)
fen.entry_a.grid(row=1,column=2)
fen.entry_b=tk.Entry(fen,textvariable=fen.b)
fen.entry_b.grid(row=2,column=2)
fen.entry_r=tk.Entry(fen,textvariable=fen.r)
fen.entry_r.grid(row=3,column=2)
image =Image.open('temp/sauvegarde.png')
largeur_image_i=image.size[0]
hauteur_image_i=image.size[1]
ratio=min([xscale/largeur_image_i,yscale/hauteur_image_i])
largeur = int(largeur_image_i*ratio)
hauteur = int(hauteur_image_i*ratio)
imagebis=image.resize((largeur,hauteur),Image.BILINEAR)
imagebis.save('temp/affichage.png')
photo = ImageTk.PhotoImage(imagebis)
figure=tk.Canvas(fen,width=xscale,height=yscale)
figure.grid(row=1,column=1,rowspan=4)
item=figure.create_image(0,0,anchor=tk.NW,image=photo)
tk.mainloop()

<|endoftext|>


<commit_before>import itertools
import string
import sys
import re
import socket

def socket_used(address='127.0.0.1', port=6633):
  s = socket.socket()
  try:
    s.connect((address, port))
    s.close()
    return True
  except Exception, e:
    return False

class ControllerConfig(object):
  _port_gen = itertools.count(6633)
  _controller_count_gen = itertools.count(1)

  def __init__(self, cmdline="", address="127.0.0.1", port=None, cwd=None, sync=None, controller_type=None, label=None):
    '''
    Store metadata for the controller.
      - cmdline is an array of command line tokens.

        Note: if you need to pass in the address and port to controller's
        command line, use the aliases __address__ and __port__ to have the
        values interpolated automatically
      - address and port are the sockets switches will bind to
      - controller_type: help us figure out by specifying controller_type as a
        string that represents the controller. If it is not specified, this
        method will try to guess if it is either pox or floodlight.
    '''
    if cmdline == "":
      raise RuntimeError("Must specify boot parameters.")
    self.cmdline = cmdline

    self.address = address
    if (re.match("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", address) or
        address == "localhost"):
      # Normal TCP socket
      if not port:
        port = self._port_gen.next()
      while socket_used(port=port):
        print "Socket %d in use... trying next" % port
        port += 1
      self.port = port
      self.server_info = (self.address, self.port)
    else:
      # Unix domain socket
      self.port = None
      self.server_info = address

    # TODO(sam): we should either call them all controller_type or all 'name'
    # we only accept strings
    self.name = ""
    if isinstance(controller_type,str):
      self.name = controller_type
    elif "pox" in self.cmdline:
      self.name = "pox"
    elif "floodlight" in self.cmdline:
      self.name = "floodlight"

    self.cwd = cwd
    if not cwd:
        sys.stderr.write("""
        =======================================================================
        WARN - no working directory defined for controller with command line
        %s
        The controller is run in the STS base directory. This may result
        in unintended consequences (i.e., POX not logging correctly).
        =======================================================================
        \n""" % (self.cmdline) )

    self.sync = sync
    if label:
      self.label = label
    else:
      self.label = "c"+str(self._controller_count_gen.next())

  @property
  def uuid(self):
    return self.server_info

  @property
  def expanded_cmdline(self):
    return map(lambda(x): string.replace(x, "__port__", str(self.port)),
           map(lambda(x): string.replace(x, "__address__", str(self.address)),
             self.cmdline.split()))

  def __repr__(self):
    attributes = ("cmdline", "address", "port", "cwd", "sync")

    pairs = ( (attr, getattr(self, attr)) for attr in attributes)
    quoted = ( "%s=%s" % (attr, repr(value)) for (attr, value) in pairs if value)

    return self.__class__.__name__  + "(" + ", ".join(quoted) + ")"
<commit_msg>socket_used now opens a listen socket, not a connect()<commit_after>import itertools
import string
import sys
import re
import socket

def socket_used(address='127.0.0.1', port=6633):
  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  try:
    s.bind((address, port))
    s.listen(1)
    s.close()
    return False
  except Exception, e:
    # TODO(cs): catch specific errors
    return True

class ControllerConfig(object):
  _port_gen = itertools.count(6633)
  _controller_count_gen = itertools.count(1)

  def __init__(self, cmdline="", address="127.0.0.1", port=None, cwd=None, sync=None, controller_type=None, label=None):
    '''
    Store metadata for the controller.
      - cmdline is an array of command line tokens.

        Note: if you need to pass in the address and port to controller's
        command line, use the aliases __address__ and __port__ to have the
        values interpolated automatically
      - address and port are the sockets switches will bind to
      - controller_type: help us figure out by specifying controller_type as a
        string that represents the controller. If it is not specified, this
        method will try to guess if it is either pox or floodlight.
    '''
    if cmdline == "":
      raise RuntimeError("Must specify boot parameters.")
    self.cmdline = cmdline

    self.address = address
    if (re.match("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", address) or
        address == "localhost"):
      # Normal TCP socket
      if not port:
        port = self._port_gen.next()
      while socket_used(port=port):
        print "Socket %d in use... trying next" % port
        port += 1
      self.port = port
      self.server_info = (self.address, self.port)
    else:
      # Unix domain socket
      self.port = None
      self.server_info = address

    # TODO(sam): we should either call them all controller_type or all 'name'
    # we only accept strings
    self.name = ""
    if isinstance(controller_type,str):
      self.name = controller_type
    elif "pox" in self.cmdline:
      self.name = "pox"
    elif "floodlight" in self.cmdline:
      self.name = "floodlight"

    self.cwd = cwd
    if not cwd:
        sys.stderr.write("""
        =======================================================================
        WARN - no working directory defined for controller with command line
        %s
        The controller is run in the STS base directory. This may result
        in unintended consequences (i.e., POX not logging correctly).
        =======================================================================
        \n""" % (self.cmdline) )

    self.sync = sync
    if label:
      self.label = label
    else:
      self.label = "c"+str(self._controller_count_gen.next())

  @property
  def uuid(self):
    return self.server_info

  @property
  def expanded_cmdline(self):
    return map(lambda(x): string.replace(x, "__port__", str(self.port)),
           map(lambda(x): string.replace(x, "__address__", str(self.address)),
             self.cmdline.split()))

  def __repr__(self):
    attributes = ("cmdline", "address", "port", "cwd", "sync")

    pairs = ( (attr, getattr(self, attr)) for attr in attributes)
    quoted = ( "%s=%s" % (attr, repr(value)) for (attr, value) in pairs if value)

    return self.__class__.__name__  + "(" + ", ".join(quoted) + ")"
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# VagrIRC Initialization Bot

# Written in 2015 by Daniel Oaks <daniel@danieloaks.net>
#
# To the extent possible under law, the author(s) have dedicated all copyright
# and related and neighboring rights to this software to the public domain
# worldwide. This software is distributed without any warranty.
#
# You should have received a copy of the CC0 Public Domain Dedication along
# with this software. If not, see
# <http://creativecommons.org/publicdomain/zero/1.0/>.

import socket
from time import sleep

from envelope import RFC1459Message

class User:
    """User, connects and executes commands."""
    def __init__(self, nick, username, commands):
        self._new_data = ''

        # user info
        self.nick = nick
        self.username = username
        self.realname = '*'
        self.commands = commands

        # connection info
        self.network = '127.0.0.1'
        self.port = 6667

        # allowed commands
        self.command_map = {
            'nickserv': self.cmd_nickserv,
        }

    # send/receive
    def recv(self, number_of_bytes=4096):
        """Receive bytes from the IRC server."""
        try:
            new_data = self.sock.recv(number_of_bytes).decode('utf-8', 'replace')
        except socket.timeout as ex:
            error = ex.args[0]

            if error == 'timed out':
                return []
            else:
                raise ex

        self._new_data += new_data
        raw_messages = []
        message_buffer = ''

        for char in self._new_data:
            if char in ('\r', '\n'):
                if len(message_buffer):
                    print(' ->', message_buffer)
                    raw_messages.append(message_buffer)
                    message_buffer = ''
                continue

            message_buffer += char

        # convert to actual rfc1459 messages
        messages = []

        for msg in raw_messages:
            messages.append(RFC1459Message.from_message(msg))

        return messages

    def send_raw(self, data):
        """Send the given data to the IRC server."""
        print('<- ', data.rstrip())
        self.sock.send(data.encode('utf-8', 'ignore'))

    def send(self, message):
        """Send the given message to the IRC server."""
        if isinstance(message, RFC1459Message):
            message = message.to_message()
        self.send_raw('{}\r\n'.format(message))

    # init
    def run(self):
        """Connect, execute commands, and then disconnect."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.network, self.port))
        self.sock.settimeout(0.3)

        # should really be async but we're lazy
        self.recv()

        self.send('NICK {}'.format(self.nick))
        self.send('USER {} 0 * :{}'.format(self.username, self.realname))

        # main event loop
        online = True
        while online:
            messages = self.recv()
            for message in messages:
                if message.verb.lower() == '001':
                    self.rpl_welcome()
                    online = False

        self.recv()
        sleep(1.5)
        self.recv()
        self.quit()

    def rpl_welcome(self):
        # yay lazy event loop!
        for name, info in self.commands:
            cmd = self.command_map.get(name, None)
            if cmd:
                cmd(info)
                self.recv()

    def quit(self):
        self.send('QUIT')

    # irc commands
    def privmsg(self, target, msg):
        """Send a PRIVMSG."""
        self.send('PRIVMSG {} :{}'.format(target, msg))

    # command list
    def cmd_nickserv(self, info):
        self.privmsg('NickServ', info)
<commit_msg>init: Respond to PINGs properly, send using messages instead of dodgy raw text<commit_after>#!/usr/bin/env python3
# VagrIRC Initialization Bot

# Written in 2015 by Daniel Oaks <daniel@danieloaks.net>
#
# To the extent possible under law, the author(s) have dedicated all copyright
# and related and neighboring rights to this software to the public domain
# worldwide. This software is distributed without any warranty.
#
# You should have received a copy of the CC0 Public Domain Dedication along
# with this software. If not, see
# <http://creativecommons.org/publicdomain/zero/1.0/>.

import socket
from time import sleep

from envelope import RFC1459Message

class User:
    """User, connects and executes commands."""
    def __init__(self, nick, username, commands):
        self._new_data = ''

        # user info
        self.nick = nick
        self.username = username
        self.realname = '*'
        self.commands = commands

        # connection info
        self.network = '127.0.0.1'
        self.port = 6667

        # allowed commands
        self.command_map = {
            'nickserv': self.cmd_nickserv,
        }

    # send/receive
    def recv(self, number_of_bytes=4096):
        """Receive bytes from the IRC server."""
        try:
            new_data = self.sock.recv(number_of_bytes).decode('utf-8', 'replace')
        except socket.timeout as ex:
            error = ex.args[0]

            if error == 'timed out':
                return []
            else:
                raise ex

        self._new_data += new_data
        raw_messages = []
        message_buffer = ''

        for char in self._new_data:
            if char in ('\r', '\n'):
                if len(message_buffer):
                    print(' ->', message_buffer)
                    raw_messages.append(message_buffer)
                    message_buffer = ''
                continue

            message_buffer += char

        # convert to actual rfc1459 messages
        messages = []

        for msg in raw_messages:
            messages.append(RFC1459Message.from_message(msg))

        return messages

    def send_raw(self, data):
        """Send the given data to the IRC server."""
        print('<- ', data.rstrip())
        self.sock.send(data.encode('utf-8', 'ignore'))

    def send(self, message):
        """Send the given message to the IRC server."""
        if isinstance(message, RFC1459Message):
            message = message.to_message()
        self.send_raw('{}\r\n'.format(message))

    def send_message(self, verb, **kwargs):
        """Send the given message to the IRC server."""
        msg = RFC1459Message.from_data(verb, **kwargs)
        self.send(msg)

    # init
    def run(self):
        """Connect, execute commands, and then disconnect."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.network, self.port))
        self.sock.settimeout(0.3)

        # should really be async but we're lazy
        self.recv()

        self.send_message('NICK', params=[self.nick])
        self.send_message('USER', params=[self.username, 0, '*', self.realname])

        # main event loop
        online = True
        while online:
            messages = self.recv()
            for message in messages:
                if message.verb == '001':
                    self.rpl_welcome()
                    online = False
                elif message.verb.lower() == 'ping':
                    self.send_message('PONG', params=message.params)

        self.recv()
        sleep(1.5)
        self.recv()
        self.quit()

    def rpl_welcome(self):
        # yay lazy event loop!
        for name, info in self.commands:
            cmd = self.command_map.get(name, None)
            if cmd:
                cmd(info)
                self.recv()

    def quit(self):
        self.send_message('QUIT')

    # irc commands
    def privmsg(self, target, msg):
        """Send a PRIVMSG."""
        self.send_message('PRIVMSG', params=[target, msg])

    # command list
    def cmd_nickserv(self, info):
        self.privmsg('NickServ', info)
<|endoftext|>


<commit_before>"""
Created on April 3, 2017

@author: Akshay Narayan

This code is shared under The MIT License
-----------------------------------------

The MIT License (MIT)

Copyright (c) <year> <copyright holders>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import logging
import numpy
import json
import random
import time
import copy
import MalmoPython

class SimpleMalmoEnvironment:

    def __init__(self):
        log = logging.getLogger('SimpleMalmoEnvironment.init')
        self.actions = ["move 1", "turn 1", "turn -1", "attack 1", "discardCurrentItem"]
        self.landmark_types = ["redstone_block", "emerald_block", "lapis_block", "cobblestone", "gold_block",
                               "quartz_block"]
        self.size = [7, 7]
        self.landmarks = [[1, 2], [2, 5], [5, 6], [5, 2]]

    def generate_malmo_environment_xml(self):
        log = logging.getLogger('SimpleMalmoEnvironment.generateMalmoEnvironmentXML')
        xml_string = '''<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
        <Mission xmlns="http://ProjectMalmo.microsoft.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
            <About>
                Simple Malmo Environment
            </About>
            <ServerSection>
                <ServerInitialConditions>
                    <Time><StartTime>1</StartTime></Time>
                    <Weather>clear</Weather>
                </ServerInitialConditions>
                <ServerHandlers>
                    <FlatWorldGenerator generatorString="3;7,220*1,5*3,2;3;,biome_1" forceReset="true" />
                <DrawingDecorator>

                    <DrawCuboid x1="0" y1="46" z1="0" x2="''' + str(self.size[0]) + '''" y2="50" z2="''' + str(
                    self.size[1]) + '''" type="air" />
                    <DrawCuboid x1="0" y1="45" z1="0" x2="''' + str(self.size[1]) + '''" y2="45" z2="''' + str(
                    self.size[1]) + '''" type="sandstone" />
                    ''' + self.draw_landmarks() + self.drawObstacles() + '''
                </DrawingDecorator>
                <ServerQuitFromTimeUp timeLimitMs="720000"/>
                <ServerQuitWhenAnyAgentFinishes/>
                </ServerHandlers>
            </ServerSection>
            <AgentSection mode="Survival">
                <Name>ButterFingers</Name>
                <AgentStart>
                    <Placement x="4" y="46.0" z="2" pitch="15" yaw="0"/>
                    <Inventory>
                    </Inventory>
                </AgentStart>
                <AgentHandlers>
                    <DiscreteMovementCommands/>
                    <MissionQuitCommands/>
                    <InventoryCommands/>
                    <ObservationFromFullStats/>
                    <ObservationFromRay/>
                    <ObservationFromFullInventory/>
                    <VideoProducer want_depth="false" viewpoint="1">
                        <Width>640</Width>
                        <Height>480</Height>
                    </VideoProducer>
                    <RewardForSendingCommand reward="-1" />
                </AgentHandlers>
                </AgentSection>
        </Mission>'''

        log.debug("Final mission XML String: %s", xml_string)

        return xml_string

    def draw_landmarks(self):
        log = logging.getLogger('SimpleMalmoEnvironment.drawlandmarks')

        landmarks = self.landmarks

        log.debug("Input landmarks: %s", landmarks)

        landmark_xml = ''''''

        for i, l in enumerate(landmarks):
            x, z = l
            landmark_xml += '''<DrawBlock x="''' + str(x) + '''" y="45" z="''' + str(z) + '''" type="''' + self.landmark_types[i % len(self.landmark_typesn)] + '''"/>'''

        return landmark_xml

<commit_msg>add draw_obstacle<commit_after>"""
Created on April 3, 2017

@author: Akshay Narayan

This code is shared under The MIT License
-----------------------------------------

The MIT License (MIT)

Copyright (c) <year> <copyright holders>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import logging
import numpy
import json
import random
import time
import copy
import MalmoPython

class SimpleMalmoEnvironment:

    def __init__(self):
        log = logging.getLogger('SimpleMalmoEnvironment.init')
        self.actions = ["move 1", "turn 1", "turn -1", "attack 1", "discardCurrentItem"]
        self.landmark_types = ["redstone_block", "emerald_block", "lapis_block", "cobblestone", "gold_block",
                               "quartz_block"]
        self.size = [7, 7]
        self.landmarks = [[1, 2], [2, 5], [5, 6], [5, 2]]
        self.walls = [[2, 2, 2], [3, 2, 2], [3, 3, 2], [4, 3, 2], [1, 4, 2], [2, 5, 2], [2, 6, 2]],

    def generate_malmo_environment_xml(self):
        log = logging.getLogger('SimpleMalmoEnvironment.generateMalmoEnvironmentXML')
        xml_string = '''<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
        <Mission xmlns="http://ProjectMalmo.microsoft.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
            <About>
                Simple Malmo Environment
            </About>
            <ServerSection>
                <ServerInitialConditions>
                    <Time><StartTime>1</StartTime></Time>
                    <Weather>clear</Weather>
                </ServerInitialConditions>
                <ServerHandlers>
                    <FlatWorldGenerator generatorString="3;7,220*1,5*3,2;3;,biome_1" forceReset="true" />
                <DrawingDecorator>

                    <DrawCuboid x1="0" y1="46" z1="0" x2="''' + str(self.size[0]) + '''" y2="50" z2="''' + str(
                    self.size[1]) + '''" type="air" />
                    <DrawCuboid x1="0" y1="45" z1="0" x2="''' + str(self.size[1]) + '''" y2="45" z2="''' + str(
                    self.size[1]) + '''" type="sandstone" />
                    ''' + self.draw_landmarks() + self.draw_obstacles() + '''
                </DrawingDecorator>
                <ServerQuitFromTimeUp timeLimitMs="720000"/>
                <ServerQuitWhenAnyAgentFinishes/>
                </ServerHandlers>
            </ServerSection>
            <AgentSection mode="Survival">
                <Name>ButterFingers</Name>
                <AgentStart>
                    <Placement x="4" y="46.0" z="2" pitch="15" yaw="0"/>
                    <Inventory>
                    </Inventory>
                </AgentStart>
                <AgentHandlers>
                    <DiscreteMovementCommands/>
                    <MissionQuitCommands/>
                    <InventoryCommands/>
                    <ObservationFromFullStats/>
                    <ObservationFromRay/>
                    <ObservationFromFullInventory/>
                    <VideoProducer want_depth="false" viewpoint="1">
                        <Width>640</Width>
                        <Height>480</Height>
                    </VideoProducer>
                    <RewardForSendingCommand reward="-1" />
                </AgentHandlers>
                </AgentSection>
        </Mission>'''

        log.debug("Final mission XML String: %s", xml_string)

        return xml_string

    def draw_landmarks(self):
        log = logging.getLogger('SimpleMalmoEnvironment.drawlandmarks')

        landmarks = self.landmarks

        log.debug("Input landmarks: %s", landmarks)

        landmark_xml = ''''''

        for i, l in enumerate(landmarks):
            x, z = l
            landmark_xml += '''<DrawBlock x="''' + str(x) + '''" y="45" z="''' + str(z) + '''" type="''' + self.landmark_types[i % len(self.landmark_typesn)] + '''"/>'''

        return landmark_xml

    def draw_obstacles(self):
        log = logging.getLogger('SimpleMalmoEnvironment.drawobstacles')

        log.debug("Input walls: %s", walls)

        obstacle_xml = ''''''
        for w in self.walls:
            log.debug("Adding obstacle: %s", w)
            x, y, direction = list(w)
            obstacle_string = self.drawObstacle(x, y, direction)
            log.debug("Obstacle string received: %s", obstacle_string)
            obstacle_xml = obstacle_xml + obstacle_string

        log.debug("Obstacle string obtained: %s", obstacle_xml)

        return obstacle_xml<|endoftext|>