<commit_before># Copyright (c) 2017, CNRS-LAAS
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging
import types

from itertools import cycle
from typing import List, Optional, Tuple, Union

import matplotlib.cm
import matplotlib.colors
import matplotlib.pyplot
import numpy as np

import fire_rs.geodata.display as gdd
from fire_rs.geodata.geo_data import GeoData
from fire_rs.planning.planning import Planner

class TrajectoryDisplayExtension(gdd.DisplayExtension):
    """Extension to GeoDataDisplay that an observation trajectories."""

    def __init__(self, base_display: 'gdd.GeoDataDisplayBase', plan_trajectory):
        super().__init__(base_display, self.__class__.__name__)
        self.plan_trajectory = plan_trajectory

    def draw_waypoints(self, *args, **kwargs):
        """Draw path waypoints in a GeoDataDisplay figure."""
        color = kwargs.get('color', 'C0')
        waypoints = self.plan_trajectory.as_waypoints()
        x = [wp.x for wp in waypoints]
        y = [wp.y for wp in waypoints]
        self._base_display.drawings.append(self._base_display.axis.scatter(x[::2], y[::2], s=7, c=color, marker='D'))
        self._base_display.drawings.append(self._base_display.axis.scatter(x[1::2], y[1::2], s=7, c=color, marker='>'))

    def draw_flighttime_path(self, *args, colorbar_time_range: 'Optional[Tuple[float, float]]' = None,
                             **kwargs):
        """Draw trajectory in a GeoDataDisplay figure.

        The color of the trajectory will reflect the time taken by the trajectory.
        Optional argument colorbar_time_range may be a tuple of start and end times in seconds.
        If the Optional argument with_colorbar is set to True, a color bar will be displayed of the trajectory.
        """
        if len(self.plan_trajectory) < 2:
            return
        sampled_waypoints = self.plan_trajectory.sampled(step_size=5)
        x = [wp.x for wp in sampled_waypoints]
        y = [wp.y for wp in sampled_waypoints]
        color_range = np.linspace(self.plan_trajectory.start_time() / 60, self.plan_trajectory.end_time() / 60, len(x))
        color_norm = matplotlib.colors.Normalize(vmin=color_range[0], vmax=color_range[-1])
        if colorbar_time_range is not None:
            color_norm = matplotlib.colors.Normalize(vmin=colorbar_time_range[0] / 60, vmax=colorbar_time_range[1] / 60)
            self._base_display.drawings.append(
                self._base_display.axis.scatter(x, y, s=1, edgecolors='none', c=color_range,
                                                norm=color_norm, cmap=matplotlib.cm.gist_rainbow,
                                                zorder=self._base_display.FOREGROUND_LAYER))
        if kwargs.get('with_colorbar', False):
            cb = self._base_display.figure.colorbar(self._base_display.drawings[-1], ax=self._base_display.axis,
                                                    shrink=0.65, aspect=20)
            cb.set_label("Flight time [min]")
            self._base_display.colorbars.append(cb)

    def draw_solid_path(self, *args, **kwargs):
        """Draw trajectory in a GeoDataDisplay figure with solid color

        kwargs:
            color: desired color. Default: C0.
        """
        if len(self.plan_trajectory) < 2:
            return
        sampled_waypoints = self.plan_trajectory.sampled(step_size=5)
        color = kwargs.get('color', 'C0')
        size = kwargs.get('size', 1)
        x = [wp.x for wp in sampled_waypoints]
        y = [wp.y for wp in sampled_waypoints]
        self._base_display.drawings.append(
            self._base_display.axis.plot(x, y, linewidth=size, c=color, zorder=self._base_display.FOREGROUND_LAYER))
        # TODO: implement legend

    def draw_segments(self, *args, **kwargs):
        '''Draw observation segments with start and end points in a GeoDataDisplay figure.'''
        if len(self.plan_trajectory) < 2:
            return
        color = kwargs.get('color', 'C0')
        py_segments = self.plan_trajectory.segments
        start_x = [s.start.x for s in py_segments]
        start_y = [s.start.y for s in py_segments]
        end_x = [s.end.x for s in py_segments]
        end_y = [s.end.y for s in py_segments]

        self._base_display.drawings.append(
            self._base_display.axis.scatter(start_x, start_y, s=10, edgecolor='black', c=color, marker='h',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))
        self._base_display.drawings.append(
            self._base_display.axis.scatter(end_x, end_y, s=10, edgecolor='black', c=color, marker='>',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))

        start_base = py_segments[0]
        finish_base = py_segments[-1]

        self._base_display.drawings.append(self._base_display.axis.scatter(
            start_base.start.x, start_base.start.y, s=10, edgecolor='black', c=color, marker='o',
            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))
        self._base_display.drawings.append(
            self._base_display.axis.scatter(finish_base.start.x, finish_base.start.y, s=10,
                                            edgecolor='black', c=color, marker='o',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))

        # Draw lines between segment bounds
        for i in range(len(py_segments)):
            self._base_display.drawings.append(
                self._base_display.axis.plot([start_x[i], end_x[i]], [start_y[i], end_y[i]], c=color, linewidth=2,
                                             zorder=self._base_display.FOREGROUND_LAYER))

    def draw_observedcells(self, observations, **kwargs):
        """Plot observed cells as points"""
        for ptt in observations:
            self._base_display.axis.scatter(ptt[0][0], ptt[0][1], s=4, c=(0., 1., 0., .5),
                                            zorder=self._base_display.BACKGROUND_OVERLAY_LAYER,
                                            edgecolors='none', marker='s')

    def draw_observation_map(self, obs_map: 'Optional[GeoData]' = None, layer='observed',
                             color='green', **kwargs):
        o_map = np.array(obs_map[layer]) if obs_map is not None else np.array(self._base_display.geodata[layer])
        o_map[~np.isnan(o_map)] = 1

        # define the colors
        cmap = matplotlib.colors.ListedColormap([color])

        shade = gdd.plot_ignition_shade(self._base_display.axis, self._base_display.x_mesh, self._base_display.y_mesh,
                                        np.around(o_map.T[::-1, ...] / 60., 1),
                                        dx=self._base_display.geodata.cell_width,
                                        dy=self._base_display.geodata.cell_height,
                                        image_scale=self._base_display.image_scale, cmap=cmap, **kwargs)
        self._base_display.drawings.append(shade)

def plot_plan_trajectories(plan, geodatadisplay, time_range: 'Optional[Tuple[float, float]]' = None,
                           colors: 'Optional[List]' = None, sizes: 'Optional[List[int]]' = None, show=False):
    """Plot the trajectories of a plan."""
    if not colors:
        colors = ["red", "green", "blue", "black", "magenta"]
    colors = cycle(colors)
    if not sizes:
        sizes = [1, 1]
    sizes = cycle(sizes)
    for traj, color, size in zip(plan.trajectories(), colors, sizes):
        if time_range:
            traj = traj.slice(time_range)
        geodatadisplay.TrajectoryDisplayExtension.plan_trajectory = traj
        geodatadisplay.TrajectoryDisplayExtension.draw_solid_path(color=color, size=size)
        geodatadisplay.TrajectoryDisplayExtension.draw_segments(color=color)
    if show:
        geodatadisplay.figure.show()

def plot_plan_with_background(planner: 'Planner', geodatadisplay, time_range, output_options_plot,
                              plan: 'Union[str, int]' = 'final'):
    """Plot a plan trajectories with background geographic information in a geodata display."""
    # Draw background layers
    for layer in output_options_plot['background']:
        if layer == 'elevation_shade':
            geodatadisplay.draw_elevation_shade(
                with_colorbar=output_options_plot.get('colorbar', True), layer='elevation')
        if layer == 'elevation_planning_shade':
            geodatadisplay.draw_elevation_shade(
                with_colorbar=output_options_plot.get('colorbar', True), layer='elevation_planning')
        elif layer == 'ignition_shade':
            geodatadisplay.draw_ignition_shade(
                with_colorbar=output_options_plot.get('colorbar', True))
        elif layer == 'observedcells':
            geodatadisplay.TrajectoryDisplayExtension.draw_observation_map(
                planner.expected_observed_map(layer_name="expected_observed"),
                layer='expected_observed', color='green', alpha=0.9)
            geodatadisplay.TrajectoryDisplayExtension.draw_observation_map(
                planner.expected_ignited_map(layer_name="expected_ignited"),
                layer='expected_ignited', color='red', alpha=0.9)
        elif layer == 'ignition_contour':
            try:
                geodatadisplay.draw_ignition_contour(with_labels=True)
            except ValueError as e:
                logging.warning(e)
        elif layer == 'wind_quiver':
            geodatadisplay.draw_wind_quiver()

    # Plot the plan
    if plan == 'final':
        plot_plan_trajectories(planner.search_result.final_plan(), geodatadisplay,
                               time_range=time_range, show=True)
    elif plan == 'initial':
        plot_plan_trajectories(planner.search_result.initial_plan(), geodatadisplay,
                               time_range=time_range, show=True)
    else:
        plot_plan_trajectories(planner.search_result.intermediate_plans[plan], geodatadisplay,
                               time_range=time_range, show=True)
<commit_msg>Draw differently modifiable, frozen and base maneuvers<commit_after># Copyright (c) 2017, CNRS-LAAS
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging
import types

from itertools import cycle
from typing import List, Optional, Tuple, Union

import matplotlib.cm
import matplotlib.colors
import matplotlib.pyplot
import numpy as np

import fire_rs.geodata.display as gdd
from fire_rs.geodata.geo_data import GeoData
from fire_rs.planning.planning import Planner

class TrajectoryDisplayExtension(gdd.DisplayExtension):
    """Extension to GeoDataDisplay that an observation trajectories."""

    def __init__(self, base_display: 'gdd.GeoDataDisplayBase', plan_trajectory):
        super().__init__(base_display, self.__class__.__name__)
        self.plan_trajectory = plan_trajectory

    def draw_waypoints(self, *args, **kwargs):
        """Draw path waypoints in a GeoDataDisplay figure."""
        color = kwargs.get('color', 'C0')
        waypoints = self.plan_trajectory.as_waypoints()
        x = [wp.x for wp in waypoints]
        y = [wp.y for wp in waypoints]
        self._base_display.drawings.append(self._base_display.axis.scatter(x[::2], y[::2], s=7, c=color, marker='D'))
        self._base_display.drawings.append(self._base_display.axis.scatter(x[1::2], y[1::2], s=7, c=color, marker='>'))

    def draw_flighttime_path(self, *args, colorbar_time_range: 'Optional[Tuple[float, float]]' = None,
                             **kwargs):
        """Draw trajectory in a GeoDataDisplay figure.

        The color of the trajectory will reflect the time taken by the trajectory.
        Optional argument colorbar_time_range may be a tuple of start and end times in seconds.
        If the Optional argument with_colorbar is set to True, a color bar will be displayed of the trajectory.
        """
        if len(self.plan_trajectory) < 2:
            return
        sampled_waypoints = self.plan_trajectory.sampled(step_size=5)
        x = [wp.x for wp in sampled_waypoints]
        y = [wp.y for wp in sampled_waypoints]
        color_range = np.linspace(self.plan_trajectory.start_time() / 60, self.plan_trajectory.end_time() / 60, len(x))
        color_norm = matplotlib.colors.Normalize(vmin=color_range[0], vmax=color_range[-1])
        if colorbar_time_range is not None:
            color_norm = matplotlib.colors.Normalize(vmin=colorbar_time_range[0] / 60, vmax=colorbar_time_range[1] / 60)
            self._base_display.drawings.append(
                self._base_display.axis.scatter(x, y, s=1, edgecolors='none', c=color_range,
                                                norm=color_norm, cmap=matplotlib.cm.gist_rainbow,
                                                zorder=self._base_display.FOREGROUND_LAYER))
        if kwargs.get('with_colorbar', False):
            cb = self._base_display.figure.colorbar(self._base_display.drawings[-1], ax=self._base_display.axis,
                                                    shrink=0.65, aspect=20)
            cb.set_label("Flight time [min]")
            self._base_display.colorbars.append(cb)

    def draw_solid_path(self, *args, **kwargs):
        """Draw trajectory in a GeoDataDisplay figure with solid color

        kwargs:
            color: desired color. Default: C0.
        """
        if len(self.plan_trajectory) < 2:
            return
        sampled_waypoints = self.plan_trajectory.sampled(step_size=5)
        color = kwargs.get('color', 'C0')
        size = kwargs.get('size', 1)
        x = [wp.x for wp in sampled_waypoints]
        y = [wp.y for wp in sampled_waypoints]
        self._base_display.drawings.append(
            self._base_display.axis.plot(x, y, linewidth=size, c=color, zorder=self._base_display.FOREGROUND_LAYER))
        # TODO: implement legend

    def draw_segments(self, *args, **kwargs):
        '''Draw observation segments with start and end points in a GeoDataDisplay figure.'''
        if len(self.plan_trajectory) < 2:
            return
        color = kwargs.get('color', 'C0')
        py_segments = self.plan_trajectory.segments[:]
        py_modifi_segments = [s for i, s in enumerate(py_segments) if self.plan_trajectory.can_modify(i)]
        py_frozen_segments = [s for i, s in enumerate(py_segments) if not self.plan_trajectory.can_modify(i)]

        # Plot modifiable segments
        start_x_m = [s.start.x for s in py_modifi_segments]
        start_y_m = [s.start.y for s in py_modifi_segments]
        end_x_m = [s.end.x for s in py_modifi_segments]
        end_y_m = [s.end.y for s in py_modifi_segments]

        self._base_display.drawings.append(
            self._base_display.axis.scatter(start_x_m, start_y_m, s=10, edgecolor='black', c=color, marker='o',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))
        self._base_display.drawings.append(
            self._base_display.axis.scatter(end_x_m, end_y_m, s=10, edgecolor='black', c=color, marker='>',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))

        # Plot frozen segments (all but the bases)
        start_x_f = [s.start.x for s in py_frozen_segments[1:len(py_frozen_segments) - 1]]
        start_y_f = [s.start.y for s in py_frozen_segments[1:len(py_frozen_segments) - 1]]
        end_x_f = [s.end.x for s in py_frozen_segments[1:len(py_frozen_segments) - 1]]
        end_y_f = [s.end.y for s in py_frozen_segments[1:len(py_frozen_segments) - 1]]

        self._base_display.drawings.append(
            self._base_display.axis.scatter(start_x_f, start_y_f, s=10, edgecolor='black', c='black', marker='o',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))
        self._base_display.drawings.append(
            self._base_display.axis.scatter(end_x_f, end_y_f, s=10, edgecolor='black', c='black', marker='>',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))

        start_base = py_segments[0]
        finish_base = py_segments[-1]

        self._base_display.drawings.append(self._base_display.axis.scatter(
            start_base.start.x, start_base.start.y, s=10, edgecolor=color, c=color, marker='D',
            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))
        self._base_display.drawings.append(
            self._base_display.axis.scatter(finish_base.start.x, finish_base.start.y, s=10,
                                            edgecolor=color, c=color, marker='D',
                                            zorder=self._base_display.FOREGROUND_OVERLAY_LAYER))

        start_x = [s.start.x for s in py_segments]
        start_y = [s.start.y for s in py_segments]
        end_x = [s.end.x for s in py_segments]
        end_y = [s.end.y for s in py_segments]

        # Draw lines between segment bounds
        for i in range(len(py_segments)):
            self._base_display.drawings.append(
                self._base_display.axis.plot([start_x[i], end_x[i]], [start_y[i], end_y[i]], c=color, linewidth=2,
                                             zorder=self._base_display.FOREGROUND_LAYER))

    def draw_observedcells(self, observations, **kwargs):
        """Plot observed cells as points"""
        for ptt in observations:
            self._base_display.axis.scatter(ptt[0][0], ptt[0][1], s=4, c=(0., 1., 0., .5),
                                            zorder=self._base_display.BACKGROUND_OVERLAY_LAYER,
                                            edgecolors='none', marker='s')

    def draw_observation_map(self, obs_map: 'Optional[GeoData]' = None, layer='observed',
                             color='green', **kwargs):
        o_map = np.array(obs_map[layer]) if obs_map is not None else np.array(self._base_display.geodata[layer])
        o_map[~np.isnan(o_map)] = 1

        # define the colors
        cmap = matplotlib.colors.ListedColormap([color])

        shade = gdd.plot_ignition_shade(self._base_display.axis, self._base_display.x_mesh, self._base_display.y_mesh,
                                        np.around(o_map.T[::-1, ...] / 60., 1),
                                        dx=self._base_display.geodata.cell_width,
                                        dy=self._base_display.geodata.cell_height,
                                        image_scale=self._base_display.image_scale, cmap=cmap, **kwargs)
        self._base_display.drawings.append(shade)

def plot_plan_trajectories(plan, geodatadisplay, time_range: 'Optional[Tuple[float, float]]' = None,
                           colors: 'Optional[List]' = None, sizes: 'Optional[List[int]]' = None, show=False):
    """Plot the trajectories of a plan."""
    if not colors:
        colors = ["red", "green", "blue", "black", "magenta"]
    colors = cycle(colors)
    if not sizes:
        sizes = [1, 1]
    sizes = cycle(sizes)
    for traj, color, size in zip(plan.trajectories(), colors, sizes):
        if time_range:
            traj = traj.slice(time_range)
        geodatadisplay.TrajectoryDisplayExtension.plan_trajectory = traj
        geodatadisplay.TrajectoryDisplayExtension.draw_solid_path(color=color, size=size)
        geodatadisplay.TrajectoryDisplayExtension.draw_segments(color=color)
    if show:
        geodatadisplay.figure.show()

def plot_plan_with_background(planner: 'Planner', geodatadisplay, time_range, output_options_plot,
                              plan: 'Union[str, int]' = 'final'):
    """Plot a plan trajectories with background geographic information in a geodata display."""
    # Draw background layers
    for layer in output_options_plot['background']:
        if layer == 'elevation_shade':
            geodatadisplay.draw_elevation_shade(
                with_colorbar=output_options_plot.get('colorbar', True), layer='elevation')
        if layer == 'elevation_planning_shade':
            geodatadisplay.draw_elevation_shade(
                with_colorbar=output_options_plot.get('colorbar', True), layer='elevation_planning')
        elif layer == 'ignition_shade':
            geodatadisplay.draw_ignition_shade(
                with_colorbar=output_options_plot.get('colorbar', True))
        elif layer == 'observedcells':
            geodatadisplay.TrajectoryDisplayExtension.draw_observation_map(
                planner.expected_observed_map(layer_name="expected_observed"),
                layer='expected_observed', color='green', alpha=0.9)
            geodatadisplay.TrajectoryDisplayExtension.draw_observation_map(
                planner.expected_ignited_map(layer_name="expected_ignited"),
                layer='expected_ignited', color='red', alpha=0.9)
        elif layer == 'ignition_contour':
            try:
                geodatadisplay.draw_ignition_contour(with_labels=True)
            except ValueError as e:
                logging.warning(e)
        elif layer == 'wind_quiver':
            geodatadisplay.draw_wind_quiver()

    # Plot the plan
    if plan == 'final':
        plot_plan_trajectories(planner.search_result.final_plan(), geodatadisplay,
                               time_range=time_range, show=True)
    elif plan == 'initial':
        plot_plan_trajectories(planner.search_result.initial_plan(), geodatadisplay,
                               time_range=time_range, show=True)
    else:
        plot_plan_trajectories(planner.search_result.intermediate_plans[plan], geodatadisplay,
                               time_range=time_range, show=True)
<|endoftext|>


<commit_before>
import bz2
import base64
import shutil
import os

import osnap.osnapy_win
import osnap.osnapy_mac
import osnap.util
import osnap.const
import osnap.write_timestamp
import osnap.launchwin
import osnap.launchmac

def create_osnapy(version, clean_cache=False, verbose=False):
    if verbose:
        print('creating osnapy')
    if osnap.util.is_windows():
        osnap.osnapy_win.create_python_win(version, clean_cache, verbose)
    elif osnap.util.is_mac():
        osnap.osnapy_mac.create_python_mac(version, clean_cache, verbose)
    else:
        raise NotImplementedError

# todo: add an optional version
def add_package(package, verbose=False):
    if verbose:
        print('adding %s to python environment' % str(package))
    if osnap.util.is_windows():
        osnap.osnapy_win.add_package_win(package, verbose)
    elif osnap.util.is_mac():
        osnap.osnapy_mac.add_package_mac(package, verbose)
    else:
        raise NotImplementedError

def add_packages_from_requirements_file(requirements_file_path = 'requirements.txt', verbose=False):
    with open(requirements_file_path) as f:
        for l in f:
            l = l.strip()
            if len(l) > 0:
                add_package(l, verbose=verbose)

def unpack_launcher(verbose=False):
    zip_file = osnap.util.get_launch_name() + '.zip'
    launch_name = osnap.util.get_launch_name()

    if os.path.exists(launch_name):
        if verbose:
            print('removing directory : %s' % launch_name)
        shutil.rmtree(launch_name)

    if verbose:
        print('unpacking %s' % launch_name)
    with open(zip_file, 'wb') as out_file:
        if osnap.util.is_windows():
            launch_code = osnap.launchwin.launchwin
        elif osnap.util.is_mac():
            launch_code = osnap.launchmac.launchmac
        else:
            raise NotImplementedError
        out_file.write(bz2.decompress(base64.b16decode(launch_code)))
    shutil.unpack_archive(zip_file, launch_name)

    # make launcher executable
    if verbose:
        print('making %s executable' % launch_name)
    mod = 0o755
    for root, dirs, files in os.walk(launch_name):
        for dir in dirs:
            os.chmod(os.path.join(root, dir), mod)
        for f in files:
            os.chmod(os.path.join(root, f), mod)

    if verbose:
        print('cleanup : removing %s' % zip_file)
    os.remove(zip_file)
<commit_msg>add default requirements file path<commit_after>
import bz2
import base64
import shutil
import os

import osnap.osnapy_win
import osnap.osnapy_mac
import osnap.util
import osnap.const
import osnap.write_timestamp
import osnap.launchwin
import osnap.launchmac

def create_osnapy(version, clean_cache=False, verbose=False):
    if verbose:
        print('creating osnapy')
    if osnap.util.is_windows():
        osnap.osnapy_win.create_python_win(version, clean_cache, verbose)
    elif osnap.util.is_mac():
        osnap.osnapy_mac.create_python_mac(version, clean_cache, verbose)
    else:
        raise NotImplementedError

# todo: add an optional version
def add_package(package, verbose=False):
    if verbose:
        print('adding %s to python environment' % str(package))
    if osnap.util.is_windows():
        osnap.osnapy_win.add_package_win(package, verbose)
    elif osnap.util.is_mac():
        osnap.osnapy_mac.add_package_mac(package, verbose)
    else:
        raise NotImplementedError

def add_packages_from_requirements_file(requirements_file_path='requirements.txt', verbose=False):
    with open(requirements_file_path) as f:
        for l in f:
            l = l.strip()
            if len(l) > 0:
                add_package(l, verbose=verbose)

def unpack_launcher(verbose=False):
    zip_file = osnap.util.get_launch_name() + '.zip'
    launch_name = osnap.util.get_launch_name()

    if os.path.exists(launch_name):
        if verbose:
            print('removing directory : %s' % launch_name)
        shutil.rmtree(launch_name)

    if verbose:
        print('unpacking %s' % launch_name)
    with open(zip_file, 'wb') as out_file:
        if osnap.util.is_windows():
            launch_code = osnap.launchwin.launchwin
        elif osnap.util.is_mac():
            launch_code = osnap.launchmac.launchmac
        else:
            raise NotImplementedError
        out_file.write(bz2.decompress(base64.b16decode(launch_code)))
    shutil.unpack_archive(zip_file, launch_name)

    # make launcher executable
    if verbose:
        print('making %s executable' % launch_name)
    mod = 0o755
    for root, dirs, files in os.walk(launch_name):
        for dir in dirs:
            os.chmod(os.path.join(root, dir), mod)
        for f in files:
            os.chmod(os.path.join(root, f), mod)

    if verbose:
        print('cleanup : removing %s' % zip_file)
    os.remove(zip_file)
<|endoftext|>


<commit_before>from datetime import datetime
import re
from typing import Iterator

from pywikibot import Page, Site
from pywikibot.proofreadpage import ProofreadPage, IndexPage
from tools.catscan import PetScan
from tools.bots import CanonicalBot, BotException

def search_for_refs(text):
    ref = []
    if re.search(r'<ref>', text):
        ref.append('ref')
    elif re.search(r'\{\{CRef\|\|', text):
        ref.append('ref')
    hit = re.findall('[Gg]roup ?= ?(\"[^\"]*\"|[^\"> ]{1,10})', text)
    if hit:
        for entry in hit:
            group = entry.strip("\"")
            if group not in ref:
                ref.append(group)
    hit = re.findall(r'\{\{CRef\|([^\|]{1,10})\|', text)
    if hit:
        for entry in hit:
            if entry not in ref:
                ref.append(entry)
    return ref

class GlCreateMagazine(CanonicalBot):
    def __init__(self, wiki, debug):
        CanonicalBot.__init__(self, wiki, debug)
        self.searcher_pages = PetScan()
        self.searcher_indexes = PetScan()
        self.regex_page = re.compile(r'Die_Gartenlaube_\((\d{4})\)_([^\.]*?)\.(?:jpg|JPG)')
        self.regex_index = re.compile(r'Die_Gartenlaube_\((\d{4})\)')
        self.regex_magazine_in_index = \
            re.compile(r'((?:Heft|Halbheft) (?:\{\{0\}\})?\d{1,2}:.*?(?:\n\n|\Z))', re.DOTALL)
        self.regex_page_in_magazine = re.compile(r'_([_\w]{1,9}).(?:jpg|JPG)')
        self.regex_number_in_index = re.compile(r'(?:Heft|Halbheft) (?:\{\{0\}\})?(\d{1,2}):?')
        self.new_data_model = datetime(year=2017, month=11, day=11, hour=12)
        self.lemmas = None

    def __enter__(self):
        CanonicalBot.__enter__(self)
        if not self.data:
            self.data.assign_dict({'pages': {}, 'indexes': {}})
        return self

    def task(self):
        self.lemmas = self.search_pages()
        temp_data_pages = {}
        self.process_indexes()
        self.process_pages(temp_data_pages)
        temp_data_magazines = self.process_actual_pages(temp_data_pages)
        self.make_magazines(temp_data_magazines)
        return True

    def process_pages(self, temp_data):
        for idx, lemma in enumerate(self.lemmas):
            try:
                hit = self.regex_page.search(lemma['title'])
                year = hit.group(1)
                page = hit.group(2)
                if year not in self.data['pages'].keys():
                    self.data['pages'][year] = {}
                proofread_lemma = ProofreadPage(self.wiki, 'Seite:{}'.format(lemma['title']))
                if self.debug:
                    self.logger.debug('{idx}/{sum} Page {page}({year}) '
                                      'has quality level {level} _ Seite:{title}'
                                      .format(idx=idx + 1,
                                              sum=len(self.lemmas),
                                              page=page,
                                              year=year,
                                              level=proofread_lemma.quality_level,
                                              title=lemma['title']))
                ref = search_for_refs(proofread_lemma.text)
                page_dict = {'q': proofread_lemma.quality_level}
                if ref:
                    self.logger.debug('There are refs ({refs}) @ {year}, {page}'.format(refs=ref,
                                                                                page=page,
                                                                                year=year))
                    page_dict.update({'r': ref})
                self.data['pages'][year][page] = page_dict
                if year not in temp_data.keys():
                    temp_data[year] = []
                temp_data[year].append(page)
            except Exception as error:  # pylint: disable=broad-except
                self.logger.error("wasn't able to process {}, error: {}"
                                  .format(lemma['title'], error))

    def process_indexes(self):
        for index_lemma, index_page in self._get_indexes():
            self.logger.debug("[[Index:{}]]".format(index_lemma))
            magazines = self.regex_magazine_in_index.findall(index_page.text)
            hit_year = self.regex_index.search(index_lemma)
            year = hit_year.group(1)
            if year not in self.data['indexes'].keys():
                self.data['indexes'][year] = {}
            for magazine in magazines:
                pages = self.regex_page_in_magazine.findall(magazine)
                hit_number = self.regex_number_in_index.findall(magazine)
                number = int(hit_number[0])
                self.data['indexes'][year][number] = pages

    def process_actual_pages(self, dictionary_of_new_pages):
        tempdata_magzines = {}
        for year in dictionary_of_new_pages:
            set_of_pages = set(dictionary_of_new_pages[year])
            tempdata_magzines[year] = set()
            try:
                dictionary_of_magazines = self.data['indexes'][year]
            except KeyError:
                raise BotException('The list of indexes is incorrect, {year} is missing.'
                                   .format(year=year))
            for magazine in dictionary_of_magazines:
                set_of_potential_pages = set(dictionary_of_magazines[magazine])
                if set_of_potential_pages.intersection(set_of_pages):
                    tempdata_magzines[year].add(magazine)
        return tempdata_magzines

    def make_magazines(self, dictionary_of_magazines_by_year):
        for idx_year, year in enumerate(dictionary_of_magazines_by_year):
            magazines = dictionary_of_magazines_by_year[year]
            self.logger.debug('make_mag_year {idx}/{len}'
                              .format(idx=idx_year + 1, len=len(dictionary_of_magazines_by_year)))
            for idx_mag, magazine in enumerate(magazines):
                self.logger.debug('make_mag_mag {idx}/{len} ... issue:{year}/{mag}'
                                  .format(idx=idx_mag + 1, len=len(magazines),
                                          year=year, mag=magazine))
                if year == '1986' and magazine == '31':
                    self.logger.warning('There is magazine 1986, 31, '
                                        'this is special, no creating here')
                    continue
                if self.debug:
                    lemma = Page(self.wiki, 'Benutzer:THEbotIT/Test')
                else:
                    lemma = Page(self.wiki, 'Die Gartenlaube ({year})/Heft {magazine:d}'
                                 .format(year=year, magazine=int(magazine)))
                new_text = self.make_magazine(year, magazine)
                if new_text:
                    if new_text != lemma.text:
                        self.logger.debug('Print [[Die Gartenlaube ({year})/Heft {magazine}]].'
                                          .format(year=year, magazine=magazine))
                        if lemma.text != '':
                            lemma.text = new_text
                            lemma.save('Automatische Aktualisierung des Heftes', botflag=True)
                        else:
                            lemma.text = new_text
                            lemma.save('automatische Hefterstellung', botflag=True)
                    else:
                        self.logger.debug('Keine Änderung im Text ({year}/{magazine}).'
                                          .format(year=year, magazine=magazine))

    def make_magazine(self, year, magazine):
        last_magazine = True
        try:
            for key in self.data['indexes'][year].keys():
                if int(key) > int(magazine):
                    last_magazine = False
                    break
        except KeyError:
            raise BotException('The list of indexes is incorrect, {year} is missing.'
                               .format(year=year))
        try:
            list_of_pages = self.data['indexes'][year][magazine]
        except KeyError:
            raise BotException('The list of indexes is incorrect, '
                               'year:{year} or mag:{mag} is missing.'
                               .format(year=year, mag=magazine))
        quality = 4
        for page in list_of_pages:
            try:
                if self.data['pages'][year][page]['q'] == 0:
                    page_quality = 4
                else:
                    page_quality = self.data['pages'][year][page]['q']
                if page_quality < quality:
                    quality = page_quality
                if quality < 3:
                    self.logger.debug('The quality of {year}/{magazine} is too poor.'
                                      .format(year=year, magazine=magazine))
                    return None
            except KeyError:
                self.logger.warning('The list of pages is incorrect, '
                                    'year:{year} or page:{page} is missing.'
                                    .format(year=year, page=page))
                return None
        return self.make_magazine_text(year, magazine, quality, list_of_pages, last_magazine)

    @staticmethod
    def convert_page_no(page: str):
        while True:
            if page[0] == "0":
                page = page[1:]
            else:
                break
        return page.replace('_', ' ')

    def make_magazine_text(self, year, magazine, quality, list_of_pages, last):
        # pylint: disable=too-many-arguments,too-many-branches
        magazine = int(magazine)
        year = int(year)
        string_list = list()
        string_list.append('<!--Diese Seite wurde automatisch durch einen Bot erstellt. '
                           'Wenn du einen Fehler findest oder eine Änderung wünscht, '
                           'benachrichtige bitte den Betreiber, THE IT, des Bots.-->\n'
                           '{{Textdaten\n')
        if magazine > 1:
            string_list.append('|VORIGER=Die Gartenlaube ({year:d})/Heft {magazine:d}\n'
                               .format(year=year, magazine=magazine - 1))
        else:
            string_list.append('|VORIGER=\n')
        if last:
            string_list.append('|NÄCHSTER=\n')
        else:
            string_list.append('|NÄCHSTER=Die Gartenlaube ({year:d})/Heft {magazine:d}\n'
                               .format(year=year, magazine=magazine + 1))
        string_list.append("|AUTOR=Verschiedene\n|TITEL=[[Die Gartenlaube]]\n"
                           "|SUBTITEL=''Illustrirtes Familienblatt''\n|HERKUNFT=off\n")
        if year < 1863:
            string_list.append('|HERAUSGEBER=[[Ferdinand Stolle]]\n')
        elif (year < 1878) or (year == 1878 and magazine < 14):
            string_list.append('|HERAUSGEBER=[[Ernst Keil]]\n')
        elif year < 1885:
            string_list.append('|HERAUSGEBER=Ernst Ziel\n')
        else:
            string_list.append('|HERAUSGEBER=Adolf Kröner\n')
        string_list.append('|ENTSTEHUNGSJAHR={year:d}\n|ERSCHEINUNGSJAHR={year:d}\n'
                           '|ERSCHEINUNGSORT=Leipzig\n|VERLAG=Ernst Keil\n'
                           '|WIKIPEDIA=Die Gartenlaube\n'.format(year=year))
        if year == 1873:
            extension = "JPG"
        else:
            extension = "jpg"
        string_list.append('|BILD=Die Gartenlaube ({year:d}) {page1}.{extension}\n'
                           .format(year=year, page1=list_of_pages[0], extension=extension))
        string_list.append('|QUELLE=[[commons:category:Gartenlaube ({year})|commons]]\n'
                           .format(year=year))
        if quality == 4:
            string_list.append('|BEARBEITUNGSSTAND=fertig\n')
        else:
            string_list.append('|BEARBEITUNGSSTAND=korrigiert\n')
        string_list.append('|INDEXSEITE=Die Gartenlaube ({year})\n}}}}\n\n'
                           '{{{{BlockSatzStart}}}}\n__TOC__\n'.format(year=year))
        ref = []
        for page in list_of_pages:
            page_format = self.convert_page_no(page)
            string_list.append(
                '{{{{SeitePR|{page_format}|Die Gartenlaube ({year}) {page}.{extension}}}}}\n'
                .format(year=year, page_format=page_format, page=page, extension=extension))
            try:
                page_dict = self.data['pages'][str(year)][page]
                if 'r' in page_dict.keys():
                    if 'ref' in page_dict['r']:
                        if 'ref' not in ref:
                            ref.append('ref')
                    for ref_type in page_dict['r']:
                        if (ref_type != 'ref') and (ref_type not in ref):
                            ref.append(ref_type)
            except KeyError:
                self.logger.error('The list of pages is incorrect, '
                                  'year:{year} or page:{page} is missing.'
                                  .format(year=year, page=page))
                return None
        if 'ref' in ref:
            string_list.append('{{references|x}}\n')
        for ref_type in ref:
            if ref_type != 'ref':
                string_list.append('{{{{references|TIT|{ref}}}}}\n'.format(ref=ref_type))
        string_list.append('{{{{BlockSatzEnd}}}}\n\n[[Kategorie:Deutschland]]\n'
                           '[[Kategorie:Neuhochdeutsch]]\n[[Kategorie:Illustrierte Werke]]\n'
                           '[[Kategorie:Die Gartenlaube ({year:d}) Hefte| {magazine:02d}]]\n'
                           .format(year=year, magazine=magazine))
        string_list.append('[[Kategorie:{year}0er Jahre]]\n\n'.format(year=str(year)[0:3]))
        return ''.join(string_list)

    def _get_indexes(self) -> Iterator[IndexPage]:
        self.searcher_indexes.add_positive_category('Die Gartenlaube')
        self.searcher_indexes.add_positive_category('Index')
        self.searcher_indexes.set_regex_filter(r'.*Die Gartenlaube \(\d{4}\)')
        self.searcher_indexes.set_timeout(60)
        for index in self.searcher_indexes.run():
            yield index["title"], IndexPage(self.wiki, 'Index:{}'.format(index['title']))

    def search_pages(self):
        self.searcher_pages.add_positive_category('Die Gartenlaube')
        self.searcher_pages.add_namespace('Seite')
        self.searcher_pages.set_search_depth(1)
        self.searcher_pages.set_timeout(60)
        if self.last_run_successful or self.debug:
            delta = (self.timestamp.start_of_run - self.timestamp.last_run).days
            if self.debug:
                delta = 10
            start_of_search = self.create_timestamp_for_search(delta)
            self.searcher_pages.last_change_after(start_of_search)
            self.logger.info('The date {} is set to the argument "after".'
                             .format(start_of_search.strftime("%d.%m.%Y")))
        return self.searcher_pages.run()

if __name__ == "__main__":
    WIKI = Site(code='de', fam='wikisource', user='THEbotIT')
    with GlCreateMagazine(wiki=WIKI, debug=True) as bot:
        bot.run()
<commit_msg>bugfix for str in quality of proofread page<commit_after>from datetime import datetime
import re
from typing import Iterator

from pywikibot import Page, Site
from pywikibot.proofreadpage import ProofreadPage, IndexPage
from tools.catscan import PetScan
from tools.bots import CanonicalBot, BotException

def search_for_refs(text):
    ref = []
    if re.search(r'<ref>', text):
        ref.append('ref')
    elif re.search(r'\{\{CRef\|\|', text):
        ref.append('ref')
    hit = re.findall('[Gg]roup ?= ?(\"[^\"]*\"|[^\"> ]{1,10})', text)
    if hit:
        for entry in hit:
            group = entry.strip("\"")
            if group not in ref:
                ref.append(group)
    hit = re.findall(r'\{\{CRef\|([^\|]{1,10})\|', text)
    if hit:
        for entry in hit:
            if entry not in ref:
                ref.append(entry)
    return ref

class GlCreateMagazine(CanonicalBot):
    def __init__(self, wiki, debug):
        CanonicalBot.__init__(self, wiki, debug)
        self.searcher_pages = PetScan()
        self.searcher_indexes = PetScan()
        self.regex_page = re.compile(r'Die_Gartenlaube_\((\d{4})\)_([^\.]*?)\.(?:jpg|JPG)')
        self.regex_index = re.compile(r'Die_Gartenlaube_\((\d{4})\)')
        self.regex_magazine_in_index = \
            re.compile(r'((?:Heft|Halbheft) (?:\{\{0\}\})?\d{1,2}:.*?(?:\n\n|\Z))', re.DOTALL)
        self.regex_page_in_magazine = re.compile(r'_([_\w]{1,9}).(?:jpg|JPG)')
        self.regex_number_in_index = re.compile(r'(?:Heft|Halbheft) (?:\{\{0\}\})?(\d{1,2}):?')
        self.new_data_model = datetime(year=2018, month=7, day=1, hour=14)
        self.lemmas = None

    def __enter__(self):
        CanonicalBot.__enter__(self)
        if not self.data:
            self.data.assign_dict({'pages': {}, 'indexes': {}})
        return self

    def task(self):
        self.lemmas = self.search_pages()
        temp_data_pages = {}
        self.process_indexes()
        self.process_pages(temp_data_pages)
        temp_data_magazines = self.process_actual_pages(temp_data_pages)
        self.make_magazines(temp_data_magazines)
        return True

    def process_pages(self, temp_data):
        for idx, lemma in enumerate(self.lemmas):
            try:
                hit = self.regex_page.search(lemma['title'])
                year = hit.group(1)
                page = hit.group(2)
                if year not in self.data['pages'].keys():
                    self.data['pages'][year] = {}
                proofread_lemma = ProofreadPage(self.wiki, 'Seite:{}'.format(lemma['title']))
                if self.debug:
                    self.logger.debug('{idx}/{sum} Page {page}({year}) '
                                      'has quality level {level} _ Seite:{title}'
                                      .format(idx=idx + 1,
                                              sum=len(self.lemmas),
                                              page=page,
                                              year=year,
                                              level=proofread_lemma.quality_level,
                                              title=lemma['title']))
                ref = search_for_refs(proofread_lemma.text)
                page_dict = {'q': int(proofread_lemma.quality_level)}
                if ref:
                    self.logger.debug('There are refs ({refs}) @ {year}, {page}'.format(refs=ref,
                                                                                page=page,
                                                                                year=year))
                    page_dict.update({'r': ref})
                self.data['pages'][year][page] = page_dict
                if year not in temp_data.keys():
                    temp_data[year] = []
                temp_data[year].append(page)
            except Exception as error:  # pylint: disable=broad-except
                self.logger.error("wasn't able to process {}, error: {}"
                                  .format(lemma['title'], error))

    def process_indexes(self):
        for index_lemma, index_page in self._get_indexes():
            self.logger.debug("[[Index:{}]]".format(index_lemma))
            magazines = self.regex_magazine_in_index.findall(index_page.text)
            hit_year = self.regex_index.search(index_lemma)
            year = hit_year.group(1)
            if year not in self.data['indexes'].keys():
                self.data['indexes'][year] = {}
            for magazine in magazines:
                pages = self.regex_page_in_magazine.findall(magazine)
                hit_number = self.regex_number_in_index.findall(magazine)
                number = int(hit_number[0])
                self.data['indexes'][year][number] = pages

    def process_actual_pages(self, dictionary_of_new_pages):
        tempdata_magzines = {}
        for year in dictionary_of_new_pages:
            set_of_pages = set(dictionary_of_new_pages[year])
            tempdata_magzines[year] = set()
            try:
                dictionary_of_magazines = self.data['indexes'][year]
            except KeyError:
                raise BotException('The list of indexes is incorrect, {year} is missing.'
                                   .format(year=year))
            for magazine in dictionary_of_magazines:
                set_of_potential_pages = set(dictionary_of_magazines[magazine])
                if set_of_potential_pages.intersection(set_of_pages):
                    tempdata_magzines[year].add(magazine)
        return tempdata_magzines

    def make_magazines(self, dictionary_of_magazines_by_year):
        for idx_year, year in enumerate(dictionary_of_magazines_by_year):
            magazines = dictionary_of_magazines_by_year[year]
            self.logger.debug('make_mag_year {idx}/{len}'
                              .format(idx=idx_year + 1, len=len(dictionary_of_magazines_by_year)))
            for idx_mag, magazine in enumerate(magazines):
                self.logger.debug('make_mag_mag {idx}/{len} ... issue:{year}/{mag}'
                                  .format(idx=idx_mag + 1, len=len(magazines),
                                          year=year, mag=magazine))
                if year == '1986' and magazine == '31':
                    self.logger.warning('There is magazine 1986, 31, '
                                        'this is special, no creating here')
                    continue
                if self.debug:
                    lemma = Page(self.wiki, 'Benutzer:THEbotIT/Test')
                else:
                    lemma = Page(self.wiki, 'Die Gartenlaube ({year})/Heft {magazine:d}'
                                 .format(year=year, magazine=int(magazine)))
                new_text = self.make_magazine(year, magazine)
                if new_text:
                    if hash(new_text.strip()) != hash(lemma.text.strip()):
                        self.logger.debug('Print [[Die Gartenlaube ({year})/Heft {magazine}]].'
                                          .format(year=year, magazine=magazine))
                        if lemma.text != '':
                            lemma.text = new_text
                            lemma.save('Automatische Aktualisierung des Heftes', botflag=True)
                        else:
                            lemma.text = new_text
                            lemma.save('automatische Hefterstellung', botflag=True)
                    else:
                        self.logger.debug('Keine Änderung im Text ({year}/{magazine}).'
                                          .format(year=year, magazine=magazine))

    def make_magazine(self, year, magazine):
        last_magazine = True
        try:
            for key in self.data['indexes'][year].keys():
                if int(key) > int(magazine):
                    last_magazine = False
                    break
        except KeyError:
            raise BotException('The list of indexes is incorrect, {year} is missing.'
                               .format(year=year))
        try:
            list_of_pages = self.data['indexes'][year][magazine]
        except KeyError:
            raise BotException('The list of indexes is incorrect, '
                               'year:{year} or mag:{mag} is missing.'
                               .format(year=year, mag=magazine))
        quality = 4
        for page in list_of_pages:
            try:
                if self.data['pages'][year][page]['q'] == 0:
                    page_quality = 4
                else:
                    page_quality = self.data['pages'][year][page]['q']
                if page_quality < quality:
                    quality = page_quality
                if quality < 3:
                    self.logger.debug('The quality of {year}/{magazine} is too poor.'
                                      .format(year=year, magazine=magazine))
                    return None
            except KeyError:
                self.logger.warning('The list of pages is incorrect, '
                                    'year:{year} or page:{page} is missing.'
                                    .format(year=year, page=page))
                return None
        return self.make_magazine_text(year, magazine, quality, list_of_pages, last_magazine)

    @staticmethod
    def convert_page_no(page: str):
        while True:
            if page[0] == "0":
                page = page[1:]
            else:
                break
        return page.replace('_', ' ')

    def make_magazine_text(self, year, magazine, quality, list_of_pages, last):
        # pylint: disable=too-many-arguments,too-many-branches
        magazine = int(magazine)
        year = int(year)
        string_list = list()
        string_list.append('<!--Diese Seite wurde automatisch durch einen Bot erstellt. '
                           'Wenn du einen Fehler findest oder eine Änderung wünscht, '
                           'benachrichtige bitte den Betreiber, THE IT, des Bots.-->\n'
                           '{{Textdaten\n')
        if magazine > 1:
            string_list.append('|VORIGER=Die Gartenlaube ({year:d})/Heft {magazine:d}\n'
                               .format(year=year, magazine=magazine - 1))
        else:
            string_list.append('|VORIGER=\n')
        if last:
            string_list.append('|NÄCHSTER=\n')
        else:
            string_list.append('|NÄCHSTER=Die Gartenlaube ({year:d})/Heft {magazine:d}\n'
                               .format(year=year, magazine=magazine + 1))
        string_list.append("|AUTOR=Verschiedene\n|TITEL=[[Die Gartenlaube]]\n"
                           "|SUBTITEL=''Illustrirtes Familienblatt''\n|HERKUNFT=off\n")
        if year < 1863:
            string_list.append('|HERAUSGEBER=[[Ferdinand Stolle]]\n')
        elif (year < 1878) or (year == 1878 and magazine < 14):
            string_list.append('|HERAUSGEBER=[[Ernst Keil]]\n')
        elif year < 1885:
            string_list.append('|HERAUSGEBER=Ernst Ziel\n')
        else:
            string_list.append('|HERAUSGEBER=Adolf Kröner\n')
        string_list.append('|ENTSTEHUNGSJAHR={year:d}\n|ERSCHEINUNGSJAHR={year:d}\n'
                           '|ERSCHEINUNGSORT=Leipzig\n|VERLAG=Ernst Keil\n'
                           '|WIKIPEDIA=Die Gartenlaube\n'.format(year=year))
        if year == 1873:
            extension = "JPG"
        else:
            extension = "jpg"
        string_list.append('|BILD=Die Gartenlaube ({year:d}) {page1}.{extension}\n'
                           .format(year=year, page1=list_of_pages[0], extension=extension))
        string_list.append('|QUELLE=[[commons:category:Gartenlaube ({year})|commons]]\n'
                           .format(year=year))
        if quality == 4:
            string_list.append('|BEARBEITUNGSSTAND=fertig\n')
        else:
            string_list.append('|BEARBEITUNGSSTAND=korrigiert\n')
        string_list.append('|INDEXSEITE=Die Gartenlaube ({year})\n}}}}\n\n'
                           '{{{{BlockSatzStart}}}}\n__TOC__\n'.format(year=year))
        ref = []
        for page in list_of_pages:
            page_format = self.convert_page_no(page)
            string_list.append(
                '{{{{SeitePR|{page_format}|Die Gartenlaube ({year}) {page}.{extension}}}}}\n'
                .format(year=year, page_format=page_format, page=page, extension=extension))
            try:
                page_dict = self.data['pages'][str(year)][page]
                if 'r' in page_dict.keys():
                    if 'ref' in page_dict['r']:
                        if 'ref' not in ref:
                            ref.append('ref')
                    for ref_type in page_dict['r']:
                        if (ref_type != 'ref') and (ref_type not in ref):
                            ref.append(ref_type)
            except KeyError:
                self.logger.error('The list of pages is incorrect, '
                                  'year:{year} or page:{page} is missing.'
                                  .format(year=year, page=page))
                return None
        if 'ref' in ref:
            string_list.append('{{references|x}}\n')
        for ref_type in ref:
            if ref_type != 'ref':
                string_list.append('{{{{references|TIT|{ref}}}}}\n'.format(ref=ref_type))
        string_list.append('{{{{BlockSatzEnd}}}}\n\n[[Kategorie:Deutschland]]\n'
                           '[[Kategorie:Neuhochdeutsch]]\n[[Kategorie:Illustrierte Werke]]\n'
                           '[[Kategorie:Die Gartenlaube ({year:d}) Hefte| {magazine:02d}]]\n'
                           .format(year=year, magazine=magazine))
        string_list.append('[[Kategorie:{year}0er Jahre]]\n\n'.format(year=str(year)[0:3]))
        return ''.join(string_list)

    def _get_indexes(self) -> Iterator[IndexPage]:
        self.searcher_indexes.add_positive_category('Die Gartenlaube')
        self.searcher_indexes.add_positive_category('Index')
        self.searcher_indexes.set_regex_filter(r'.*Die Gartenlaube \(\d{4}\)')
        self.searcher_indexes.set_timeout(60)
        for index in self.searcher_indexes.run():
            yield index["title"], IndexPage(self.wiki, 'Index:{}'.format(index['title']))

    def search_pages(self):
        self.searcher_pages.add_positive_category('Die Gartenlaube')
        self.searcher_pages.add_namespace('Seite')
        self.searcher_pages.set_search_depth(1)
        self.searcher_pages.set_timeout(60)
        if self.last_run_successful or self.debug:
            delta = (self.timestamp.start_of_run - self.timestamp.last_run).days
            if self.debug:
                delta = 10
            start_of_search = self.create_timestamp_for_search(delta)
            self.searcher_pages.last_change_after(start_of_search)
            self.logger.info('The date {} is set to the argument "after".'
                             .format(start_of_search.strftime("%d.%m.%Y")))
        return self.searcher_pages.run()

if __name__ == "__main__":
    WIKI = Site(code='de', fam='wikisource', user='THEbotIT')
    with GlCreateMagazine(wiki=WIKI, debug=True) as bot:
        bot.run()
<|endoftext|>


<commit_before># Copyright (c) 2016 SUSE Linux LLC
# All Rights Reserved.
#
# Author: Bo Maryniuk <bo@suse.de>

from yum.plugins import TYPE_CORE
from yum import config
import os
import hashlib

CK_PATH = "/var/cache/salt/minion/rpmdb.cookie"
RPM_PATH = "/var/lib/rpm/Packages"

requires_api_version = '2.5'
plugin_type = TYPE_CORE

def _get_mtime():
    """
    Get the modified time of the RPM Database.

    Returns:
        Unix ticks
    """
    return os.path.exists(RPM_PATH) and int(os.path.getmtime(RPM_PATH)) or 0

def _get_checksum():
    """
    Get the checksum of the RPM Database.

    Returns:
        hexdigest
    """
    digest = hashlib.md5()
    with open(RPM_PATH, "rb") as rpm_db_fh:
        while True:
            buff = rpm_db_fh.read(0x1000)
            if not buff:
                break
            digest.update(buff)
    return digest.hexdigest()

def posttrans_hook(conduit):
    """
    Hook after the package installation transaction.

    :param conduit:
    :return:
    """
    # Integrate Yum with Salt
    if 'SALT_RUNNING' not in os.environ:
        with open(CK_PATH, 'w') as ck_fh:
            ck_fh.write('{chksum} {mtime}\n'.format(chksum=_get_checksum(), mtime=_get_mtime()))
<commit_msg>Use SHA256 algorithm for yumnotify plugin<commit_after># Copyright (c) 2016 SUSE Linux LLC
# All Rights Reserved.
#
# Author: Bo Maryniuk <bo@suse.de>

from yum.plugins import TYPE_CORE
from yum import config
import os
import hashlib

CK_PATH = "/var/cache/salt/minion/rpmdb.cookie"
RPM_PATH = "/var/lib/rpm/Packages"

requires_api_version = '2.5'
plugin_type = TYPE_CORE

def _get_mtime():
    """
    Get the modified time of the RPM Database.

    Returns:
        Unix ticks
    """
    return os.path.exists(RPM_PATH) and int(os.path.getmtime(RPM_PATH)) or 0

def _get_checksum():
    """
    Get the checksum of the RPM Database.

    Returns:
        hexdigest
    """

    digest = hashlib.sha256()
    with open(RPM_PATH, "rb") as rpm_db_fh:
        while True:
            buff = rpm_db_fh.read(0x1000)
            if not buff:
                break
            digest.update(buff)
    return digest.hexdigest()

def posttrans_hook(conduit):
    """
    Hook after the package installation transaction.

    :param conduit:
    :return:
    """
    # Integrate Yum with Salt
    if 'SALT_RUNNING' not in os.environ:
        with open(CK_PATH, 'w') as ck_fh:
            ck_fh.write('{chksum} {mtime}\n'.format(chksum=_get_checksum(), mtime=_get_mtime()))
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from distutils.command.install import INSTALL_SCHEMES
from os.path import dirname, join, abspath
from setuptools import setup
from setuptools.command.install import install

for scheme in INSTALL_SCHEMES.values():
    scheme['data'] = scheme['purelib']

setup_args = {
    'cmdclass': {'install': install},
    'name': 'selenium',
    'version': "3.13.0",
    'license': 'Apache 2.0',
    'description': 'Python bindings for Selenium',
    'long_description': open(join(abspath(dirname(__file__)), "README.rst")).read(),
    'url': 'https://github.com/SeleniumHQ/selenium/',
    'classifiers': ['Development Status :: 5 - Production/Stable',
                    'Intended Audience :: Developers',
                    'License :: OSI Approved :: Apache Software License',
                    'Operating System :: POSIX',
                    'Operating System :: Microsoft :: Windows',
                    'Operating System :: MacOS :: MacOS X',
                    'Topic :: Software Development :: Testing',
                    'Topic :: Software Development :: Libraries',
                    'Programming Language :: Python',
                    'Programming Language :: Python :: 2.7',
                    'Programming Language :: Python :: 3.4',
                    'Programming Language :: Python :: 3.5',
                    'Programming Language :: Python :: 3.6'],
    'package_dir': {
        'selenium': 'selenium',
        'selenium.common': 'selenium/common',
        'selenium.webdriver': 'selenium/webdriver',
    },
    'packages': ['selenium',
                 'selenium.common',
                 'selenium.webdriver',
                 'selenium.webdriver.android',
                 'selenium.webdriver.chrome',
                 'selenium.webdriver.common',
                 'selenium.webdriver.common.html5',
                 'selenium.webdriver.support',
                 'selenium.webdriver.firefox',
                 'selenium.webdriver.ie',
                 'selenium.webdriver.edge',
                 'selenium.webdriver.opera',
                 'selenium.webdriver.phantomjs',
                 'selenium.webdriver.remote',
                 'selenium.webdriver.support', ],
    'package_data': {
        'selenium.webdriver.firefox': ['*.xpi', 'webdriver_prefs.json'],
        'selenium.webdriver.remote': ['getAttribute.js', 'isDisplayed.js'],
    },
    'data_files': [('selenium/webdriver/firefox/x86', ['selenium/webdriver/firefox/x86/x_ignore_nofocus.so']),
                   ('selenium/webdriver/firefox/amd64', ['selenium/webdriver/firefox/amd64/x_ignore_nofocus.so']),
                   ('selenium/webdriver/remote', ['selenium/webdriver/remote/getAttribute.js']),
                   ('selenium/webdriver/remote', ['selenium/webdriver/remote/isDisplayed.js'])],
    'include_package_data': True,
    'zip_safe': False
}

setup(**setup_args)
<commit_msg>Remove package_data and data_files options from setup.py (#5982)<commit_after>#!/usr/bin/env python

# Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from distutils.command.install import INSTALL_SCHEMES
from os.path import dirname, join, abspath
from setuptools import setup
from setuptools.command.install import install

for scheme in INSTALL_SCHEMES.values():
    scheme['data'] = scheme['purelib']

setup_args = {
    'cmdclass': {'install': install},
    'name': 'selenium',
    'version': "3.13.0",
    'license': 'Apache 2.0',
    'description': 'Python bindings for Selenium',
    'long_description': open(join(abspath(dirname(__file__)), "README.rst")).read(),
    'url': 'https://github.com/SeleniumHQ/selenium/',
    'classifiers': ['Development Status :: 5 - Production/Stable',
                    'Intended Audience :: Developers',
                    'License :: OSI Approved :: Apache Software License',
                    'Operating System :: POSIX',
                    'Operating System :: Microsoft :: Windows',
                    'Operating System :: MacOS :: MacOS X',
                    'Topic :: Software Development :: Testing',
                    'Topic :: Software Development :: Libraries',
                    'Programming Language :: Python',
                    'Programming Language :: Python :: 2.7',
                    'Programming Language :: Python :: 3.4',
                    'Programming Language :: Python :: 3.5',
                    'Programming Language :: Python :: 3.6'],
    'package_dir': {
        'selenium': 'selenium',
        'selenium.common': 'selenium/common',
        'selenium.webdriver': 'selenium/webdriver',
    },
    'packages': ['selenium',
                 'selenium.common',
                 'selenium.webdriver',
                 'selenium.webdriver.android',
                 'selenium.webdriver.chrome',
                 'selenium.webdriver.common',
                 'selenium.webdriver.common.html5',
                 'selenium.webdriver.support',
                 'selenium.webdriver.firefox',
                 'selenium.webdriver.ie',
                 'selenium.webdriver.edge',
                 'selenium.webdriver.opera',
                 'selenium.webdriver.phantomjs',
                 'selenium.webdriver.remote',
                 'selenium.webdriver.support', ],
    'include_package_data': True,
    'zip_safe': False
}

setup(**setup_args)
<|endoftext|>


<commit_before>from paddle.v2.fluid import framework as framework
from . import core
import collections
import pdb

__all__ = ['append_backward']

def _rename_arg_(op_desc_list, old_name, new_name, begin_idx=None,
                 end_idx=None):
    if begin_idx is None:
        begin_idx = 0
    if end_idx is None:
        end_idx = len(op_desc_list)
    for i in range(begin_idx, end_idx):
        op_desc = op_desc_list[i]
        if isinstance(op_desc, tuple):
            op_desc = op_desc[0]
        op_desc.rename_input(old_name, new_name)
        op_desc.rename_output(old_name, new_name)

def _create_op_desc_(op_type, inputs, outputs, attrs):
    op_desc = core.OpDesc()
    op_desc.set_type(op_type)
    for para, args in inputs.iteritems():
        op_desc.set_input(para, args)
    for para, args in outputs.iteritems():
        op_desc.set_output(para, args)
    for name, val in attrs.iteritems():
        if isinstance(val, framework.Block):
            op_desc.set_block_attr(name, val.desc)
        else:
            op_desc.set_attr(name, val)
    return op_desc

def _infer_var_data_type_(var_name, block):
    grad_var = block.desc.find_var(var_name.encode("ascii"))
    fwd_name = _strip_grad_suffix_(var_name.encode("ascii"))
    if block.desc.has_var_recursive(fwd_name):
        fwd_var = block.desc.find_var_recursive(fwd_name.encode("ascii"))
        grad_var.set_dtype(fwd_var.dtype())
    else:
        grad_var.set_dtype(core.DataType.FP32)

def _is_all_in_set_(cands, s):
    for c in cands:
        if not c in s:
            return False
    return True

def _strip_grad_suffix_(name):
    pos = name.find(core.grad_var_suffix())
    return name[:pos] if pos != -1 else name

def _append_grad_suffix_(name):
    return name + core.grad_var_suffix()

def _append_backward_ops_(target,
                          block,
                          target_block,
                          no_grad_set,
                          callback=None):
    grad_op_descs = []
    grad_to_var = dict()
    program = block.program
    for each_op in reversed(block.ops):
        grad_sub_block_list = []
        if each_op.has_attr("sub_block"):
            sub_block_idx = each_op.block_attr("sub_block")
            sub_block = program.block(sub_block_idx)
            grad_sub_block = program.create_block(parent_idx=sub_block_idx)
            sub_grad_to_var = _append_backward_ops_(
                target, sub_block, grad_sub_block, no_grad_set, callback)
            grad_to_var = dict(grad_to_var, **sub_grad_to_var)
            grad_sub_block_list.append(grad_sub_block.desc)
        grad_op_desc, op_grad_to_var = core.get_grad_op_desc(
            each_op.desc, no_grad_set[block.idx], grad_sub_block_list)
        grad_op_descs.append(grad_op_desc)
        grad_to_var = dict(grad_to_var, **op_grad_to_var)
    # grad_op_descs = [[op1_g1, op1_g2], [op2_g], ...]
    # flatten grad_op_descs
    grad_op_descs = [op for sublist in grad_op_descs for op in sublist]  # ?????

    pending_sum_ops = []
    var_rename_count = collections.defaultdict(int)
    var_inputs = collections.defaultdict(list)
    for idx, op_desc in enumerate(grad_op_descs):
        for var_name in op_desc.input_arg_names():
            if len(var_inputs[var_name]) > 1:
                pending_sum_ops.append((_create_op_desc_(
                    op_type="sum",
                    inputs={"X": var_inputs[var_name]},
                    outputs={"Out": [var_name]},
                    attrs={}), idx))
                var_inputs[var_name] = [var_name]
        for var_name in op_desc.output_arg_names():
            if var_name in op_desc.input_arg_names():
                # in place operator
                continue
            if var_name == core.empty_var_name() or len(var_inputs[
                    var_name]) == 0:
                # it's the first time we get the variable
                var_inputs[var_name] = [var_name]
            else:
                if len(var_inputs[var_name]) == 1:
                    new_name = var_name + "@RENAME@" + \
                        str(var_rename_count[var_name])
                    var_rename_count[var_name] = var_rename_count[var_name] + 1
                    # rename original var_name
                    var_inputs[var_name][0] = new_name
                    _rename_arg_(grad_op_descs, var_name, new_name, 0, idx)
                    _rename_arg_(pending_sum_ops, var_name, new_name)

                new_name = var_name + "@RENAME@" + \
                    str(var_rename_count[var_name])
                var_rename_count[var_name] = var_rename_count[var_name] + 1
                op_desc.rename_output(var_name, new_name)
                var_inputs[var_name].append(new_name)
    for var_name, inputs in var_inputs.iteritems():
        if len(inputs) > 1:
            pending_sum_ops.append((_create_op_desc_(
                op_type="sum",
                inputs={"X": inputs},
                outputs={"Out": [var_name]},
                attrs={}), len(grad_op_descs)))
    # sum_op descs are sorted according to their insert position
    for p in reversed(pending_sum_ops):
        grad_op_descs.insert(p[1], p[0])
    # Remove ops whose outputs are all in no_grad_set
    grad_op_descs = filter(
        lambda op_desc: not _is_all_in_set_(op_desc.output_arg_names(), no_grad_set[block.idx]),
        grad_op_descs)
    # Insert fill_zeros_like_op
    to_insert = []
    for idx, op_desc in enumerate(grad_op_descs):
        for arg in op_desc.input_arg_names():
            if core.grad_var_suffix() in arg and arg in no_grad_set[block.idx]:
                to_insert.append((arg, idx))
    for ele in reversed(to_insert):
        arg = ele[0]
        fill_zeros_like_op = _create_op_desc_(
            "fill_zeros_like", {"X": [_strip_grad_suffix_(arg)]}, {"Y": [arg]},
            {})
        grad_op_descs.insert(ele[1], fill_zeros_like_op)

    if target_block.idx == 0:
        grad_target_name = _append_grad_suffix_(target.name)
        # target_block.desc.var(grad_target_name.encode("ascii"))
        grad_op_descs.insert(
            0,
            _create_op_desc_(
                op_type="fill_constant",
                inputs={},
                outputs={"Out": [grad_target_name]},
                attrs={"shape": [1],
                       "value": 1.0,
                       "dtype": target.dtype}))
    for op_desc in grad_op_descs:
        new_op_desc = target_block.desc.append_op()
        new_op_desc.copy_from(op_desc)

    return grad_to_var

def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map):
    for op_idx in range(start_op_idx, block.desc.op_size()):
        op_desc = block.desc.op(op_idx)
        if op_desc.has_attr("sub_block"):
            sub_block = block.program.block(op_desc.block_attr("sub_block"))
            _append_backward_vars_(sub_block, 0, grad_to_var, grad_info_map)
        new_vars = set()
        # create new gradient variables
        for grad_var_name in op_desc.output_arg_names():
            grad_var_name = grad_var_name.encode("ascii")
            if block.desc.has_var_recursive(
                    grad_var_name) or grad_var_name == core.empty_var_name():
                continue
            block.desc.var(grad_var_name)
            new_vars.add(grad_var_name)
            if not grad_to_var.has_key(grad_var_name):
                continue
            grad_info_map[grad_to_var[grad_var_name]] = (grad_var_name, block)
        # infer_shape and infer_type
        op_desc.infer_var_type(block.desc)
        op_desc.infer_shape(block.desc)
        for arg in op_desc.output_arg_names():
            if arg in new_vars:
                _infer_var_data_type_(arg, block)

def append_backward(loss, parameter_list=None, no_grad_set=None):
    """
    Create and add gradient Operators in BlockDesc to compute
    gradients of `loss` for parameters in parameter_list

    :param loss: an variable generated by cost function.
    :type loss: Variable
    :param no_grad_set: variable that should not create gradient
    :type no_grad_set: set
    :param parameter_list: parameters that need to compute gradient and
    update to optimize the lost.
    :type: list
    :return: list of (parameters, gradients) pair.
    :rtype: list[Variable]
    """
    assert isinstance(loss, framework.Variable)

    program = loss.block.program
    if no_grad_set is None:
        no_grad_set = dict()
        assert isinstance(program, framework.Program)
        for block in program.blocks:
            assert isinstance(block, framework.Block)
            block_no_grad_set = set()
            for var in block.vars.itervalues():
                assert isinstance(var, framework.Variable)
                if var.stop_gradient:
                    block_no_grad_set.add(_append_grad_suffix_(var.name))
            no_grad_set[block.idx] = block_no_grad_set
    else:
        # FIX ME
        no_grad_set = {0: no_grad_set}

    grad_info_map = dict()
    root_block = program.block(0)

    fwd_op_num = root_block.desc.op_size()
    current_block_idx = program.current_block_idx
    grad_to_var = _append_backward_ops_(loss, root_block, root_block,
                                        no_grad_set)
    _append_backward_vars_(root_block, fwd_op_num, grad_to_var, grad_info_map)
    program.current_block_idx = current_block_idx
    program.sync_with_cpp()

    if parameter_list is not None:
        parameters = parameter_list
    else:
        params = program.global_block().all_parameters()
        parameters = [param.name for param in params]
    params_and_grads = []
    for param in parameters:
        if param not in grad_info_map:
            raise ValueError("param %s is not in map" % param)
        grad_info = grad_info_map[param]
        grad_block = grad_info[1]
        if not grad_block.has_var(grad_info[0]):
            raise ValueError("grad block[{0}] did not have grad var {1}".format(
                grad_info[1], grad_info[0]))
        # Get the param var from the global block
        param_var = program.global_block().var(param)
        grad_var = grad_block.var(grad_info[0])
        if loss.block.has_var(grad_info[0]):
            params_and_grads.append((param_var, grad_var))
        else:
            params_and_grads.append((param_var, None))
    return params_and_grads
<commit_msg>code clean<commit_after>from paddle.v2.fluid import framework as framework
from . import core
import collections

__all__ = ['append_backward']

def _rename_arg_(op_desc_list, old_name, new_name, begin_idx=None,
                 end_idx=None):
    if begin_idx is None:
        begin_idx = 0
    if end_idx is None:
        end_idx = len(op_desc_list)
    for i in range(begin_idx, end_idx):
        op_desc = op_desc_list[i]
        if isinstance(op_desc, tuple):
            op_desc = op_desc[0]
        op_desc.rename_input(old_name, new_name)
        op_desc.rename_output(old_name, new_name)

def _create_op_desc_(op_type, inputs, outputs, attrs):
    op_desc = core.OpDesc()
    op_desc.set_type(op_type)
    for para, args in inputs.iteritems():
        op_desc.set_input(para, args)
    for para, args in outputs.iteritems():
        op_desc.set_output(para, args)
    for name, val in attrs.iteritems():
        if isinstance(val, framework.Block):
            op_desc.set_block_attr(name, val.desc)
        else:
            op_desc.set_attr(name, val)
    return op_desc

def _infer_var_data_type_(var_name, block):
    grad_var = block.desc.find_var(var_name.encode("ascii"))
    fwd_name = _strip_grad_suffix_(var_name.encode("ascii"))
    if block.desc.has_var_recursive(fwd_name):
        fwd_var = block.desc.find_var_recursive(fwd_name.encode("ascii"))
        grad_var.set_dtype(fwd_var.dtype())
    else:
        grad_var.set_dtype(core.DataType.FP32)

def _all_in_set_(cands, s):
    for c in cands:
        if not c in s:
            return False
    return True

def _strip_grad_suffix_(name):
    pos = name.find(core.grad_var_suffix())
    return name[:pos] if pos != -1 else name

def _append_grad_suffix_(name):
    return name + core.grad_var_suffix()

def _addup_repetitive_outputs_(op_descs):
    # In backward part, an variable my be the output of more than one ops.
    # In this case, the variable should be the accumulation of all the outputs.
    # We adopt adding `sum_op`s to implement the accumulate.
    pending_sum_ops = []
    var_rename_count = collections.defaultdict(int)
    renamed_vars = collections.defaultdict(list)
    for idx, op_desc in enumerate(op_descs):
        for var_name in op_desc.input_arg_names():
            if len(renamed_vars[var_name]) > 1:
                pending_sum_ops.append(
                    (_create_op_desc_("sum", {"X": renamed_vars[var_name]},
                                      {"Out": [var_name]}, {}), idx))
                renamed_vars[var_name] = [var_name]
        for var_name in op_desc.output_arg_names():
            if var_name == core.empty_var_name(
            ) or var_name in op_desc.input_arg_names():
                # empty variable or inplace op
                continue
            if len(renamed_vars[var_name]) == 0:
                # it's the first time we get the variable
                renamed_vars[var_name] = [var_name]
            else:
                if len(renamed_vars[var_name]) == 1:
                    new_name = var_name + "@RENAME@" + \
                        str(var_rename_count[var_name])
                    var_rename_count[var_name] += 1
                    # rename original var_name
                    renamed_vars[var_name][0] = new_name
                    _rename_arg_(op_descs, var_name, new_name, 0, idx)
                    _rename_arg_(pending_sum_ops, var_name, new_name)

                new_name = var_name + "@RENAME@" + \
                    str(var_rename_count[var_name])
                var_rename_count[var_name] += 1
                op_desc.rename_output(var_name, new_name)
                renamed_vars[var_name].append(new_name)
    for var_name, inputs in renamed_vars.iteritems():
        if len(inputs) > 1:
            pending_sum_ops.append((_create_op_desc_(
                "sum", {"X": inputs}, {"Out": [var_name]}, {}), len(op_descs)))
    # sum_op descs are sorted according to their insert position
    for p in reversed(pending_sum_ops):
        op_descs.insert(p[1], p[0])

    return op_descs

def _remove_no_grad_branch_(op_descs, no_grad_set):
    # Remove ops whose outputs are all in no_grad_dict
    op_descs = filter(
        lambda op_desc: not _all_in_set_(op_desc.output_arg_names(), no_grad_set),
        op_descs)
    # Insert fill_zeros_like_op
    to_insert = []
    for idx, op_desc in enumerate(op_descs):
        for arg in op_desc.input_arg_names():
            if core.grad_var_suffix() in arg and arg in no_grad_set:
                to_insert.append((_create_op_desc_("fill_zeros_like", {
                    "X": [_strip_grad_suffix_(arg)]
                }, {"Y": [arg]}, {}), idx))

    map(lambda p: op_descs.insert(p[1], p[0]), reversed(to_insert))

    return op_descs

def _append_backward_ops_(target,
                          block,
                          target_block,
                          no_grad_dict,
                          grad_to_var,
                          callback=None):
    grad_op_descs = []
    program = block.program
    for op in reversed(block.ops):
        grad_sub_block_list = []
        # If the op has its own sub-block, deal with the sub-block first
        if op.has_attr("sub_block"):
            sub_block = program.block(op.block_attr("sub_block"))
            grad_sub_block = program.create_block(parent_idx=sub_block.idx)
            _append_backward_ops_(target, sub_block, grad_sub_block,
                                  no_grad_dict, grad_to_var, callback)
            grad_sub_block_list.append(grad_sub_block.desc)

        grad_op_desc, op_grad_to_var = core.get_grad_op_desc(
            op.desc, no_grad_dict[block.idx], grad_sub_block_list)
        grad_op_descs.extend(grad_op_desc)
        grad_to_var.update(op_grad_to_var)

    grad_op_descs = _addup_repetitive_outputs_(grad_op_descs)

    grad_op_descs = _remove_no_grad_branch_(grad_op_descs,
                                            no_grad_dict[block.idx])

    if target_block.idx == 0:
        grad_op_descs.insert(
            0,
            _create_op_desc_("fill_constant", {}, {
                "Out": [_append_grad_suffix_(target.name)]
            }, {"shape": [1],
                "value": 1.0,
                "dtype": target.dtype}))
    # append op_desc in grad_op_descs to target_block
    for op_desc in grad_op_descs:
        new_op_desc = target_block.desc.append_op()
        new_op_desc.copy_from(op_desc)

def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map):
    for op_idx in range(start_op_idx, block.desc.op_size()):
        op_desc = block.desc.op(op_idx)
        if op_desc.has_attr("sub_block"):
            sub_block = block.program.block(op_desc.block_attr("sub_block"))
            _append_backward_vars_(sub_block, 0, grad_to_var, grad_info_map)
        new_vars = set()
        # create new gradient variables
        for grad_var_name in op_desc.output_arg_names():
            grad_var_name = grad_var_name.encode("ascii")
            if block.desc.has_var_recursive(
                    grad_var_name) or grad_var_name == core.empty_var_name():
                continue
            block.desc.var(grad_var_name)
            new_vars.add(grad_var_name)
            if not grad_to_var.has_key(grad_var_name):
                continue
            grad_info_map[grad_to_var[grad_var_name]] = (grad_var_name, block)
        # infer_shape and infer_type
        op_desc.infer_var_type(block.desc)
        op_desc.infer_shape(block.desc)
        for arg in op_desc.output_arg_names():
            if arg in new_vars:
                _infer_var_data_type_(arg, block)

def append_backward(loss, parameter_list=None, no_grad_dict=None):
    """
    Create and add gradient Operators in BlockDesc to compute
    gradients of `loss` for parameters in parameter_list

    :param loss: an variable generated by cost function.
    :type loss: Variable
    :param no_grad_dict: variable that should not create gradient
    :type no_grad_dict: set
    :param parameter_list: parameters that need to compute gradient and
    update to optimize the lost.
    :type: list
    :return: list of (parameters, gradients) pair.
    :rtype: list[Variable]
    """
    assert isinstance(loss, framework.Variable)

    program = loss.block.program
    if no_grad_dict is None:
        no_grad_dict = dict()
        assert isinstance(program, framework.Program)
        for block in program.blocks:
            assert isinstance(block, framework.Block)
            block_no_grad_set = set()
            for var in block.vars.itervalues():
                assert isinstance(var, framework.Variable)
                if var.stop_gradient:
                    block_no_grad_set.add(_append_grad_suffix_(var.name))
            no_grad_dict[block.idx] = block_no_grad_set
    elif isinstance(no_grad_dict, set):
        no_grad_dict = {0: no_grad_dict}

    grad_info_map = dict()
    root_block = program.block(0)

    fwd_op_num = root_block.desc.op_size()
    current_block_idx = program.current_block_idx
    grad_to_var = dict()

    _append_backward_ops_(loss, root_block, root_block, no_grad_dict,
                          grad_to_var)
    _append_backward_vars_(root_block, fwd_op_num, grad_to_var, grad_info_map)

    program.current_block_idx = current_block_idx
    program.sync_with_cpp()

    if parameter_list is not None:
        parameters = parameter_list
    else:
        params = program.global_block().all_parameters()
        parameters = [param.name for param in params]
    params_and_grads = []
    for param in parameters:
        if param not in grad_info_map:
            raise ValueError("param %s is not in map" % param)
        grad_info = grad_info_map[param]
        grad_block = grad_info[1]
        if not grad_block.has_var(grad_info[0]):
            raise ValueError("grad block[{0}] did not have grad var {1}".format(
                grad_info[1], grad_info[0]))
        # Get the param var from the global block
        param_var = program.global_block().var(param)
        grad_var = grad_block.var(grad_info[0])
        if loss.block.has_var(grad_info[0]):
            params_and_grads.append((param_var, grad_var))
        else:
            params_and_grads.append((param_var, None))
    return params_and_grads
<|endoftext|>