<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import calendar
from datetime import timedelta ,datetime
from time import strptime
from glob import glob
import re
import string
import os

from ocw.dataset import Dataset
import ocw.utils as utils

import netCDF4
import numpy
import numpy.ma as ma

LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
TIME_NAMES = ['time', 'times', 'date', 'dates', 'julian']

def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
    ''' Determine if one of a set of variable names are in a NetCDF Dataset.

    Looks for an occurrence of a valid_var_name in the NetCDF variable data.
    This is useful for automatically determining the names of the lat, lon,
    and time variable names inside of a dataset object.

    :param valid_var_names: The possible variable names to search for in
        the netCDF object.
    :type valid_var_names: List of Strings
    :param netcdf: The netCDF Dataset object in which to check for
        valid_var_names.
    :type netcdf: netcdf4.Dataset
    :param netcdf_var: The relevant variable name to search over in the
        netcdf object. This is used to narrow down the search for valid
        variable names by first checking the desired variable's dimension
        values for one or more of the valid variable names.

    :returns: The variable from valid_var_names that it locates in
        the netCDF object.

    :raises ValueError: When unable to locate a single matching variable
        name in the NetCDF Dataset from the supplied list of valid variable
        names.
    '''

    # Check for valid variable names in netCDF variable dimensions
    dimensions = netcdf.variables[netcdf_var].dimensions
    dims_lower = [dim.encode().lower() for dim in dimensions]

    intersect = set(valid_var_names).intersection(dims_lower)

    if len(intersect) == 1:
        # Retrieve the name of the dimension where we found the matching
        # variable name
        index = dims_lower.index(intersect.pop())
        dimension_name = dimensions[index].encode()

        # Locate all of the variables that share the dimension that we matched
        # earlier. If the dimension's name matches then that variable is
        # potentially what we want to return to the user.
        possible_vars = []
        for var in netcdf.variables.keys():
            var_dimensions = netcdf.variables[var].dimensions

            # Skip any dimensions are > 1D
            if len(var_dimensions) != 1:
                continue

            if var_dimensions[0].encode() == dimension_name:
                possible_vars.append(var)

        # If there are multiple variables with matching dimension names then we
        # aren't able to determining the correct variable name using the
        # variable dimensions. We need to try a different approach. Otherwise,
        # we're done!
        if len(possible_vars) == 1:
            return possible_vars[0]

    # Check for valid variable names in netCDF variable names
    variables = netcdf.variables.keys()
    vars_lower = [var.encode().lower() for var in variables]

    intersect = set(valid_var_names).intersection(vars_lower)

    if len(intersect) == 1:
        index = vars_lower.index(intersect.pop())
        return variables[index]

    # If we couldn't locate a single matching valid variable then we're unable
    # to automatically determine the variable names for the user.
    error = (
        "Unable to locate a single matching variable name from the "
        "supplied list of valid variable names. "
    )
    raise ValueError(error)

def load_WRF_2d_files(file_path=None,
                      filename_pattern=None,
                      filelist=None,
                      variable_name='T2',
                      name=''):
    ''' Load multiple WRF (or nuWRF) original output files containing 2D fields such as precipitation and surface variables into a Dataset.
    The dataset can be spatially subset.
    :param file_path: Directory to the NetCDF file to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the NetCDF file to load.
    :type filename_pattern: :list:`string`
    :param filelist: A list of filenames
    :type filelist: :list:`string`
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    if not filelist:
        WRF_files = []
        for pattern in filename_pattern:
            WRF_files.extend(glob(file_path + pattern))
    else:
        WRF_files = [line.rstrip('\n') for line in open(filelist)]

    WRF_files.sort()

    file_object_first = netCDF4.Dataset(WRF_files[0])
    lats = file_object_first.variables['XLAT'][0,:]
    lons = file_object_first.variables['XLONG'][0,:]

    times = []
    nfile = len(WRF_files)
    for ifile, file in enumerate(WRF_files):
        print 'Reading file '+str(ifile+1)+'/'+str(nfile), file
        file_object = netCDF4.Dataset(file)
        time_struct_parsed = strptime(file[-19:],"%Y-%m-%d_%H:%M:%S")
        for ihour in numpy.arange(24):
            times.append(datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour))
        values0= file_object.variables[variable_name][:]
        if ifile == 0:
            values = file_object.variables[variable_name][:]
            variable_unit = file_object.variables[variable_name].units
        else:
            values = numpy.concatenate((values, file_object.variables[variable_name][:]))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)

def load_file(file_path,
              variable_name,
              variable_unit = None,
              elevation_index=0,
              name='',
              lat_name=None,
              lon_name=None,
              time_name=None):
    ''' Load a NetCDF file into a Dataset.

    :param file_path: Path to the NetCDF file to load.
    :type file_path: :mod:`string`

    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`

    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`

    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :type elevation_index: :class:`int`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`

    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`

    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`

    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`

    :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
        or when the lat/lon/time variable name cannot be determined
        automatically.
    '''

    try:
        netcdf = netCDF4.Dataset(file_path, mode='r')
    except RuntimeError:
        err = "Dataset filepath is invalid. Please ensure it is correct."
        raise ValueError(err)
    except:
        err = (
            "The given file cannot be loaded. Please ensure that it is a valid "
            "NetCDF file. If problems persist, report them to the project's "
            "mailing list."
        )
        raise ValueError(err)

    if not lat_name:
        lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
    if not lon_name:
        lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
    if not time_name:
        time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)

    lats = netcdf.variables[lat_name][:]
    lons = netcdf.variables[lon_name][:]
    time_raw_values = netcdf.variables[time_name][:]
    times = utils.decode_time_values(netcdf, time_name)
    times = numpy.array(times)
    values = ma.array(netcdf.variables[variable_name][:])
    variable_unit = netcdf.variables[variable_name].units

    # If the values are 4D then we need to strip out the elevation index
    if len(values.shape) == 4:
        # Determine the set of possible elevation dimension names excluding
        # the list of names that are used for the lat, lon, and time values.
        dims = netcdf.variables[variable_name].dimensions
        dimension_names = [dim_name.encode() for dim_name in dims]
        lat_lon_time_var_names = [lat_name, lon_name, time_name]

        elev_names = set(dimension_names) - set(lat_lon_time_var_names)

        # Grab the index value for the elevation values
        level_index = dimension_names.index(elev_names.pop())

        # Strip out the elevation values so we're left with a 3D array.
        if level_index == 0:
            values = values [elevation_index,:,:,:]
        elif level_index == 1:
            values = values [:,elevation_index,:,:]
        elif level_index == 2:
            values = values [:,:,elevation_index,:]
        else:
            values = values [:,:,:,elevation_index]

    origin = {
        'source': 'local',
        'path': file_path,
        'lat_name': lat_name,
        'lon_name': lon_name,
        'time_name': time_name
    }
    if elevation_index != 0: origin['elevation_index'] = elevation_index

    return Dataset(lats, lons, times, values, variable=variable_name,
                   units=variable_unit, name=name, origin=origin)

def load_multiple_files(file_path,
                        variable_name,
                        dataset_name='model',
                        variable_unit=None,
                        lat_name=None,
                        lon_name=None,
                        time_name=None):
    ''' load multiple netcdf files with common filename pattern and return an array of OCW datasets

    :param file_path: directory name and common file name patterns where the NetCDF files to load are stored.
    :type file_path: :mod:`string`
    :param dataset_name: a name of dataset when reading a single file
    :type dataset_name: :mod:'string'
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`
    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`
    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`
    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`
    :returns: An array of OCW Dataset objects, an array of dataset names
    :rtype: :class:`list`
    '''

    data_filenames = []
    data_filenames.extend(glob(file_path))
    data_filenames.sort()

    # number of files
    ndata = len(data_filenames)
    if ndata == 1:
        data_name = [dataset_name]
    else:
        data_name = []
        data_filenames_reversed = []
        for element in data_filenames:
            data_filenames_reversed.append(element[::-1])
        prefix = os.path.commonprefix(data_filenames)
        postfix = os.path.commonprefix(data_filenames_reversed)[::-1]
        for element in data_filenames:
            data_name.append(element.replace(prefix,'').replace(postfix,''))

    datasets = []
    for ifile,filename in enumerate(data_filenames):
        datasets.append(load_file(filename, variable_name, variable_unit, name=data_name[ifile],
                        lat_name=lat_name, lon_name=lon_name, time_name=time_name))

    return datasets, data_name
<commit_msg>code clean up<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import calendar
from datetime import timedelta ,datetime
from time import strptime
from glob import glob
import re
import string
import os

from ocw.dataset import Dataset
import ocw.utils as utils

import netCDF4
import numpy
import numpy.ma as ma

LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
TIME_NAMES = ['time', 'times', 'date', 'dates', 'julian']

def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
    ''' Determine if one of a set of variable names are in a NetCDF Dataset.

    Looks for an occurrence of a valid_var_name in the NetCDF variable data.
    This is useful for automatically determining the names of the lat, lon,
    and time variable names inside of a dataset object.

    :param valid_var_names: The possible variable names to search for in
        the netCDF object.
    :type valid_var_names: List of Strings
    :param netcdf: The netCDF Dataset object in which to check for
        valid_var_names.
    :type netcdf: netcdf4.Dataset
    :param netcdf_var: The relevant variable name to search over in the
        netcdf object. This is used to narrow down the search for valid
        variable names by first checking the desired variable's dimension
        values for one or more of the valid variable names.

    :returns: The variable from valid_var_names that it locates in
        the netCDF object.

    :raises ValueError: When unable to locate a single matching variable
        name in the NetCDF Dataset from the supplied list of valid variable
        names.
    '''

    # Check for valid variable names in netCDF variable dimensions
    dimensions = netcdf.variables[netcdf_var].dimensions
    dims_lower = [dim.encode().lower() for dim in dimensions]

    intersect = set(valid_var_names).intersection(dims_lower)

    if len(intersect) == 1:
        # Retrieve the name of the dimension where we found the matching
        # variable name
        index = dims_lower.index(intersect.pop())
        dimension_name = dimensions[index].encode()

        # Locate all of the variables that share the dimension that we matched
        # earlier. If the dimension's name matches then that variable is
        # potentially what we want to return to the user.
        possible_vars = []
        for var in netcdf.variables.keys():
            var_dimensions = netcdf.variables[var].dimensions

            # Skip any dimensions are > 1D
            if len(var_dimensions) != 1:
                continue

            if var_dimensions[0].encode() == dimension_name:
                possible_vars.append(var)

        # If there are multiple variables with matching dimension names then we
        # aren't able to determining the correct variable name using the
        # variable dimensions. We need to try a different approach. Otherwise,
        # we're done!
        if len(possible_vars) == 1:
            return possible_vars[0]

    # Check for valid variable names in netCDF variable names
    variables = netcdf.variables.keys()
    vars_lower = [var.encode().lower() for var in variables]

    intersect = set(valid_var_names).intersection(vars_lower)

    if len(intersect) == 1:
        index = vars_lower.index(intersect.pop())
        return variables[index]

    # If we couldn't locate a single matching valid variable then we're unable
    # to automatically determine the variable names for the user.
    error = (
        "Unable to locate a single matching variable name from the "
        "supplied list of valid variable names. "
    )
    raise ValueError(error)

def load_WRF_2d_files(file_path=None,
                      filename_pattern=None,
                      filelist=None,
                      variable_name='T2',
                      name=''):
    ''' Load multiple WRF (or nuWRF) original output files containing 2D fields such as precipitation and surface variables into a Dataset.
    The dataset can be spatially subset.
    :param file_path: Directory to the NetCDF file to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the NetCDF file to load.
    :type filename_pattern: :list:`string`
    :param filelist: A list of filenames
    :type filelist: :list:`string`
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    if not filelist:
        WRF_files = []
        for pattern in filename_pattern:
            WRF_files.extend(glob(file_path + pattern))
    else:
        WRF_files = [line.rstrip('\n') for line in open(filelist)]

    WRF_files.sort()

    file_object_first = netCDF4.Dataset(WRF_files[0])
    lats = file_object_first.variables['XLAT'][0,:]
    lons = file_object_first.variables['XLONG'][0,:]

    times = []
    nfile = len(WRF_files)
    for ifile, file in enumerate(WRF_files):
        print 'Reading file '+str(ifile+1)+'/'+str(nfile), file
        file_object = netCDF4.Dataset(file)
        time_struct_parsed = strptime(file[-19:],"%Y-%m-%d_%H:%M:%S")
        for ihour in numpy.arange(24):
            times.append(datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour))
        values0= file_object.variables[variable_name][:]
        if ifile == 0:
            values = values0
            variable_unit = file_object.variables[variable_name].units
        else:
            values = numpy.concatenate((values, values0))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)

def load_file(file_path,
              variable_name,
              variable_unit = None,
              elevation_index=0,
              name='',
              lat_name=None,
              lon_name=None,
              time_name=None):
    ''' Load a NetCDF file into a Dataset.

    :param file_path: Path to the NetCDF file to load.
    :type file_path: :mod:`string`

    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`

    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`

    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :type elevation_index: :class:`int`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`

    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`

    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`

    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`

    :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
        or when the lat/lon/time variable name cannot be determined
        automatically.
    '''

    try:
        netcdf = netCDF4.Dataset(file_path, mode='r')
    except RuntimeError:
        err = "Dataset filepath is invalid. Please ensure it is correct."
        raise ValueError(err)
    except:
        err = (
            "The given file cannot be loaded. Please ensure that it is a valid "
            "NetCDF file. If problems persist, report them to the project's "
            "mailing list."
        )
        raise ValueError(err)

    if not lat_name:
        lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
    if not lon_name:
        lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
    if not time_name:
        time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)

    lats = netcdf.variables[lat_name][:]
    lons = netcdf.variables[lon_name][:]
    time_raw_values = netcdf.variables[time_name][:]
    times = utils.decode_time_values(netcdf, time_name)
    times = numpy.array(times)
    values = ma.array(netcdf.variables[variable_name][:])
    variable_unit = netcdf.variables[variable_name].units

    # If the values are 4D then we need to strip out the elevation index
    if len(values.shape) == 4:
        # Determine the set of possible elevation dimension names excluding
        # the list of names that are used for the lat, lon, and time values.
        dims = netcdf.variables[variable_name].dimensions
        dimension_names = [dim_name.encode() for dim_name in dims]
        lat_lon_time_var_names = [lat_name, lon_name, time_name]

        elev_names = set(dimension_names) - set(lat_lon_time_var_names)

        # Grab the index value for the elevation values
        level_index = dimension_names.index(elev_names.pop())

        # Strip out the elevation values so we're left with a 3D array.
        if level_index == 0:
            values = values [elevation_index,:,:,:]
        elif level_index == 1:
            values = values [:,elevation_index,:,:]
        elif level_index == 2:
            values = values [:,:,elevation_index,:]
        else:
            values = values [:,:,:,elevation_index]

    origin = {
        'source': 'local',
        'path': file_path,
        'lat_name': lat_name,
        'lon_name': lon_name,
        'time_name': time_name
    }
    if elevation_index != 0: origin['elevation_index'] = elevation_index

    return Dataset(lats, lons, times, values, variable=variable_name,
                   units=variable_unit, name=name, origin=origin)

def load_multiple_files(file_path,
                        variable_name,
                        dataset_name='model',
                        variable_unit=None,
                        lat_name=None,
                        lon_name=None,
                        time_name=None):
    ''' load multiple netcdf files with common filename pattern and return an array of OCW datasets

    :param file_path: directory name and common file name patterns where the NetCDF files to load are stored.
    :type file_path: :mod:`string`
    :param dataset_name: a name of dataset when reading a single file
    :type dataset_name: :mod:'string'
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`
    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`
    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`
    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`
    :returns: An array of OCW Dataset objects, an array of dataset names
    :rtype: :class:`list`
    '''

    data_filenames = []
    data_filenames.extend(glob(file_path))
    data_filenames.sort()

    # number of files
    ndata = len(data_filenames)
    if ndata == 1:
        data_name = [dataset_name]
    else:
        data_name = []
        data_filenames_reversed = []
        for element in data_filenames:
            data_filenames_reversed.append(element[::-1])
        prefix = os.path.commonprefix(data_filenames)
        postfix = os.path.commonprefix(data_filenames_reversed)[::-1]
        for element in data_filenames:
            data_name.append(element.replace(prefix,'').replace(postfix,''))

    datasets = []
    for ifile,filename in enumerate(data_filenames):
        datasets.append(load_file(filename, variable_name, variable_unit, name=data_name[ifile],
                        lat_name=lat_name, lon_name=lon_name, time_name=time_name))

    return datasets, data_name
<|endoftext|>


<commit_before><commit_msg>Added basic test for lib's get_logger fn<commit_after>"""Tests for the lib module."""

import unittest
import logging

import sdnperf.lib.lib as lib

class TestGetLogger(unittest.TestCase):

    """Tests for the lib method that builds loggers."""

    @unittest.expectedFailure
    def test_type(self):
        """Confirm that the object returned is of type Logger."""
        logger = lib.get_logger()
        assert type(logger) is logging.Logger
<|endoftext|>


<commit_before>import math
from pandas import DataFrame
import time
from etl import ETLUtils
from tripadvisor.fourcity import extractor
from tripadvisor.fourcity import movielens_extractor

__author__ = 'fpena'

class ReviewsDatasetAnalyzer:
    """
    This class is used to analyze the dataset that contains a list of reviews.
    Some statistics can be obtained thanks to this class, such as the sparsity
    of the dataset and the number of users that have rated the same item. This
    statistics will show how complete is the dataset.
    """

    def __init__(self, reviews):

        if not reviews:
            raise ValueError('Can not analyze an empty list')

        self.reviews = reviews
        self.user_ids = extractor.get_groupby_list(self.reviews, 'user_id')
        self.num_users = len(self.user_ids)

    def calculate_sparsity(self):
        """
        Returns the percentage of missing ratings in the list of reviews of this
        ReviewsDatasetAnalyzer

        :return: the rate of missing ratings
        (i.e. number of missing ratings / (number of items * number of users))
        :raise ValueError: in case an empty list is given
        """
        if not self.reviews:
            raise ValueError('Can not determine the sparsity for an empty list')

        user_ids = extractor.get_groupby_list(self.reviews, 'user_id')
        item_ids = extractor.get_groupby_list(self.reviews, 'offering_id')

        non_missing_reviews = 0.
        total_expected_reviews = len(user_ids) * len(item_ids)

        for user in user_ids:
            user_reviews = ETLUtils.filter_records(self.reviews, 'user_id', [user])
            user_items = extractor.get_groupby_list(user_reviews, 'offering_id')

            non_missing_reviews += len(set(item_ids).intersection(set(user_items)))

        return 1 - non_missing_reviews / total_expected_reviews

    def calculate_sparsity_approx(self):
        """
        Returns the approximate percentage of missing ratings in the list of
        reviews of this ReviewsDatasetAnalyzer. This method is an approximation
        because it counts two reviews from the same user to the same item as
        two, when the correct count should be one. This method was created to
        calculate the sparsity in very big datasets where calculating the exact
        sparsity can be a very slow process.

        :return: the rate of approximate missing ratings
        (i.e. number of missing ratings / (number of reviews))
        :raise ValueError: in case an empty list is given
        """
        if not self.reviews:
            raise ValueError('Can not determine the sparsity for an empty list')

        user_ids = extractor.get_groupby_list(self.reviews, 'user_id')
        item_ids = extractor.get_groupby_list(self.reviews, 'offering_id')
        total_expected_reviews = float(len(user_ids) * len(item_ids))

        return 1 - float(len(self.reviews)) / total_expected_reviews

    def count_items_in_common(self):
        """
        Counts the number of items each user has in common with every other user
        and stores the results in a dictionary.

        :return: a dictionary with the count of the number of times users have
        a certain number of items in common. For example, the dictionary
        {0:4, 1:10, 2:6, 3:3, 4:1} means that there a 4 users who have 0 items
        in common with the rest of users, there are 10 users who have rated 1
        item in common with the rest of users, 6 users who have rated 6 items in
        common with the rest of the users, and so on.
        """
        common_item_counts = {}
        user_dictionary = extractor.initialize_users(self.reviews, False)

        for i in range(self.num_users):
            for j in range(i+1, self.num_users):

                user1 = self.user_ids[i]
                user2 = self.user_ids[j]

                num_common_items = len(extractor.get_common_items(
                    user_dictionary, user1, user2))

                if num_common_items in common_item_counts:
                    common_item_counts[num_common_items] += 1
                else:
                    common_item_counts[num_common_items] = 1

        return common_item_counts

    def analyze_common_items_count(self, common_item_counts, cumulative=False):
        """
        Converts the dictionary returned by
        :func:`~reviews_dataset_analyzer.ReviewsDatasetAnalyzer.count_items_in_common`
        into a dictionary with the same keys but instead of counts the values
        have percentages

        :param common_item_counts: a dictionary with the count of the number of
        items the users have in common
        :param cumulative: indicates whether the percentage is cumulative or not
        :return: a dictionary with the same keys but instead of counts the
        values have percentages
        """
        num_combinations = ReviewsDatasetAnalyzer.nCr(self.num_users, 2)
        common_item_percentages = {}

        cumulative_percentage = 0.

        for key in common_item_counts.keys():
            percentage = float(common_item_counts[key]) / num_combinations

            if cumulative:
                cumulative_percentage += percentage
                common_item_percentages[key] = cumulative_percentage
            else:
                common_item_percentages[key] = percentage

        return common_item_percentages

    def count_reviews_by_user(self):
        data_frame = DataFrame(self.reviews)
        counts = data_frame.groupby('user_id').size()
        return counts

    def count_reviews_by_item(self):
        data_frame = DataFrame(self.reviews)
        counts = data_frame.groupby('offering_id').size()
        return counts

    @staticmethod
    def nCr(n, r):
        f = math.factorial
        return f(n) / f(r) / f(n-r)

# file_path = '/Users/fpena/tmp/filtered_reviews_multi_non_sparse_shuffled.json'
# reviews = ETLUtils.load_json_file(file_path)
# reviews = extractor.pre_process_reviews()
# reviews = movielens_extractor.get_ml_100K_dataset()
#
# reviewsDatasetAnalyzer = ReviewsDatasetAnalyzer(reviews)
# print(reviewsDatasetAnalyzer.count_reviews_by_user())
# print(reviewsDatasetAnalyzer.count_reviews_by_item())
# common_item_counts = reviewsDatasetAnalyzer.count_items_in_common()
# print(common_item_counts)
# print (time.strftime("%H:%M:%S"))
# print('Sparsity', reviewsDatasetAnalyzer.calculate_sparsity())
# print (time.strftime("%H:%M:%S"))
# print('Sparsity Approx', reviewsDatasetAnalyzer.calculate_sparsity_approx())
# print (time.strftime("%H:%M:%S"))
# print(reviewsDatasetAnalyzer.analyze_common_items_count(common_item_counts))
# print(reviewsDatasetAnalyzer.analyze_common_items_count(common_item_counts, True))

# for i in range(10):
#     for j in range(i+1, 10):
#         print(i, j)
<commit_msg>Added a new method to the ReviewsDatasetAnalyzer class<commit_after>import math
from pandas import DataFrame
from etl import ETLUtils
from tripadvisor.fourcity import extractor

__author__ = 'fpena'

class ReviewsDatasetAnalyzer:
    """
    This class is used to analyze the dataset that contains a list of reviews.
    Some statistics can be obtained thanks to this class, such as the sparsity
    of the dataset and the number of users that have rated the same item. This
    statistics will show how complete is the dataset.
    """

    def __init__(self, reviews):

        if not reviews:
            raise ValueError('Can not analyze an empty list')

        self.reviews = reviews
        self.user_ids = extractor.get_groupby_list(self.reviews, 'user_id')
        self.item_ids = extractor.get_groupby_list(self.reviews, 'offering_id')
        self.num_reviews = len(self.reviews)
        self.num_users = len(self.user_ids)
        self.num_items = len(self.item_ids)
        self.data_frame = DataFrame(self.reviews)
        self.users_count = self.data_frame.groupby('user_id').size()
        self.items_count = self.data_frame.groupby('offering_id').size()

    def calculate_sparsity(self):
        """
        Returns the percentage of missing ratings in the list of reviews of this
        ReviewsDatasetAnalyzer

        :return: the rate of missing ratings
        (i.e. number of missing ratings / (number of items * number of users))
        :raise ValueError: in case an empty list is given
        """
        if not self.reviews:
            raise ValueError('Can not determine the sparsity for an empty list')

        user_ids = extractor.get_groupby_list(self.reviews, 'user_id')
        item_ids = extractor.get_groupby_list(self.reviews, 'offering_id')

        non_missing_reviews = 0.
        total_expected_reviews = len(user_ids) * len(item_ids)

        for user in user_ids:
            user_reviews = ETLUtils.filter_records(self.reviews, 'user_id', [user])
            user_items = extractor.get_groupby_list(user_reviews, 'offering_id')

            non_missing_reviews += len(set(item_ids).intersection(set(user_items)))

        return 1 - non_missing_reviews / total_expected_reviews

    def calculate_sparsity_approx(self):
        """
        Returns the approximate percentage of missing ratings in the list of
        reviews of this ReviewsDatasetAnalyzer. This method is an approximation
        because it counts two reviews from the same user to the same item as
        two, when the correct count should be one. This method was created to
        calculate the sparsity in very big datasets where calculating the exact
        sparsity can be a very slow process.

        :return: the rate of approximate missing ratings
        (i.e. number of missing ratings / (number of reviews))
        :raise ValueError: in case an empty list is given
        """
        if not self.reviews:
            raise ValueError('Can not determine the sparsity for an empty list')

        user_ids = extractor.get_groupby_list(self.reviews, 'user_id')
        item_ids = extractor.get_groupby_list(self.reviews, 'offering_id')
        total_expected_reviews = float(len(user_ids) * len(item_ids))

        return 1 - float(len(self.reviews)) / total_expected_reviews

    def count_items_in_common(self):
        """
        Counts the number of items each user has in common with every other user
        and stores the results in a dictionary.

        :return: a dictionary with the count of the number of times users have
        a certain number of items in common. For example, the dictionary
        {0:4, 1:10, 2:6, 3:3, 4:1} means that there a 4 users who have 0 items
        in common with the rest of users, there are 10 users who have rated 1
        item in common with the rest of users, 6 users who have rated 6 items in
        common with the rest of the users, and so on.
        """
        common_item_counts = {}
        user_dictionary = extractor.initialize_users(self.reviews, False)

        for i in range(self.num_users):
            for j in range(i+1, self.num_users):

                user1 = self.user_ids[i]
                user2 = self.user_ids[j]

                num_common_items = len(extractor.get_common_items(
                    user_dictionary, user1, user2))

                if num_common_items in common_item_counts:
                    common_item_counts[num_common_items] += 1
                else:
                    common_item_counts[num_common_items] = 1

        return common_item_counts

    def analyze_common_items_count(self, common_item_counts, cumulative=False):
        """
        Converts the dictionary returned by
        :func:`~reviews_dataset_analyzer.ReviewsDatasetAnalyzer.count_items_in_common`
        into a dictionary with the same keys but instead of counts the values
        have percentages

        :param common_item_counts: a dictionary with the count of the number of
        items the users have in common
        :param cumulative: indicates whether the percentage is cumulative or not
        :return: a dictionary with the same keys but instead of counts the
        values have percentages
        """
        num_combinations = ReviewsDatasetAnalyzer.nCr(self.num_users, 2)
        common_item_percentages = {}

        cumulative_percentage = 0.

        for key in common_item_counts.keys():
            percentage = float(common_item_counts[key]) / num_combinations

            if cumulative:
                cumulative_percentage += percentage
                common_item_percentages[key] = cumulative_percentage
            else:
                common_item_percentages[key] = percentage

        return common_item_percentages

    def summarize_reviews_by_field(self, field):

        counts = self.data_frame.groupby(field).size()

        values = counts.values
        values_data_frame = DataFrame(values, columns=['frequency'])
        values_count = values_data_frame.groupby('frequency').size()

        return values_count

    @staticmethod
    def nCr(n, r):
        f = math.factorial
        return f(n) / f(r) / f(n-r)

file_path = '/Users/fpena/tmp/filtered_reviews_multi_non_sparse_shuffled.json'
my_reviews = ETLUtils.load_json_file(file_path)
# my_reviews = extractor.pre_process_reviews()
# my_reviews = movielens_extractor.get_ml_100K_dataset()
#
reviewsDatasetAnalyzer = ReviewsDatasetAnalyzer(my_reviews)
# my_counts = reviewsDatasetAnalyzer.summarize_reviews_by_field('user_id')
# file_name = '/Users/fpena/UCC/Thesis/projects/yelp/notebooks/test.ipynb'
# my_load_reviews_code =\
#     'file_path = \'/Users/fpena/tmp/filtered_reviews_multi_non_sparse_shuffled.json\'\n' +\
#     'reviews = ETLUtils.load_json_file(file_path)\n'
# my_load_reviews_code =\
#     'from tripadvisor.fourcity import movielens_extractor\n' +\
#     'reviews = movielens_extractor.get_ml_100K_dataset()'
# ReviewsDatasetAnalyzer.generate_report(my_reviews, 'Fourcity TripAdvisor', file_name, my_load_reviews_code)
# print(my_counts)
# print(reviewsDatasetAnalyzer.count_reviews_by_item())
common_item_counts = reviewsDatasetAnalyzer.count_items_in_common()
# print(common_item_counts)
# print (time.strftime("%H:%M:%S"))
# print('Sparsity', reviewsDatasetAnalyzer.calculate_sparsity())
# print (time.strftime("%H:%M:%S"))
# print('Sparsity Approx', reviewsDatasetAnalyzer.calculate_sparsity_approx())
# print (time.strftime("%H:%M:%S"))
# print(reviewsDatasetAnalyzer.analyze_common_items_count(common_item_counts))
# print(reviewsDatasetAnalyzer.analyze_common_items_count(common_item_counts, True))

<|endoftext|>


<commit_before>import tempfile
import requests
import os
import signal
import json
import time
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

#django
from django.core.files import File
from django.core.exceptions import ObjectDoesNotExist

#db models
from urlocators.models import Page, Locator, make_url_id

#===========================================================#
#=================== Drivers helpers =======================#
class Helpers:
    '''
    '''
    @staticmethod
    def save_page_source(url, source, **kwargs):
        '''
        '''
        #set url id
        url_id=make_url_id(url)
        #get job
        try:
            job=kwargs['job']
        except KeyError:
            raise Exception('[-] Job is required to save page source')
        #build url relations
        try:
            locs=Locator.objects.get(url_id=url_id)
        except ObjectDoesNotExist:
            locs=Locator()
            locs.url=url
            locs.save()
        try:
            page=Page.objects.get(addr=locs.id, job=job)
        except ObjectDoesNotExist:
            page=Page()
            #build html file
            page.job=job
            page.addr=locs
            fp=tempfile.TemporaryFile()
            fp.write(source.encode())
            fp.seek(0)
            file_html=File(fp)
            page.html=file_html
            page.save()
            fp.close()
            #close temp file
        print('[+] Done saving page [{}]'.format(url[:150]))
        if ('return_page' in kwargs
            and kwargs['return_page']):
            return page

#===========================================================#
#==================== Drivers base classes =================#
class BaseSeleniumBrowser:
    '''
    '''
    def __init__(self, driver_name, remote_server):
        '''
        '''
        self._driver_name=driver_name
        self._driver=webdriver
        self.remote_server=remote_server
        self.host=''
        self.pid=None
        self.browser=None
        self.proxy=None

    def set_host(self, host):
        '''
        '''
        self.host=host

    def build_driver(self, proxy_port=None):
        '''
        '''
        #set browser global waits
        max_implicity=5
        max_timeout=30
        if proxy_port:
            proxy_addr='--proxy=127.0.0.1:{}'.format(proxy_port)
            service_args=[proxy_addr, '--ignore-ssl-errors=yes']
            self.browser=getattr(self._driver, self._driver_name)(service_args=service_args)
        else:
            self.browser=getattr(self._driver, self._driver_name)()
        #phantom window spec
        if self._driver_name == 'PhantomJS':
            self.browser.set_window_size(1124, 850)
            self.pid=self.browser.service.process.pid
        #waits
        self.browser.implicitly_wait(max_implicity)
        self.browser.set_page_load_timeout(max_timeout)

    def set_cookies(self, **cookies):
        '''
        '''
        if not self.host:
            raise Exception('Host is required for cookie seting')
        self.browser.get(self.host)
        for key,value in cookies.items():
            self.browser.add_cookie({'name':key, 'value':value})

    def get_page_source(self):
        '''
        '''
        return self.browser.page_source

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser.page_source
        url=self.browser.current_url
        return Helpers.save_page_source(url, source, **kwargs)

    def back(self, job,  **kwargs):
        '''
        '''
        self.browser.back()

    def get(self, url):
        '''
        '''
        self.browser.get(url)

    def close(self):
        '''
        '''
        if self.browser:
            if self._driver_name == 'PhantomJS':
                self.browser.service.process.send_signal(signal.SIGTERM)
            self.browser.quit()
        if self.proxy:
            self.proxy.close()
        return

    @property
    def current_url(self):
        '''
        '''
        return self.browser.current_url

    def execute_script(self, page_data, action_data):
        '''
        '''
        field_name=action_data['field_name']
        script=action_data['script']
        value=self.browser.execute_script(script)
        if isinstance(value, (list,dict)):
            value=json.dumps(value)
        page_data['page_data'].append(
                {field_name:[value,], 'index':-1})
        return

    def wait_3_seconds(self, **kwargs):
        '''
        '''
        time.sleep(3)

    def switch_to_frame(self, **kwargs):
        '''
        '''
        raise NotImplemented('Not implemented yet')

class BaseRequests:
    '''
    '''
    def __init__(self):
        '''
        '''
        self._headers=None
        self._cookies=None
        self.browser={}
        self.proxy=None

    def build_driver(self, proxy=None):
        '''
        '''
        if proxy:
            raise NotImplemented('[+] Proxy not available in lean requests ')
        session=requests.Session()
        session.headers=self._headers
        self.browser['session']=session

    def set_cookies(self, **cookies):
        '''
        '''
        raise NotImplemented('[-] Not implemeted yet::must be a CookieJar')

    def get_page_source(self):
        '''
        '''
        return self.browser['source']

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser['source']
        url=self.browser['url']
        Helpers.save_page_source(url, source, **kwargs)

    def back(self, **kwargs):
        '''
        '''
        raise TypeError('Requests driver does not have history')

    def get(self, url, allow_redirects=False):
        '''
        '''
        result=self.browser['session'].get(url, allow_redirects=allow_redirects)
        if result.status_code != 200:
            msg='LeanRequests GET fail. Headers [{}]'.format(result.headers)
            raise Exception(msg)
        self.browser['url']=url
        self.browser['source']=result.text
        self.browser['headers']=result.headers

    def close(self):
        '''
        '''
        if not self.browser.get('session'):return
        self.browser['session'].close()
        self.browser={}

    @property
    def current_url(self):
        '''
        '''
        return self.browser['url']

    def get_header_field(self, page_data, action_data):
        '''
        '''
        header_field=action_data['header_field']
        header_value=self.browser['headers'].get(header_field, 'None')
        page_data['page_data'].append(
                {header_field:[header_value,], 'index':-1})
        return

    def wait_3_seconds(self, **kwargs):
        '''
        '''
        time.sleep(3)

class DriverChoices:
    '''
    '''
    _choices=[]

    @classmethod
    def register(cls, driverClass):
        '''
        '''
        cls._choices.append(driverClass.__name__)

    @classmethod
    def get_all(cls):
        '''
        '''
        return cls._choices
<commit_msg>Fixing connection issues in gevent mode<commit_after>import tempfile
import requests
import os
import signal
import json
import time
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

#django
from django.core.files import File
from django.core.exceptions import ObjectDoesNotExist
from django.db

#db models
from urlocators.models import Page, Locator, make_url_id

#===========================================================#
#=================== Drivers helpers =======================#
class Helpers:
    '''
    '''
    @staticmethod
    def save_page_source(url, source, **kwargs):
        '''
        '''
        #set url id
        url_id=make_url_id(url)
        #get job
        try:
            job=kwargs['job']
        except KeyError:
            raise Exception('[-] Job is required to save page source')
        #build url relations
        try:
            locs=Locator.objects.get(url_id=url_id)
        except:
            db.close_old_connections()
            locs=Locator.objects.get(url_id=url_id)
        except ObjectDoesNotExist:
            locs=Locator()
            locs.url=url
            locs.save()
        try:
            page=Page.objects.get(addr=locs.id, job=job)
        except ObjectDoesNotExist:
            page=Page()
            #build html file
            page.job=job
            page.addr=locs
            fp=tempfile.TemporaryFile()
            fp.write(source.encode())
            fp.seek(0)
            file_html=File(fp)
            page.html=file_html
            page.save()
            fp.close()
            #close temp file
        print('[+] Done saving page [{}]'.format(url[:150]))
        if ('return_page' in kwargs
            and kwargs['return_page']):
            return page

#===========================================================#
#==================== Drivers base classes =================#
class BaseSeleniumBrowser:
    '''
    '''
    def __init__(self, driver_name, remote_server):
        '''
        '''
        self._driver_name=driver_name
        self._driver=webdriver
        self.remote_server=remote_server
        self.host=''
        self.pid=None
        self.browser=None
        self.proxy=None

    def set_host(self, host):
        '''
        '''
        self.host=host

    def build_driver(self, proxy_port=None):
        '''
        '''
        #set browser global waits
        max_implicity=5
        max_timeout=30
        if proxy_port:
            proxy_addr='--proxy=127.0.0.1:{}'.format(proxy_port)
            service_args=[proxy_addr, '--ignore-ssl-errors=yes']
            self.browser=getattr(self._driver, self._driver_name)(service_args=service_args)
        else:
            self.browser=getattr(self._driver, self._driver_name)()
        #phantom window spec
        if self._driver_name == 'PhantomJS':
            self.browser.set_window_size(1124, 850)
            self.pid=self.browser.service.process.pid
        #waits
        self.browser.implicitly_wait(max_implicity)
        self.browser.set_page_load_timeout(max_timeout)

    def set_cookies(self, **cookies):
        '''
        '''
        if not self.host:
            raise Exception('Host is required for cookie seting')
        self.browser.get(self.host)
        for key,value in cookies.items():
            self.browser.add_cookie({'name':key, 'value':value})

    def get_page_source(self):
        '''
        '''
        return self.browser.page_source

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser.page_source
        url=self.browser.current_url
        return Helpers.save_page_source(url, source, **kwargs)

    def back(self, job,  **kwargs):
        '''
        '''
        self.browser.back()

    def get(self, url):
        '''
        '''
        self.browser.get(url)

    def close(self):
        '''
        '''
        if self.browser:
            if self._driver_name == 'PhantomJS':
                self.browser.service.process.send_signal(signal.SIGTERM)
            self.browser.quit()
        if self.proxy:
            self.proxy.close()
        return

    @property
    def current_url(self):
        '''
        '''
        return self.browser.current_url

    def execute_script(self, page_data, action_data):
        '''
        '''
        field_name=action_data['field_name']
        script=action_data['script']
        value=self.browser.execute_script(script)
        if isinstance(value, (list,dict)):
            value=json.dumps(value)
        page_data['page_data'].append(
                {field_name:[value,], 'index':-1})
        return

    def wait_3_seconds(self, **kwargs):
        '''
        '''
        time.sleep(3)

    def switch_to_frame(self, **kwargs):
        '''
        '''
        raise NotImplemented('Not implemented yet')

class BaseRequests:
    '''
    '''
    def __init__(self):
        '''
        '''
        self._headers=None
        self._cookies=None
        self.browser={}
        self.proxy=None

    def build_driver(self, proxy=None):
        '''
        '''
        if proxy:
            raise NotImplemented('[+] Proxy not available in lean requests ')
        session=requests.Session()
        session.headers=self._headers
        self.browser['session']=session

    def set_cookies(self, **cookies):
        '''
        '''
        raise NotImplemented('[-] Not implemeted yet::must be a CookieJar')

    def get_page_source(self):
        '''
        '''
        return self.browser['source']

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser['source']
        url=self.browser['url']
        Helpers.save_page_source(url, source, **kwargs)

    def back(self, **kwargs):
        '''
        '''
        raise TypeError('Requests driver does not have history')

    def get(self, url, allow_redirects=False):
        '''
        '''
        result=self.browser['session'].get(url, allow_redirects=allow_redirects)
        if result.status_code != 200:
            msg='LeanRequests GET fail. Headers [{}]'.format(result.headers)
            raise Exception(msg)
        self.browser['url']=url
        self.browser['source']=result.text
        self.browser['headers']=result.headers

    def close(self):
        '''
        '''
        if not self.browser.get('session'):return
        self.browser['session'].close()
        self.browser={}

    @property
    def current_url(self):
        '''
        '''
        return self.browser['url']

    def get_header_field(self, page_data, action_data):
        '''
        '''
        header_field=action_data['header_field']
        header_value=self.browser['headers'].get(header_field, 'None')
        page_data['page_data'].append(
                {header_field:[header_value,], 'index':-1})
        return

    def wait_3_seconds(self, **kwargs):
        '''
        '''
        time.sleep(3)

class DriverChoices:
    '''
    '''
    _choices=[]

    @classmethod
    def register(cls, driverClass):
        '''
        '''
        cls._choices.append(driverClass.__name__)

    @classmethod
    def get_all(cls):
        '''
        '''
        return cls._choices
<|endoftext|>


<commit_before>import tempfile
import requests
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

#django
from django.core.files import File
from django.core.exceptions import ObjectDoesNotExist

#db models
from urlocators.models import Page, Locator, make_url_id

#===========================================================#
#=================== Drivers helpers =======================#
class Helpers:
    '''
    '''
    @staticmethod
    def save_page_source(url, source, **kwargs):
        '''
        '''
        #set url id
        url_id=make_url_id(url)
        #get job
        try:
            job=kwargs['job']
        except KeyError:
            raise Exception('[-] Job is required to save page source')
        #build url relations
        try:
            locs=Locator.objects.get(url_id=url_id)
        except ObjectDoesNotExist:
            locs=Locator()
            locs.url=url
            locs.save()
        try:
            page=Page.objects.get(addr=locs.id)
        except ObjectDoesNotExist:
            page=Page()
            #build html file
            page.job=job
            page.addr=locs
            fp=tempfile.TemporaryFile()
            fp.write(source.encode())
            fp.seek(0)
            file_html=File(fp)
            page.html=file_html
            page.save()
            fp.close()
            #close temp file
        print('[+] Done saving page [{}]'.format(url[:150]))

#===========================================================#
#==================== Drivers base classes =================#
class BaseSeleniumBrowser:
    '''
    '''
    def __init__(self, driver_name, remote_server):
        '''
        '''
        self._driver_name=driver_name
        self._driver=webdriver
        self.remote_server=remote_server
        self.host=''

    def set_host(self, host):
        '''
        '''
        self.host=host

    def build_driver(self, **kwargs):
        '''
        '''
        if kwargs:
            self.browser=getattr(self._driver, self._driver_name)(**kwargs)
        else:
            self.browser=getattr(self._driver, self._driver_name)()
        if self._driver_name == 'PhantomJS':
            self.browser.set_window_size(1124, 850)

    def set_cookies(self, **cookies):
        '''
        '''
        if not self.host:
            raise Exception('Host is required for cookie seting')
        self.browser.get(self.host)
        for key,value in cookies.items():
            self.browser.add_cookie({'name':key, 'value':value})

    def get_page_source(self):
        '''
        '''
        return self.browser.page_source

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser.page_source
        url=self.browser.current_url
        Helpers.save_page_source(url, source, **kwargs)

    def back(self, **kwargs):
        '''
        '''
        self.browser.back()

    def get(self, url):
        '''
        '''
        self.browser.get(url)

    def close(self):
        '''
        '''
        if not self.browser:return
        self.browser.quit()

    @property
    def current_url(self):
        '''
        '''
        return self.browser.current_url

    def wait_for_element(self, target_element, eltype, timeout=5):
        '''
        wait for html element to load
        for now, only working with xpath pattern
        '''
        timeout=timeout
        eltypeDict={'xpath':By.XPATH,}
        target=(eltypeDict[eltype], target_element)
        element_present = EC.presence_of_element_located(target)
        WebDriverWait(self.browser, timeout).until(element_present)

class BaseRequests:
    '''
    '''
    def __init__(self):
        '''
        '''
        self._headers=None
        self._cookies=None
        self.browser={}

    def build_driver(self, **kwargs):
        '''
        '''
        session=requests.Session()
        session.headers=self._headers
        self.browser['session']=session

    def set_cookies(self, **cookies):
        '''
        '''
        raise NotImplemented('[-] Not implemeted yet::must be a CookieJar')

    def get_page_source(self):
        '''
        '''
        return self.browser['source']

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser['source']
        url=self.browser['url']
        Helpers.save_page_source(url, source, **kwargs)

    def back(self, **kwargs):
        '''
        '''
        raise TypeError('Requests driver does not have history')

    def get(self, url):
        '''
        '''
        result=self.browser['session'].get(url)
        if result.status_code != 200:
            msg='LeanRequests GET fail. Headers [{}]'.format(result.headers)
            raise Exception(msg)
        self.browser['url']=url
        self.browser['source']=result.content

    def close(self):
        '''
        '''
        if not self.browser.get('session'):return
        self.browser['session'].close()
        self.browser={}

    @property
    def current_url(self):
        '''
        '''
        return self.browser['url']

    #dummy - will improve this
    def wait_for_element(self, target_element, eltype, timeout=5):
        '''
        '''
        return

class DriverChoices:
    '''
    '''
    _choices=[]

    @classmethod
    def register(cls, driverClass):
        '''
        '''
        cls._choices.append(driverClass.__name__)

    @classmethod
    def get_all(cls):
        '''
        '''
        return cls._choices
<commit_msg>Bug fix page source lean request<commit_after>import tempfile
import requests
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

#django
from django.core.files import File
from django.core.exceptions import ObjectDoesNotExist

#db models
from urlocators.models import Page, Locator, make_url_id

#===========================================================#
#=================== Drivers helpers =======================#
class Helpers:
    '''
    '''
    @staticmethod
    def save_page_source(url, source, **kwargs):
        '''
        '''
        #set url id
        url_id=make_url_id(url)
        #get job
        try:
            job=kwargs['job']
        except KeyError:
            raise Exception('[-] Job is required to save page source')
        #build url relations
        try:
            locs=Locator.objects.get(url_id=url_id)
        except ObjectDoesNotExist:
            locs=Locator()
            locs.url=url
            locs.save()
        try:
            page=Page.objects.get(addr=locs.id)
        except ObjectDoesNotExist:
            page=Page()
            #build html file
            page.job=job
            page.addr=locs
            fp=tempfile.TemporaryFile()
            fp.write(source.encode())
            fp.seek(0)
            file_html=File(fp)
            page.html=file_html
            page.save()
            fp.close()
            #close temp file
        print('[+] Done saving page [{}]'.format(url[:150]))

#===========================================================#
#==================== Drivers base classes =================#
class BaseSeleniumBrowser:
    '''
    '''
    def __init__(self, driver_name, remote_server):
        '''
        '''
        self._driver_name=driver_name
        self._driver=webdriver
        self.remote_server=remote_server
        self.host=''

    def set_host(self, host):
        '''
        '''
        self.host=host

    def build_driver(self, **kwargs):
        '''
        '''
        if kwargs:
            self.browser=getattr(self._driver, self._driver_name)(**kwargs)
        else:
            self.browser=getattr(self._driver, self._driver_name)()
        if self._driver_name == 'PhantomJS':
            self.browser.set_window_size(1124, 850)

    def set_cookies(self, **cookies):
        '''
        '''
        if not self.host:
            raise Exception('Host is required for cookie seting')
        self.browser.get(self.host)
        for key,value in cookies.items():
            self.browser.add_cookie({'name':key, 'value':value})

    def get_page_source(self):
        '''
        '''
        return self.browser.page_source

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser.page_source
        url=self.browser.current_url
        Helpers.save_page_source(url, source, **kwargs)

    def back(self, **kwargs):
        '''
        '''
        self.browser.back()

    def get(self, url):
        '''
        '''
        self.browser.get(url)

    def close(self):
        '''
        '''
        if not self.browser:return
        self.browser.quit()

    @property
    def current_url(self):
        '''
        '''
        return self.browser.current_url

    def wait_for_element(self, target_element, eltype, timeout=5):
        '''
        wait for html element to load
        for now, only working with xpath pattern
        '''
        timeout=timeout
        eltypeDict={'xpath':By.XPATH,}
        target=(eltypeDict[eltype], target_element)
        element_present = EC.presence_of_element_located(target)
        WebDriverWait(self.browser, timeout).until(element_present)

class BaseRequests:
    '''
    '''
    def __init__(self):
        '''
        '''
        self._headers=None
        self._cookies=None
        self.browser={}

    def build_driver(self, **kwargs):
        '''
        '''
        session=requests.Session()
        session.headers=self._headers
        self.browser['session']=session

    def set_cookies(self, **cookies):
        '''
        '''
        raise NotImplemented('[-] Not implemeted yet::must be a CookieJar')

    def get_page_source(self):
        '''
        '''
        return self.browser['source']

    def page_source(self, **kwargs):
        '''
        '''
        source=self.browser['source']
        url=self.browser['url']
        Helpers.save_page_source(url, source, **kwargs)

    def back(self, **kwargs):
        '''
        '''
        raise TypeError('Requests driver does not have history')

    def get(self, url):
        '''
        '''
        result=self.browser['session'].get(url)
        if result.status_code != 200:
            msg='LeanRequests GET fail. Headers [{}]'.format(result.headers)
            raise Exception(msg)
        self.browser['url']=url
        self.browser['source']=result.text

    def close(self):
        '''
        '''
        if not self.browser.get('session'):return
        self.browser['session'].close()
        self.browser={}

    @property
    def current_url(self):
        '''
        '''
        return self.browser['url']

    #dummy - will improve this
    def wait_for_element(self, target_element, eltype, timeout=5):
        '''
        '''
        return

class DriverChoices:
    '''
    '''
    _choices=[]

    @classmethod
    def register(cls, driverClass):
        '''
        '''
        cls._choices.append(driverClass.__name__)

    @classmethod
    def get_all(cls):
        '''
        '''
        return cls._choices
<|endoftext|>


<commit_before>import sys
import re

def translate_words(words, dictionary, after_sep = ' '):
    translated = []
    for word in words:
        if word in dictionary:
            translated.append(dictionary[word])
        else:
            translated.append(word)

    return after_sep.join(translated)

dictionary = {}
for line in open(sys.argv[1]):
    before, after = line.split()
    dictionary[before] = after

SPLIT_PATTERN = re.compile(r'".*?"|\S+')

for line in sys.stdin:
    translated = []
    for word in SPLIT_PATTERN.findall(line):
        unquoted = word.strip('"')
        if word in dictionary:
            translated.append(dictionary[word])
        elif word != unquoted:
            zipped = translate_words(unquoted.split(), dictionary, '')
            translated.append(zipped)
            dictionary[word] = zipped
        else:
            translated.append(word)

    print(' '.join(translated))
<commit_msg>先頭に `!` をつけたクオート単語群は変換しない<commit_after>import sys
import re

def translate_words(words, dictionary, after_sep = ' '):
    translated = []
    for word in words:
        if word in dictionary:
            translated.append(dictionary[word])
        else:
            translated.append(word)

    return after_sep.join(translated)

dictionary = {}
for line in open(sys.argv[1]):
    before, after = line.split()
    dictionary[before] = after

SPLIT_PATTERN = re.compile(r'[!]".*?"|".*?"|\S+')

for line in sys.stdin:
    translated = []
    for word in SPLIT_PATTERN.findall(line):
        unquoted = word.strip('"')
        is_banged = (re.match('^[!]".*?"', word) is not None)
        if word in dictionary:
            translated.append(dictionary[word])
        elif is_banged:
            translated.append(word[1:])
        elif word != unquoted:
            zipped = translate_words(unquoted.split(), dictionary, '')
            translated.append(zipped)
            dictionary[word] = zipped
        else:
            translated.append(word)

    print(' '.join(translated))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
import json
import random
import urllib2
import datetime

try:
    import irclib
    import ircbot
except ImportError:
    print >>sys.stderr, "This bot needs python-irclib."
    print >>sys.stderr, "http://python-irclib.sourceforge.net"

try:
    import twitter
except ImportError:
    print >>sys.stderr, "This bot needs python-twitter."
    print >>sys.stderr, "http://code.google.com/p/python-twitter/"

NICK = "ec09bot"
SERVERS = [("irc.freenode.net", 6667)]
CHANNEL = "#ec09"

class EC09Bot(ircbot.SingleServerIRCBot):
    def __init__(self, *args):
        ircbot.SingleServerIRCBot.__init__(self, SERVERS, NICK, NICK)

        self.connection.add_global_handler("pubmsg", self._on_pubmsg)

        self.twitter = twitter.Api()
        self.batima_cache = []
        self.batima_cache_age_max = 30*60 # seconds
        self.batima_cache_time = datetime.datetime.now()

    def start(self):
        self._connect()
        self.connection.join(CHANNEL)
        irclib.SimpleIRCClient.start(self)

    def _on_pubmsg(self, connection, event):
        sendernick, _ = event.source().split('!', 1)
        message = event.arguments()[0].strip()

        if not message.startswith("!"):
            return

        command = message[1:]
        handler = getattr(self, "command_" + command, None)

        if callable(handler):
            self.connection.privmsg(CHANNEL, "%s: %s" % (sendernick, handler()))

    def command_bandeco(self):
        bandeco_url = \
            "http://www.students.ic.unicamp.br/~ra091187/bandeco_svg/json.php"

        try:
            u = urllib2.urlopen(bandeco_url, timeout = 5)
            st = u.read()
        except urllib2.URLError:
            return "Indisponível :("

        d = json.loads(st.strip("bandeco()"), encoding = "latin-1")
        return "%(prato)s, com suco de %(suco)s e sobremesa %(sobremesa)s" % d

    def command_batima(self):
        age = (datetime.datetime.now() - self.batima_cache_time).seconds
        if not self.batima_cache or age > self.batima_cache_age_max:
            self._rebuild_batima_cache()

        if not self.batima_cache:
        	return "FODEU! SERÁ QUE O TWITTER CAIU?"

        return random.choice(self.batima_cache)

    def _rebuild_batima_cache(self):
        status = self.twitter.GetUserTimeline("falasdobatima", count = 50)
        tweets = [s.GetText() for s in status]

        # Try not very hard to exclude replies
        self.batima_cache = [t.encode("utf-8") for t in tweets if t[0] != "@"]
        self.batima_cache_time = datetime.datetime.now()

if __name__ == "__main__":
    bot = EC09Bot()
    bot.start()
<commit_msg>!bandeco should work now.<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
import json
import random
import urllib2
import datetime

try:
    import irclib
    import ircbot
except ImportError:
    print >>sys.stderr, "This bot needs python-irclib."
    print >>sys.stderr, "http://python-irclib.sourceforge.net"

try:
    import twitter
except ImportError:
    print >>sys.stderr, "This bot needs python-twitter."
    print >>sys.stderr, "http://code.google.com/p/python-twitter/"

NICK = "ec09bot"
SERVERS = [("irc.freenode.net", 6667)]
CHANNEL = "#ec09"

class EC09Bot(ircbot.SingleServerIRCBot):
    def __init__(self, *args):
        ircbot.SingleServerIRCBot.__init__(self, SERVERS, NICK, NICK)

        self.connection.add_global_handler("pubmsg", self._on_pubmsg)

        self.twitter = twitter.Api()
        self.batima_cache = []
        self.batima_cache_age_max = 30*60 # seconds
        self.batima_cache_time = datetime.datetime.now()

    def start(self):
        self._connect()
        self.connection.join(CHANNEL)
        irclib.SimpleIRCClient.start(self)

    def _on_pubmsg(self, connection, event):
        sendernick, _ = event.source().split('!', 1)
        message = event.arguments()[0].strip()

        if not message.startswith("!"):
            return

        command = message[1:]
        handler = getattr(self, "command_" + command, None)

        if callable(handler):
            self.connection.privmsg(CHANNEL, "%s: %s" % (sendernick, handler()))

    def command_bandeco(self):
        bandeco_url = \
            "http://www.students.ic.unicamp.br/~ra091187/bandeco_svg/json.php"

        try:
            u = urllib2.urlopen(bandeco_url, timeout = 5)
            st = u.read()
        except urllib2.URLError:
            return "Indisponível :("

        st = st[len("bandeco("):-2]

        d = json.loads(st)
        d.update((k, v.lower()) for k, v in d.items())

        bandeco_str = \
            "%(prato)s, com suco de %(suco)s, " \
            "salada %(salada)s e sobremesa %(sobremesa)s." % d

        bandeco_str = bandeco_str[0].upper() + bandeco_str[1:]
        return bandeco_str.encode("utf-8")

    def command_batima(self):
        age = (datetime.datetime.now() - self.batima_cache_time).seconds
        if not self.batima_cache or age > self.batima_cache_age_max:
            self._rebuild_batima_cache()

        if not self.batima_cache:
            return "FODEU! SERÁ QUE O TWITTER CAIU?"

        return random.choice(self.batima_cache)

    def _rebuild_batima_cache(self):
        status = self.twitter.GetUserTimeline("falasdobatima", count = 50)
        tweets = [s.GetText() for s in status]

        # Try not very hard to exclude replies
        self.batima_cache = [t.encode("utf-8") for t in tweets if t[0] != "@"]
        self.batima_cache_time = datetime.datetime.now()

if __name__ == "__main__":
    bot = EC09Bot()
    bot.start()
<|endoftext|>


<commit_before>"""
@brief      test log(time=20s)

skip this test for regular run
"""

import sys
import os
import unittest
import re

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

try:
    import pyquickhelper
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    if "PYQUICKHELPER" in os.environ and len(os.environ["PYQUICKHELPER"]) > 0:
        sys.path.append(os.environ["PYQUICKHELPER"])
    import pyquickhelper

from src.pymyinstall.installhelper.module_install import ModuleInstall
from pyquickhelper import fLOG

class TestDownloadSkData(unittest.TestCase):

    def test_install_skata(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2:
            # skdata is not available (yet)
            continue

        fold = os.path.abspath(os.path.split(__file__)[0])
        temp = os.path.join(fold, "temp_download_skdata")
        if not os.path.exists(temp):
            os.mkdir(temp)
        for _ in os.listdir(temp):
            if os.path.isfile(os.path.join(temp, _)):
                os.remove(os.path.join(temp, _))

        if sys.platform.startswith("win"):
            fLOG("install", "skdata")
            m = ModuleInstall("skdata", "wheel2", fLOG=fLOG)
            exe = m.download(
                temp_folder=temp,
                file_save=os.path.join(
                    temp,
                    "out_page.html"), source="2")
            assert os.path.exists(exe)

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix unit test<commit_after>"""
@brief      test log(time=20s)

skip this test for regular run
"""

import sys
import os
import unittest
import re

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

try:
    import pyquickhelper
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    if "PYQUICKHELPER" in os.environ and len(os.environ["PYQUICKHELPER"]) > 0:
        sys.path.append(os.environ["PYQUICKHELPER"])
    import pyquickhelper

from src.pymyinstall.installhelper.module_install import ModuleInstall
from pyquickhelper import fLOG

class TestDownloadSkData(unittest.TestCase):

    def test_install_skata(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2:
            # skdata is not available (yet)
            return

        fold = os.path.abspath(os.path.split(__file__)[0])
        temp = os.path.join(fold, "temp_download_skdata")
        if not os.path.exists(temp):
            os.mkdir(temp)
        for _ in os.listdir(temp):
            if os.path.isfile(os.path.join(temp, _)):
                os.remove(os.path.join(temp, _))

        if sys.platform.startswith("win"):
            fLOG("install", "skdata")
            m = ModuleInstall("skdata", "wheel2", fLOG=fLOG)
            exe = m.download(
                temp_folder=temp,
                file_save=os.path.join(
                    temp,
                    "out_page.html"), source="2")
            assert os.path.exists(exe)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This file is part of Tumulus.
#
# Copyright (C) 2013 OKso (http://okso.me)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class Element(object):

    def __init__(self, tagname, components, **kwargs):
        self.tagname = tagname
        self.components = components
        if 'class_' in kwargs:
            kwargs['class'] = kwargs.pop('class_')
        self.args = kwargs

    def html(self):
        components_html = ''
        for c in self.components:
            if hasattr(c, 'html'):
                components_html += c.html() + '\n'
            elif type(c) in (str, ):
                components_html += c + '\n'
        return ''.join((
            '<{} '.format(self.tagname),
            ' '.join(key + '="{}"'.format(self.args[key])
                     for key in self.args),
            '>\n{}\n</{}>'.format(components_html, self.tagname),
            ))

    __str__ = build = html

    def plugins(self):
        '''
            Returns a flattened list of all plugins used by page components.
        '''
        plugins = []
        for c in self.components:
            if hasattr(c, 'plugins'):
                plugins += c.plugins()
        return plugins

    def __iter__(self):
        'Hack to be returned to CherryPy with no prior conversion'
        class TagIterator(object):

            def __init__(self, parent):
                self.stop = False
                self.parent = parent

            def next(self):
                if self.stop:
                    raise StopIteration
                else:
                    self.stop = True
                    return str(self.parent)

        return TagIterator(self)

class EmptyElement(Element):

    def __init__(self, tagname, **kwargs):
        self.tagname = tagname
        if 'class_' in kwargs:
            kwargs['class'] = kwargs.pop('class_')
        self.args = kwargs

    def html(self):
        return ''.join((
            '<{} '.format(self.tagname),
            ' '.join(key + '="' + self.args[key] + '"' for key in self.args),
            ' />',
            ))

    __str__ = build = html

class HTMLElement(Element):

    def build(self):
        return '<!doctype html>\n' + Element.build(self)
<commit_msg>Adding BeautifulSoup support, enh on plugins<commit_after># -*- coding: utf-8 -*-

# This file is part of Tumulus.
#
# Copyright (C) 2013 OKso (http://okso.me)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from bs4 import BeautifulSoup, Tag

class Element(object):

    def __init__(self, tagname, components=None, **kwargs):
        self.tagname = tagname
        self.components = components if components else []
        if 'class_' in kwargs:
            kwargs['class'] = kwargs.pop('class_')
        self.args = kwargs

    def html(self):
        components_html = ''
        for c in self.components:
            if hasattr(c, 'html'):
                components_html += c.html() + '\n'
            elif type(c) in (str, ):
                components_html += c + '\n'
        return ''.join((
            '<{} '.format(self.tagname),
            ' '.join(key + '="{}"'.format(self.args[key])
                     for key in self.args),
            '>\n{}\n</{}>'.format(components_html, self.tagname),
            ))

    __str__ = build = html

    def soup(self):
        '''
            Returns HTML as a BeautifulSoup element.
        '''
        components_soup = Tag(name=self.tagname)
        for c in self.components:
            if hasattr(c, 'soup'):
                components_soup.append(c.soup())
            elif hasattr(c, 'html'):
                components_soup.append(BeautifulSoup(c.html()))
            elif type(c) in (str, ):
                components_soup.append(BeautifulSoup(str(c)))
            else:
                # Component should not be integrated
                pass
        return components_soup

    def plugins(self):
        '''
            Returns a flattened list of all plugins used by page components.
        '''
        plugins = []
        for c in self.components:
            if hasattr(c, 'plugins'):
                plugins += c.plugins()
            elif hasattr(c, 'is_plugin') and c.is_plugin:
                plugins.append(c)
        return plugins

    def __iter__(self):
        'Hack to be returned to CherryPy with no prior conversion'
        class TagIterator(object):

            def __init__(self, parent):
                self.stop = False
                self.parent = parent

            def next(self):
                if self.stop:
                    raise StopIteration
                else:
                    self.stop = True
                    return str(self.parent)

        return TagIterator(self)

class EmptyElement(Element):

    def __init__(self, tagname, **kwargs):
        self.tagname = tagname
        if 'class_' in kwargs:
            kwargs['class'] = kwargs.pop('class_')
        self.args = kwargs

    def html(self):
        return ''.join((
            '<{} '.format(self.tagname),
            ' '.join(key + '="' + self.args[key] + '"' for key in self.args),
            ' />',
            ))

    __str__ = build = html

class HTMLElement(Element):

    def build(self):
        return '<!doctype html>\n' + Element.build(self)
<|endoftext|>