<commit_before>#!/usr/bin/env python
from __future__ import division
from __future__ import print_function

import os
import re
import argparse

from wand.image import Image

parser = argparse.ArgumentParser(add_help=False)

parser.add_argument("-d", "--directory",
    help="Path to input directory. Defaults to current directory")
parser.add_argument("-r", "--recursive",
    help="Recurse through the file system and process images.",
    action='store_true')
parser.add_argument("-w", "--width",
    help="Target width")
parser.add_argument("-h", "--height",
    help="Target height")
parser.add_argument("-g", "--gravity",
    help="Crop focus. Options include NorthWest, North, NorthEast, West,"
         "Center, East, SouthWest, South, SouthEast")

args = parser.parse_args()

class ProcessImages(object):
    '''
    Runs through a given directory looking for filenames that don't end in
    "-WidthxHeight.ext" (-1074x483.jpg, for example), resizes, and then crops
    to the desired dimensions.

    Usage:
    ./generate.py --directory='/path-to-images' --width=500 --height=500 --gravity='center'
    '''
    def __init__(self, input_directory, target_width, target_height,
            output_directory='', gravity=''):
        self.input_directory = os.path.abspath(input_directory)
        self.output_directory = self.input_directory
        self.target_width = target_width
        self.target_height = target_height
        self.gravity = gravity
        self.all_images = []
        self.uncropped = []
        self.execute();

    def get_images(self):
        '''
        Get all the image paths in the given directory.
        '''
        all_images = []
        files = os.listdir(self.input_directory)
        formats = ['.jpg', '.jpeg', '.png', '.gif', 'tiff', 'bmp']

        for f in files:
            file_ext = os.path.splitext(f)[1]
            if file_ext in formats:
                self.all_images.append(f)

    def remove_previously_processed(self):
        '''
        Remove all the previously cropped images to get the original image.
        Matches everything after the hyphen in the filename. For example,
        "-300x275.jpg" in this file: "timeline7-300x275.jpg"
        '''
        pattern = re.compile(
            '([a-zA-Z0-9\.\-\_+])+(-([0-9]+)x([0-9]+)(.jpg|.png|.jpeg|.gif))'
        )

        for image in self.all_images:
            if not pattern.match(image):
                image = '{0}/{1}'.format(self.input_directory, image)
                self.uncropped.append(image)

    def aspect_ratio(self, width, height):
        '''Returns the aspect ratio'''
        return width / height

    def get_orientation(self, image):
        '''Determine image orientation'''
        if self.aspect_ratio(image.width, image.height) >= 1:
            return "landscape"
        else:
            return "portrait"

    def calculate_height(self, width, aspect_ratio):
        '''Calcuates the height to preserve aspect ratio'''
        return width / aspect_ratio

    def calculate_width(self, height, aspect_ratio):
        '''Calcuates the width to preserve aspect ratio'''
        return height * aspect_ratio

    def calculate_landscape_dimensions(self, min_width, min_height, aspect_ratio):
        '''
        Caluculate landscape dimensions so that the min_width and min_height are
        not exceeded
        '''
        dimensions = {}
        dimensions['width'] = self.calculate_width(min_height, aspect_ratio)

        if dimensions['width'] >= min_width:
            dimensions['height'] = self.calculate_height(dimensions['width'],
                aspect_ratio)
        else:
            dimensions['width'] = min_width
            dimensions['height'] = self.calculate_height(dimensions['width'],
                aspect_ratio)

        dimensions['width'] = int(round(dimensions['width']))
        dimensions['height'] = int(round(dimensions['height']))

        return dimensions

    def calculate_portrait_dimensions(self, min_width, min_height, aspect_ratio):
        '''
        Caluculate portrait dimensions so that the min_width and min_height are
        not exceeded
        '''
        dimensions = {}
        dimensions['height'] = self.calculate_height(min_width, aspect_ratio)

        if dimensions['height'] >= min_height:
            dimensions['width'] = self.calculate_width(dimensions['height'],
                aspect_ratio)
        else:
            dimensions['height'] = min_height
            dimensions['width'] = self.calculate_width(dimensions['height'],
                aspect_ratio)

        dimensions['width'] = int(round(dimensions['width']))
        dimensions['height'] = int(round(dimensions['height']))

        return dimensions

    def scale_image(self, image, min_width, min_height):
        '''
        Get scale dimensions, resize, and return the image instance
        '''
        aspect_ratio = self.aspect_ratio(image.width, image.height)

        if self.get_orientation(image) == "landscape":
            dimensions = self.calculate_landscape_dimensions(min_width,
                min_height, aspect_ratio)
        else:
            dimensions = self.calculate_portrait_dimensions(min_width,
                min_height, aspect_ratio)

        image.resize(dimensions['width'], dimensions['height'])

        return image

    def crop_image(self, image, width, height):
        '''
        Checks if the image is larger than the desired dimensions, then crops
        '''
        if image.width >= width and image.height >= height:
            image.crop(width=width, height=height, gravity=self.gravity)
            return image

    def scale_crop_save_images(self):
        '''Scale, crop, and save the images'''
        for img in self.uncropped:
            image_name = os.path.splitext(os.path.split(img)[1])[0]
            image_ext = os.path.splitext(os.path.split(img)[1])[1]

            print("Processing:", img)

            with Image(filename=img) as image:
                with image.clone() as clone:
                    scaled = self.scale_image(clone, self.target_width,
                        self.target_height)
                    cropped = self.crop_image(scaled, width=self.target_width,
                        height=self.target_height)
                    cropped.save(filename='{0}/{1}-{2}x{3}{4}'.format(
                        self.output_directory, image_name, self.target_width,
                        self.target_height, image_ext))
                    cropped.size

    def execute(self):
        '''Execute on instantiation'''
        self.get_images()
        self.remove_previously_processed()
        self.scale_crop_save_images()

def check_gravity(gravity):
    '''Type check gravity'''
    gravity_types = ["northwest", "north", "northeast", "west", "center",
                     "east", "southwest", "south", "southeast"]
    gravity = gravity.lower()

    if not gravity in gravity_types:
        raise ValueError

def main():
    '''Do some error checking before getting too far'''
    try:
        width = int(args.width)
    except ValueError as error:
        print("Width must be a number \n")
    except TypeError as error:
        print("Width must be a number \n")
        return

    try:
        height = int(args.height)
    except ValueError as error:
        print("Height must be a number \n")
    except TypeError as error:
        print("Height must be a number \n")
        return

    try:
        if args.directory:
            if os.path.isdir(os.path.abspath(args.directory)):
                input_directory = args.directory
            else:
                raise IOError
        else:
           input_directory = os.curdir
    except IOError as error:
        print("IOError: Pass a valid directory. \n")
        return

    try:
        if args.gravity:
            gravity = check_gravity(args.gravity)
        else:
            gravity = 'center'
    except ValueError as error:
        print("Gravity must be northwest, north, northeast, west, center, east,"
              " southwest, south, southeast \n")
        return

    if args.recursive:
        input_directories = []
        for path, directories, files in os.walk(input_directory):
            input_directories.append(os.path.abspath(path))

        for input_directory in input_directories:
            process_images = ProcessImages(input_directory, width, height,
                                           gravity=gravity)
    else:
        process_images = ProcessImages(input_directory, width, height,
                                       gravity=gravity)

if __name__ == '__main__':
    main()
<commit_msg>added error handling for corrupt images<commit_after>#!/usr/bin/env python
from __future__ import division
from __future__ import print_function

import os
import re
import argparse

from wand.image import Image
from wand.exceptions import CoderError, CorruptImageError

parser = argparse.ArgumentParser(add_help=False)

parser.add_argument("-d", "--directory",
    help="Path to input directory. Defaults to current directory")
parser.add_argument("-r", "--recursive",
    help="Recurse through the file system and process images.",
    action='store_true')
parser.add_argument("-w", "--width",
    help="Target width")
parser.add_argument("-h", "--height",
    help="Target height")
parser.add_argument("-g", "--gravity",
    help="Crop focus. Options include NorthWest, North, NorthEast, West,"
         "Center, East, SouthWest, South, SouthEast")

args = parser.parse_args()

class ProcessImages(object):
    '''
    Runs through a given directory looking for filenames that don't end in
    "-WidthxHeight.ext" (-1074x483.jpg, for example), resizes, and then crops
    to the desired dimensions.

    Usage:
    ./generate.py --directory='/path-to-images' --width=500 --height=500 --gravity='center'
    '''
    def __init__(self, input_directory, target_width, target_height,
            output_directory='', gravity=''):
        self.input_directory = os.path.abspath(input_directory)
        self.output_directory = self.input_directory
        self.target_width = target_width
        self.target_height = target_height
        self.gravity = gravity
        self.all_images = []
        self.uncropped = []
        self.execute();

    def get_images(self):
        '''
        Get all the image paths in the given directory.
        '''
        all_images = []
        files = os.listdir(self.input_directory)
        formats = ['.jpg', '.jpeg', '.png', '.gif', 'tiff', 'bmp']

        for f in files:
            file_ext = os.path.splitext(f)[1]
            if file_ext in formats:
                self.all_images.append(f)

    def remove_previously_processed(self):
        '''
        Remove all the previously cropped images to get the original image.
        Matches everything after the hyphen in the filename. For example,
        "-300x275.jpg" in this file: "timeline7-300x275.jpg"
        '''
        pattern = re.compile(
            '([a-zA-Z0-9\.\-\_+])+(-([0-9]+)x([0-9]+)(.jpg|.png|.jpeg|.gif))'
        )

        for image in self.all_images:
            if not pattern.match(image):
                image = '{0}/{1}'.format(self.input_directory, image)
                self.uncropped.append(image)

    def aspect_ratio(self, width, height):
        '''Returns the aspect ratio'''
        return width / height

    def get_orientation(self, image):
        '''Determine image orientation'''
        if self.aspect_ratio(image.width, image.height) >= 1:
            return "landscape"
        else:
            return "portrait"

    def calculate_height(self, width, aspect_ratio):
        '''Calcuates the height to preserve aspect ratio'''
        return width / aspect_ratio

    def calculate_width(self, height, aspect_ratio):
        '''Calcuates the width to preserve aspect ratio'''
        return height * aspect_ratio

    def calculate_landscape_dimensions(self, min_width, min_height, aspect_ratio):
        '''
        Caluculate landscape dimensions so that the min_width and min_height are
        not exceeded
        '''
        dimensions = {}
        dimensions['width'] = self.calculate_width(min_height, aspect_ratio)

        if dimensions['width'] >= min_width:
            dimensions['height'] = self.calculate_height(dimensions['width'],
                aspect_ratio)
        else:
            dimensions['width'] = min_width
            dimensions['height'] = self.calculate_height(dimensions['width'],
                aspect_ratio)

        dimensions['width'] = int(round(dimensions['width']))
        dimensions['height'] = int(round(dimensions['height']))

        return dimensions

    def calculate_portrait_dimensions(self, min_width, min_height, aspect_ratio):
        '''
        Caluculate portrait dimensions so that the min_width and min_height are
        not exceeded
        '''
        dimensions = {}
        dimensions['height'] = self.calculate_height(min_width, aspect_ratio)

        if dimensions['height'] >= min_height:
            dimensions['width'] = self.calculate_width(dimensions['height'],
                aspect_ratio)
        else:
            dimensions['height'] = min_height
            dimensions['width'] = self.calculate_width(dimensions['height'],
                aspect_ratio)

        dimensions['width'] = int(round(dimensions['width']))
        dimensions['height'] = int(round(dimensions['height']))

        return dimensions

    def scale_image(self, image, min_width, min_height):
        '''
        Get scale dimensions, resize, and return the image instance
        '''
        aspect_ratio = self.aspect_ratio(image.width, image.height)

        if self.get_orientation(image) == "landscape":
            dimensions = self.calculate_landscape_dimensions(min_width,
                min_height, aspect_ratio)
        else:
            dimensions = self.calculate_portrait_dimensions(min_width,
                min_height, aspect_ratio)

        image.resize(dimensions['width'], dimensions['height'])

        return image

    def crop_image(self, image, width, height):
        '''
        Checks if the image is larger than the desired dimensions, then crops
        '''
        if image.width >= width and image.height >= height:
            image.crop(width=width, height=height, gravity=self.gravity)
            return image

    def scale_crop_save_images(self):
        '''Scale, crop, and save the images'''
        for img in self.uncropped:
            image_name = os.path.splitext(os.path.split(img)[1])[0]
            image_ext = os.path.splitext(os.path.split(img)[1])[1]

            print("Processing:", img)

            try:
                with Image(filename=img) as image:
                    with image.clone() as clone:
                        scaled = self.scale_image(clone, self.target_width,
                            self.target_height)
                        cropped = self.crop_image(scaled, width=self.target_width,
                            height=self.target_height)
                        cropped.save(filename='{0}/{1}-{2}x{3}{4}'.format(
                            self.output_directory, image_name, self.target_width,
                            self.target_height, image_ext))
                        cropped.size
            except (CoderError, CorruptImageError) as error:
                print("Error: {0}".format(error))
            except:
                print("Unexpected error: {0}".format(sys.exc_info()[0]))

    def execute(self):
        '''Execute on instantiation'''
        self.get_images()
        self.remove_previously_processed()
        self.scale_crop_save_images()

def check_gravity(gravity):
    '''Type check gravity'''
    gravity_types = ["northwest", "north", "northeast", "west", "center",
                     "east", "southwest", "south", "southeast"]
    gravity = gravity.lower()

    if not gravity in gravity_types:
        raise ValueError

def main():
    '''Do some error checking before getting too far'''
    try:
        width = int(args.width)
    except ValueError as error:
        print("Width must be a number \n")
    except TypeError as error:
        print("Width must be a number \n")
        return

    try:
        height = int(args.height)
    except ValueError as error:
        print("Height must be a number \n")
    except TypeError as error:
        print("Height must be a number \n")
        return

    try:
        if args.directory:
            if os.path.isdir(os.path.abspath(args.directory)):
                input_directory = args.directory
            else:
                raise IOError
        else:
           input_directory = os.curdir
    except IOError as error:
        print("IOError: Pass a valid directory. \n")
        return

    try:
        if args.gravity:
            gravity = check_gravity(args.gravity)
        else:
            gravity = 'center'
    except ValueError as error:
        print("Gravity must be northwest, north, northeast, west, center, east,"
              " southwest, south, southeast \n")
        return

    if args.recursive:
        input_directories = []
        for path, directories, files in os.walk(input_directory):
            input_directories.append(os.path.abspath(path))

        for input_directory in input_directories:
            process_images = ProcessImages(input_directory, width, height,
                                           gravity=gravity)
    else:
        process_images = ProcessImages(input_directory, width, height,
                                       gravity=gravity)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Create metrics for Datasets.
This may be used for APIs, views with visualizations, etc.
"""
#from django.db.models.functions import TruncMonth  # 1.10
from collections import OrderedDict

from django.db import models
from django.db.models import Q

from dv_apps.utils.date_helper import get_month_name_abbreviation
from dv_apps.dvobjects.models import DvObject, DTYPE_DATASET, DTYPE_DATAFILE
from dv_apps.datafiles.models import Datafile
from dv_apps.guestbook.models import GuestBookResponse, RESPONSE_TYPE_DOWNLOAD
from dv_apps.metrics.stats_util_base import StatsMakerBase, TruncYearMonth

class StatsMakerFiles(StatsMakerBase):

    def __init__(self, **kwargs):
        """
        Start and end dates are optional.

        start_date = string in YYYY-MM-DD format
        end_date = string in YYYY-MM-DD format
        """
        super(StatsMakerFiles, self).__init__(**kwargs)

    # ----------------------------
    #  Datafile counts - single number
    # ----------------------------
    def get_datafile_count(self, **extra_filters):
        """
        Return the Datafile count
        """
        if self.was_error_found():
            return self.get_error_msg_return()

        filter_params = self.get_date_filter_params()

        # Add extra filters, if they exist
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        return True, Datafile.objects.filter(**filter_params).count()

    def get_datafile_count_published(self):
        """
        Return the count of published Dataverses
        """
        return self.get_datafile_count(**self.get_is_published_filter_param())

    def get_datafile_count_unpublished(self):
        """
        Return the count of published Dataverses
        """
        return self.get_datafile_count(**self.get_is_NOT_published_filter_param())

    # ----------------------------
    #  Monthly download counts
    # ----------------------------
    def get_file_downloads_by_month_published(self):
        """File downloads by month for published files"""

        params = self.get_is_published_filter_param(dvobject_var_name='datafile')

        return self.get_file_downloads_by_month(**params)

    def get_file_downloads_by_month_unpublished(self):
        """File downloads by month for unpublished files"""

        params = self.get_is_NOT_published_filter_param(dvobject_var_name='datafile')

        return self.get_file_downloads_by_month(**params)

    def get_file_download_start_point(self, **extra_filters):
        """Get the startpoint when keeping a running total of file downloads"""

        start_point_filters = self.get_running_total_base_date_filters(date_var_name='responsetime')
        if start_point_filters is None:
            return 0

        if extra_filters:
            for k, v in extra_filters.items():
                start_point_filters[k] = v

        return GuestBookResponse.objects.filter(**start_point_filters).count()

    def get_file_downloads_by_month(self, **extra_filters):
        """
        Using the GuestBookResponse object, find the number of file
        downloads per month
        """
        if self.was_error_found():
            return self.get_error_msg_return()

        filter_params = self.get_date_filter_params(date_var_name='responsetime')
        filter_params['downloadtype'] = RESPONSE_TYPE_DOWNLOAD

        # Add extra filters, if they exist
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        file_counts_by_month = GuestBookResponse.objects.filter(**filter_params\
            ).annotate(month_yyyy_dd=TruncYearMonth('responsetime')\
            ).values('month_yyyy_dd'\
            ).annotate(cnt=models.Count('id')\
            ).values('month_yyyy_dd', 'cnt'\
            ).order_by('%smonth_yyyy_dd' % self.time_sort)

        formatted_records = []  # move from a queryset to a []
        file_running_total = self.get_file_download_start_point(**extra_filters)
        for d in file_counts_by_month:
            file_running_total += d['cnt']
            d['running_total'] = file_running_total

            # Add year and month numbers
            d['year_num'] = int(d['month_yyyy_dd'][0:4])
            month_num = int(d['month_yyyy_dd'][5:])
            d['month_num'] = month_num

            # Add month name
            month_name_found, month_name = get_month_name_abbreviation(month_num)
            if month_name_found:
                d['month_name'] = month_name
            else:
                # Log it!!!!!!
                pass

            formatted_records.append(d)

        return True, formatted_records

    # ----------------------------
    #  Monthly files added
    # ----------------------------
    def get_file_count_by_month_published(self):
        """Published file counts by month"""

        return self.get_file_count_by_month(**self.get_is_published_filter_param())

    def get_file_downloads_by_month_unpublished(self):
        """Unpublished file counts by month"""

        return self.get_file_downloads_by_month(**self.get_is_NOT_published_filter_param())

    def get_file_count_start_point(self, **extra_filters):
        """Get the startpoint when keeping a running total of file downloads"""

        start_point_filters = self.get_running_total_base_date_filters()
        if start_point_filters is None:
            return 0

        if extra_filters:
            for k, v in extra_filters.items():
                start_point_filters[k] = v

        return Datafile.objects.select_related('dvobject').filter(**start_point_filters).count()

    def get_file_count_by_month(self, date_param='dvobject__createdate', **extra_filters):
        """
        File counts by month
        """
        # Was an error found earlier?
        #
        if self.was_error_found():
            return self.get_error_msg_return()

        # -----------------------------------
        # (1) Build query filters
        # -----------------------------------

        # Exclude records where dates are null
        #   - e.g. a record may not have a publication date
        if date_param == 'dvobject__createdate':
            exclude_params = {}
        else:
            exclude_params = { '%s__isnull' % date_param : True}

        # Retrieve the date parameters
        #
        filter_params = self.get_date_filter_params()

        # Add extra filters from kwargs
        #
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        # -----------------------------------
        # (2) Construct query
        # -----------------------------------

        # add exclude filters date filters
        #
        file_counts_by_month = Datafile.objects.select_related('dvobject'\
                            ).exclude(**exclude_params\
                            ).filter(**filter_params)

        # annotate query adding "month_yyyy_dd" and "cnt"
        #
        file_counts_by_month = file_counts_by_month.annotate(\
            month_yyyy_dd=TruncYearMonth('%s' % date_param)\
            ).values('month_yyyy_dd'\
            ).annotate(cnt=models.Count('dvobject_id')\
            ).values('month_yyyy_dd', 'cnt'\
            ).order_by('%smonth_yyyy_dd' % self.time_sort)

        #print file_counts_by_month.query

        # -----------------------------------
        # (3) Format results
        # -----------------------------------
        running_total = self.get_file_count_start_point(**extra_filters)   # hold the running total count
        formatted_records = []  # move from a queryset to a []

        for d in file_counts_by_month:
            # running total
            running_total += d['cnt']
            d['running_total'] = running_total

            # Add year and month numbers
            d['year_num'] = int(d['month_yyyy_dd'][0:4])
            month_num = int(d['month_yyyy_dd'][5:])
            d['month_num'] = month_num

            # Add month name
            month_name_found, month_name = get_month_name_abbreviation(month_num)
            if month_name_found:
                d['month_name'] = month_name
            else:
                # Log it!!!!!!
                pass

            # Add formatted record
            formatted_records.append(d)

        return True, formatted_records

    '''
    def get_number_of_datafile_types(self):
        """Return the number of distinct contenttypes found in Datafile objects"""
        if self.was_error_found():
            return self.get_error_msg_return()

        # Retrieve the date parameters
        #
        filter_params = self.get_date_filter_params('dvobject__createdate')

        datafile_counts_by_type = Datafile.objects.select_related('dvobject'\
                    ).filter(**filter_params\
                    ).values('contenttype'\
                    ).distinct().count()

        return True, dict(datafile_counts_by_type=datafile_counts_by_type)
    '''

    # ----------------------------
    #  Datafile counts by content type.
    #   e.g. how many .csv files, how many excel files, etc.
    # ----------------------------

    def get_datafile_content_type_counts_published(self):
        """Return datafile counts by 'content type' for published files"""

        return self.get_datafile_content_type_counts(\
            **self.get_is_published_filter_param())

    def get_datafile_content_type_counts_unpublished(self):
        """Return datafile counts by 'content type' for unpublished files"""

        return self.get_datafile_content_type_counts(\
            **self.get_is_NOT_published_filter_param())

    def get_datafile_content_type_counts(self, **extra_filters):
        """
        Return datafile counts by 'content type'

        "datafile_content_type_counts": [
                {
                    "total_count": 1584,
                    "contenttype": "text/tab-separated-values",
                    "type_count": 187,
                    "percent_string": "11.8%"
                },
                {
                    "total_count": 1584,
                    "contenttype": "image/jpeg",
                    "type_count": 182,
                    "percent_string": "11.5%"
                },
                {
                    "total_count": 1584,
                    "contenttype": "text/plain",
                    "type_count": 147,
                    "percent_string": "9.3%"
                }
            ]
        """
        if self.was_error_found():
            return self.get_error_msg_return()

        # Retrieve the date parameters
        #
        filter_params = self.get_date_filter_params('dvobject__createdate')

        # Add extra filters
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        datafile_counts_by_type = Datafile.objects.select_related('dvobject'\
                    ).filter(**filter_params\
                    ).values('contenttype'\
                    ).order_by('contenttype'\
                    ).annotate(type_count=models.Count('contenttype')\
                    ).order_by('-type_count')

        # Count all dataverses
        #
        total_count = sum([rec.get('type_count', 0) for rec in datafile_counts_by_type])
        total_count = total_count + 0.0

        # Format the records, adding 'total_count' and 'percent_string' to each one
        #
        formatted_records = []
        #num = 0
        for rec in datafile_counts_by_type:

            if total_count > 0:
                float_percent = rec.get('type_count', 0) / total_count
                rec['percent_string'] = '{0:.1%}'.format(float_percent)
                rec['total_count'] = int(total_count)
                #num+=1
                #rec['num'] = num
            formatted_records.append(rec)

        return True, formatted_records

    def get_files_per_dataset(self):
        """
        To do
        """

        # Pull file counts under each dataset
        files_per_dataset = DvObject.objects.filter(dtype=DTYPE_DATAFILE\
                    ).filter(**filter_params\
                    ).values('owner'\
                    ).annotate(parent_count=models.Count('owner')\
                    ).order_by('-parent_count')

        # Bin this data
<commit_msg>short content type added<commit_after>"""
Create metrics for Datasets.
This may be used for APIs, views with visualizations, etc.
"""
#from django.db.models.functions import TruncMonth  # 1.10
from collections import OrderedDict

from django.db import models
from django.db.models import Q

from dv_apps.utils.date_helper import get_month_name_abbreviation
from dv_apps.dvobjects.models import DvObject, DTYPE_DATASET, DTYPE_DATAFILE
from dv_apps.datafiles.models import Datafile
from dv_apps.guestbook.models import GuestBookResponse, RESPONSE_TYPE_DOWNLOAD
from dv_apps.metrics.stats_util_base import StatsMakerBase, TruncYearMonth

class StatsMakerFiles(StatsMakerBase):

    def __init__(self, **kwargs):
        """
        Start and end dates are optional.

        start_date = string in YYYY-MM-DD format
        end_date = string in YYYY-MM-DD format
        """
        super(StatsMakerFiles, self).__init__(**kwargs)

    # ----------------------------
    #  Datafile counts - single number
    # ----------------------------
    def get_datafile_count(self, **extra_filters):
        """
        Return the Datafile count
        """
        if self.was_error_found():
            return self.get_error_msg_return()

        filter_params = self.get_date_filter_params()

        # Add extra filters, if they exist
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        return True, Datafile.objects.filter(**filter_params).count()

    def get_datafile_count_published(self):
        """
        Return the count of published Dataverses
        """
        return self.get_datafile_count(**self.get_is_published_filter_param())

    def get_datafile_count_unpublished(self):
        """
        Return the count of published Dataverses
        """
        return self.get_datafile_count(**self.get_is_NOT_published_filter_param())

    # ----------------------------
    #  Monthly download counts
    # ----------------------------
    def get_file_downloads_by_month_published(self):
        """File downloads by month for published files"""

        params = self.get_is_published_filter_param(dvobject_var_name='datafile')

        return self.get_file_downloads_by_month(**params)

    def get_file_downloads_by_month_unpublished(self):
        """File downloads by month for unpublished files"""

        params = self.get_is_NOT_published_filter_param(dvobject_var_name='datafile')

        return self.get_file_downloads_by_month(**params)

    def get_file_download_start_point(self, **extra_filters):
        """Get the startpoint when keeping a running total of file downloads"""

        start_point_filters = self.get_running_total_base_date_filters(date_var_name='responsetime')
        if start_point_filters is None:
            return 0

        if extra_filters:
            for k, v in extra_filters.items():
                start_point_filters[k] = v

        return GuestBookResponse.objects.filter(**start_point_filters).count()

    def get_file_downloads_by_month(self, **extra_filters):
        """
        Using the GuestBookResponse object, find the number of file
        downloads per month
        """
        if self.was_error_found():
            return self.get_error_msg_return()

        filter_params = self.get_date_filter_params(date_var_name='responsetime')
        filter_params['downloadtype'] = RESPONSE_TYPE_DOWNLOAD

        # Add extra filters, if they exist
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        file_counts_by_month = GuestBookResponse.objects.filter(**filter_params\
            ).annotate(month_yyyy_dd=TruncYearMonth('responsetime')\
            ).values('month_yyyy_dd'\
            ).annotate(cnt=models.Count('id')\
            ).values('month_yyyy_dd', 'cnt'\
            ).order_by('%smonth_yyyy_dd' % self.time_sort)

        print 'file_counts_by_month.query', file_counts_by_month.query

        formatted_records = []  # move from a queryset to a []
        file_running_total = self.get_file_download_start_point(**extra_filters)
        for d in file_counts_by_month:
            file_running_total += d['cnt']
            d['running_total'] = file_running_total

            # Add year and month numbers
            d['year_num'] = int(d['month_yyyy_dd'][0:4])
            month_num = int(d['month_yyyy_dd'][5:])
            d['month_num'] = month_num

            # Add month name
            month_name_found, month_name = get_month_name_abbreviation(month_num)
            if month_name_found:
                d['month_name'] = month_name
            else:
                # Log it!!!!!!
                pass

            formatted_records.append(d)

        return True, formatted_records

    # ----------------------------
    #  Monthly files added
    # ----------------------------
    def get_file_count_by_month_published(self):
        """Published file counts by month"""

        return self.get_file_count_by_month(**self.get_is_published_filter_param())

    def get_file_downloads_by_month_unpublished(self):
        """Unpublished file counts by month"""

        return self.get_file_downloads_by_month(**self.get_is_NOT_published_filter_param())

    def get_file_count_start_point(self, **extra_filters):
        """Get the startpoint when keeping a running total of file downloads"""

        start_point_filters = self.get_running_total_base_date_filters()
        if start_point_filters is None:
            return 0

        if extra_filters:
            for k, v in extra_filters.items():
                start_point_filters[k] = v

        return Datafile.objects.select_related('dvobject').filter(**start_point_filters).count()

    def get_file_count_by_month(self, date_param='dvobject__createdate', **extra_filters):
        """
        File counts by month
        """
        # Was an error found earlier?
        #
        if self.was_error_found():
            return self.get_error_msg_return()

        # -----------------------------------
        # (1) Build query filters
        # -----------------------------------

        # Exclude records where dates are null
        #   - e.g. a record may not have a publication date
        if date_param == 'dvobject__createdate':
            exclude_params = {}
        else:
            exclude_params = { '%s__isnull' % date_param : True}

        # Retrieve the date parameters
        #
        filter_params = self.get_date_filter_params()

        # Add extra filters from kwargs
        #
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        # -----------------------------------
        # (2) Construct query
        # -----------------------------------

        # add exclude filters date filters
        #
        file_counts_by_month = Datafile.objects.select_related('dvobject'\
                            ).exclude(**exclude_params\
                            ).filter(**filter_params)

        # annotate query adding "month_yyyy_dd" and "cnt"
        #
        file_counts_by_month = file_counts_by_month.annotate(\
            month_yyyy_dd=TruncYearMonth('%s' % date_param)\
            ).values('month_yyyy_dd'\
            ).annotate(cnt=models.Count('dvobject_id')\
            ).values('month_yyyy_dd', 'cnt'\
            ).order_by('%smonth_yyyy_dd' % self.time_sort)

        #print file_counts_by_month.query

        # -----------------------------------
        # (3) Format results
        # -----------------------------------
        running_total = self.get_file_count_start_point(**extra_filters)   # hold the running total count
        formatted_records = []  # move from a queryset to a []

        for d in file_counts_by_month:
            # running total
            running_total += d['cnt']
            d['running_total'] = running_total

            # Add year and month numbers
            d['year_num'] = int(d['month_yyyy_dd'][0:4])
            month_num = int(d['month_yyyy_dd'][5:])
            d['month_num'] = month_num

            # Add month name
            month_name_found, month_name = get_month_name_abbreviation(month_num)
            if month_name_found:
                d['month_name'] = month_name
            else:
                # Log it!!!!!!
                pass

            # Add formatted record
            formatted_records.append(d)

        return True, formatted_records

    '''
    def get_number_of_datafile_types(self):
        """Return the number of distinct contenttypes found in Datafile objects"""
        if self.was_error_found():
            return self.get_error_msg_return()

        # Retrieve the date parameters
        #
        filter_params = self.get_date_filter_params('dvobject__createdate')

        datafile_counts_by_type = Datafile.objects.select_related('dvobject'\
                    ).filter(**filter_params\
                    ).values('contenttype'\
                    ).distinct().count()

        return True, dict(datafile_counts_by_type=datafile_counts_by_type)
    '''

    # ----------------------------
    #  Datafile counts by content type.
    #   e.g. how many .csv files, how many excel files, etc.
    # ----------------------------

    def get_datafile_content_type_counts_published(self):
        """Return datafile counts by 'content type' for published files"""

        return self.get_datafile_content_type_counts(\
            **self.get_is_published_filter_param())

    def get_datafile_content_type_counts_unpublished(self):
        """Return datafile counts by 'content type' for unpublished files"""

        return self.get_datafile_content_type_counts(\
            **self.get_is_NOT_published_filter_param())

    def get_datafile_content_type_counts(self, **extra_filters):
        """
        Return datafile counts by 'content type'

        "datafile_content_type_counts": [
                {
                    "total_count": 1584,
                    "contenttype": "text/tab-separated-values",
                    "type_count": 187,
                    "percent_string": "11.8%"
                },
                {
                    "total_count": 1584,
                    "contenttype": "image/jpeg",
                    "type_count": 182,
                    "percent_string": "11.5%"
                },
                {
                    "total_count": 1584,
                    "contenttype": "text/plain",
                    "type_count": 147,
                    "percent_string": "9.3%"
                }
            ]
        """
        if self.was_error_found():
            return self.get_error_msg_return()

        # Retrieve the date parameters
        #
        filter_params = self.get_date_filter_params('dvobject__createdate')

        # Add extra filters
        if extra_filters:
            for k, v in extra_filters.items():
                filter_params[k] = v

        datafile_counts_by_type = Datafile.objects.select_related('dvobject'\
                    ).filter(**filter_params\
                    ).values('contenttype'\
                    ).order_by('contenttype'\
                    ).annotate(type_count=models.Count('contenttype')\
                    ).order_by('-type_count')

        # Count all dataverses
        #
        total_count = sum([rec.get('type_count', 0) for rec in datafile_counts_by_type])
        total_count = total_count + 0.0

        # Format the records, adding 'total_count' and 'percent_string' to each one
        #
        formatted_records = []
        #num = 0
        for rec in datafile_counts_by_type:

            if total_count > 0:
                float_percent = rec.get('type_count', 0) / total_count
                rec['percent_string'] = '{0:.1%}'.format(float_percent)
                rec['total_count'] = int(total_count)

                contenttype_parts = rec['contenttype'].split('/')
                if len(contenttype_parts) > 1:
                    rec['short_content_type'] = '/'.join(contenttype_parts[1:])
                else:
                    rec['short_content_type'] = rec['contenttype']
                #num+=1
                #rec['num'] = num
            formatted_records.append(rec)

        return True, formatted_records

    def get_files_per_dataset(self):
        """
        To do
        """

        # Pull file counts under each dataset
        files_per_dataset = DvObject.objects.filter(dtype=DTYPE_DATAFILE\
                    ).filter(**filter_params\
                    ).values('owner'\
                    ).annotate(parent_count=models.Count('owner')\
                    ).order_by('-parent_count')

        # Bin this data
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from gzip import GzipFile
from StringIO import StringIO
from time import sleep, time
from urllib2 import build_opener, URLError

try:
    import oauth2 as oauth
except ImportError:
    oauth = None

from earwigbot import exceptions
from earwigbot.wiki.copyvios.markov import MarkovChain, MarkovChainIntersection
from earwigbot.wiki.copyvios.parsers import ArticleTextParser, HTMLTextParser
from earwigbot.wiki.copyvios.result import CopyvioCheckResult
from earwigbot.wiki.copyvios.search import YahooBOSSSearchEngine

__all__ = ["CopyvioMixIn"]

class CopyvioMixIn(object):
    """
    **EarwigBot: Wiki Toolset: Copyright Violation MixIn**

    This is a mixin that provides two public methods, :py:meth:`copyvio_check`
    and :py:meth:`copyvio_compare`. The former checks the page for copyright
    violations using a search engine API, and the latter compares the page
    against a given URL. Credentials for the search engine API are stored in
    the :py:class:`~earwigbot.wiki.site.Site`'s config.
    """

    def __init__(self, site):
        self._search_config = site._search_config
        self._exclusions_db = self._search_config["exclusions_db"]
        self._opener = build_opener()
        self._opener.addheaders = site._opener.addheaders

    def _open_url_ignoring_errors(self, url):
        """Open a URL using self._opener and return its content, or None.

        Will decompress the content if the headers contain "gzip" as its
        content encoding, and will return None if URLError is raised while
        opening the URL. IOErrors while gunzipping a compressed response are
        ignored, and the original content is returned.
        """
        try:
            response = self._opener.open(url)
        except URLError:
            return None
        result = response.read()

        if response.headers.get("Content-Encoding") == "gzip":
            stream = StringIO(result)
            gzipper = GzipFile(fileobj=stream)
            try:
                result = gzipper.read()
            except IOError:
                pass

        return result

    def _select_search_engine(self):
        """Return a function that can be called to do web searches.

        The function takes one argument, a search query, and returns a list of
        URLs, ranked by importance. The underlying logic depends on the
        *engine* argument within our config; for example, if *engine* is
        "Yahoo! BOSS", we'll use YahooBOSSSearchEngine for querying.

        Raises UnknownSearchEngineError if the 'engine' listed in our config is
        unknown to us, and UnsupportedSearchEngineError if we are missing a
        required package or module, like oauth2 for "Yahoo! BOSS".
        """
        engine = self._search_config["engine"]
        credentials = self._search_config["credentials"]

        if engine == "Yahoo! BOSS":
            if not oauth:
                e = "The package 'oauth2' could not be imported"
                raise exceptions.UnsupportedSearchEngineError(e)
            return YahooBOSSSearchEngine(credentials)

        raise exceptions.UnknownSearchEngineError(engine)

    def _copyvio_compare_content(self, article, url):
        """Return a number comparing an article and a URL.

        The *article* is a Markov chain, whereas the *url* is just a string
        that we'll try to open and read ourselves.
        """
        html = self._open_url_ignoring_errors(url)
        if not html:
            return 0

        source = MarkovChain(HTMLTextParser(html).strip())
        delta = MarkovChainIntersection(article, source)
        return float(delta.size()) / article.size(), (source, delta)

    def copyvio_check(self, min_confidence=0.5, max_queries=-1,
                      interquery_sleep=1):
        """Check the page for copyright violations.

        Returns a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object
        with information on the results of the check.

        *max_queries* is self-explanatory; we will never make more than this
        number of queries in a given check. If it's lower than 0, we will not
        limit the number of queries.

        *interquery_sleep* is the minimum amount of time we will sleep between
        search engine queries, in seconds.

        Raises :py:exc:`~earwigbot.exceptions.CopyvioCheckError` or subclasses
        (:py:exc:`~earwigbot.exceptions.UnknownSearchEngineError`,
        :py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors.
        """
        searcher = self._select_search_engine()
        self._exclusions_db.sync(self.site.name)
        handled_urls = []
        best_confidence = 0
        best_match = None
        num_queries = 0
        empty = MarkovChain("")
        best_chains = (empty, MarkovChainIntersection(empty, empty))
        parser = ArticleTextParser(self.get())
        clean = parser.strip()
        chunks = parser.chunk(self._search_config["nltk_dir"], max_queries)
        article_chain = MarkovChain(clean)
        last_query = time()

        if article_chain.size() < 20:  # Auto-fail very small articles
            return CopyvioCheckResult(False, best_confidence, best_match,
                                      num_queries, article_chain, best_chains)

        while (chunks and best_confidence < min_confidence and
               (max_queries < 0 or num_queries < max_queries)):
            chunk = chunks.pop(0)
            log = u"[[{0}]] -> querying {1} for {2!r}"
            self._logger.debug(log.format(self.title, searcher.name, chunk))
            urls = searcher.search(chunk)
            urls = [url for url in urls if url not in handled_urls]
            for url in urls:
                handled_urls.append(url)
                if self._exclusions_db.check(self.site.name, url):
                    continue
                conf, chains = self._copyvio_compare_content(article_chain, url)
                if conf > best_confidence:
                    best_confidence = conf
                    best_match = url
                    best_chains = chains
            num_queries += 1
            diff = time() - last_query
            if diff < interquery_sleep:
                sleep(interquery_sleep - diff)
            last_query = time()

        if best_confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2}; using {3} queries)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          best_match, num_queries))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; using {2} queries)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          num_queries))

        return CopyvioCheckResult(is_violation, best_confidence, best_match,
                                  num_queries, article_chain, best_chains)

    def copyvio_compare(self, url, min_confidence=0.5):
        """Check the page like :py:meth:`copyvio_check` against a specific URL.

        This is essentially a reduced version of the above - a copyivo
        comparison is made using Markov chains and the result is returned in a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object -
        but without using a search engine, since the suspected "violated" URL
        is supplied from the start.

        Its primary use is to generate a result when the URL is retrieved from
        a cache, like the one used in EarwigBot's Toolserver site. After a
        search is done, the resulting URL is stored in a cache for 24 hours so
        future checks against that page will not require another set of
        time-and-money-consuming search engine queries. However, the comparison
        itself (which includes the article's and the source's content) cannot
        be stored for data retention reasons, so a fresh comparison is made
        using this function.

        Since no searching is done, neither
        :py:exc:`~earwigbot.exceptions.UnknownSearchEngineError` nor
        :py:exc:`~earwigbot.exceptions.SearchQueryError` will be raised.
        """
        content = self.get()
        clean = ArticleTextParser(content).strip()
        article_chain = MarkovChain(clean)
        confidence, chains = self._copyvio_compare_content(article_chain, url)

        if confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2})"
            self._logger.debug(log.format(self.title, confidence, url))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; URL: {2})"
            self._logger.debug(log.format(self.title, confidence, url))

        return CopyvioCheckResult(is_violation, confidence, url, 0,
                                  article_chain, chains)
<commit_msg>Institute a timeout so we don't try to open these suspicious URLs forever.<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from gzip import GzipFile
from socket import timeout
from StringIO import StringIO
from time import sleep, time
from urllib2 import build_opener, URLError

try:
    import oauth2 as oauth
except ImportError:
    oauth = None

from earwigbot import exceptions
from earwigbot.wiki.copyvios.markov import MarkovChain, MarkovChainIntersection
from earwigbot.wiki.copyvios.parsers import ArticleTextParser, HTMLTextParser
from earwigbot.wiki.copyvios.result import CopyvioCheckResult
from earwigbot.wiki.copyvios.search import YahooBOSSSearchEngine

__all__ = ["CopyvioMixIn"]

class CopyvioMixIn(object):
    """
    **EarwigBot: Wiki Toolset: Copyright Violation MixIn**

    This is a mixin that provides two public methods, :py:meth:`copyvio_check`
    and :py:meth:`copyvio_compare`. The former checks the page for copyright
    violations using a search engine API, and the latter compares the page
    against a given URL. Credentials for the search engine API are stored in
    the :py:class:`~earwigbot.wiki.site.Site`'s config.
    """

    def __init__(self, site):
        self._search_config = site._search_config
        self._exclusions_db = self._search_config["exclusions_db"]
        self._opener = build_opener()
        self._opener.addheaders = site._opener.addheaders

    def _open_url_ignoring_errors(self, url):
        """Open a URL using self._opener and return its content, or None.

        Will decompress the content if the headers contain "gzip" as its
        content encoding, and will return None if URLError is raised while
        opening the URL. IOErrors while gunzipping a compressed response are
        ignored, and the original content is returned.
        """
        try:
            response = self._opener.open(url, timeout=5)
        except (URLError, timeout):
            return None
        result = response.read()

        if response.headers.get("Content-Encoding") == "gzip":
            stream = StringIO(result)
            gzipper = GzipFile(fileobj=stream)
            try:
                result = gzipper.read()
            except IOError:
                pass

        return result

    def _select_search_engine(self):
        """Return a function that can be called to do web searches.

        The function takes one argument, a search query, and returns a list of
        URLs, ranked by importance. The underlying logic depends on the
        *engine* argument within our config; for example, if *engine* is
        "Yahoo! BOSS", we'll use YahooBOSSSearchEngine for querying.

        Raises UnknownSearchEngineError if the 'engine' listed in our config is
        unknown to us, and UnsupportedSearchEngineError if we are missing a
        required package or module, like oauth2 for "Yahoo! BOSS".
        """
        engine = self._search_config["engine"]
        credentials = self._search_config["credentials"]

        if engine == "Yahoo! BOSS":
            if not oauth:
                e = "The package 'oauth2' could not be imported"
                raise exceptions.UnsupportedSearchEngineError(e)
            return YahooBOSSSearchEngine(credentials)

        raise exceptions.UnknownSearchEngineError(engine)

    def _copyvio_compare_content(self, article, url):
        """Return a number comparing an article and a URL.

        The *article* is a Markov chain, whereas the *url* is just a string
        that we'll try to open and read ourselves.
        """
        html = self._open_url_ignoring_errors(url)
        if not html:
            return 0

        source = MarkovChain(HTMLTextParser(html).strip())
        delta = MarkovChainIntersection(article, source)
        return float(delta.size()) / article.size(), (source, delta)

    def copyvio_check(self, min_confidence=0.5, max_queries=-1,
                      interquery_sleep=1):
        """Check the page for copyright violations.

        Returns a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object
        with information on the results of the check.

        *max_queries* is self-explanatory; we will never make more than this
        number of queries in a given check. If it's lower than 0, we will not
        limit the number of queries.

        *interquery_sleep* is the minimum amount of time we will sleep between
        search engine queries, in seconds.

        Raises :py:exc:`~earwigbot.exceptions.CopyvioCheckError` or subclasses
        (:py:exc:`~earwigbot.exceptions.UnknownSearchEngineError`,
        :py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors.
        """
        searcher = self._select_search_engine()
        self._exclusions_db.sync(self.site.name)
        handled_urls = []
        best_confidence = 0
        best_match = None
        num_queries = 0
        empty = MarkovChain("")
        best_chains = (empty, MarkovChainIntersection(empty, empty))
        parser = ArticleTextParser(self.get())
        clean = parser.strip()
        chunks = parser.chunk(self._search_config["nltk_dir"], max_queries)
        article_chain = MarkovChain(clean)
        last_query = time()

        if article_chain.size() < 20:  # Auto-fail very small articles
            return CopyvioCheckResult(False, best_confidence, best_match,
                                      num_queries, article_chain, best_chains)

        while (chunks and best_confidence < min_confidence and
               (max_queries < 0 or num_queries < max_queries)):
            chunk = chunks.pop(0)
            log = u"[[{0}]] -> querying {1} for {2!r}"
            self._logger.debug(log.format(self.title, searcher.name, chunk))
            urls = searcher.search(chunk)
            urls = [url for url in urls if url not in handled_urls]
            for url in urls:
                handled_urls.append(url)
                if self._exclusions_db.check(self.site.name, url):
                    continue
                conf, chains = self._copyvio_compare_content(article_chain, url)
                if conf > best_confidence:
                    best_confidence = conf
                    best_match = url
                    best_chains = chains
            num_queries += 1
            diff = time() - last_query
            if diff < interquery_sleep:
                sleep(interquery_sleep - diff)
            last_query = time()

        if best_confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2}; using {3} queries)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          best_match, num_queries))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; using {2} queries)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          num_queries))

        return CopyvioCheckResult(is_violation, best_confidence, best_match,
                                  num_queries, article_chain, best_chains)

    def copyvio_compare(self, url, min_confidence=0.5):
        """Check the page like :py:meth:`copyvio_check` against a specific URL.

        This is essentially a reduced version of the above - a copyivo
        comparison is made using Markov chains and the result is returned in a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object -
        but without using a search engine, since the suspected "violated" URL
        is supplied from the start.

        Its primary use is to generate a result when the URL is retrieved from
        a cache, like the one used in EarwigBot's Toolserver site. After a
        search is done, the resulting URL is stored in a cache for 24 hours so
        future checks against that page will not require another set of
        time-and-money-consuming search engine queries. However, the comparison
        itself (which includes the article's and the source's content) cannot
        be stored for data retention reasons, so a fresh comparison is made
        using this function.

        Since no searching is done, neither
        :py:exc:`~earwigbot.exceptions.UnknownSearchEngineError` nor
        :py:exc:`~earwigbot.exceptions.SearchQueryError` will be raised.
        """
        content = self.get()
        clean = ArticleTextParser(content).strip()
        article_chain = MarkovChain(clean)
        confidence, chains = self._copyvio_compare_content(article_chain, url)

        if confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2})"
            self._logger.debug(log.format(self.title, confidence, url))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; URL: {2})"
            self._logger.debug(log.format(self.title, confidence, url))

        return CopyvioCheckResult(is_violation, confidence, url, 0,
                                  article_chain, chains)
<|endoftext|>


<commit_before>import inspect
from collections import defaultdict, namedtuple
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional

import pydantic

from ninja import UploadedFile
from ninja.params import File
from ninja.compatibility.util import get_origin as get_collection_origin

if TYPE_CHECKING:
    from pydantic.fields import ModelField  # pragma: no cover

from ninja import params
from ninja.signature.utils import get_path_param_names, get_typed_signature

__all__ = [
    "FuncParam",
    "ViewSignature",
    "is_pydantic_model",
    "is_collection_type",
    "detect_collection_fields",
]

FuncParam = namedtuple("FuncParam", ["name", "source", "annotation", "is_collection"])

class ViewSignature:
    def __init__(self, path: str, view_func: Callable) -> None:
        self.view_func = view_func
        self.signature = get_typed_signature(self.view_func)
        self.path_params_names = get_path_param_names(path)
        self.docstring = inspect.cleandoc(view_func.__doc__ or "")
        self.has_kwargs = False

        self.params = []
        for name, arg in self.signature.parameters.items():
            if name == "request":
                # TODO: maybe better assert that 1st param is request or check by type?
                # maybe even have attribute like `has_request`
                # so that users can ignore passing request if not needed
                continue

            if arg.kind == arg.VAR_KEYWORD:
                # Skipping **kwargs
                self.has_kwargs = True
                continue

            if arg.kind == arg.VAR_POSITIONAL:
                # Skipping *args
                continue

            func_param = self._get_param_type(name, arg)
            self.params.append(func_param)

        if hasattr(view_func, "_ninja_contribute_args"):
            # _ninja_contribute_args is a special attribute
            # which allows developers to create custom function params
            # inside decorators or other functions
            for p_name, p_type, p_source in view_func._ninja_contribute_args:  # type: ignore
                self.params.append(FuncParam(p_name, p_source, p_type, False))

        self.models = self._create_models()

    def _create_models(self) -> List[Any]:
        grouping: Dict[Any, List[FuncParam]] = defaultdict(list)
        for param in self.params:
            d_type = type(param.source)
            grouping[d_type].append(param)

        result = []
        for cls, args in grouping.items():
            cls_name: str = cls.__name__ + "Params"
            attrs = {i.name: i.source for i in args}
            attrs["_in"] = cls._in()

            if len(args) == 1:
                if cls._in() == "body" or is_pydantic_model(args[0].annotation):
                    attrs["_single_attr"] = args[0].name

            elif cls._in() == "query":
                pydantic_models = [
                    arg for arg in args if is_pydantic_model(arg.annotation)
                ]
                if pydantic_models:
                    mixed_attrs = {}
                    for modeled_attr in pydantic_models:
                        for (
                            attr_name,
                            field,
                        ) in modeled_attr.annotation.__fields__.items():
                            mixed_attrs[attr_name] = modeled_attr.name
                            if hasattr(field, "alias"):
                                mixed_attrs[field.alias] = modeled_attr.name

                    attrs["_mixed_attrs"] = mixed_attrs

            # adding annotations:
            attrs["__annotations__"] = {i.name: i.annotation for i in args}

            # collection fields:
            attrs["_collection_fields"] = detect_collection_fields(args)

            base_cls = cls._model
            model_cls = type(cls_name, (base_cls,), attrs)
            # TODO: https://pydantic-docs.helpmanual.io/usage/models/#dynamic-model-creation - check if anything special in create_model method that I did not use
            result.append(model_cls)
        return result

    def _get_param_type(self, name: str, arg: inspect.Parameter) -> FuncParam:
        # _EMPTY = self.signature.empty
        annotation = arg.annotation

        if annotation == self.signature.empty:
            if arg.default == self.signature.empty:
                annotation = str
            else:
                if isinstance(arg.default, params.Param):
                    annotation = type(arg.default.default)
                else:
                    annotation = type(arg.default)

        if annotation == type(None) or annotation == type(Ellipsis):  # noqa
            annotation = str

        is_collection = is_collection_type(annotation)

        if annotation == UploadedFile or (
            is_collection and annotation.__args__[0] == UploadedFile
        ):
            # People often forgot to mark UploadedFile as a File, so we better assign it automatically
            if arg.default == self.signature.empty or arg.default is None:
                default = arg.default == self.signature.empty and ... or arg.default
                return FuncParam(name, File(default), annotation, is_collection)

        # 1) if type of the param is defined as one of the Param's subclasses - we just use that definition
        if isinstance(arg.default, params.Param):
            param_source = arg.default

        # 2) if param name is a part of the path parameter
        elif name in self.path_params_names:
            assert (
                arg.default == self.signature.empty
            ), f"'{name}' is a path param, default not allowed"
            param_source = params.Path(...)

        # 3) if param is a collection or annotation is part of pydantic model:
        elif is_collection or is_pydantic_model(annotation):
            if arg.default == self.signature.empty:
                param_source = params.Body(...)
            else:
                param_source = params.Body(arg.default)

        # 4) the last case is query param
        else:
            if arg.default == self.signature.empty:
                param_source = params.Query(...)
            else:
                param_source = params.Query(arg.default)

        return FuncParam(name, param_source, annotation, is_collection)

def is_pydantic_model(cls: Any) -> bool:
    try:
        return issubclass(cls, pydantic.BaseModel)
    except TypeError:
        return False

def is_collection_type(annotation: Any) -> bool:
    origin = get_collection_origin(annotation)
    return origin in (
        List,
        list,
        set,
        tuple,
    )  # TODO: I guess we should handle only list

def detect_pydantic_model_collection_fields(model: pydantic.BaseModel) -> List[str]:
    "Extracts collection fields aliases from collection fields"

    def _list_field_name(field: "ModelField") -> Optional[str]:
        if is_collection_type(field.outer_type_):
            return str(field.alias)
        return None

    return list(filter(None, map(_list_field_name, model.__fields__.values())))

def detect_collection_fields(args: List[FuncParam]) -> List[str]:
    """
    QueryDict has values that are always lists, so we need to help django ninja to understand
    better the input parameters if it's a list or a single value
    This method detects attributes that should be treated by ninja as lists and returns this list as a result
    """
    result = [i.name for i in args if i.is_collection]

    if len(args) == 1 and is_pydantic_model(args[0].annotation):
        result += detect_pydantic_model_collection_fields(args[0].annotation)

    return result
<commit_msg>Make isort happier?<commit_after>import inspect
from collections import defaultdict, namedtuple
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional

import pydantic

from ninja import UploadedFile, params
from ninja.compatibility.util import get_origin as get_collection_origin
from ninja.params import File
from ninja.signature.utils import get_path_param_names, get_typed_signature

if TYPE_CHECKING:
    from pydantic.fields import ModelField  # pragma: no cover

__all__ = [
    "FuncParam",
    "ViewSignature",
    "is_pydantic_model",
    "is_collection_type",
    "detect_collection_fields",
]

FuncParam = namedtuple("FuncParam", ["name", "source", "annotation", "is_collection"])

class ViewSignature:
    def __init__(self, path: str, view_func: Callable) -> None:
        self.view_func = view_func
        self.signature = get_typed_signature(self.view_func)
        self.path_params_names = get_path_param_names(path)
        self.docstring = inspect.cleandoc(view_func.__doc__ or "")
        self.has_kwargs = False

        self.params = []
        for name, arg in self.signature.parameters.items():
            if name == "request":
                # TODO: maybe better assert that 1st param is request or check by type?
                # maybe even have attribute like `has_request`
                # so that users can ignore passing request if not needed
                continue

            if arg.kind == arg.VAR_KEYWORD:
                # Skipping **kwargs
                self.has_kwargs = True
                continue

            if arg.kind == arg.VAR_POSITIONAL:
                # Skipping *args
                continue

            func_param = self._get_param_type(name, arg)
            self.params.append(func_param)

        if hasattr(view_func, "_ninja_contribute_args"):
            # _ninja_contribute_args is a special attribute
            # which allows developers to create custom function params
            # inside decorators or other functions
            for p_name, p_type, p_source in view_func._ninja_contribute_args:  # type: ignore
                self.params.append(FuncParam(p_name, p_source, p_type, False))

        self.models = self._create_models()

    def _create_models(self) -> List[Any]:
        grouping: Dict[Any, List[FuncParam]] = defaultdict(list)
        for param in self.params:
            d_type = type(param.source)
            grouping[d_type].append(param)

        result = []
        for cls, args in grouping.items():
            cls_name: str = cls.__name__ + "Params"
            attrs = {i.name: i.source for i in args}
            attrs["_in"] = cls._in()

            if len(args) == 1:
                if cls._in() == "body" or is_pydantic_model(args[0].annotation):
                    attrs["_single_attr"] = args[0].name

            elif cls._in() == "query":
                pydantic_models = [
                    arg for arg in args if is_pydantic_model(arg.annotation)
                ]
                if pydantic_models:
                    mixed_attrs = {}
                    for modeled_attr in pydantic_models:
                        for (
                            attr_name,
                            field,
                        ) in modeled_attr.annotation.__fields__.items():
                            mixed_attrs[attr_name] = modeled_attr.name
                            if hasattr(field, "alias"):
                                mixed_attrs[field.alias] = modeled_attr.name

                    attrs["_mixed_attrs"] = mixed_attrs

            # adding annotations:
            attrs["__annotations__"] = {i.name: i.annotation for i in args}

            # collection fields:
            attrs["_collection_fields"] = detect_collection_fields(args)

            base_cls = cls._model
            model_cls = type(cls_name, (base_cls,), attrs)
            # TODO: https://pydantic-docs.helpmanual.io/usage/models/#dynamic-model-creation - check if anything special in create_model method that I did not use
            result.append(model_cls)
        return result

    def _get_param_type(self, name: str, arg: inspect.Parameter) -> FuncParam:
        # _EMPTY = self.signature.empty
        annotation = arg.annotation

        if annotation == self.signature.empty:
            if arg.default == self.signature.empty:
                annotation = str
            else:
                if isinstance(arg.default, params.Param):
                    annotation = type(arg.default.default)
                else:
                    annotation = type(arg.default)

        if annotation == type(None) or annotation == type(Ellipsis):  # noqa
            annotation = str

        is_collection = is_collection_type(annotation)

        if annotation == UploadedFile or (
            is_collection and annotation.__args__[0] == UploadedFile
        ):
            # People often forgot to mark UploadedFile as a File, so we better assign it automatically
            if arg.default == self.signature.empty or arg.default is None:
                default = arg.default == self.signature.empty and ... or arg.default
                return FuncParam(name, File(default), annotation, is_collection)

        # 1) if type of the param is defined as one of the Param's subclasses - we just use that definition
        if isinstance(arg.default, params.Param):
            param_source = arg.default

        # 2) if param name is a part of the path parameter
        elif name in self.path_params_names:
            assert (
                arg.default == self.signature.empty
            ), f"'{name}' is a path param, default not allowed"
            param_source = params.Path(...)

        # 3) if param is a collection or annotation is part of pydantic model:
        elif is_collection or is_pydantic_model(annotation):
            if arg.default == self.signature.empty:
                param_source = params.Body(...)
            else:
                param_source = params.Body(arg.default)

        # 4) the last case is query param
        else:
            if arg.default == self.signature.empty:
                param_source = params.Query(...)
            else:
                param_source = params.Query(arg.default)

        return FuncParam(name, param_source, annotation, is_collection)

def is_pydantic_model(cls: Any) -> bool:
    try:
        return issubclass(cls, pydantic.BaseModel)
    except TypeError:
        return False

def is_collection_type(annotation: Any) -> bool:
    origin = get_collection_origin(annotation)
    return origin in (
        List,
        list,
        set,
        tuple,
    )  # TODO: I guess we should handle only list

def detect_pydantic_model_collection_fields(model: pydantic.BaseModel) -> List[str]:
    "Extracts collection fields aliases from collection fields"

    def _list_field_name(field: "ModelField") -> Optional[str]:
        if is_collection_type(field.outer_type_):
            return str(field.alias)
        return None

    return list(filter(None, map(_list_field_name, model.__fields__.values())))

def detect_collection_fields(args: List[FuncParam]) -> List[str]:
    """
    QueryDict has values that are always lists, so we need to help django ninja to understand
    better the input parameters if it's a list or a single value
    This method detects attributes that should be treated by ninja as lists and returns this list as a result
    """
    result = [i.name for i in args if i.is_collection]

    if len(args) == 1 and is_pydantic_model(args[0].annotation):
        result += detect_pydantic_model_collection_fields(args[0].annotation)

    return result
<|endoftext|>


<commit_before>import json
import urllib2
from flask import Flask, render_template, jsonify, request

app = Flask(__name__)

@app.route('/')
def index():
    return render_template("gh-index.html")

@app.route('/calculate')
def calculate():
    requestdata = request.args.get("a")
    data1 = urllib2.urlopen(
        "https://api.github.com/users/%s/repos?per_page=100&page=1"
        % requestdata)
    data2 = urllib2.urlopen(
        "https://api.github.com/users/%s/repos?per_page=100&page=2"
        % requestdata)
    data3 = urllib2.urlopen(
        "https://api.github.com/users/%s/repos?per_page=100&page=3"
        % requestdata)

    repolist = []
    countlist = []
    readdata1 = data1.read()
    readdata2 = data2.read()
    readdata3 = data3.read()
    jsondata1 = json.loads(readdata1)
    jsondata2 = json.loads(readdata2)
    jsondata3 = json.loads(readdata3)
    jsondata = jsondata1 + jsondata2 + jsondata3

    for repo in jsondata:
        for k, v in repo.iteritems():
            if k == "stargazers_count" and v != 0:
                repolist.append(v)

    if repolist == []:
        return jsonify(result="No stars found.")
    else:
        sortedlist = sorted(repolist)

        print sortedlist
        for item in sortedlist:
            if len(sortedlist[sortedlist.index(item):]) >= item:
                countlist.append(item)

        print countlist
        return jsonify(result=str(max(countlist)))

if __name__ == '__main__':
    app.run(port=8000, host='127.0.0.1', debug=True)
<commit_msg>more dry<commit_after>import json
import urllib2
from flask import Flask, render_template, jsonify, request

app = Flask(__name__)

# load the html page
@app.route('/')
def index():
    return render_template("gh-index.html")

@app.route('/calculate')
def calculate():

    # create some empty lists
    data = []
    repolist = []
    jsondata = []
    countlist = []

    # take the input from the form and make the api call
    # the api breaks results into pages. iterate through pages
    requestdata = request.args.get("a")
    for i in range(1, 3):
        data.append(urllib2.urlopen(
                    "https://api.github.com/users/{}/repos?per_page=100&page={}"
                    .format(requestdata, i)))

    # turn the api call data into json
    for i in data:
        jsondata.append(json.loads(i.read()))

    # get the stargazers counts from the json
    for call in jsondata:
        for repo in call:
            for k, v in repo.iteritems():
                print(k, v)
                if k == "stargazers_count" and v != 0:
                    repolist.append(v)

    # sometimes there are no stars :(
    if repolist == []:
        return jsonify(result="No stars found.")
    else:
        sortedlist = sorted(repolist)

        # calculate the h-index
        print sortedlist
        for item in sortedlist:
            if len(sortedlist[sortedlist.index(item):]) >= item:
                countlist.append(item)

        # return the h-index value
        print countlist
        return jsonify(result=str(max(countlist)))

if __name__ == '__main__':
    app.run(port=8000, host='127.0.0.1', debug=True)
<|endoftext|>


<commit_before>from vcs.exceptions import VCSError
from vcs.nodes import FileNode
from pygments.formatters import HtmlFormatter
from pygments import highlight

try:
    import cStringIO as StringIO
except ImportError:
    import StringIO

def annotate_highlight(filenode, annotate_from_changeset_func=None,
        order=None, headers=None, **options):
    """
    Returns html portion containing annotated table with 3 columns: line
    numbers, changeset information and pygmentized line of code.

    :param filenode: FileNode object
    :param annotate_from_changeset_func: function taking changeset and returning
      single annotate cell; needs break line at the end
    :param order: ordered sequence of ``ls`` (line numbers column), ``annotate``
      (annotate column), ``code`` (code column); Default is ``['ls', 'annotate',
      'code']``
    :param headers: dictionary with headers (keys are whats in ``order``
      parameter)
    """
    options['linenos'] = True
    formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
        headers=headers,
        annotate_from_changeset_func=annotate_from_changeset_func, **options)
    lexer = filenode.lexer
    highlighted = highlight(filenode.content, lexer, formatter)
    return highlighted

class AnnotateHtmlFormatter(HtmlFormatter):

    def __init__(self, filenode, annotate_from_changeset_func=None,
            order=None, **options):
        """
        If ``annotate_from_changeset_func`` is passed it should be a function
        which returns string from the given changeset. For example, we may pass
        following function as ``annotate_from_changeset_func``::

            def changeset_to_anchor(changeset):
                return '%s\n' %\
                       (changeset.id, changeset.id)

        :param annotate_from_changeset_func: see above
        :param order: (default: ``['ls', 'annotate', 'code']``); order of
          columns;
        :param options: standard pygment's HtmlFormatter options, there is
          extra option tough, ``headers``. For instance we can pass::

             formatter = AnnotateHtmlFormatter(filenode, headers={
                'ls': '#',
                'annotate': 'Annotate',
                'code': 'Code',
             })

        """
        super(AnnotateHtmlFormatter, self).__init__(**options)
        self.annotate_from_changeset_func = annotate_from_changeset_func
        self.order = order or ('ls', 'annotate', 'code')
        headers = options.pop('headers', None)
        if headers and not ('ls' in headers and 'annotate' in headers and
            'code' in headers):
            raise ValueError("If headers option dict is specified it must "
                "all 'ls', 'annotate' and 'code' keys")
        self.headers = headers
        if isinstance(filenode, FileNode):
            self.filenode = filenode
        else:
            raise VCSError("This formatter expect FileNode parameter, not %r"
                % type(filenode))

    def annotate_from_changeset(self, changeset):
        """
        Returns full html line for single changeset per annotated line.
        """
        if self.annotate_from_changeset_func:
            return self.annotate_from_changeset_func(changeset)
        else:
            return ''.join((changeset.id, '\n'))

    def _wrap_tablelinenos(self, inner):
        dummyoutfile = StringIO.StringIO()
        lncount = 0
        for t, line in inner:
            if t:
                lncount += 1
            dummyoutfile.write(line)

        fl = self.linenostart
        mw = len(str(lncount + fl - 1))
        sp = self.linenospecial
        st = self.linenostep
        la = self.lineanchors
        aln = self.anchorlinenos
        if sp:
            lines = []

            for i in range(fl, fl+lncount):
                if i % st == 0:
                    if i % sp == 0:
                        if aln:
                            lines.append('%*d' %
                                         (la, i, mw, i))
                        else:
                            lines.append('%*d' % (mw, i))
                    else:
                        if aln:
                            lines.append('%*d' % (la, i, mw, i))
                        else:
                            lines.append('%*d' % (mw, i))
                else:
                    lines.append('')
            ls = '\n'.join(lines)
        else:
            lines = []
            for i in range(fl, fl+lncount):
                if i % st == 0:
                    if aln:
                        lines.append('%*d' % (la, i, mw, i))
                    else:
                        lines.append('%*d' % (mw, i))
                else:
                    lines.append('')
            ls = '\n'.join(lines)

        annotate_changesets = [tup[1] for tup in self.filenode.annotate]
        # If pygments cropped last line break we need do that too
        if len(annotate_changesets) > len(ls.splitlines()):
            annotate_changesets = annotate_changesets[:-1]
        annotate = ''.join((self.annotate_from_changeset(changeset)
            for changeset in annotate_changesets))
        # in case you wonder about the seemingly redundant
 here: since the
        # content in the other cell also is wrapped in a div, some browsers in
        # some configurations seem to mess up the formatting...
        '''
        yield 0, ('
' % self.cssclass +
                  '

' +
                  ls + '

	' +
                  '')
        yield 0, dummyoutfile.getvalue()
        yield 0, '

'

        '''
        headers_row = []
        if self.headers:
            headers_row = ['
']
            for key in self.order:
                td = ''.join(('', self.headers[key], '	'))
                headers_row.append(td)
            headers_row.append('
')

        body_row_start = ['
']
        for key in self.order:
            if key == 'ls':
                body_row_start.append(
                    '

' +
                    ls + '

	')
            elif key == 'annotate':
                body_row_start.append(
                    '

' +
                    annotate + '

	')
            elif key == 'code':
                body_row_start.append('')
        yield 0, ('
' % self.cssclass +
                  ''.join(headers_row) +
                  ''.join(body_row_start)
                  )
        yield 0, dummyoutfile.getvalue()
        yield 0, '

'

<commit_msg>fixed bug when annotation raises UnicodeDecode error, CstringIO implementation brokes that.<commit_after>from vcs.exceptions import VCSError
from vcs.nodes import FileNode
from pygments.formatters import HtmlFormatter
from pygments import highlight

import StringIO

def annotate_highlight(filenode, annotate_from_changeset_func=None,
        order=None, headers=None, **options):
    """
    Returns html portion containing annotated table with 3 columns: line
    numbers, changeset information and pygmentized line of code.

    :param filenode: FileNode object
    :param annotate_from_changeset_func: function taking changeset and returning
      single annotate cell; needs break line at the end
    :param order: ordered sequence of ``ls`` (line numbers column), ``annotate``
      (annotate column), ``code`` (code column); Default is ``['ls', 'annotate',
      'code']``
    :param headers: dictionary with headers (keys are whats in ``order``
      parameter)
    """
    options['linenos'] = True
    formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
        headers=headers,
        annotate_from_changeset_func=annotate_from_changeset_func, **options)
    lexer = filenode.lexer
    highlighted = highlight(filenode.content, lexer, formatter)
    return highlighted

class AnnotateHtmlFormatter(HtmlFormatter):

    def __init__(self, filenode, annotate_from_changeset_func=None,
            order=None, **options):
        """
        If ``annotate_from_changeset_func`` is passed it should be a function
        which returns string from the given changeset. For example, we may pass
        following function as ``annotate_from_changeset_func``::

            def changeset_to_anchor(changeset):
                return '%s\n' %\
                       (changeset.id, changeset.id)

        :param annotate_from_changeset_func: see above
        :param order: (default: ``['ls', 'annotate', 'code']``); order of
          columns;
        :param options: standard pygment's HtmlFormatter options, there is
          extra option tough, ``headers``. For instance we can pass::

             formatter = AnnotateHtmlFormatter(filenode, headers={
                'ls': '#',
                'annotate': 'Annotate',
                'code': 'Code',
             })

        """
        super(AnnotateHtmlFormatter, self).__init__(**options)
        self.annotate_from_changeset_func = annotate_from_changeset_func
        self.order = order or ('ls', 'annotate', 'code')
        headers = options.pop('headers', None)
        if headers and not ('ls' in headers and 'annotate' in headers and
            'code' in headers):
            raise ValueError("If headers option dict is specified it must "
                "all 'ls', 'annotate' and 'code' keys")
        self.headers = headers
        if isinstance(filenode, FileNode):
            self.filenode = filenode
        else:
            raise VCSError("This formatter expect FileNode parameter, not %r"
                % type(filenode))

    def annotate_from_changeset(self, changeset):
        """
        Returns full html line for single changeset per annotated line.
        """
        if self.annotate_from_changeset_func:
            return self.annotate_from_changeset_func(changeset)
        else:
            return ''.join((changeset.id, '\n'))

    def _wrap_tablelinenos(self, inner):
        dummyoutfile = StringIO.StringIO()
        lncount = 0
        for t, line in inner:
            if t:
                lncount += 1
            dummyoutfile.write(line)

        fl = self.linenostart
        mw = len(str(lncount + fl - 1))
        sp = self.linenospecial
        st = self.linenostep
        la = self.lineanchors
        aln = self.anchorlinenos
        if sp:
            lines = []

            for i in range(fl, fl + lncount):
                if i % st == 0:
                    if i % sp == 0:
                        if aln:
                            lines.append('%*d' %
                                         (la, i, mw, i))
                        else:
                            lines.append('%*d' % (mw, i))
                    else:
                        if aln:
                            lines.append('%*d' % (la, i, mw, i))
                        else:
                            lines.append('%*d' % (mw, i))
                else:
                    lines.append('')
            ls = '\n'.join(lines)
        else:
            lines = []
            for i in range(fl, fl + lncount):
                if i % st == 0:
                    if aln:
                        lines.append('%*d' % (la, i, mw, i))
                    else:
                        lines.append('%*d' % (mw, i))
                else:
                    lines.append('')
            ls = '\n'.join(lines)

        annotate_changesets = [tup[1] for tup in self.filenode.annotate]
        # If pygments cropped last line break we need do that too
        if len(annotate_changesets) > len(ls.splitlines()):
            annotate_changesets = annotate_changesets[:-1]
        annotate = ''.join((self.annotate_from_changeset(changeset)
            for changeset in annotate_changesets))
        # in case you wonder about the seemingly redundant
 here: since the
        # content in the other cell also is wrapped in a div, some browsers in
        # some configurations seem to mess up the formatting...
        '''
        yield 0, ('
' % self.cssclass +
                  '

' +
                  ls + '

	' +
                  '')
        yield 0, dummyoutfile.getvalue()
        yield 0, '

'

        '''
        headers_row = []
        if self.headers:
            headers_row = ['
']
            for key in self.order:
                td = ''.join(('', self.headers[key], '	'))
                headers_row.append(td)
            headers_row.append('
')

        body_row_start = ['
']
        for key in self.order:
            if key == 'ls':
                body_row_start.append(
                    '

' +
                    ls + '

	')
            elif key == 'annotate':
                body_row_start.append(
                    '

' +
                    annotate + '

	')
            elif key == 'code':
                body_row_start.append('')
        yield 0, ('
' % self.cssclass +
                  ''.join(headers_row) +
                  ''.join(body_row_start)
                  )
        yield 0, dummyoutfile.getvalue()
        yield 0, '

'

<|endoftext|>