<commit_before>import csv
import traceback
import simplejson
from django.db import models
from django.contrib.auth.models import User

class SmartModel(models.Model):
    """
    Useful abstract base class that adds the concept of something being active,
    having a user that created or modified the item and creation and modification
    dates.
    """
    is_active = models.BooleanField(default=True,
                                    help_text="Whether this item is active, use this instead of deleting")

    created_by = models.ForeignKey(User, related_name="%(class)s_creations",
                                   help_text="The user which originally created this item")
    created_on = models.DateTimeField(auto_now_add=True,
                                      help_text="When this item was originally created")

    modified_by = models.ForeignKey(User, related_name="%(class)s_modifications",
                                    help_text="The user which last modified this item")
    modified_on = models.DateTimeField(auto_now=True,
                                       help_text="When this item was last modified")

    class Meta:
        abstract = True

    @classmethod
    def prepare_fields(cls, field_dict, import_params=None, user=None):
        return field_dict

    @classmethod
    def create_instance(cls, field_dict):
        return cls.objects.create(**field_dict)

    @classmethod
    def import_csv(cls, task, log=None):

        from xlrd import XLRDError
        filename = task.csv_file.file
        user = task.created_by

        import_params = None

        # additional parameters are optional
        if task.import_params:
            try:
                import_params = simplejson.loads(task.import_params)
            except:
                pass

        try:
            records = cls.import_xls(filename, user, import_params, log)
        except XLRDError:
            records = cls.import_raw_csv(filename, user, import_params, log)

        return records

    @classmethod
    def normalize_value(cls, val):
        # remove surrounding whitespace
        val = val.strip()

        # if surrounded by double quotes, remove those
        if val and val[0] == '"' and val[-1] == '"':
            val = val[1:-1]

        # if surrounded by single quotes, remove those
        if val and val[0] == "'" and val[-1] == "'":
            val = val[1:-1]

        return val

    @classmethod
    def import_xls(cls, filename, user, import_params, log=None):
        from xlrd import open_workbook
        workbook = open_workbook(filename.name, 'rb')

        records = []

        for sheet in workbook.sheets():

            # read our header
            header = []
            for col in range(sheet.ncols):
                header.append(str(sheet.cell(0, col).value))
            header = [cls.normalize_value(_).lower() for _ in header]

            # read our rows
            line_number = 1
            for row in range(sheet.nrows - 1):
                field_values = []
                for col in range(sheet.ncols):
                    field_values.append(str(sheet.cell(row + 1, col).value))

                field_values = [cls.normalize_value(_) for _ in field_values]
                field_values = dict(zip(header, field_values))
                field_values['created_by'] = user
                field_values['modified_by'] = user

                try:
                    field_values = cls.prepare_fields(field_values, import_params, user)
                    record = cls.create_instance(field_values)
                    if record:
                        records.append(record)
                except Exception as e:
                    if log:
                        traceback.print_exc(100, log)
                    raise Exception("Line %d: %s\n\n%s" % (line_number, str(e), field_values))
                line_number += 1
            # only care about the first sheet
            break

        return records

    @classmethod
    def import_raw_csv(cls, filename, user, import_params, log=None):
        # our alternative codec, by default we are the crazy windows encoding
        ascii_codec = 'cp1252'

        # read the entire file, look for mac_roman characters
        reader = open(filename.name, "rb")
        for byte in reader.read():
            # these are latin accented characterse in mac_roman, if we see them then our alternative
            # encoding should be mac_roman
            if ord(byte) in [0x81, 0x8d, 0x8f, 0x90, 0x9d]:
                ascii_codec = 'mac_roman'
                break
        reader.close()

        reader = open(filename.name, "rU")

        def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
            csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
            for row in csv_reader:
                encoded = []
                for cell in row:
                    try:
                        cell = unicode(cell)
                    except:
                        cell = unicode(cell.decode(ascii_codec))

                    encoded.append(cell)

                yield encoded

        reader = unicode_csv_reader(reader)

        # read in our header
        line_number = 0

        header = reader.next()
        line_number += 1
        while header is not None and len(header[0]) > 1 and header[0][0] == "#":
            header = reader.next()
            line_number += 1

        # do some sanity checking to make sure they uploaded the right kind of file
        if len(header) < 1:
            raise Exception("Invalid header for import file")

        # normalize our header names, removing quotes and spaces
        header = [cls.normalize_value(_).lower() for _ in header]

        records = []
        for row in reader:
            # trim all our values
            row = [cls.normalize_value(_) for _ in row]

            line_number += 1

            # make sure there are same number of fields
            if len(row) != len(header):
                raise Exception("Line %d: The number of fields for this row is incorrect. Expected %d but found %d." % (line_number, len(header), len(row)))

            field_values = dict(zip(header, row))
            field_values['created_by'] = user
            field_values['modified_by'] = user
            try:
                field_values = cls.prepare_fields(field_values, import_params, user)
                record = cls.create_instance(field_values)
                if record:
                    records.append(record)
            except Exception as e:
                if log:
                    traceback.print_exc(100, log)
                raise Exception("Line %d: %s\n\n%s" % (line_number, str(e), field_values))
        return records

class ActiveManager(models.Manager):
    """
    A manager that only selects items which are still active.
    """
    def get_query_set(self):
        """
        Where the magic happens, we automatically throw on an extra is_active = True to every filter
        """
        return super(ActiveManager, self).get_query_set().filter(is_active=True)
<commit_msg>Use unicode on XLS import instead of str<commit_after>import csv
import traceback
import simplejson
from django.db import models
from django.contrib.auth.models import User

class SmartModel(models.Model):
    """
    Useful abstract base class that adds the concept of something being active,
    having a user that created or modified the item and creation and modification
    dates.
    """
    is_active = models.BooleanField(default=True,
                                    help_text="Whether this item is active, use this instead of deleting")

    created_by = models.ForeignKey(User, related_name="%(class)s_creations",
                                   help_text="The user which originally created this item")
    created_on = models.DateTimeField(auto_now_add=True,
                                      help_text="When this item was originally created")

    modified_by = models.ForeignKey(User, related_name="%(class)s_modifications",
                                    help_text="The user which last modified this item")
    modified_on = models.DateTimeField(auto_now=True,
                                       help_text="When this item was last modified")

    class Meta:
        abstract = True

    @classmethod
    def prepare_fields(cls, field_dict, import_params=None, user=None):
        return field_dict

    @classmethod
    def create_instance(cls, field_dict):
        return cls.objects.create(**field_dict)

    @classmethod
    def import_csv(cls, task, log=None):

        from xlrd import XLRDError
        filename = task.csv_file.file
        user = task.created_by

        import_params = None

        # additional parameters are optional
        if task.import_params:
            try:
                import_params = simplejson.loads(task.import_params)
            except:
                pass

        try:
            records = cls.import_xls(filename, user, import_params, log)
        except XLRDError:
            records = cls.import_raw_csv(filename, user, import_params, log)

        return records

    @classmethod
    def normalize_value(cls, val):
        # remove surrounding whitespace
        val = val.strip()

        # if surrounded by double quotes, remove those
        if val and val[0] == '"' and val[-1] == '"':
            val = val[1:-1]

        # if surrounded by single quotes, remove those
        if val and val[0] == "'" and val[-1] == "'":
            val = val[1:-1]

        return val

    @classmethod
    def import_xls(cls, filename, user, import_params, log=None):
        from xlrd import open_workbook
        workbook = open_workbook(filename.name, 'rb')

        records = []

        for sheet in workbook.sheets():

            # read our header
            header = []
            for col in range(sheet.ncols):
                header.append(unicode(sheet.cell(0, col).value))
            header = [cls.normalize_value(_).lower() for _ in header]

            # read our rows
            line_number = 1
            for row in range(sheet.nrows - 1):
                field_values = []
                for col in range(sheet.ncols):
                    field_values.append(unicode(sheet.cell(row + 1, col).value))

                field_values = [cls.normalize_value(_) for _ in field_values]
                field_values = dict(zip(header, field_values))
                field_values['created_by'] = user
                field_values['modified_by'] = user

                try:
                    field_values = cls.prepare_fields(field_values, import_params, user)
                    record = cls.create_instance(field_values)
                    if record:
                        records.append(record)
                except Exception as e:
                    if log:
                        traceback.print_exc(100, log)
                    raise Exception("Line %d: %s\n\n%s" % (line_number, str(e), field_values))
                line_number += 1
            # only care about the first sheet
            break

        return records

    @classmethod
    def import_raw_csv(cls, filename, user, import_params, log=None):
        # our alternative codec, by default we are the crazy windows encoding
        ascii_codec = 'cp1252'

        # read the entire file, look for mac_roman characters
        reader = open(filename.name, "rb")
        for byte in reader.read():
            # these are latin accented characterse in mac_roman, if we see them then our alternative
            # encoding should be mac_roman
            if ord(byte) in [0x81, 0x8d, 0x8f, 0x90, 0x9d]:
                ascii_codec = 'mac_roman'
                break
        reader.close()

        reader = open(filename.name, "rU")

        def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
            csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
            for row in csv_reader:
                encoded = []
                for cell in row:
                    try:
                        cell = unicode(cell)
                    except:
                        cell = unicode(cell.decode(ascii_codec))

                    encoded.append(cell)

                yield encoded

        reader = unicode_csv_reader(reader)

        # read in our header
        line_number = 0

        header = reader.next()
        line_number += 1
        while header is not None and len(header[0]) > 1 and header[0][0] == "#":
            header = reader.next()
            line_number += 1

        # do some sanity checking to make sure they uploaded the right kind of file
        if len(header) < 1:
            raise Exception("Invalid header for import file")

        # normalize our header names, removing quotes and spaces
        header = [cls.normalize_value(_).lower() for _ in header]

        records = []
        for row in reader:
            # trim all our values
            row = [cls.normalize_value(_) for _ in row]

            line_number += 1

            # make sure there are same number of fields
            if len(row) != len(header):
                raise Exception("Line %d: The number of fields for this row is incorrect. Expected %d but found %d." % (line_number, len(header), len(row)))

            field_values = dict(zip(header, row))
            field_values['created_by'] = user
            field_values['modified_by'] = user
            try:
                field_values = cls.prepare_fields(field_values, import_params, user)
                record = cls.create_instance(field_values)
                if record:
                    records.append(record)
            except Exception as e:
                if log:
                    traceback.print_exc(100, log)
                raise Exception("Line %d: %s\n\n%s" % (line_number, str(e), field_values))
        return records

class ActiveManager(models.Manager):
    """
    A manager that only selects items which are still active.
    """
    def get_query_set(self):
        """
        Where the magic happens, we automatically throw on an extra is_active = True to every filter
        """
        return super(ActiveManager, self).get_query_set().filter(is_active=True)
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from users import views

urlpatterns = patterns('',
    url(r'^$', views.home, name='home'),
    url(r'^list/$', views.ListAllView.as_view(), name='listAll'),
    url(r'^list/(?P<email>[ a-zA-Z0-9_@\.]+)/$', views.ListOneView.as_view(), name='listOne'),
    #url(r'^users/', include('users.urls')),
)

<commit_msg>Update urls.py<commit_after>from django.conf.urls import patterns, include, url
from users import views

urlpatterns = patterns('',
    url(r'^$', views.home, name='home'),
    url(r'^list/$', views.ListAllView.as_view(), name='listAll'),
    url(r'^list/(?P<email>[ a-zA-Z0-9_@\.]+)/$', views.ListOneView.as_view(), name='listOne'),
)

<|endoftext|>


<commit_before>"""
Base type readers
"""

class Error(Exception):
    pass

class ReaderError(Error):
    pass

class BaseTypeReader(object):
    target_type = None
    reader_name = None
    is_value_type = False
    is_generic_type = False

    def __init__(self, stream=None, version=None):
        self.stream = stream
        self.version = version

    def __str__(self):
        return self.reader_name

    def read(self):
        raise ReaderError("Unimplemented type reader: '%s'" % self.reader_name)

    @classmethod
    def read_from(cls, stream):
        local_reader = cls.__init__(stream)
        return local_reader.read()

    def init(self):
        pass

class ValueTypeReader(BaseTypeReader):
    is_value_type = True

class GenericTypeReader(BaseTypeReader):
    generic_target_type = None
    generic_reader_name = None
    target_type = None
    reader_name = None
    is_value_type = False
    is_generic_type = True
    name_index = 0
    arguments = []
    readers = []

    def init(self):
        for arg in self.arguments:
            reader = self.stream.type_reader_manager.get_type_reader_by_type(arg)
            self.readers.append(reader)

    @classmethod
    def create(cls, type_spec):
        args = [arg.full_name for arg in type_spec.generic_params]
        class_name = cls.generic_reader_name.rpartition('.')[2]
        class_name = class_name.partition('`')[0]
        class_name += str(cls.name_index)
        cls.name_index += 1
        reader_name = type_spec.full_name
        target_type = cls.generic_target_type
        target_type += ','.join(['[' + arg + ']' for arg in args])
        # really need to figure out how to do this more cleanly
        body = dict(cls.__dict__)
        body['target_type'] = target_type
        body['reader_name'] = reader_name
        body['arguments'] = args
        class_ = type(class_name, cls.__bases__, body)
        return class_

class GenericValueTypeReader(GenericTypeReader):
    is_value_type = True
<commit_msg>make sure arguments and readers lists in generic types don't end up shared<commit_after>"""
Base type readers
"""

class Error(Exception):
    pass

class ReaderError(Error):
    pass

class BaseTypeReader(object):
    target_type = None
    reader_name = None
    is_value_type = False
    is_generic_type = False

    def __init__(self, stream=None, version=None):
        self.stream = stream
        self.version = version

    def __str__(self):
        return self.reader_name

    def read(self):
        raise ReaderError("Unimplemented type reader: '%s'" % self.reader_name)

    @classmethod
    def read_from(cls, stream):
        local_reader = cls.__init__(stream)
        return local_reader.read()

    def init(self):
        pass

class ValueTypeReader(BaseTypeReader):
    is_value_type = True

class GenericTypeReader(BaseTypeReader):
    generic_target_type = None
    generic_reader_name = None
    target_type = None
    reader_name = None
    is_value_type = False
    is_generic_type = True
    name_index = 0
    arguments = None
    readers = None

    def init(self):
        for arg in self.arguments:
            reader = self.stream.type_reader_manager.get_type_reader_by_type(arg)
            self.readers.append(reader)

    @classmethod
    def create(cls, type_spec):
        args = [arg.full_name for arg in type_spec.generic_params]
        class_name = cls.generic_reader_name.rpartition('.')[2]
        class_name = class_name.partition('`')[0]
        class_name += str(cls.name_index)
        cls.name_index += 1
        reader_name = type_spec.full_name
        target_type = cls.generic_target_type
        target_type += ','.join(['[' + arg + ']' for arg in args])
        # really need to figure out how to do this more cleanly
        body = dict(cls.__dict__)
        body['target_type'] = target_type
        body['reader_name'] = reader_name
        body['arguments'] = args
        body['readers'] = []
        class_ = type(class_name, cls.__bases__, body)
        return class_

class GenericValueTypeReader(GenericTypeReader):
    is_value_type = True
<|endoftext|>


<commit_before>"""
Courseware API Mixins.
"""

from babel.numbers import get_currency_symbol

from common.djangoapps.course_modes.models import CourseMode
from common.djangoapps.student.models import CourseEnrollmentCelebration, UserCelebration
from lms.djangoapps.courseware.utils import can_show_verified_upgrade, verified_upgrade_deadline_link
from lms.djangoapps.courseware.toggles import COURSEWARE_MFE_MILESTONES_STREAK_DISCOUNT
from openedx.features.course_duration_limits.access import get_user_course_expiration_date
from openedx.features.discounts.applicability import can_show_streak_discount_coupon
from common.djangoapps.track import segment

def get_celebrations_dict(user, enrollment, course, browser_timezone):
    """
    Returns a dict of celebrations that should be performed.
    """
    if not enrollment:
        return {
            'first_section': False,
            'streak_length_to_celebrate': None,
            'streak_discount_enabled': False,
        }

    streak_length_to_celebrate = UserCelebration.perform_streak_updates(
        user, course.id, browser_timezone
    )
    celebrations = {
        'first_section': CourseEnrollmentCelebration.should_celebrate_first_section(enrollment),
        'streak_length_to_celebrate': streak_length_to_celebrate,
        'streak_discount_enabled': False,
    }

    if streak_length_to_celebrate:
        # We only want to offer the streak discount
        # if the course has not ended, is upgradeable and the user is not an enterprise learner
        if can_show_streak_discount_coupon(user, course):
            celebrations['streak_discount_enabled'] = COURSEWARE_MFE_MILESTONES_STREAK_DISCOUNT.is_enabled()

            # Send course streak coupon event
            course_key = str(course.id)
            modes_dict = CourseMode.modes_for_course_dict(course_id=course_key, include_expired=False)
            verified_mode = modes_dict.get('verified', None)
            if verified_mode:
                segment.track(
                    user_id=user.id,
                    event_name='edx.bi.course.streak_discount_enabled',
                    properties={
                        'course_id': str(course_key),
                        'sku': verified_mode.sku,
                    }
                )

    return celebrations

def serialize_upgrade_info(user, course_overview, enrollment):
    """
    Return verified mode upgrade information, or None.

    This is used in a few API views to provide consistent upgrade info to frontends.
    """
    if not can_show_verified_upgrade(user, enrollment):
        return None

    mode = CourseMode.verified_mode_for_course(course=course_overview)
    return {
        'access_expiration_date': get_user_course_expiration_date(user, course_overview),
        'currency': mode.currency.upper(),
        'currency_symbol': get_currency_symbol(mode.currency.upper()),
        'price': mode.min_price,
        'sku': mode.sku,
        'upgrade_url': verified_upgrade_deadline_link(user, course_overview),
    }
<commit_msg>fix: [AA-950] Fix gating feature from WaffleFlag (#28652)<commit_after>"""
Courseware API Mixins.
"""

from babel.numbers import get_currency_symbol

from common.djangoapps.course_modes.models import CourseMode
from common.djangoapps.student.models import CourseEnrollmentCelebration, UserCelebration
from lms.djangoapps.courseware.utils import can_show_verified_upgrade, verified_upgrade_deadline_link
from lms.djangoapps.courseware.toggles import COURSEWARE_MFE_MILESTONES_STREAK_DISCOUNT
from openedx.features.course_duration_limits.access import get_user_course_expiration_date
from openedx.features.discounts.applicability import can_show_streak_discount_coupon
from common.djangoapps.track import segment

def get_celebrations_dict(user, enrollment, course, browser_timezone):
    """
    Returns a dict of celebrations that should be performed.
    """
    if not enrollment:
        return {
            'first_section': False,
            'streak_length_to_celebrate': None,
            'streak_discount_enabled': False,
        }

    streak_length_to_celebrate = UserCelebration.perform_streak_updates(
        user, course.id, browser_timezone
    )
    celebrations = {
        'first_section': CourseEnrollmentCelebration.should_celebrate_first_section(enrollment),
        'streak_length_to_celebrate': streak_length_to_celebrate,
        'streak_discount_enabled': False,
    }

    if streak_length_to_celebrate:
        # We only want to offer the streak discount
        # if the course has not ended, is upgradeable and the user is not an enterprise learner
        discount_enabled = COURSEWARE_MFE_MILESTONES_STREAK_DISCOUNT.is_enabled()

        if can_show_streak_discount_coupon(user, course) and discount_enabled:
            # Send course streak coupon event
            course_key = str(course.id)
            modes_dict = CourseMode.modes_for_course_dict(course_id=course_key, include_expired=False)
            verified_mode = modes_dict.get('verified', None)
            if verified_mode:
                celebrations['streak_discount_enabled'] = discount_enabled
                segment.track(
                    user_id=user.id,
                    event_name='edx.bi.course.streak_discount_enabled',
                    properties={
                        'course_id': str(course_key),
                        'sku': verified_mode.sku,
                    }
                )

    return celebrations

def serialize_upgrade_info(user, course_overview, enrollment):
    """
    Return verified mode upgrade information, or None.

    This is used in a few API views to provide consistent upgrade info to frontends.
    """
    if not can_show_verified_upgrade(user, enrollment):
        return None

    mode = CourseMode.verified_mode_for_course(course=course_overview)
    return {
        'access_expiration_date': get_user_course_expiration_date(user, course_overview),
        'currency': mode.currency.upper(),
        'currency_symbol': get_currency_symbol(mode.currency.upper()),
        'price': mode.min_price,
        'sku': mode.sku,
        'upgrade_url': verified_upgrade_deadline_link(user, course_overview),
    }
<|endoftext|>


<commit_before>"""
Global configuration variables
"""
import sys
from numbers import Number
# Make it safe to assume Python 3 when testing version
if sys.version_info.major < 3:
    raise RuntimeError("theano_shim requires Python 3. You are using {}."
                       .format(sys.version))
from collections import OrderedDict
if sys.version_info.minor >= 5:
    from typing import Union
import numpy as np

# These forms will retrieve theano or theano.tensor, whether or not cf.use_theano == True,
# as long as any module in the stack has loaded them
# These functions should only be used within this package, for cases where we need to treat
# Theano arguments even if use_theano == False. Type-checking is an example of such a use case.
def _gettheano():
    if 'theano' in sys.modules:
        return sys.modules['theano']
    else:
        raise RuntimeError("Tried to access theano, but it has not been loaded.")
def _getT():
    if 'theano' in sys.modules:
        return sys.modules['theano'].tensor
    else:
        raise RuntimeError("Tried to access theano.tensor, but Theano has not been loaded.")

def make_32bit_var(module, var):
    """
    Modify a module's variable so that it's a single precision float.
    The original value is saved under the variable name 'var64',
    where 'var' is the provided variable name.
    """
    value = getattr(module, var)
    dname = var + '64'
    if not hasattr(module, dname):
        # Save the 64bit value before overwriting the variable
        setattr(module, dname, value)
    # Overwrite with 32bit value
    setattr(module, var, np.float32(value))

class Config:
    # TODO: Make singleton
    use_theano = False
    inf = None
    _floatX = 'float64'  # Use this if Theano is not loaded

    # Unified support for type hints; supported in Python >=3.5
    if sys.version_info.minor >= 5:
        @property
        def Numeric(self):
            if self.use_theano:
                return Union[np.ndarray, T.TensorVariable]
            else:
                return Union[np.ndarray]

    theano_updates = OrderedDict()
        # Stores a Theano update dictionary

    lib = None
        # DEPRECATION WARNING: lib will soon be removed
    RandomStreams = None

    # TerminatingTypes is a tuple of all types which may be iterable but
    # are treated as a single unit. This is used when recursively expanding
    # an argument in core._expand_args.
    # For example, if the sparse module is loaded, sparse types are added to TerminatingTypes,
    # since we will never store a Theano object as an element of a SciPy sparse matrix
    _TerminatingTypes = (bytes,str,np.ndarray)

    def add_terminating_types(self, type_list):
        self._TerminatingTypes = tuple( set(type_list).union( self._TerminatingTypes ) )
    @property
    def TerminatingTypes(self):
        if 'theano' in sys.modules:
            dyntypes = ((self.SymbolicType,) + self.GraphTypes
                        + self.ConstantTypes)
        else:
            dyntypes = self.ConstantTypes
        return self._TerminatingTypes + dyntypes

    @staticmethod
    def make_constants_32bit():
        # TODO: Have a maintainable list of these constants
        make_32bit_var(np, 'pi')

    @property
    def SymbolicType(self):
        # FIXME: What about ScalarVariable ? SharedVariable ?
        return _getT().TensorVariable
    @property
    def GraphTypes(self):
        return (_gettheano().gof.Variable, self.RandomStreamType)
    @property
    def ConstantType(self):
        return _gettheano().gof.Constant
    @property
    def TensorConstantType(self):
        return _getT().TensorConstant
    @property
    def ScalarConstantType(self):
        return _gettheano().scalar.ScalarConstant
    @property
    def SharedType(self):
        return _getT().sharedvar.SharedVariable
    @property
    def ConstantTypes(self):
        if 'theano' in sys.modules:
            return (self.ConstantType, self.ScalarConstantType,
                    self.TensorConstantType, Number)
        else:
            return (Number,)
    @property
    def RandomStreamType(self):
        return _getT().shared_randomstreams.RandomStreams
    @property
    def RandomStateType(self):
        return _getT().shared_randomstreams.RandomStateSharedVariable
    @property
    def CompiledType(self):
        return _gettheano().compile.function_module.Function

    @property
    def floatX(self):
        if 'theano' in sys.modules:
            # Update internal floatX, in case theano's was changed
            self._floatX = _gettheano().config.floatX
            return self._floatX
        else:
            # TODO: Read theanorc ?
            return self._floatX  # plain python float

    @floatX.setter
    def floatX(self, floatX):
        if floatX not in ['float32', 'float64']:
            raise ValueError("'floatX' value must be a string, either "
                             "'float32' or 'float64'.")
        if 'theano' in sys.modules:
            _gettheano().config.floatX = floatX
            self._floatX = floatX
        else:
            self._floatX = floatX
        if floatX == 'float32':
            # Make NumPy constants 32-bit so they don't trigger
            # upcasts when used in expressions
            self.make_constants_32bit()

config = Config()
<commit_msg>Add compute_test_value option to config<commit_after>"""
Global configuration variables
"""
import sys
from warnings import warn
from numbers import Number
# Make it safe to assume Python 3 when testing version
if sys.version_info.major < 3:
    raise RuntimeError("theano_shim requires Python 3. You are using {}."
                       .format(sys.version))
from collections import OrderedDict
if sys.version_info.minor >= 5:
    from typing import Union
import numpy as np

# These forms will retrieve theano or theano.tensor, whether or not cf.use_theano == True,
# as long as any module in the stack has loaded them
# These functions should only be used within this package, for cases where we need to treat
# Theano arguments even if use_theano == False. Type-checking is an example of such a use case.
def _gettheano():
    if 'theano' in sys.modules:
        return sys.modules['theano']
    else:
        raise RuntimeError("Tried to access theano, but it has not been loaded.")
def _getT():
    if 'theano' in sys.modules:
        return sys.modules['theano'].tensor
    else:
        raise RuntimeError("Tried to access theano.tensor, but Theano has not been loaded.")

def make_32bit_var(module, var):
    """
    Modify a module's variable so that it's a single precision float.
    The original value is saved under the variable name 'var64',
    where 'var' is the provided variable name.
    """
    value = getattr(module, var)
    dname = var + '64'
    if not hasattr(module, dname):
        # Save the 64bit value before overwriting the variable
        setattr(module, dname, value)
    # Overwrite with 32bit value
    setattr(module, var, np.float32(value))

class Config:
    # TODO: Make singleton
    use_theano = False
    inf = None
    _floatX = 'float64'  # Use this if Theano is not loaded

    # Unified support for type hints; supported in Python >=3.5
    if sys.version_info.minor >= 5:
        @property
        def Numeric(self):
            if self.use_theano:
                return Union[np.ndarray, T.TensorVariable]
            else:
                return Union[np.ndarray]

    theano_updates = OrderedDict()
        # Stores a Theano update dictionary

    lib = None
        # DEPRECATION WARNING: lib will soon be removed
    RandomStreams = None

    # TerminatingTypes is a tuple of all types which may be iterable but
    # are treated as a single unit. This is used when recursively expanding
    # an argument in core._expand_args.
    # For example, if the sparse module is loaded, sparse types are added to TerminatingTypes,
    # since we will never store a Theano object as an element of a SciPy sparse matrix
    _TerminatingTypes = (bytes,str,np.ndarray)

    def add_terminating_types(self, type_list):
        self._TerminatingTypes = tuple( set(type_list).union( self._TerminatingTypes ) )
    @property
    def TerminatingTypes(self):
        if 'theano' in sys.modules:
            dyntypes = ((self.SymbolicType,) + self.GraphTypes
                        + self.ConstantTypes)
        else:
            dyntypes = self.ConstantTypes
        return self._TerminatingTypes + dyntypes

    @staticmethod
    def make_constants_32bit():
        # TODO: Have a maintainable list of these constants
        make_32bit_var(np, 'pi')

    @property
    def SymbolicType(self):
        # FIXME: What about ScalarVariable ? SharedVariable ?
        return _getT().TensorVariable
    @property
    def GraphTypes(self):
        return (_gettheano().gof.Variable, self.RandomStreamType)
    @property
    def ConstantType(self):
        return _gettheano().gof.Constant
    @property
    def TensorConstantType(self):
        return _getT().TensorConstant
    @property
    def ScalarConstantType(self):
        return _gettheano().scalar.ScalarConstant
    @property
    def SharedType(self):
        return _getT().sharedvar.SharedVariable
    @property
    def ConstantTypes(self):
        if 'theano' in sys.modules:
            return (self.ConstantType, self.ScalarConstantType,
                    self.TensorConstantType, Number)
        else:
            return (Number,)
    @property
    def RandomStreamType(self):
        return _getT().shared_randomstreams.RandomStreams
    @property
    def RandomStateType(self):
        return _getT().shared_randomstreams.RandomStateSharedVariable
    @property
    def CompiledType(self):
        return _gettheano().compile.function_module.Function

    @property
    def floatX(self):
        if 'theano' in sys.modules:
            # Update internal floatX, in case theano's was changed
            self._floatX = _gettheano().config.floatX
            return self._floatX
        else:
            # TODO: Read theanorc ?
            return self._floatX  # plain python float

    @floatX.setter
    def floatX(self, floatX):
        if floatX not in ['float32', 'float64']:
            raise ValueError("'floatX' value must be a string, either "
                             "'float32' or 'float64'.")
        if 'theano' in sys.modules:
            _gettheano().config.floatX = floatX
            self._floatX = floatX
        else:
            self._floatX = floatX
        if floatX == 'float32':
            # Make NumPy constants 32-bit so they don't trigger
            # upcasts when used in expressions
            self.make_constants_32bit()

    @property
    def compute_test_value(self):
        if 'theano' in sys.modules:
            return _gettheano().config.copmute_test_value
        else:
            return "Theano is not loaded"

    @compute_test_value.setter
    def compute_test_value(self, flag):
        if 'theano' in sys.modules:
            _gettheano().config.compute_test_value = flag
        else:
            warn("Setting `compute_test_value` has no effect when Theano "
                 "is not loaded.")

config = Config()
<|endoftext|>


<commit_before>__author__ = 'max'

import time
import sys
import argparse
from lasagne_nlp.utils import utils
import lasagne_nlp.utils.data_processor as data_processor
import theano.tensor as T
import theano
from lasagne_nlp.networks.networks import *
import lasagne.nonlinearities as nonlinearities

def main():
    parser = argparse.ArgumentParser(description='Tuning with bi-directional RNN')
    parser.add_argument('--fine_tune', action='store_true', help='Fine tune the word embeddings')
    parser.add_argument('--embedding', choices=['word2vec', 'senna'], help='Embedding for words', required=True)
    parser.add_argument('--embedding_dict', default='data/word2vec/GoogleNews-vectors-negative300.bin',
                        help='path for embedding dict')
    parser.add_argument('--batch_size', type=int, default=10, help='Number of sentences in each batch')
    parser.add_argument('--num_units', type=int, default=100, help='Number of hidden units in RNN')
    parser.add_argument('--oov', choices=['random', 'embedding'], help='Embedding for oov word', required=True)
    parser.add_argument('--update', choices=['sgd', 'momentum', 'nesterov'], help='update algorithm', default='sgd')
    parser.add_argument('--regular', choices=['none', 'l2', 'dropout'], help='regularization for training',
                        required=True)
    parser.add_argument('--train')  # "data/POS-penn/wsj/split1/wsj1.train.original"
    parser.add_argument('--dev')  # "data/POS-penn/wsj/split1/wsj1.dev.original"
    parser.add_argument('--test')  # "data/POS-penn/wsj/split1/wsj1.test.original"

    args = parser.parse_args()

    def construct_input_layer(input_var, fine_tune, embedd_table, max_length, embedd_dim):
        if fine_tune:
            alphabet_size, _ = embedd_table.shape
            layer_input = lasagne.layers.InputLayer(shape=(None, max_length), input_var=input_var, name='input')
            layer_embedding = lasagne.layers.EmbeddingLayer(layer_input, input_size=alphabet_size,
                                                            output_size=embedd_dim,
                                                            W=embedd_table, name='embedding')
            return layer_embedding
        else:
            layer_input = lasagne.layers.InputLayer(shape=(None, max_length, embedd_dim), input_var=input_var,
                                                    name='input')
            return layer_input

    logger = utils.get_logger("BiRNN")
    fine_tune = args.fine_tune
    oov = args.oov
    regular = args.regular
    embedding = args.embedding
    embedding_path = args.embedding_dict
    train_path = args.train
    dev_path = args.dev
    test_path = args.test
    update_algo = args.update

    X_train, Y_train, mask_train, X_dev, Y_dev, mask_dev, X_test, Y_test, mask_test, \
    embedd_table, num_labels = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, oov=oov,
                                                                             fine_tune=fine_tune, embedding=embedding,
                                                                             embedding_path=embedding_path)

    logger.info("constructing network...")
    # create variables
    target_var = T.imatrix(name='targets')
    mask_var = T.matrix(name='masks', dtype=theano.config.floatX)
    if fine_tune:
        input_var = T.imatrix(name='inputs')
        num_data, max_length = X_train.shape
        _, embedd_dim = embedd_table.shape
    else:
        input_var = T.tensor3(name='inputs', dtype=theano.config.floatX)
        num_data, max_length, embedd_dim = X_train.shape

    # construct input and mask layers
    layer_input = construct_input_layer(input_var, fine_tune, embedd_table, max_length, embedd_dim)
    layer_mask = lasagne.layers.InputLayer(shape=(None, max_length), input_var=mask_var, name='mask')

    # construct bi-rnn
    num_units = args.num_units
    bi_rnn = build_BiRNN(layer_input, num_units, mask=layer_mask)

    # reshape bi-rnn to [batch * max_length, embedd_dim]
    bi_rnn = lasagne.layers.reshape(bi_rnn, (-1, [2]))

    # drop out layer?
    if regular == 'dropout':
        bi_rnn = lasagne.layers.DropoutLayer(bi_rnn, p=0.5)

    # construct output layer (dense layer with softmax)
    layer_output = lasagne.layers.DenseLayer(bi_rnn, num_units=num_labels, nonlinearity=nonlinearities.softmax)

    # get output of bi-rnn shape=[batch * max_length, #label]
    prediction_train = lasagne.layers.get_output(layer_output)
    prediction_eval = lasagne.layers.get_output(layer_output, deterministic=True)
    # flat target_var to vector
    target_var_flatten = target_var.flatten()
    # flat mask_var to vector
    mask_var_flatten = mask_var.flatten()

    # compute loss
    num_loss = mask_var_flatten.sum(dtype=theano.config.floatX)
    # for training, we use mean of loss over number of labels
    loss_train = lasagne.objectives.categorical_crossentropy(prediction_train, target_var_flatten)
    loss_train = (loss_train * mask_var_flatten).sum(dtype=theano.config.floatX) / num_loss
    # l2 regularization?
    if regular == 'l2':
        gamma = 1e-4
        l2_penalty = lasagne.regularization.regularize_network_params(layer_output, lasagne.regularization.l2)
        loss_train = loss_train + gamma * l2_penalty

    loss_eval = lasagne.objectives.categorical_crossentropy(prediction_eval, target_var_flatten)
    loss_eval = (loss_eval * mask_var_flatten).sum(dtype=theano.config.floatX) / num_loss

    # compute number of correct labels
    corr_train = lasagne.objectives.categorical_accuracy(prediction_train, target_var_flatten)
    corr_train = (corr_train * mask_var_flatten).sum(dtype=theano.config.floatX)

    corr_eval = lasagne.objectives.categorical_accuracy(prediction_eval, target_var_flatten)
    corr_eval = (corr_eval * mask_var_flatten).sum(dtype=theano.config.floatX)

    # Create update expressions for training.
    # hyper parameters to tune: learning rate, momentum, regularization.
    batch_size = args.batch_size
    learning_rate = 0.1
    decay_rate = 0.1
    momentum = 0.9
    params = lasagne.layers.get_all_params(layer_output, trainable=True)
    updates = utils.create_updates(loss_train, params, update_algo, learning_rate, momentum=momentum)

    # Compile a function performing a training step on a mini-batch
    train_fn = theano.function([input_var, target_var, mask_var], [loss_train, corr_train, num_loss], updates=updates)
    # Compile a second function evaluating the loss and accuracy of network
    eval_fn = theano.function([input_var, target_var, mask_var], [loss_eval, corr_eval, num_loss])

    # Finally, launch the training loop.
    logger.info("Start training: %s (#training data: %d, batch size: %d)..." % (update_algo, num_data, batch_size))
    num_batches = num_data / batch_size
    num_epochs = 1000
    best_loss = 1e+12
    stop_count = 0
    lr = learning_rate
    for epoch in range(1, num_epochs + 1):
        print 'Epoch %d (learning rate=%.4f): ' % (epoch, lr)
        train_err = 0.0
        train_corr = 0.0
        train_total = 0
        start_time = time.time()
        num_back = 0
        train_batches = 0
        for batch in utils.iterate_minibatches(X_train, Y_train, mask_train, batch_size, shuffle=True):
            inputs, targets, masks = batch
            err, corr, num = train_fn(inputs, targets, masks)
            train_err += err * num
            train_corr += corr
            train_total += num
            train_batches += 1
            time_ave = (time.time() - start_time) / train_batches
            time_left = (num_batches - train_batches) * time_ave

            # update log
            sys.stdout.write("\b" * num_back)
            log_info = 'train: %d/%d loss: %.4f, acc: %.2f%%, time left (estimated): %.2fs' % (
                min(train_batches * batch_size, num_data), num_data,
                train_err / train_total, train_corr * 100 / train_total, time_left)
            sys.stdout.write(log_info)
            num_back = len(log_info)

        # evaluate performance on dev data
        dev_err = 0.0
        dev_corr = 0.0
        dev_total = 0
        for batch in utils.iterate_minibatches(X_dev, Y_dev, mask_dev, batch_size):
            inputs, targets, masks = batch
            err, corr, num = eval_fn(inputs, targets, masks)
            dev_err += err * num
            dev_corr += corr
            dev_total += num
        print '\ndev loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (
            dev_err / dev_total, dev_corr, dev_total, dev_corr * 100 / dev_total)

        if best_loss < dev_err:
            stop_count += 1
        else:
            best_loss = dev_err
            stop_count = 0

        # stop if dev acc decrease 3 time straightly.
        if stop_count == 3:
            break

        # re-compile a function with new learning rate for training
        lr = learning_rate / (1.0 + epoch * decay_rate)
        updates = utils.create_updates(loss_train, params, update_algo, lr, momentum=momentum)
        train_fn = theano.function([input_var, target_var, mask_var], [loss_train, corr_train, num_loss],
                                   updates=updates)

    # evaluate on test data.
    logger.info("evaluating test performance...")
    test_err = 0.0
    test_corr = 0.0
    test_total = 0
    for batch in utils.iterate_minibatches(X_test, Y_test, mask_test, batch_size):
        inputs, targets, masks = batch
        err, corr, num = eval_fn(inputs, targets, masks)
        test_err += err * num
        test_corr += corr
        test_total += num
    print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (
        test_err / test_total, test_corr, test_total, test_corr * 100 / test_total)

if __name__ == '__main__':
    main()
<commit_msg>add grad_clipping=5<commit_after>__author__ = 'max'

import time
import sys
import argparse
from lasagne_nlp.utils import utils
import lasagne_nlp.utils.data_processor as data_processor
import theano.tensor as T
import theano
from lasagne_nlp.networks.networks import *
import lasagne.nonlinearities as nonlinearities

def main():
    parser = argparse.ArgumentParser(description='Tuning with bi-directional RNN')
    parser.add_argument('--fine_tune', action='store_true', help='Fine tune the word embeddings')
    parser.add_argument('--embedding', choices=['word2vec', 'senna'], help='Embedding for words', required=True)
    parser.add_argument('--embedding_dict', default='data/word2vec/GoogleNews-vectors-negative300.bin',
                        help='path for embedding dict')
    parser.add_argument('--batch_size', type=int, default=10, help='Number of sentences in each batch')
    parser.add_argument('--num_units', type=int, default=100, help='Number of hidden units in RNN')
    parser.add_argument('--oov', choices=['random', 'embedding'], help='Embedding for oov word', required=True)
    parser.add_argument('--update', choices=['sgd', 'momentum', 'nesterov'], help='update algorithm', default='sgd')
    parser.add_argument('--regular', choices=['none', 'l2', 'dropout'], help='regularization for training',
                        required=True)
    parser.add_argument('--train')  # "data/POS-penn/wsj/split1/wsj1.train.original"
    parser.add_argument('--dev')  # "data/POS-penn/wsj/split1/wsj1.dev.original"
    parser.add_argument('--test')  # "data/POS-penn/wsj/split1/wsj1.test.original"

    args = parser.parse_args()

    def construct_input_layer(input_var, fine_tune, embedd_table, max_length, embedd_dim):
        if fine_tune:
            alphabet_size, _ = embedd_table.shape
            layer_input = lasagne.layers.InputLayer(shape=(None, max_length), input_var=input_var, name='input')
            layer_embedding = lasagne.layers.EmbeddingLayer(layer_input, input_size=alphabet_size,
                                                            output_size=embedd_dim,
                                                            W=embedd_table, name='embedding')
            return layer_embedding
        else:
            layer_input = lasagne.layers.InputLayer(shape=(None, max_length, embedd_dim), input_var=input_var,
                                                    name='input')
            return layer_input

    logger = utils.get_logger("BiRNN")
    fine_tune = args.fine_tune
    oov = args.oov
    regular = args.regular
    embedding = args.embedding
    embedding_path = args.embedding_dict
    train_path = args.train
    dev_path = args.dev
    test_path = args.test
    update_algo = args.update

    X_train, Y_train, mask_train, X_dev, Y_dev, mask_dev, X_test, Y_test, mask_test, \
    embedd_table, num_labels = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, oov=oov,
                                                                             fine_tune=fine_tune, embedding=embedding,
                                                                             embedding_path=embedding_path)

    logger.info("constructing network...")
    # create variables
    target_var = T.imatrix(name='targets')
    mask_var = T.matrix(name='masks', dtype=theano.config.floatX)
    if fine_tune:
        input_var = T.imatrix(name='inputs')
        num_data, max_length = X_train.shape
        _, embedd_dim = embedd_table.shape
    else:
        input_var = T.tensor3(name='inputs', dtype=theano.config.floatX)
        num_data, max_length, embedd_dim = X_train.shape

    # construct input and mask layers
    layer_input = construct_input_layer(input_var, fine_tune, embedd_table, max_length, embedd_dim)
    layer_mask = lasagne.layers.InputLayer(shape=(None, max_length), input_var=mask_var, name='mask')

    # construct bi-rnn
    num_units = args.num_units
    bi_rnn = build_BiRNN(layer_input, num_units, mask=layer_mask, grad_clipping=5)

    # reshape bi-rnn to [batch * max_length, embedd_dim]
    bi_rnn = lasagne.layers.reshape(bi_rnn, (-1, [2]))

    # drop out layer?
    if regular == 'dropout':
        bi_rnn = lasagne.layers.DropoutLayer(bi_rnn, p=0.5)

    # construct output layer (dense layer with softmax)
    layer_output = lasagne.layers.DenseLayer(bi_rnn, num_units=num_labels, nonlinearity=nonlinearities.softmax)

    # get output of bi-rnn shape=[batch * max_length, #label]
    prediction_train = lasagne.layers.get_output(layer_output)
    prediction_eval = lasagne.layers.get_output(layer_output, deterministic=True)
    # flat target_var to vector
    target_var_flatten = target_var.flatten()
    # flat mask_var to vector
    mask_var_flatten = mask_var.flatten()

    # compute loss
    num_loss = mask_var_flatten.sum(dtype=theano.config.floatX)
    # for training, we use mean of loss over number of labels
    loss_train = lasagne.objectives.categorical_crossentropy(prediction_train, target_var_flatten)
    loss_train = (loss_train * mask_var_flatten).sum(dtype=theano.config.floatX) / num_loss
    # l2 regularization?
    if regular == 'l2':
        gamma = 1e-6
        l2_penalty = lasagne.regularization.regularize_network_params(layer_output, lasagne.regularization.l2)
        loss_train = loss_train + gamma * l2_penalty

    loss_eval = lasagne.objectives.categorical_crossentropy(prediction_eval, target_var_flatten)
    loss_eval = (loss_eval * mask_var_flatten).sum(dtype=theano.config.floatX) / num_loss

    # compute number of correct labels
    corr_train = lasagne.objectives.categorical_accuracy(prediction_train, target_var_flatten)
    corr_train = (corr_train * mask_var_flatten).sum(dtype=theano.config.floatX)

    corr_eval = lasagne.objectives.categorical_accuracy(prediction_eval, target_var_flatten)
    corr_eval = (corr_eval * mask_var_flatten).sum(dtype=theano.config.floatX)

    # Create update expressions for training.
    # hyper parameters to tune: learning rate, momentum, regularization.
    batch_size = args.batch_size
    learning_rate = 0.1
    decay_rate = 0.1
    momentum = 0.9
    params = lasagne.layers.get_all_params(layer_output, trainable=True)
    updates = utils.create_updates(loss_train, params, update_algo, learning_rate, momentum=momentum)

    # Compile a function performing a training step on a mini-batch
    train_fn = theano.function([input_var, target_var, mask_var], [loss_train, corr_train, num_loss], updates=updates)
    # Compile a second function evaluating the loss and accuracy of network
    eval_fn = theano.function([input_var, target_var, mask_var], [loss_eval, corr_eval, num_loss])

    # Finally, launch the training loop.
    logger.info("Start training: %s (#training data: %d, batch size: %d)..." % (update_algo, num_data, batch_size))
    num_batches = num_data / batch_size
    num_epochs = 1000
    best_loss = 1e+12
    stop_count = 0
    lr = learning_rate
    for epoch in range(1, num_epochs + 1):
        print 'Epoch %d (learning rate=%.4f): ' % (epoch, lr)
        train_err = 0.0
        train_corr = 0.0
        train_total = 0
        start_time = time.time()
        num_back = 0
        train_batches = 0
        for batch in utils.iterate_minibatches(X_train, Y_train, mask_train, batch_size, shuffle=True):
            inputs, targets, masks = batch
            err, corr, num = train_fn(inputs, targets, masks)
            train_err += err * num
            train_corr += corr
            train_total += num
            train_batches += 1
            time_ave = (time.time() - start_time) / train_batches
            time_left = (num_batches - train_batches) * time_ave

            # update log
            sys.stdout.write("\b" * num_back)
            log_info = 'train: %d/%d loss: %.4f, acc: %.2f%%, time left (estimated): %.2fs' % (
                min(train_batches * batch_size, num_data), num_data,
                train_err / train_total, train_corr * 100 / train_total, time_left)
            sys.stdout.write(log_info)
            num_back = len(log_info)

        # evaluate performance on dev data
        dev_err = 0.0
        dev_corr = 0.0
        dev_total = 0
        for batch in utils.iterate_minibatches(X_dev, Y_dev, mask_dev, batch_size):
            inputs, targets, masks = batch
            err, corr, num = eval_fn(inputs, targets, masks)
            dev_err += err * num
            dev_corr += corr
            dev_total += num
        print '\ndev loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (
            dev_err / dev_total, dev_corr, dev_total, dev_corr * 100 / dev_total)

        if best_loss < dev_err:
            stop_count += 1
        else:
            best_loss = dev_err
            stop_count = 0

        # stop if dev acc decrease 3 time straightly.
        if stop_count == 3:
            break

        # re-compile a function with new learning rate for training
        lr = learning_rate / (1.0 + epoch * decay_rate)
        updates = utils.create_updates(loss_train, params, update_algo, lr, momentum=momentum)
        train_fn = theano.function([input_var, target_var, mask_var], [loss_train, corr_train, num_loss],
                                   updates=updates)

    # evaluate on test data.
    logger.info("evaluating test performance...")
    test_err = 0.0
    test_corr = 0.0
    test_total = 0
    for batch in utils.iterate_minibatches(X_test, Y_test, mask_test, batch_size):
        inputs, targets, masks = batch
        err, corr, num = eval_fn(inputs, targets, masks)
        test_err += err * num
        test_corr += corr
        test_total += num
    print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (
        test_err / test_total, test_corr, test_total, test_corr * 100 / test_total)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from django import forms
from django.template.loader import render_to_string
from amcat.tools.selection.webscripts.webscript import WebScript
from amcat.model.project import Project
from django.http import HttpResponse
from amcat.tools.table.table3 import ObjectColumn, ObjectTable
from amcat.tools.table.tableoutput import table2csv
import csv

class ExportArticles(forms.Form):
    columns = forms.MultipleChoiceField(
        choices=(
            ('articleid', 'Article ID'),
            ('date','Date'),
            ('mediumid','Medium ID'),
            ('mediumname','Medium Name'),
            ('projectid','Project ID'),
            ('projectname','Project Name'),
            ('pagenr','Page number'),
            ('section','Section'),
            ('length','Length'),
            ('url','url'),
            ('parentid','Parent Article ID'),
            ('externalid','External ID'),
            ('additionalMetadata','Additional Metadata'),
            ('headline','Headline'),
            ('text','Article Text')
        ),
        initial = ('articleid', 'date', 'mediumid', 'projectid')
    )
    limit = forms.ChoiceField(choices=((100,'100'),(1000,'1.000'),(10000,'10.000'),(100000,'100.000'),('nolimit', 'No limit')), initial=1000)
    limitTextLength = forms.BooleanField(initial=True, required=False)

class ExportArticles(WebScript):
    name = "Export Articles"
    template = "navigator/selection/webscripts/exportArticlesForm.html"
    form = ExportArticles
    displayLocation = ('ShowSummary', 'ShowArticleTable')
    id = 'ExportArticles'
    supportedOutputTypes = ('csv-tab', 'csv-semicolon', 'csv-comma')

    def run(self):
        length = self.ownForm.cleaned_data['limit']
        if length == 'nolimit': length = -1
        articles = self.getArticles(start=0, length=length, highlight=False)

        cols = self.ownForm.cleaned_data['columns']

        if self.ownForm.cleaned_data['limitTextLength']:
            textLambda = lambda a:a.text[:31900]
        else:
            textLambda = lambda a:a.text

        colDict = { # mapping of names to article object attributes
            'articleid': ObjectColumn("id", lambda a: a.id),
            'date': ObjectColumn('Date', lambda a: a.date.strftime('%Y-%m-%d %H:%M')),
            'mediumid': ObjectColumn('Medium ID', lambda a:a.medium_id),
            'mediumname': ObjectColumn('Medium Name', lambda a:a.medium.name),
            'projectid': ObjectColumn('Project ID', lambda a:a.project_id),
            'projectname': ObjectColumn('Project Name', lambda a:a.project.name),
            'pagenr': ObjectColumn('Page number', lambda a:a.pagenr),
            'section': ObjectColumn('Section', lambda a:a.section),
            'length': ObjectColumn('Length', lambda a:a.length),
            'url': ObjectColumn('url', lambda a:a.url),
            'parentid': ObjectColumn('Parent Article ID', lambda a:a.parent_id),
            'externalid': ObjectColumn('External ID', lambda a:a.externalid),
            'additionalMetadata': ObjectColumn('Additional Metadata', lambda a:a.metastring),
            'headline': ObjectColumn('Headline', lambda a:a.headline),
            'text': ObjectColumn('Article Text', textLambda)
        }
        columns = [colDict[col] for col in cols]

        table = ObjectTable(articles, columns)

        response = HttpResponse(mimetype='text/csv')
        response['Content-Disposition'] = 'attachment; filename=exportArticles.csv'

        output = self.generalForm.cleaned_data['output']
        if output == 'csv-tab':
            delimiter = '\t'
        elif output == 'csv-semicolon':
            delimiter = ';'
        elif output == 'csv-comma':
            delimiter = ':'

        table2csv(table, csvwriter=csv.writer(response, dialect='excel', delimiter=delimiter), writecolnames=True, writerownames=False, tabseparated=False)

        return response

        <commit_msg>added excel output<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from django import forms
from django.template.loader import render_to_string
from amcat.tools.selection.webscripts.webscript import WebScript
from amcat.model.project import Project
from django.http import HttpResponse
from amcat.tools.table.table3 import ObjectColumn, ObjectTable
from amcat.tools.table.tableoutput import table2csv
import csv

from openpyxl.workbook import Workbook
from openpyxl.writer.dump_worksheet import ExcelDumpWriter
import zipfile
from cStringIO import StringIO

class ExportArticles(forms.Form):
    columns = forms.MultipleChoiceField(
        choices=(
            ('articleid', 'Article ID'),
            ('date','Date'),
            ('mediumid','Medium ID'),
            ('mediumname','Medium Name'),
            ('projectid','Project ID'),
            ('projectname','Project Name'),
            ('pagenr','Page number'),
            ('section','Section'),
            ('length','Length'),
            ('url','url'),
            ('parentid','Parent Article ID'),
            ('externalid','External ID'),
            ('additionalMetadata','Additional Metadata'),
            ('headline','Headline'),
            ('text','Article Text')
        ),
        initial = ('articleid', 'date', 'mediumid', 'projectid')
    )
    limit = forms.ChoiceField(choices=((100,'100'),(1000,'1.000'),(10000,'10.000'),(100000,'100.000'),('nolimit', 'No limit')), initial=1000)
    limitTextLength = forms.BooleanField(initial=True, required=False)

class ExportArticles(WebScript):
    name = "Export Articles"
    template = "navigator/selection/webscripts/exportArticlesForm.html"
    form = ExportArticles
    displayLocation = ('ShowSummary', 'ShowArticleTable')
    id = 'ExportArticles'
    supportedOutputTypes = ('csv-tab', 'csv-semicolon', 'csv-comma','xlsx')

    def run(self):
        length = self.ownForm.cleaned_data['limit']
        if length == 'nolimit': length = -1
        articles = self.getArticles(start=0, length=length, highlight=False)

        cols = self.ownForm.cleaned_data['columns']

        if self.ownForm.cleaned_data['limitTextLength']:
            textLambda = lambda a:a.text[:31900]
        else:
            textLambda = lambda a:a.text

        output = self.generalForm.cleaned_data['output']

        colDict = { # mapping of names to article object attributes
            'articleid': ObjectColumn("id", lambda a: a.id),
            'date': ObjectColumn('Date', lambda a: a.date.strftime('%Y-%m-%d %H:%M')),
            'mediumid': ObjectColumn('Medium ID', lambda a:a.medium_id),
            'mediumname': ObjectColumn('Medium Name', lambda a:a.medium.name),
            'projectid': ObjectColumn('Project ID', lambda a:a.project_id),
            'projectname': ObjectColumn('Project Name', lambda a:a.project.name),
            'pagenr': ObjectColumn('Page number', lambda a:a.pagenr),
            'section': ObjectColumn('Section', lambda a:a.section),
            'length': ObjectColumn('Length', lambda a:a.length),
            'url': ObjectColumn('url', lambda a:a.url),
            'parentid': ObjectColumn('Parent Article ID', lambda a:a.parent_id),
            'externalid': ObjectColumn('External ID', lambda a:a.externalid),
            'additionalMetadata': ObjectColumn('Additional Metadata', lambda a:a.metastring),
            'headline': ObjectColumn('Headline', lambda a:a.headline),
            'text': ObjectColumn('Article Text', textLambda)
        }
        if output == 'xlsx':
            colDict['date'] = ObjectColumn('Date', lambda a: a.date)
        columns = [colDict[col] for col in cols]

        table = ObjectTable(articles, columns)

        if output == 'csv-tab':
            delimiter = '\t'
        elif output == 'csv-semicolon':
            delimiter = ';'
        elif output == 'csv-comma':
            delimiter = ':'

        if output == 'xlsx':
            wb = Workbook(optimized_write = True)
            ws = wb.create_sheet()
            for row in table.getRows():
                ws.append([table.getValue(row, column) for column in table.getColumns()])
            writer = ExcelDumpWriter(wb)
            response = HttpResponse(mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
            response['Content-Disposition'] = 'attachment; filename=exportArticles.xlsx'
            # need to do a little bit more work here, since the openpyxl library only supports writing to a filename, while we need a buffer here..
            buffer = StringIO()
            zf = zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED)
            writer.write_data(zf)
            zf.close()
            buffer.flush()
            buffervalue = buffer.getvalue()
            buffer.close()
            response.write(buffervalue)
        else:
            response = HttpResponse(mimetype='text/csv')
            response['Content-Disposition'] = 'attachment; filename=exportArticles.csv'
            table2csv(table, csvwriter=csv.writer(response, dialect='excel', delimiter=delimiter), writecolnames=True, writerownames=False, tabseparated=False)

        return response

        <|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: iso-8859-1 -*-
#
# progressbar  - Text progressbar library for python.
# Copyright (c) 2005 Nilton Volpato
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

"""Text progressbar library for python.

This library provides a text mode progressbar. This is typically used
to display the progress of a long running operation, providing a
visual clue that processing is underway.

The ProgressBar class manages the progress, and the format of the line
is given by a number of widgets. A widget is an object that may
display diferently depending on the state of the progress. There are
three types of widget:
- a string, which always shows itself;
- a ProgressBarWidget, which may return a diferent value every time
it's update method is called; and
- a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it
expands to fill the remaining width of the line.

The progressbar module is very easy to use, yet very powerful. And
automatically supports features like auto-resizing when available.
"""

from __future__ import division

__author__ = "Nilton Volpato"
__author_email__ = "first-name dot last-name @ gmail.com"
__date__ = "2006-05-07"
__version__ = "2.3-dev"

import sys, time, os
import datetime
from array import array
try:
    from fcntl import ioctl
    import termios
except ImportError:
    pass
import signal
try:
    basestring
except NameError:
    basestring = (str,)

class ProgressBarWidget(object):
    """This is an element of ProgressBar formatting.

    The ProgressBar object will call it's update value when an update
    is needed. It's size may change between call, but the results will
    not be good if the size changes drastically and repeatedly.

    If the class constant TIME_SENSITIVE is True, then the progressbar
    will be updated at least once every second, provided update() is called.
    """
    TIME_SENSITIVE = False
    def update(self, pbar):
        """Returns the string representing the widget.

        The parameter pbar is a reference to the calling ProgressBar,
        where one can access attributes of the class for knowing how
        the update must be made.

        At least this function must be overriden."""
        pass

class ProgressBarWidgetHFill(ProgressBarWidget):
    """This is a variable width element of ProgressBar formatting.

    The ProgressBar object will call it's update value, informing the
    width this object must the made. This is like TeX \\hfill, it will
    expand to fill the line. You can use more than one in the same
    line, and they will all have the same width, and together will
    fill the line.
    """
    def update(self, pbar, width):
        """Returns the string representing the widget.

        The parameter pbar is a reference to the calling ProgressBar,
        where one can access attributes of the class for knowing how
        the update must be made. The parameter width is the total
        horizontal width the widget must have.

        At least this function must be overriden."""
        pass

class ETA(ProgressBarWidget):
    "Widget for the Estimated Time of Arrival"
    TIME_SENSITIVE = True
    def format_time(self, seconds):
        return str(datetime.timedelta(seconds=int(seconds)))
    def update(self, pbar):
        if pbar.currval == 0:
            return 'ETA:  --:--:--'
        elif pbar.finished:
            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
        else:
            elapsed = pbar.seconds_elapsed
            eta = elapsed * pbar.maxval / pbar.currval - elapsed
            return 'ETA:  %s' % self.format_time(eta)

class FileTransferSpeed(ProgressBarWidget):
    "Widget for showing the transfer speed (useful for file transfers)."
    def __init__(self, unit='B'):
        self.unit = unit
        self.fmt = '%6.2f %s'
        self.prefixes = ['', 'K', 'M', 'G', 'T', 'P']
    def update(self, pbar):
        if pbar.seconds_elapsed < 2e-6:#== 0:
            bps = 0.0
        else:
            bps = pbar.currval / pbar.seconds_elapsed
        spd = bps
        for u in self.prefixes:
            if spd < 1000:
                break
            spd /= 1000
        return self.fmt % (spd, u + self.unit + '/s')

class RotatingMarker(ProgressBarWidget):
    "A rotating marker for filling the bar of progress."
    def __init__(self, markers='|/-\\'):
        self.markers = markers
        self.curmark = -1
    def update(self, pbar):
        if pbar.finished:
            return self.markers[0]
        self.curmark = (self.curmark + 1) % len(self.markers)
        return self.markers[self.curmark]

class Percentage(ProgressBarWidget):
    "Just the percentage done."
    def update(self, pbar):
        return '%3d%%' % pbar.percentage()

class SimpleProgress(ProgressBarWidget):
    "Returns what is already done and the total, e.g.: '5 of 47'"
    def __init__(self, sep=' of '):
        self.sep = sep
    def update(self, pbar):
        return '%d%s%d' % (pbar.currval, self.sep, pbar.maxval)

class Bar(ProgressBarWidgetHFill):
    "The bar of progress. It will stretch to fill the line."
    def __init__(self, marker='#', left='|', right='|', fillchar=' '):
        self.marker = marker
        self.left = left
        self.right = right
        self.fillchar = fillchar
    def _format_marker(self, pbar):
        if isinstance(self.marker, basestring):
            return self.marker
        else:
            return self.marker.update(pbar)
    def update(self, pbar, width):
        percent = pbar.percentage()
        cwidth = width - len(self.left) - len(self.right)
        marked_width = int(percent * cwidth // 100)
        m = self._format_marker(pbar)
        bar = (self.left +
               (m * marked_width).ljust(cwidth, self.fillchar) +
               self.right)
        return bar

class ReverseBar(Bar):
    "The reverse bar of progress, or bar of regress. :)"
    def update(self, pbar, width):
        percent = pbar.percentage()
        cwidth = width - len(self.left) - len(self.right)
        marked_width = int(percent * cwidth // 100)
        m = self._format_marker(pbar)
        bar = (self.left + (m*marked_width).rjust(cwidth) + self.right)
        return bar

default_widgets = [Percentage(), ' ', Bar()]
class ProgressBar(object):
    """This is the ProgressBar class, it updates and prints the bar.

    A common way of using it is like:
    >>> pbar = ProgressBar().start()
    >>> for i in xrange(100):
    ...    # do something
    ...    pbar.update(i+1)
    ...
    >>> pbar.finish()

    You can also use a progressbar as an iterator:
    >>> progress = ProgressBar()
    >>> for i in progress(some_iterable):
    ...    # do something
    ...

    But anything you want to do is possible (well, almost anything).
    You can supply different widgets of any type in any order. And you
    can even write your own widgets! There are many widgets already
    shipped and you should experiment with them.

    The term_width parameter must be an integer or None. In the latter case
    it will try to guess it, if it fails it will default to 80 columns.

    When implementing a widget update method you may access any
    attribute or function of the ProgressBar object calling the
    widget's update method. The most important attributes you would
    like to access are:
    - currval: current value of the progress, 0 <= currval <= maxval
    - maxval: maximum (and final) value of the progress
    - finished: True if the bar has finished (reached 100%), False o/w
    - start_time: the time when start() method of ProgressBar was called
    - seconds_elapsed: seconds elapsed since start_time
    - percentage(): percentage of the progress [0..100]. This is a method.

    The attributes above are unlikely to change between different versions,
    the other ones may change or cease to exist without notice, so try to rely
    only on the ones documented above if you are extending the progress bar.
    """

    __slots__ = ('currval', 'fd', 'finished', 'last_update_time', 'maxval',
                 'next_update', 'num_intervals', 'seconds_elapsed',
                 'signal_set', 'start_time', 'term_width', 'update_interval',
                 'widgets', '_iterable', '_time_sensitive')

    _DEFAULT_MAXVAL = 100

    def __init__(self, maxval=None, widgets=default_widgets, term_width=None,
                 fd=sys.stderr):
        self.maxval = maxval
        self.widgets = widgets
        self.fd = fd
        self.signal_set = False
        if term_width is not None:
            self.term_width = term_width
        else:
            try:
                self._handle_resize(None, None)
                signal.signal(signal.SIGWINCH, self._handle_resize)
                self.signal_set = True
            except (SystemExit, KeyboardInterrupt):
                raise
            except:
                self.term_width = int(os.environ.get('COLUMNS', 80)) - 1

        self._time_sensitive = False
        for w in widgets:
          if getattr(w, 'TIME_SENSITIVE', False):
            self._time_sensitive = True
            break

        self.currval = 0
        self.finished = False
        self.start_time = None
        self.last_update_time = None
        self.seconds_elapsed = 0
        self._iterable = None

    def __call__(self, iterable):
        try:
            self.maxval = len(iterable)
        except TypeError:
            # If the iterable has no length, then rely on the value provided
            # by the user, otherwise fail.
            if not (isinstance(self.maxval, (int, long)) and self.maxval > 0):
                raise RuntimeError('Could not determine maxval from iterable. '
                                   'You must explicitly provide a maxval.')
        self._iterable = iter(iterable)
        return self

    def __iter__(self):
        return self

    def next(self):
        try:
            next = self._iterable.next()
            if self.start_time is None:
                self.start()
            else:
                self.update(self.currval + 1)
            return next
        except StopIteration:
            self.finish()
            raise

    def _handle_resize(self, signum, frame):
        h, w = array('h', ioctl(self.fd, termios.TIOCGWINSZ, '\0' * 8))[:2]
        self.term_width = w

    def percentage(self):
        "Returns the percentage of the progress."
        return self.currval * 100.0 / self.maxval

    def _format_widgets(self):
        r = []
        hfill_inds = []
        num_hfill = 0
        currwidth = 0
        for i, w in enumerate(self.widgets):
            if isinstance(w, ProgressBarWidgetHFill):
                r.append(w)
                hfill_inds.append(i)
                num_hfill += 1
            elif isinstance(w, basestring):
                r.append(w)
                currwidth += len(w)
            else:
                weval = w.update(self)
                currwidth += len(weval)
                r.append(weval)
        for iw in hfill_inds:
            widget_width = int((self.term_width - currwidth) // num_hfill)
            r[iw] = r[iw].update(self, widget_width)
        return r

    def _format_line(self):
        return ''.join(self._format_widgets()).ljust(self.term_width)

    def _next_update(self):
        return int((int(self.num_intervals *
                        (self.currval / self.maxval)) + 1) *
                   self.update_interval)

    def _need_update(self):
        """Returns true when the progressbar should print an updated line.

        You can override this method if you want finer grained control over
        updates.

        Ideally you could call self._format_line() and see if it's different
        from the previous _format_line() call, but calling _format_line() takes
        around 20 times more time than calling this implementation of
        _need_update().
        """
        return (self.currval >= self.next_update or
                (self._time_sensitive and
                 int(time.time() - self.last_update_time) > 1))

    def update(self, value):
        "Updates the progress bar to a new value."
        assert 0 <= value <= self.maxval, '0 <= %d <= %d' % (value, self.maxval)
        self.currval = value
        if not self._need_update():
            return
        if self.start_time is None:
            raise RuntimeError('You must call start() before calling update()')
        now = time.time()
        self.seconds_elapsed = now - self.start_time
        self.next_update = self._next_update()
        self.fd.write(self._format_line() + '\r')
        self.last_update_time = now

    def start(self):
        """Starts measuring time, and prints the bar at 0%.

        It returns self so you can use it like this:
        >>> pbar = ProgressBar().start()
        >>> for i in xrange(100):
        ...    # do something
        ...    pbar.update(i+1)
        ...
        >>> pbar.finish()
        """
        if self.maxval is None:
            self.maxval = self._DEFAULT_MAXVAL
        assert self.maxval > 0

        self.num_intervals = max(100, self.term_width)
        self.update_interval = self.maxval / self.num_intervals
        self.next_update = 0

        self.start_time = self.last_update_time = time.time()
        self.update(0)
        return self

    def finish(self):
        """Used to tell the progress is finished."""
        self.finished = True
        self.update(self.maxval)
        self.fd.write('\n')
        if self.signal_set:
            signal.signal(signal.SIGWINCH, signal.SIG_DFL)
<commit_msg>Fix bug http://code.google.com/p/python-progressbar/issues/detail?id=9<commit_after>#!/usr/bin/python
# -*- coding: iso-8859-1 -*-
#
# progressbar  - Text progressbar library for python.
# Copyright (c) 2005 Nilton Volpato
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

"""Text progressbar library for python.

This library provides a text mode progressbar. This is typically used
to display the progress of a long running operation, providing a
visual clue that processing is underway.

The ProgressBar class manages the progress, and the format of the line
is given by a number of widgets. A widget is an object that may
display diferently depending on the state of the progress. There are
three types of widget:
- a string, which always shows itself;
- a ProgressBarWidget, which may return a diferent value every time
it's update method is called; and
- a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it
expands to fill the remaining width of the line.

The progressbar module is very easy to use, yet very powerful. And
automatically supports features like auto-resizing when available.
"""

from __future__ import division

__author__ = "Nilton Volpato"
__author_email__ = "first-name dot last-name @ gmail.com"
__date__ = "2006-05-07"
__version__ = "2.3-dev"

import sys, time, os
import datetime
from array import array
try:
    from fcntl import ioctl
    import termios
except ImportError:
    pass
import signal
try:
    basestring
except NameError:
    basestring = (str,)

class ProgressBarWidget(object):
    """This is an element of ProgressBar formatting.

    The ProgressBar object will call it's update value when an update
    is needed. It's size may change between call, but the results will
    not be good if the size changes drastically and repeatedly.

    If the class constant TIME_SENSITIVE is True, then the progressbar
    will be updated at least once every second, provided update() is called.
    """
    TIME_SENSITIVE = False
    def update(self, pbar):
        """Returns the string representing the widget.

        The parameter pbar is a reference to the calling ProgressBar,
        where one can access attributes of the class for knowing how
        the update must be made.

        At least this function must be overriden."""
        pass

class ProgressBarWidgetHFill(ProgressBarWidget):
    """This is a variable width element of ProgressBar formatting.

    The ProgressBar object will call it's update value, informing the
    width this object must the made. This is like TeX \\hfill, it will
    expand to fill the line. You can use more than one in the same
    line, and they will all have the same width, and together will
    fill the line.
    """
    def update(self, pbar, width):
        """Returns the string representing the widget.

        The parameter pbar is a reference to the calling ProgressBar,
        where one can access attributes of the class for knowing how
        the update must be made. The parameter width is the total
        horizontal width the widget must have.

        At least this function must be overriden."""
        pass

class ETA(ProgressBarWidget):
    "Widget for the Estimated Time of Arrival"
    TIME_SENSITIVE = True
    def format_time(self, seconds):
        return str(datetime.timedelta(seconds=int(seconds)))
    def update(self, pbar):
        if pbar.currval == 0:
            return 'ETA:  --:--:--'
        elif pbar.finished:
            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
        else:
            elapsed = pbar.seconds_elapsed
            eta = elapsed * pbar.maxval / pbar.currval - elapsed
            return 'ETA:  %s' % self.format_time(eta)

class FileTransferSpeed(ProgressBarWidget):
    "Widget for showing the transfer speed (useful for file transfers)."
    def __init__(self, unit='B'):
        self.unit = unit
        self.fmt = '%6.2f %s'
        self.prefixes = ['', 'K', 'M', 'G', 'T', 'P']
    def update(self, pbar):
        if pbar.seconds_elapsed < 2e-6:#== 0:
            bps = 0.0
        else:
            bps = pbar.currval / pbar.seconds_elapsed
        spd = bps
        for u in self.prefixes:
            if spd < 1000:
                break
            spd /= 1000
        return self.fmt % (spd, u + self.unit + '/s')

class RotatingMarker(ProgressBarWidget):
    "A rotating marker for filling the bar of progress."
    def __init__(self, markers='|/-\\'):
        self.markers = markers
        self.curmark = -1
    def update(self, pbar):
        if pbar.finished:
            return self.markers[0]
        self.curmark = (self.curmark + 1) % len(self.markers)
        return self.markers[self.curmark]

class Percentage(ProgressBarWidget):
    "Just the percentage done."
    def update(self, pbar):
        return '%3d%%' % pbar.percentage()

class SimpleProgress(ProgressBarWidget):
    "Returns what is already done and the total, e.g.: '5 of 47'"
    def __init__(self, sep=' of '):
        self.sep = sep
    def update(self, pbar):
        return '%d%s%d' % (pbar.currval, self.sep, pbar.maxval)

class Bar(ProgressBarWidgetHFill):
    "The bar of progress. It will stretch to fill the line."
    def __init__(self, marker='#', left='|', right='|', fillchar=' '):
        self.marker = marker
        self.left = left
        self.right = right
        self.fillchar = fillchar
    def _format_marker(self, pbar):
        if isinstance(self.marker, basestring):
            return self.marker
        else:
            return self.marker.update(pbar)
    def update(self, pbar, width):
        percent = pbar.percentage()
        cwidth = width - len(self.left) - len(self.right)
        marked_width = int(percent * cwidth // 100)
        m = self._format_marker(pbar)
        bar = (self.left +
               (m * marked_width).ljust(cwidth, self.fillchar) +
               self.right)
        return bar

class ReverseBar(Bar):
    "The reverse bar of progress, or bar of regress. :)"
    def update(self, pbar, width):
        percent = pbar.percentage()
        cwidth = width - len(self.left) - len(self.right)
        marked_width = int(percent * cwidth // 100)
        m = self._format_marker(pbar)
        bar = (self.left + (m*marked_width).rjust(cwidth) + self.right)
        return bar

default_widgets = [Percentage(), ' ', Bar()]
class ProgressBar(object):
    """This is the ProgressBar class, it updates and prints the bar.

    A common way of using it is like:
    >>> pbar = ProgressBar().start()
    >>> for i in xrange(100):
    ...    # do something
    ...    pbar.update(i+1)
    ...
    >>> pbar.finish()

    You can also use a progressbar as an iterator:
    >>> progress = ProgressBar()
    >>> for i in progress(some_iterable):
    ...    # do something
    ...

    But anything you want to do is possible (well, almost anything).
    You can supply different widgets of any type in any order. And you
    can even write your own widgets! There are many widgets already
    shipped and you should experiment with them.

    The term_width parameter must be an integer or None. In the latter case
    it will try to guess it, if it fails it will default to 80 columns.

    When implementing a widget update method you may access any
    attribute or function of the ProgressBar object calling the
    widget's update method. The most important attributes you would
    like to access are:
    - currval: current value of the progress, 0 <= currval <= maxval
    - maxval: maximum (and final) value of the progress
    - finished: True if the bar has finished (reached 100%), False o/w
    - start_time: the time when start() method of ProgressBar was called
    - seconds_elapsed: seconds elapsed since start_time
    - percentage(): percentage of the progress [0..100]. This is a method.

    The attributes above are unlikely to change between different versions,
    the other ones may change or cease to exist without notice, so try to rely
    only on the ones documented above if you are extending the progress bar.
    """

    __slots__ = ('currval', 'fd', 'finished', 'last_update_time', 'maxval',
                 'next_update', 'num_intervals', 'seconds_elapsed',
                 'signal_set', 'start_time', 'term_width', 'update_interval',
                 'widgets', '_iterable', '_time_sensitive')

    _DEFAULT_MAXVAL = 100

    def __init__(self, maxval=None, widgets=default_widgets, term_width=None,
                 fd=sys.stderr):
        self.maxval = maxval
        self.widgets = widgets
        self.fd = fd
        self.signal_set = False
        if term_width is not None:
            self.term_width = term_width
        else:
            try:
                self._handle_resize(None, None)
                signal.signal(signal.SIGWINCH, self._handle_resize)
                self.signal_set = True
            except (SystemExit, KeyboardInterrupt):
                raise
            except:
                self.term_width = int(os.environ.get('COLUMNS', 80)) - 1

        self._time_sensitive = False
        for w in widgets:
          if getattr(w, 'TIME_SENSITIVE', False):
            self._time_sensitive = True
            break

        self.currval = 0
        self.finished = False
        self.start_time = None
        self.last_update_time = None
        self.seconds_elapsed = 0
        self._iterable = None

    def __call__(self, iterable):
        try:
            self.maxval = len(iterable)
        except TypeError:
            # If the iterable has no length, then rely on the value provided
            # by the user, otherwise fail.
            if not (isinstance(self.maxval, (int, long)) and self.maxval > 0):
                raise RuntimeError('Could not determine maxval from iterable. '
                                   'You must explicitly provide a maxval.')
        self._iterable = iter(iterable)
        return self

    def __iter__(self):
        return self

    def next(self):
        try:
            next = self._iterable.next()
            if self.start_time is None:
                self.start()
            else:
                self.update(self.currval + 1)
            return next
        except StopIteration:
            self.finish()
            raise

    def _handle_resize(self, signum, frame):
        h, w = array('h', ioctl(self.fd, termios.TIOCGWINSZ, '\0' * 8))[:2]
        self.term_width = w

    def percentage(self):
        "Returns the percentage of the progress."
        return self.currval * 100.0 / self.maxval

    def _format_widgets(self):
        r = []
        hfill_inds = []
        num_hfill = 0
        currwidth = 0
        for i, w in enumerate(self.widgets):
            if isinstance(w, ProgressBarWidgetHFill):
                r.append(w)
                hfill_inds.append(i)
                num_hfill += 1
            elif isinstance(w, basestring):
                r.append(w)
                currwidth += len(w)
            else:
                weval = w.update(self)
                currwidth += len(weval)
                r.append(weval)
        for iw in hfill_inds:
            widget_width = int((self.term_width - currwidth) // num_hfill)
            r[iw] = r[iw].update(self, widget_width)
        return r

    def _format_line(self):
        return ''.join(self._format_widgets()).ljust(self.term_width)

    def _next_update(self):
        return int((int(self.num_intervals *
                        (self.currval / self.maxval)) + 1) *
                   self.update_interval)

    def _need_update(self):
        """Returns true when the progressbar should print an updated line.

        You can override this method if you want finer grained control over
        updates.

        Ideally you could call self._format_line() and see if it's different
        from the previous _format_line() call, but calling _format_line() takes
        around 20 times more time than calling this implementation of
        _need_update().
        """
        return (self.currval >= self.next_update or self.finished or
                (self._time_sensitive and
                 int(time.time() - self.last_update_time) > 1))

    def update(self, value):
        "Updates the progress bar to a new value."
        assert 0 <= value <= self.maxval, '0 <= %d <= %d' % (value, self.maxval)
        self.currval = value
        if not self._need_update():
            return
        if self.start_time is None:
            raise RuntimeError('You must call start() before calling update()')
        now = time.time()
        self.seconds_elapsed = now - self.start_time
        self.next_update = self._next_update()
        self.fd.write(self._format_line() + '\r')
        self.last_update_time = now

    def start(self):
        """Starts measuring time, and prints the bar at 0%.

        It returns self so you can use it like this:
        >>> pbar = ProgressBar().start()
        >>> for i in xrange(100):
        ...    # do something
        ...    pbar.update(i+1)
        ...
        >>> pbar.finish()
        """
        if self.maxval is None:
            self.maxval = self._DEFAULT_MAXVAL
        assert self.maxval > 0

        self.num_intervals = max(100, self.term_width)
        self.update_interval = self.maxval / self.num_intervals
        self.next_update = 0

        self.start_time = self.last_update_time = time.time()
        self.update(0)
        return self

    def finish(self):
        """Used to tell the progress is finished."""
        self.finished = True
        self.update(self.maxval)
        self.fd.write('\n')
        if self.signal_set:
            signal.signal(signal.SIGWINCH, signal.SIG_DFL)
<|endoftext|>