<commit_before>"""
Signal Handlers for our models.

"""
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.db.models import ObjectDoesNotExist
from django.db.models.signals import (
    pre_delete, pre_save, post_delete, post_save
)
from django.dispatch import receiver
from django.utils import timezone

from notifications.signals import notification_snoozed
from redis_metrics import metric

from .custom import CustomAction
from .packages import PackageEnrollment
from .progress import DailyProgress, UserCompletedAction
from .public import Action, Behavior, Category, Goal
from .users import UserAction, UserBehavior, UserCategory, UserGoal
from .triggers import Trigger

from ..utils import clean_title, clean_notification, strip

@receiver(post_save, sender=CustomAction, dispatch_uid="coru_daily_progress")
@receiver(post_save, sender=UserBehavior, dispatch_uid="coru_daily_progress")
@receiver(post_save, sender=UserAction, dispatch_uid="coru_daily_progress")
def create_or_update_daily_progress(sender, instance, created, raw, using, **kwargs):
    """When a useraction or a userbehavior is created, we want to update the
    day's DailyProgress for the user.
    """
    if created:
        dp = DailyProgress.objects.for_today(instance.user)
        dp.update_stats()
        dp.save()

@receiver(post_save, sender=Trigger, dispatch_uid="custom-trigger-updated")
def custom_trigger_updated(sender, instance, created, raw, using, **kwargs):
    """Record metrics when a User updates their custom triggers."""
    if instance.user:
        metric('custom-trigger-updated', category="User Interactions")

@receiver(pre_delete, sender=Trigger, dispatch_uid="trigger-remove-queued-messages")
@receiver(post_save, sender=Trigger, dispatch_uid="trigger-remove-queued-messages")
def remove_queued_messages(sender, instance, *args, **kwargs):
    """If a trigger is updated, we need to remove all currently queued GCM
    Messages that are using the trigger."""
    try:
        instance.action_default.remove_queued_messages()
    except ObjectDoesNotExist:
        pass

@receiver(pre_save, sender=Action)
@receiver(pre_save, sender=Behavior)
@receiver(pre_save, sender=Goal)
@receiver(pre_save, sender=Category)
def clean_content(sender, instance, raw, using, **kwargs):
    # A mapping of model field names and the function that cleans them.
    clean_functions = {
        "title": clean_title,
        "subtitle": clean_title,
        "description": strip,
        "more_info": strip,
        "notification_text": clean_notification,
    }
    for field, func in clean_functions.items():
        if hasattr(instance, field):
            setattr(instance, field, func(getattr(instance, field)))

@receiver(post_delete, sender=Action)
@receiver(post_delete, sender=Behavior)
@receiver(post_delete, sender=Goal)
@receiver(post_delete, sender=Category)
def delete_model_icon(sender, instance, using, **kwargs):
    """Once a model instance has been deleted, this will remove its `icon` from
    the filesystem."""
    if hasattr(instance, 'icon') and instance.icon:
        instance.icon.delete()

@receiver(post_delete, sender=Action)
@receiver(post_delete, sender=Behavior)
def delete_model_image(sender, instance, using, **kwargs):
    """Once a model instance has been deleted, this will remove its `image`
    from the filesystem."""
    if hasattr(instance, 'image') and instance.image:
        instance.image.delete()

@receiver(pre_delete, sender=UserGoal, dispatch_uid="del_goal_behaviors")
def delete_goal_child_behaviors(sender, instance, using, **kwargs):
    """If a user is removing a goal, delete all of the user's selected
    behaviors that have *no other* parent goal."""
    # Get a list of all goals selected by the user, excluding the one
    # we're about to remove.
    user_goals = UserGoal.objects.filter(user=instance.user)
    user_goals = user_goals.exclude(id=instance.id)
    user_goals = user_goals.values_list('goal', flat=True)

    # Delete all the user's behaviors that lie ONLY in the goal we're
    # about to remove
    user_behaviors = instance.user.userbehavior_set.all()
    user_behaviors = user_behaviors.exclude(behavior__goals__in=user_goals)
    user_behaviors.delete()

@receiver(pre_delete, sender=UserBehavior, dispatch_uid="del_behavior_actions")
def delete_behavior_child_actions(sender, instance, using, **kwargs):
    """If a user is removing a behavior, delete all of the user's selected
    actions that are a child of this behavior."""

    user_actions = instance.user.useraction_set.filter(
        action__behavior=instance.behavior
    )
    user_actions.delete()

@receiver(post_delete, sender=UserBehavior)
def remove_behavior_reminders(sender, instance, using, **kwargs):
    """If a user deletes ALL of their UserBehavior instances, we should also
    remove the currently-queued GCMMessage for the Behavior reminder.

    """
    # NOTE: All behavior reminders use the default trigger, and we're not
    # actually connecting them to any content types, so that's null.
    if not UserBehavior.objects.filter(user=instance.user).exists():
        try:
            from notifications.models import GCMMessage
            messages = GCMMessage.objects.for_model("behavior", include_null=True)
            messages = messages.filter(user=instance.user)
            messages.delete()
        except (ImportError, ContentType.DoesNotExist):
            pass

@receiver(post_save, sender=UserAction, dispatch_uid="create-parent-userbehavior")
def create_parent_user_behavior(sender, instance, using, **kwargs):
    """If a user doens't have a UserBehavior object for the UserAction's
    parent behavior this will create one.

    """
    params = {'user': instance.user, 'behavior': instance.action.behavior}
    if not UserBehavior.objects.filter(**params).exists():
        UserBehavior.objects.create(
            user=instance.user,
            behavior=instance.action.behavior
        )

@receiver(post_save, sender=UserAction, dispatch_uid="create-relative-reminder")
def create_relative_reminder(sender, instance, created, raw, using, **kwargs):
    """When a UserAction is created, we need to look at it's default_trigger
    and see if it's a relative reminder. If so, we automatically create a
    custom trigger for the user filling in it's trigger_date based on the
    UserAction's creation date.

    """
    is_relative = (
        instance.custom_trigger is None and
        instance.default_trigger is not None and
        instance.default_trigger.is_relative
    )
    if created and is_relative:
        trigger = Trigger.objects.create(
            user=instance.user,
            name="Trigger for {}".format(instance),
            time=instance.default_trigger.time,
            trigger_date=instance.default_trigger.trigger_date,
            recurrences=instance.default_trigger.recurrences,
            start_when_selected=instance.default_trigger.start_when_selected,
            stop_on_complete=instance.default_trigger.stop_on_complete,
            relative_value=instance.default_trigger.relative_value,
            relative_units=instance.default_trigger.relative_units
        )
        trigger.trigger_date = trigger.relative_trigger_date(instance.created_on)
        trigger.save()
        instance.custom_trigger = trigger
        instance.save()

@receiver(notification_snoozed)
def reset_next_trigger_date_when_snoozed(sender, message, user,
                                         related_object, deliver_on, **kwargs):
    """If a user snoozes a notification (in the notifications app), this function
    will try to update the relevant UserAction's `next_trigger_date`.

    """
    if related_object and message.content_type.name.lower() == 'action':
        ua = related_object.useraction_set.filter(user=user).first()
        if ua and deliver_on:
            ua.next_trigger_date = deliver_on
            ua.save(update_triggers=False)

@receiver(post_delete, sender=UserAction)
def remove_action_reminders(sender, instance, using, **kwargs):
    """If a user deletes one of their UserAction instances, we should also
    remove the GCMMessage associated with it, so they don't get a
    notification.

    NOTE: GCMMessages have a generic relationship to the Action
    """
    # Remove any custom triggers associated with this object.
    try:
        if instance.custom_trigger:
            instance.custom_trigger.delete()
    except Trigger.DoesNotExist:
        # This really shouldn't happen, but sometimes it does when cleaning
        # up generated objects in our test suite
        pass

    try:
        # Remove any pending notifications
        from notifications.models import GCMMessage
        action_type = ContentType.objects.get_for_model(Action)
        messages = GCMMessage.objects.filter(
            content_type=action_type,
            object_id=instance.action.id,
            user=instance.user
        )
        messages.delete()
    except (ImportError, ContentType.DoesNotExist):
        pass

@receiver(post_save, sender=UserCompletedAction, dispatch_uid="action-completed")
def action_completed(sender, instance, created, raw, using, **kwargs):
    """Record metrics when a UserCompletedAction status is updated."""
    if instance.state:
        key = "action-{}".format(instance.state)
        metric(key, category="User Interactions")

    # Kill all of the queued messages that match this action when the trigger's
    # stop_on_complete is True
    completed = instance.state == UserCompletedAction.COMPLETED

    # Try to get the trigger (custom or default)
    trigger = instance.useraction.trigger

    if completed and trigger and trigger.stop_on_complete:
        messages = instance.user.gcmmessage_set.filter(
            object_id=instance.action.id,
            content_type=ContentType.objects.get_for_model(Action)
        )
        messages.delete()

@receiver(pre_delete, sender=UserCategory, dispatch_uid="del_cat_goals")
def delete_category_child_goals(sender, instance, using, **kwargs):
    """If a user is removing a category, delete all of the user's selected
    goals that have *no other* parent category."""
    # Get a list of all categories selected by the user, excluding the one
    # we're about to remove.
    user_categories = UserCategory.objects.filter(user=instance.user)
    user_categories = user_categories.exclude(id=instance.id)
    user_categories = user_categories.values_list('category', flat=True)

    # Delete all the user's goals that lie ONLY in the category we're
    # about to remove
    user_goals = instance.user.usergoal_set.all()
    user_goals = user_goals.exclude(goal__categories__in=user_categories)
    user_goals.delete()

@receiver(post_save, sender=UserCategory, dispatch_uid="adopt_usercategories")
@receiver(post_save, sender=UserGoal, dispatch_uid="adopt_usergoals")
@receiver(post_save, sender=UserBehavior, dispatch_uid="adopt_userbehaviors")
@receiver(post_save, sender=UserAction, dispatch_uid="adopt_useractions")
def user_adopted_content(sender, instance, created, raw, using, **kwargs):
    """Record some metrics when a user adopts a piece of behavior content."""
    if created:
        key = "{}-created".format(sender.__name__.lower())
        metric(key, category="User Interactions")

@receiver(pre_save, sender=UserAction, dispatch_uid='bust_useraction_cache')
@receiver(pre_save, sender=UserBehavior, dispatch_uid='bust_userbehavior_cache')
@receiver(pre_save, sender=UserGoal, dispatch_uid='bust_usergoal_cache')
@receiver(pre_save, sender=UserCategory, dispatch_uid='bust_usercategory_cache')
def bust_cache(sender, instance, raw, using, **kwargs):
    """This is a little messy, but whenever a user's mapping to content is saved
    we need to bust some cache values. This is mostly for the giant api endpoint
    that exposes a lot of user data (e.g. in the userprofile app).

    # TODO: extend this to bust all cache keys related to User* objects?

    """
    # A mapping of model to cache keys
    cache_key = {
        UserAction: '{}-User.get_actions',
        UserBehavior: '{}-User.get_behaviors',
        UserGoal: '{}-User.get_goals',
        UserCategory: '{}-User.get_categories',
    }
    cache_key = cache_key.get(sender, None)
    if cache_key:
        cache_key = cache_key.format(instance.user.id)
        cache.delete(cache_key)

@receiver(post_save, sender=PackageEnrollment, dispatch_uid="notifiy_for_new_package")
def notify_for_new_package(sender, instance, created, **kwargs):
    """Create and schedule a GCMMEssage for users that have a device registered,
    once they've been enrolled in a new package.

    """
    if created and instance.user.gcmdevice_set.exists():
        from notifications.models import GCMMessage
        GCMMessage.objects.create(
            user=instance.user,
            title="You've been enrolled.",
            message="Welcome to {0}".format(instance.category.title),
            deliver_on=timezone.now(),
            obj=instance,
            priority=GCMMessage.HIGH
        )
<commit_msg>updating signal handlers so deletion of objects also updates the DailyProgress stats<commit_after>"""
Signal Handlers for our models.

"""
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.db.models import ObjectDoesNotExist
from django.db.models.signals import (
    pre_delete, pre_save, post_delete, post_save
)
from django.dispatch import receiver
from django.utils import timezone

from notifications.signals import notification_snoozed
from redis_metrics import metric

from .custom import CustomAction
from .packages import PackageEnrollment
from .progress import DailyProgress, UserCompletedAction
from .public import Action, Behavior, Category, Goal
from .users import UserAction, UserBehavior, UserCategory, UserGoal
from .triggers import Trigger

from ..utils import clean_title, clean_notification, strip

@receiver(post_save, sender=CustomAction, dispatch_uid="coru_daily_progress")
@receiver(post_save, sender=UserBehavior, dispatch_uid="coru_daily_progress")
@receiver(post_save, sender=UserAction, dispatch_uid="coru_daily_progress")
def create_or_update_daily_progress(sender, instance, created, raw, using, **kwargs):
    """When a CustomAction, UserAction or UserBehavior is created, we want to
    create (if necessary) or update the day's DailyProgress for the user.
    """
    if created:
        dp = DailyProgress.objects.for_today(instance.user)
        dp.update_stats()
        dp.save()

@receiver(post_delete, sender=CustomAction, dispatch_uid="coru_daily_progress")
@receiver(post_delete, sender=UserBehavior, dispatch_uid="coru_daily_progress")
@receiver(post_delete, sender=UserAction, dispatch_uid="coru_daily_progress")
def update_daily_progress(sender, instance, using, **kwargs):
    """When a CustomAction, UserAction or UserBehavior is deleted, we want to
    update the day's DailyProgress again to reflect the change.
    """
    dp = DailyProgress.objects.for_today(instance.user)
    dp.update_stats()
    dp.save()

@receiver(post_save, sender=Trigger, dispatch_uid="custom-trigger-updated")
def custom_trigger_updated(sender, instance, created, raw, using, **kwargs):
    """Record metrics when a User updates their custom triggers."""
    if instance.user:
        metric('custom-trigger-updated', category="User Interactions")

@receiver(pre_delete, sender=Trigger, dispatch_uid="trigger-remove-queued-messages")
@receiver(post_save, sender=Trigger, dispatch_uid="trigger-remove-queued-messages")
def remove_queued_messages(sender, instance, *args, **kwargs):
    """If a trigger is updated, we need to remove all currently queued GCM
    Messages that are using the trigger."""
    try:
        instance.action_default.remove_queued_messages()
    except ObjectDoesNotExist:
        pass

@receiver(pre_save, sender=Action)
@receiver(pre_save, sender=Behavior)
@receiver(pre_save, sender=Goal)
@receiver(pre_save, sender=Category)
def clean_content(sender, instance, raw, using, **kwargs):
    # A mapping of model field names and the function that cleans them.
    clean_functions = {
        "title": clean_title,
        "subtitle": clean_title,
        "description": strip,
        "more_info": strip,
        "notification_text": clean_notification,
    }
    for field, func in clean_functions.items():
        if hasattr(instance, field):
            setattr(instance, field, func(getattr(instance, field)))

@receiver(post_delete, sender=Action)
@receiver(post_delete, sender=Behavior)
@receiver(post_delete, sender=Goal)
@receiver(post_delete, sender=Category)
def delete_model_icon(sender, instance, using, **kwargs):
    """Once a model instance has been deleted, this will remove its `icon` from
    the filesystem."""
    if hasattr(instance, 'icon') and instance.icon:
        instance.icon.delete()

@receiver(post_delete, sender=Action)
@receiver(post_delete, sender=Behavior)
def delete_model_image(sender, instance, using, **kwargs):
    """Once a model instance has been deleted, this will remove its `image`
    from the filesystem."""
    if hasattr(instance, 'image') and instance.image:
        instance.image.delete()

@receiver(pre_delete, sender=UserGoal, dispatch_uid="del_goal_behaviors")
def delete_goal_child_behaviors(sender, instance, using, **kwargs):
    """If a user is removing a goal, delete all of the user's selected
    behaviors that have *no other* parent goal."""
    # Get a list of all goals selected by the user, excluding the one
    # we're about to remove.
    user_goals = UserGoal.objects.filter(user=instance.user)
    user_goals = user_goals.exclude(id=instance.id)
    user_goals = user_goals.values_list('goal', flat=True)

    # Delete all the user's behaviors that lie ONLY in the goal we're
    # about to remove
    user_behaviors = instance.user.userbehavior_set.all()
    user_behaviors = user_behaviors.exclude(behavior__goals__in=user_goals)
    user_behaviors.delete()

@receiver(pre_delete, sender=UserBehavior, dispatch_uid="del_behavior_actions")
def delete_behavior_child_actions(sender, instance, using, **kwargs):
    """If a user is removing a behavior, delete all of the user's selected
    actions that are a child of this behavior."""

    user_actions = instance.user.useraction_set.filter(
        action__behavior=instance.behavior
    )
    user_actions.delete()

@receiver(post_delete, sender=UserBehavior)
def remove_behavior_reminders(sender, instance, using, **kwargs):
    """If a user deletes ALL of their UserBehavior instances, we should also
    remove the currently-queued GCMMessage for the Behavior reminder.

    """
    # NOTE: All behavior reminders use the default trigger, and we're not
    # actually connecting them to any content types, so that's null.
    if not UserBehavior.objects.filter(user=instance.user).exists():
        try:
            from notifications.models import GCMMessage
            messages = GCMMessage.objects.for_model("behavior", include_null=True)
            messages = messages.filter(user=instance.user)
            messages.delete()
        except (ImportError, ContentType.DoesNotExist):
            pass

@receiver(post_save, sender=UserAction, dispatch_uid="create-parent-userbehavior")
def create_parent_user_behavior(sender, instance, using, **kwargs):
    """If a user doens't have a UserBehavior object for the UserAction's
    parent behavior this will create one.

    """
    params = {'user': instance.user, 'behavior': instance.action.behavior}
    if not UserBehavior.objects.filter(**params).exists():
        UserBehavior.objects.create(
            user=instance.user,
            behavior=instance.action.behavior
        )

@receiver(post_save, sender=UserAction, dispatch_uid="create-relative-reminder")
def create_relative_reminder(sender, instance, created, raw, using, **kwargs):
    """When a UserAction is created, we need to look at it's default_trigger
    and see if it's a relative reminder. If so, we automatically create a
    custom trigger for the user filling in it's trigger_date based on the
    UserAction's creation date.

    """
    is_relative = (
        instance.custom_trigger is None and
        instance.default_trigger is not None and
        instance.default_trigger.is_relative
    )
    if created and is_relative:
        trigger = Trigger.objects.create(
            user=instance.user,
            name="Trigger for {}".format(instance),
            time=instance.default_trigger.time,
            trigger_date=instance.default_trigger.trigger_date,
            recurrences=instance.default_trigger.recurrences,
            start_when_selected=instance.default_trigger.start_when_selected,
            stop_on_complete=instance.default_trigger.stop_on_complete,
            relative_value=instance.default_trigger.relative_value,
            relative_units=instance.default_trigger.relative_units
        )
        trigger.trigger_date = trigger.relative_trigger_date(instance.created_on)
        trigger.save()
        instance.custom_trigger = trigger
        instance.save()

@receiver(notification_snoozed)
def reset_next_trigger_date_when_snoozed(sender, message, user,
                                         related_object, deliver_on, **kwargs):
    """If a user snoozes a notification (in the notifications app), this function
    will try to update the relevant UserAction's `next_trigger_date`.

    """
    if related_object and message.content_type.name.lower() == 'action':
        ua = related_object.useraction_set.filter(user=user).first()
        if ua and deliver_on:
            ua.next_trigger_date = deliver_on
            ua.save(update_triggers=False)

@receiver(post_delete, sender=UserAction)
def remove_action_reminders(sender, instance, using, **kwargs):
    """If a user deletes one of their UserAction instances, we should also
    remove the GCMMessage associated with it, so they don't get a
    notification.

    NOTE: GCMMessages have a generic relationship to the Action
    """
    # Remove any custom triggers associated with this object.
    try:
        if instance.custom_trigger:
            instance.custom_trigger.delete()
    except Trigger.DoesNotExist:
        # This really shouldn't happen, but sometimes it does when cleaning
        # up generated objects in our test suite
        pass

    try:
        # Remove any pending notifications
        from notifications.models import GCMMessage
        action_type = ContentType.objects.get_for_model(Action)
        messages = GCMMessage.objects.filter(
            content_type=action_type,
            object_id=instance.action.id,
            user=instance.user
        )
        messages.delete()
    except (ImportError, ContentType.DoesNotExist):
        pass

@receiver(post_save, sender=UserCompletedAction, dispatch_uid="action-completed")
def action_completed(sender, instance, created, raw, using, **kwargs):
    """Record metrics when a UserCompletedAction status is updated."""
    if instance.state:
        key = "action-{}".format(instance.state)
        metric(key, category="User Interactions")

    # Kill all of the queued messages that match this action when the trigger's
    # stop_on_complete is True
    completed = instance.state == UserCompletedAction.COMPLETED

    # Try to get the trigger (custom or default)
    trigger = instance.useraction.trigger

    if completed and trigger and trigger.stop_on_complete:
        messages = instance.user.gcmmessage_set.filter(
            object_id=instance.action.id,
            content_type=ContentType.objects.get_for_model(Action)
        )
        messages.delete()

@receiver(pre_delete, sender=UserCategory, dispatch_uid="del_cat_goals")
def delete_category_child_goals(sender, instance, using, **kwargs):
    """If a user is removing a category, delete all of the user's selected
    goals that have *no other* parent category."""
    # Get a list of all categories selected by the user, excluding the one
    # we're about to remove.
    user_categories = UserCategory.objects.filter(user=instance.user)
    user_categories = user_categories.exclude(id=instance.id)
    user_categories = user_categories.values_list('category', flat=True)

    # Delete all the user's goals that lie ONLY in the category we're
    # about to remove
    user_goals = instance.user.usergoal_set.all()
    user_goals = user_goals.exclude(goal__categories__in=user_categories)
    user_goals.delete()

@receiver(post_save, sender=UserCategory, dispatch_uid="adopt_usercategories")
@receiver(post_save, sender=UserGoal, dispatch_uid="adopt_usergoals")
@receiver(post_save, sender=UserBehavior, dispatch_uid="adopt_userbehaviors")
@receiver(post_save, sender=UserAction, dispatch_uid="adopt_useractions")
def user_adopted_content(sender, instance, created, raw, using, **kwargs):
    """Record some metrics when a user adopts a piece of behavior content."""
    if created:
        key = "{}-created".format(sender.__name__.lower())
        metric(key, category="User Interactions")

@receiver(pre_save, sender=UserAction, dispatch_uid='bust_useraction_cache')
@receiver(pre_save, sender=UserBehavior, dispatch_uid='bust_userbehavior_cache')
@receiver(pre_save, sender=UserGoal, dispatch_uid='bust_usergoal_cache')
@receiver(pre_save, sender=UserCategory, dispatch_uid='bust_usercategory_cache')
def bust_cache(sender, instance, raw, using, **kwargs):
    """This is a little messy, but whenever a user's mapping to content is saved
    we need to bust some cache values. This is mostly for the giant api endpoint
    that exposes a lot of user data (e.g. in the userprofile app).

    # TODO: extend this to bust all cache keys related to User* objects?

    """
    # A mapping of model to cache keys
    cache_key = {
        UserAction: '{}-User.get_actions',
        UserBehavior: '{}-User.get_behaviors',
        UserGoal: '{}-User.get_goals',
        UserCategory: '{}-User.get_categories',
    }
    cache_key = cache_key.get(sender, None)
    if cache_key:
        cache_key = cache_key.format(instance.user.id)
        cache.delete(cache_key)

@receiver(post_save, sender=PackageEnrollment, dispatch_uid="notifiy_for_new_package")
def notify_for_new_package(sender, instance, created, **kwargs):
    """Create and schedule a GCMMEssage for users that have a device registered,
    once they've been enrolled in a new package.

    """
    if created and instance.user.gcmdevice_set.exists():
        from notifications.models import GCMMessage
        GCMMessage.objects.create(
            user=instance.user,
            title="You've been enrolled.",
            message="Welcome to {0}".format(instance.category.title),
            deliver_on=timezone.now(),
            obj=instance,
            priority=GCMMessage.HIGH
        )
<|endoftext|>


<commit_before>
from sklearn.linear_model import LinearRegression
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn import preprocessing
from sklearn.base import TransformerMixin
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C, RationalQuadratic as RQ
from sklearn.base import RegressorMixin, BaseEstimator
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.preprocessing import StandardScaler
import numpy, pandas

from .attribute_dict import dicta

class LinearAndGaussianProcessRegression(
		BaseEstimator,
		RegressorMixin,
):

	def __init__(self):
		self.lr = LinearRegression()
		self.gpr = GaussianProcessRegressor()
		self.y_residual = None
		self.kernel_generator = lambda dims: C() * RBF([1.0] * dims)

	def fit(self, X, y):
		"""
		Fit linear and gaussian model.

		Parameters
		----------
		X : numpy array or sparse matrix of shape [n_samples, n_features]
			Training data
		y : numpy array of shape [n_samples, n_targets]
			Target values.

		Returns
		-------
		self : returns an instance of self.
		"""
		self.lr.fit(X, y)
		self.y_residual = y - self.lr.predict(X)
		dims = X.shape[1]
		self.gpr.kernel = self.kernel_generator(dims)
		self.gpr.fit(X, self.y_residual)
		return self

	def predict(self, X):
		"""Predict using the model

		Parameters
		----------
		X : {array-like, sparse matrix}, shape = (n_samples, n_features)
			Samples.

		Returns
		-------
		C : array, shape = (n_samples,)
			Returns predicted values.
		"""
		y_hat_lr = self.lr.predict(X=X)
		y_hat_gpr = self.gpr.predict(X)
		return y_hat_lr + y_hat_gpr

	def cross_val_scores(self, X, y, cv=3):
		total = cross_val_score(self, X, y, cv=cv)
		linear_cv_score = cross_val_score(self.lr, X, y, cv=cv)
		linear_cv_predict = cross_val_predict(self.lr, X, y, cv=cv)
		linear_cv_residual = y-linear_cv_predict
		gpr_cv_score = cross_val_score(self.gpr, X, linear_cv_residual, cv=cv)

		self.lr.fit(X, y)
		y_residual = y - self.lr.predict(X)
		gpr_cv_score2 = cross_val_score(self.gpr, X, y_residual, cv=cv)

		return dicta(
			total=total,
			linear=linear_cv_score,
			net_gpr=total-linear_cv_score,
			gpr=gpr_cv_score,
			gpr2=gpr_cv_score2,
		)

def cross_val_scores(pipe, X, y, cv=3):
	# For pipelines

	self = pipe.steps[-1][1]

	total = cross_val_score(self, X, y, cv=cv)
	linear_cv_score = cross_val_score(self.lr, X, y, cv=cv)
	linear_cv_predict = cross_val_predict(self.lr, X, y, cv=cv)
	linear_cv_residual = y-linear_cv_predict
	gpr_cv_score = cross_val_score(self.gpr, X, linear_cv_residual, cv=cv)

	self.lr.fit(X, y)
	y_residual = y - self.lr.predict(X)
	gpr_cv_score2 = cross_val_score(self.gpr, X, y_residual, cv=cv)

	return dicta(
		total=total,
		linear=linear_cv_score,
		net_gpr=total-linear_cv_score,
		gpr=gpr_cv_score,
		gpr2=gpr_cv_score2,
	)

class PartialStandardScaler(StandardScaler):

	def __init__(self, copy=True, with_mean=True, with_std=True, omit=()):
		super().__init__(copy=copy, with_mean=with_mean, with_std=with_std)
		self._omit = omit

	def fit(self, X, y=None):
		result = super().fit(X, y)
		omit = [i for i in self._omit]
		if isinstance(X, pandas.DataFrame):
			for n,k in enumerate(omit):
				if isinstance(k, str):
					omit[n] = X.columns.get_loc(k)
		for k in omit:
			if self.with_mean:
				self.mean_[k] = 0
			if self.with_std:
				self.scale_[k] = 1
		return result

class ExponentialFeatures(BaseEstimator, TransformerMixin):

	def __init__(self):
		pass

	def fit(self, X, y=None):
		"""
		Compute number of output features.

		Parameters
		----------
		X : array-like, shape (n_samples, n_features)
			The data.

		Returns
		-------
		self : instance
		"""
		return self

	def transform(self, X):
		"""Transform data to add exponential features

		Parameters
		----------
		X : array-like, shape [n_samples, n_features]
			The data to transform, row by row.

		Returns
		-------
		XP : np.ndarray shape [n_samples, NP]
			The matrix of features, where NP is the number of polynomial
			features generated from the combination of inputs.
		"""
		from sklearn.utils import check_array
		from sklearn.utils.validation import check_is_fitted, check_random_state, FLOAT_DTYPES

		from scipy.stats.stats import pearsonr

		X = check_array(X, dtype=FLOAT_DTYPES)
		n_samples, n_features = X.shape

		# allocate output data
		XP = numpy.empty((n_samples, n_features*2), dtype=X.dtype)
		XP_use = numpy.ones((n_features*2,), dtype=bool)

		for i in range(n_features):
			XP[:, i] = X[:, i]
			exp_x = numpy.exp(X[:, i])
			correlation = pearsonr(X[:, i], exp_x)[0]
			print("correlation is",correlation)
			if numpy.fabs(correlation) < 0.99:
				XP[:, i+n_features] = exp_x
			else:
				XP_use[i+n_features] = 0

		if isinstance(X, pandas.DataFrame):
			result = pandas.DataFrame(
				data=XP,
				columns=list(X.columns) + [f'exp({c})' for c in X.columns],
				index=X.index,
			)
			return result.iloc[:, XP_use]

		return XP[:,XP_use]

class InteractionFeatures(BaseEstimator, TransformerMixin):

	def __init__(self, interaction_point):
		self._interaction_name = interaction_point

	def fit(self, X, y=None):
		"""
		Compute number of output features.

		Parameters
		----------
		X : array-like, shape (n_samples, n_features)
			The data.

		Returns
		-------
		self : instance
		"""
		return self

	def transform(self, X):
		"""Transform data to add exponential features

		Parameters
		----------
		X : array-like, shape [n_samples, n_features]
			The data to transform, row by row.

		Returns
		-------
		XP : np.ndarray shape [n_samples, NP]
			The matrix of features, where NP is the number of polynomial
			features generated from the combination of inputs.
		"""
		from sklearn.utils import check_array
		from sklearn.utils.validation import check_is_fitted, check_random_state, FLOAT_DTYPES

		from scipy.stats.stats import pearsonr

		interact_point = None

		if isinstance(self._interaction_name, int):
			interact_point = self._interaction_name
		else:
			if isinstance(X, pandas.DataFrame):
				interact_point = X.columns.get_loc(self._interaction_name)
			else:
				raise TypeError("X must be DataFrame when interaction_name is string")

		if interact_point is None:
			raise TypeError('interact_point is None')

		Xa = check_array(X, dtype=FLOAT_DTYPES)
		n_samples, n_features = Xa.shape

		# allocate output data
		XP = numpy.empty((n_samples, n_features*2), dtype=Xa.dtype)
		XP_use = numpy.ones((n_features*2,), dtype=bool)

		for i in range(n_features):
			XP[:, i] = Xa[:, i]
			exp_x = Xa[:, i] * Xa[:, interact_point]
			correlation = pearsonr(Xa[:, i], exp_x)[0]
			if numpy.fabs(correlation) < 0.99:
				XP[:, i+n_features] = exp_x
			else:
				XP_use[i+n_features] = 0

		if isinstance(X, pandas.DataFrame):
			result = pandas.DataFrame(
				data=XP,
				columns=list(X.columns) + [f'{c} ~ {self._interaction_name}' for c in X.columns],
				index=X.index,
			)
			return result.iloc[:, XP_use]

		return XP[:,XP_use]

<commit_msg>more gpr<commit_after>
from sklearn.linear_model import LinearRegression as _sklearn_LinearRegression
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn import preprocessing
from sklearn.base import TransformerMixin
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C, RationalQuadratic as RQ
from sklearn.base import RegressorMixin, BaseEstimator
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.preprocessing import StandardScaler
import numpy, pandas
import scipy.stats

from .attribute_dict import dicta

class LinearRegression(_sklearn_LinearRegression):

	def fit(self, X, y, sample_weight=None):
		# print(" LR FIT on",len(X))
		super().fit(X, y, sample_weight=sample_weight)

		if isinstance(X, pandas.DataFrame):
			self.names_ = X.columns.copy()

		sse = numpy.sum(
			(self.predict(X) - y) ** 2, axis=0) / float(X.shape[0] - X.shape[1])
		se = numpy.array([
			numpy.sqrt(numpy.diagonal(sse[i] * numpy.linalg.inv(numpy.dot(X.T, X))))
			for i in range(sse.shape[0])
		])

		self.t_ = self.coef_ / se
		self.p_ = 2 * (1 - scipy.stats.t.cdf(numpy.abs(self.t_), y.shape[0] - X.shape[1]))
		# try:
		# 	print(y.values[0])
		# except AttributeError:
		# 	print(y[0])
		return self

	def predict(self, X):
		# print(" "*55,"LR PREDICT on", len(X))
		return super().predict(X)

class GaussianProcessRegressor_(GaussianProcessRegressor):

	def fit(self, X, y):
		# print(" GPR FIT on",len(X))
		q = super().fit(X,y)
		# try:
		# 	print(y.values[0])
		# except AttributeError:
		# 	print(y[0])
		return q

	def predict(self, X, return_std=False, return_cov=False):
		#print(" "*55,"GPR PREDICT on", len(X))
		return super().predict(X, return_std=return_std, return_cov=return_cov)

class LinearAndGaussianProcessRegression(
		BaseEstimator,
		RegressorMixin,
):

	def __init__(self):
		self.lr = LinearRegression()
		self.gpr = GaussianProcessRegressor_(n_restarts_optimizer=9)
		self.y_residual = None
		self.kernel_generator = lambda dims: C() * RBF([1.0] * dims)

	def fit(self, X, y):
		"""
		Fit linear and gaussian model.

		Parameters
		----------
		X : numpy array or sparse matrix of shape [n_samples, n_features]
			Training data
		y : numpy array of shape [n_samples, n_targets]
			Target values.

		Returns
		-------
		self : returns an instance of self.
		"""
		# print("META FIT on",len(X))
		self.lr.fit(X, y)
		self.y_residual = y - self.lr.predict(X)
		dims = X.shape[1]
		self.gpr.kernel = self.kernel_generator(dims)
		self.gpr.fit(X, self.y_residual)
		# print(self.y_residual.values[0])
		return self

	def predict(self, X):
		"""Predict using the model

		Parameters
		----------
		X : {array-like, sparse matrix}, shape = (n_samples, n_features)
			Samples.

		Returns
		-------
		C : array, shape = (n_samples,)
			Returns predicted values.
		"""
		y_hat_lr = self.lr.predict(X=X)
		y_hat_gpr = self.gpr.predict(X)
		return y_hat_lr + y_hat_gpr

	def cross_val_scores(self, X, y, cv=3, alt_y=None):
		total = cross_val_score(self, X, y, cv=cv)
		linear_cv_score = cross_val_score(self.lr, X, y, cv=cv)
		linear_cv_predict = cross_val_predict(self.lr, X, y, cv=cv)
		linear_cv_residual = y-linear_cv_predict
		gpr_cv_score = cross_val_score(self.gpr, X, linear_cv_residual, cv=cv)

		self.lr.fit(X, y)
		y_residual = y - self.lr.predict(X)
		gpr_cv_score2 = cross_val_score(self.gpr, X, y_residual, cv=cv)

		result = dicta(
			total=total,
			linear=linear_cv_score,
			net_gpr=total-linear_cv_score,
			gpr=gpr_cv_score,
			gpr2=gpr_cv_score2,
		)
		if alt_y is not None:
			result['gpr_alt'] = cross_val_score(self.gpr, X, alt_y, cv=cv)
			# print()
			# print(numpy.concatenate([y_residual, alt_y, y_residual-alt_y], axis=1 ))
			# print()
			# print(result['gpr_alt'])
			# print(result['gpr2'])
			# print()
		return result

	def cross_val_predicts(self, X, y, cv=3):
		total = cross_val_predict(self, X, y, cv=cv)
		linear_cv_predict = cross_val_predict(self.lr, X, y, cv=cv)
		linear_cv_residual = y-linear_cv_predict
		gpr_cv_predict_over_cv_linear = cross_val_predict(self.gpr, X, linear_cv_residual, cv=cv)

		self.lr.fit(X, y)
		linear_full_predict = self.lr.predict(X)
		y_residual = y - linear_full_predict
		gpr_cv_predict_over_full_linear = cross_val_predict(self.gpr, X, y_residual, cv=cv)

		return dicta(
			total=total,
			linear=linear_cv_predict,
			net_gpr=total-linear_cv_predict,
			gpr=gpr_cv_predict_over_cv_linear+linear_cv_predict,
			gpr2=gpr_cv_predict_over_full_linear+linear_full_predict,
		)

def cross_val_scores(pipe, X, y, cv=3):
	# For pipelines

	self = pipe.steps[-1][1]

	total = cross_val_score(self, X, y, cv=cv)
	linear_cv_score = cross_val_score(self.lr, X, y, cv=cv)
	linear_cv_predict = cross_val_predict(self.lr, X, y, cv=cv)
	linear_cv_residual = y-linear_cv_predict
	gpr_cv_score = cross_val_score(self.gpr, X, linear_cv_residual, cv=cv)

	self.lr.fit(X, y)
	y_residual = y - self.lr.predict(X)
	gpr_cv_score2 = cross_val_score(self.gpr, X, y_residual, cv=cv)

	return dicta(
		total=total,
		linear=linear_cv_score,
		net_gpr=total-linear_cv_score,
		gpr=gpr_cv_score,
		gpr2=gpr_cv_score2,
	)

class PartialStandardScaler(StandardScaler):

	def __init__(self, copy=True, with_mean=True, with_std=True, omit=()):
		super().__init__(copy=copy, with_mean=with_mean, with_std=with_std)
		self._omit = omit
		self._names = None

	def fit(self, X, y=None):
		result = super().fit(X, y)
		omit = [i for i in self._omit]
		if isinstance(X, pandas.DataFrame):
			self._names = X.columns.copy()
			for n,k in enumerate(omit):
				if isinstance(k, str):
					omit[n] = X.columns.get_loc(k)
		for k in omit:
			if self.with_mean:
				self.mean_[k] = 0
			if self.with_std:
				self.scale_[k] = 1
		return result

	def inverse_transform_by_name(self, X, name):
		ix = self._names.get_loc(name)
		if self.with_std:
			X *= self.scale_[ix]
		if self.with_mean:
			X += self.mean_[ix]
		return X

	def transform_by_name(self, X, name):
		ix = self._names.get_loc(name)
		if self.with_mean:
			X -= self.mean_[ix]
		if self.with_std:
			X /= self.scale_[ix]
		return X

class ExponentialFeatures(BaseEstimator, TransformerMixin):

	def __init__(self):
		pass

	def fit(self, X, y=None):
		"""
		Compute number of output features.

		Parameters
		----------
		X : array-like, shape (n_samples, n_features)
			The data.

		Returns
		-------
		self : instance
		"""
		return self

	def transform(self, X):
		"""Transform data to add exponential features

		Parameters
		----------
		X : array-like, shape [n_samples, n_features]
			The data to transform, row by row.

		Returns
		-------
		XP : np.ndarray shape [n_samples, NP]
			The matrix of features, where NP is the number of polynomial
			features generated from the combination of inputs.
		"""
		from sklearn.utils import check_array
		from sklearn.utils.validation import check_is_fitted, check_random_state, FLOAT_DTYPES

		from scipy.stats.stats import pearsonr

		X = check_array(X, dtype=FLOAT_DTYPES)
		n_samples, n_features = X.shape

		# allocate output data
		XP = numpy.empty((n_samples, n_features*2), dtype=X.dtype)
		XP_use = numpy.ones((n_features*2,), dtype=bool)

		for i in range(n_features):
			XP[:, i] = X[:, i]
			exp_x = numpy.exp(X[:, i])
			correlation = pearsonr(X[:, i], exp_x)[0]
			# print("correlation is",correlation)
			if numpy.fabs(correlation) < 0.99:
				XP[:, i+n_features] = exp_x
			else:
				XP_use[i+n_features] = 0

		if isinstance(X, pandas.DataFrame):
			result = pandas.DataFrame(
				data=XP,
				columns=list(X.columns) + [f'exp({c})' for c in X.columns],
				index=X.index,
			)
			return result.iloc[:, XP_use]

		return XP[:,XP_use]

class InteractionFeatures(BaseEstimator, TransformerMixin):

	def __init__(self, interaction_point):
		self._interaction_name = interaction_point

	def fit(self, X, y=None):
		"""
		Compute number of output features.

		Parameters
		----------
		X : array-like, shape (n_samples, n_features)
			The data.

		Returns
		-------
		self : instance
		"""
		return self

	def transform(self, X):
		"""Transform data to add exponential features

		Parameters
		----------
		X : array-like, shape [n_samples, n_features]
			The data to transform, row by row.

		Returns
		-------
		XP : np.ndarray shape [n_samples, NP]
			The matrix of features, where NP is the number of polynomial
			features generated from the combination of inputs.
		"""
		from sklearn.utils import check_array
		from sklearn.utils.validation import check_is_fitted, check_random_state, FLOAT_DTYPES

		from scipy.stats.stats import pearsonr

		interact_point = None

		if isinstance(self._interaction_name, int):
			interact_point = self._interaction_name
		else:
			if isinstance(X, pandas.DataFrame):
				interact_point = X.columns.get_loc(self._interaction_name)
			else:
				raise TypeError("X must be DataFrame when interaction_name is string")

		if interact_point is None:
			raise TypeError('interact_point is None')

		Xa = check_array(X, dtype=FLOAT_DTYPES)
		n_samples, n_features = Xa.shape

		# allocate output data
		XP = numpy.empty((n_samples, n_features*2), dtype=Xa.dtype)
		XP_use = numpy.ones((n_features*2,), dtype=bool)
		XP_fresh = numpy.zeros((n_features*2,), dtype=bool)

		for i in range(n_features):
			XP[:, i] = Xa[:, i]
			exp_x = Xa[:, i] * Xa[:, interact_point]
			#correlation = pearsonr(Xa[:, i], exp_x)[0]
			correlation1 = numpy.corrcoef(Xa, exp_x, rowvar=False)[-1, :-1]
			if XP_fresh.sum():
				correlation2 = numpy.corrcoef(XP[:,XP_fresh], exp_x, rowvar=False)[-1, :-1]
			else:
				correlation2 = [0]
			correlation = max( numpy.fabs(correlation1).max(), numpy.fabs(correlation2).max())
			if numpy.fabs(correlation) < 0.99:
				XP[:, i+n_features] = exp_x
				XP_fresh[i+n_features] = True
			else:
				XP_use[i+n_features] = 0

		if isinstance(X, pandas.DataFrame):
			result = pandas.DataFrame(
				data=XP,
				columns=list(X.columns) + [f'{c} ~ {self._interaction_name}' for c in X.columns],
				index=X.index,
			)
			return result.iloc[:, XP_use]

		return XP[:,XP_use]

<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import torch
from tqdm import tqdm

from nupic.research.frameworks.dynamic_sparse.models import BaseModel
from nupic.research.frameworks.stochastic_connections.binary_layers import (
    BinaryGatedConv2d,
    BinaryGatedLinear,
)
from nupic.research.frameworks.stochastic_connections.reparameterization_layers import (
    HardConcreteGatedConv2d,
    HardConcreteGatedLinear,
)

STOCHASTIC_MODULES = (
    BinaryGatedConv2d,
    BinaryGatedLinear,
    HardConcreteGatedConv2d,
    HardConcreteGatedLinear,
)

class StochasticSynapsesModel(BaseModel):

    def setup(self, config=None):
        super().setup(config)

        # Add specific defaults.
        new_defaults = dict(
            use_tqdm=False,
        )
        new_defaults = {k: v for k, v in new_defaults.items() if k not in self.__dict__}
        self.__dict__.update(new_defaults)

        # TODO: This could use a stricter condition
        # to ensure module is stochastic - not just having
        # a "l0_strength" perhaps by chance.
        self.named_stochastic_modules = [
            (name, module)
            for (name, module) in self.network.named_modules()
            if isinstance(module, STOCHASTIC_MODULES)
        ]

    @property
    def stochastic_modules(self):
        return [module for (_, module) in self.named_stochastic_modules]

    # -------------------
    # Procedural Methods
    # -------------------

    def _pre_epoch_setup(self):
        pass

    def _run_one_pass(self, loader, train=True, noise=False):

        # TODO: The logic in this function is mostly redundant to the logic in
        # `BaseModel._run_one_pass`. Could be good to merge the two.

        if self.use_tqdm:
            batches = tqdm(
                loader, leave=False, desc=("Training" if train else "Testing"))
        else:
            batches = loader

        epoch_loss = 0.
        correct = 0.

        for data, target in batches:
            data, target = data.to(self.device), target.to(self.device)
            output = self.network(data)
            loss = self.calc_loss(output, target, loader)

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

            with torch.no_grad():
                epoch_loss += torch.sum(loss).item()
                pred = output.argmax(dim=1, keepdim=True)
                correct += pred.eq(target.view_as(pred)).sum().item()

            for module in self.stochastic_modules:
                module.constrain_parameters()

        # Store loss and accuracy at each pass.
        loss = epoch_loss / len(loader.dataset)
        acc = correct / len(loader.dataset)
        self.logger.log_metrics(loss, acc, train, noise)

    def _post_epoch_updates(self, dataset=None):
        # TODO: Make custom logger to handle `nonzero_counts` functionality.
        # See 'dynamic_sparse/models/loggers.py'
        self.logger.log.update(self.nonzero_counts())

    def _post_optimize_updates(self):
        for module in self.stochastic_modules:
            module.constrain_parameters()

    # --------------------
    # Training Utils
    # --------------------

    def calc_loss(self, outputs, targets, loader):
        dataset_percent = loader.batch_size / len(loader.dataset)
        loss = self.loss_func(outputs, targets)
        loss += dataset_percent * self.regularization()
        return loss

    def regularization(self):
        reg = torch.tensor(0.).to(self.device)
        for module in self.stochastic_modules:
            reg += module.regularization()
        return reg

    def nonzero_counts(self):
        result = {}

        for name, module in self.named_stochastic_modules:

            num_inputs = 1.
            for d in module.weight_size()[1:]:
                num_inputs *= d

            e_nz_by_unit = module.get_expected_nonzeros()
            i_nz_by_unit = module.get_inference_nonzeros()

            multiplies, adds = module.count_inference_flops()

            result[name] = {
                "hist_expected_nz_by_unit": e_nz_by_unit.tolist(),
                "expected_nz": torch.sum(e_nz_by_unit).item(),
                "hist_inference_nz_by_unit": module.get_inference_nonzeros().tolist(),
                "inference_nz": torch.sum(i_nz_by_unit).item(),
                "num_input_units": num_inputs,
                "multiplies": multiplies,
                "flops": multiplies + adds,
            }

        return result
<commit_msg><commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import torch
from tqdm import tqdm

from nupic.research.frameworks.dynamic_sparse.models import BaseModel
from nupic.research.frameworks.stochastic_connections.binary_layers import (
    BinaryGatedConv2d,
    BinaryGatedLinear,
)
from nupic.research.frameworks.stochastic_connections.reparameterization_layers import (
    HardConcreteGatedConv2d,
    HardConcreteGatedLinear,
)

STOCHASTIC_MODULES = (
    BinaryGatedConv2d,
    BinaryGatedLinear,
    HardConcreteGatedConv2d,
    HardConcreteGatedLinear,
)

class StochasticSynapsesModel(BaseModel):

    def setup(self, config=None):
        super().setup(config)

        # Add specific defaults.
        new_defaults = dict(
            use_tqdm=False,
        )
        new_defaults = {k: v for k, v in new_defaults.items() if k not in self.__dict__}
        self.__dict__.update(new_defaults)

        # TODO: This could use a stricter condition
        # to ensure module is stochastic - not just having
        # a "l0_strength" perhaps by chance.
        self.named_stochastic_modules = [
            (name, module)
            for (name, module) in self.network.named_modules()
            if isinstance(module, STOCHASTIC_MODULES)
        ]

    @property
    def stochastic_modules(self):
        return [module for (_, module) in self.named_stochastic_modules]

    # -------------------
    # Procedural Methods
    # -------------------

    def _pre_epoch_setup(self):
        pass

    def _run_one_pass(self, loader, train=True, noise=False):

        # TODO: The logic in this function is mostly redundant to the logic in
        # `BaseModel._run_one_pass`. Could be good to merge the two.

        if self.use_tqdm:
            batches = tqdm(
                loader, leave=False, desc=("Training" if train else "Testing"))
        else:
            batches = loader

        epoch_loss = 0.
        correct = 0.

        for data, target in batches:
            data, target = data.to(self.device), target.to(self.device)
            output = self.network(data)
            loss = self.calc_loss(output, target, loader)

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

            with torch.no_grad():
                epoch_loss += torch.sum(loss).item()
                pred = output.argmax(dim=1, keepdim=True)
                correct += pred.eq(target.view_as(pred)).sum().item()

            for module in self.stochastic_modules:
                module.constrain_parameters()

        # Store loss and accuracy at each pass.
        loss = epoch_loss / len(loader.dataset)
        acc = correct / len(loader.dataset)
        self.logger.log_metrics(loss, acc, train, noise)

    def _post_epoch_updates(self, dataset=None):
        # TODO: Make custom logger to handle `nonzero_counts` functionality.
        # See 'dynamic_sparse/models/loggers.py'
        self.logger.log.update(self.nonzero_counts())

    def _post_optimize_updates(self):
        for module in self.stochastic_modules:
            module.constrain_parameters()

    # --------------------
    # Training Utils
    # --------------------

    def calc_loss(self, outputs, targets, loader):
        dataset_percent = loader.batch_size / len(loader.dataset)
        loss = self.loss_func(outputs, targets)
        loss += dataset_percent * self.regularization()
        return loss

    def regularization(self):
        reg = torch.tensor(0.).to(self.device)
        for module in self.stochastic_modules:
            reg += module.regularization()
        return reg

    def nonzero_counts(self):
        result = {}

        for name, module in self.named_stochastic_modules:

            num_inputs = 1.
            for d in module.weight_size()[1:]:
                num_inputs *= d

            e_nz_by_unit = module.get_expected_nonzeros()
            i_nz_by_unit = module.get_inference_nonzeros()

            multiplies, adds = module.count_inference_flops()

            result[name] = {
                "hist_expected_nz_by_unit": e_nz_by_unit.tolist(),
                "expected_nz": torch.sum(e_nz_by_unit).item(),
                "hist_inference_nz_by_unit": module.get_inference_nonzeros().tolist(),
                "inference_nz": torch.sum(i_nz_by_unit).item(),
                "num_input_units": num_inputs,
                "multiplies": multiplies,
                "flops": multiplies + adds,
            }

        return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Copyright 2013 Jacek Markowski

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import sys
import os
import platform
system = platform.system()
if system == 'Windows':
    new_line = '\r\n'
    from pyfann_win import libfann
elif system == 'Linux':
    new_line = '\n'
    from pyfann import libfann
elif system == 'Darwin':
    new_line = '\r'
else:
    new_line = '\r\n'

print sys.argv[:]
def start(file_in=sys.argv[1],
          connection_rate =float(sys.argv[2]),
          num_neurons_hidden = int(sys.argv[3]),
          desired_error =float(sys.argv[4]),
          max_iterations = int(sys.argv[5]),
          iterations_between_reports = int(sys.argv[6]),
          function_output = 'libfann.'+sys.argv[7],
          function_hidden = 'libfann.'+sys.argv[8],
          training_algorithm = 'libfann.'+sys.argv[9],
          file_out= sys.argv[10]):
    ''' Learning network'''
    title = open(os.path.join('export', '')+file_in, 'r').readline()
    sets, num_input, num_output = title.split(' ')
    num_input = int(num_input)
    num_output = int(num_output)
    print num_input, num_output
    # create training data, and ann object
    train_data = libfann.training_data()
    train_data.read_train_from_file(os.path.join('export', '')+file_in)
    ann = libfann.neural_net()
    ann.create_sparse_array(connection_rate, (num_input, num_neurons_hidden,
                                              num_output))
    ## start training the network
    ann.set_activation_function_output(eval(function_output))
    ann.set_activation_function_hidden(eval(function_hidden))
    ann.set_bit_fail_limit(0.1)
    ann.set_rprop_increase_factor(1.225)
    ann.set_rprop_decrease_factor(0.325)
    ann.set_rprop_delta_min(-10.0)
    ann.set_rprop_delta_max(10.0)
    ann.set_training_algorithm(eval(training_algorithm))
    ann.set_train_stop_function(libfann.STOPFUNC_MSE)
    ann.train_on_data(train_data, max_iterations,
                      iterations_between_reports, desired_error)
    # save network to disk
    ann.save(os.path.join('net','')+file_out)

if __name__ == "__main__":
    start()
<commit_msg>changed hidden layers from 1 to 3<commit_after># -*- coding: utf-8 -*-
"""
Copyright 2013 Jacek Markowski

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import sys
import os
import platform
system = platform.system()
if system == 'Windows':
    new_line = '\r\n'
    from pyfann_win import libfann
elif system == 'Linux':
    new_line = '\n'
    from pyfann import libfann
elif system == 'Darwin':
    new_line = '\r'
else:
    new_line = '\n'

print sys.argv[:]
def start(file_in=sys.argv[1],
          connection_rate =float(sys.argv[2]),
          num_neurons_hidden = int(sys.argv[3]),
          desired_error =float(sys.argv[4]),
          max_iterations = int(sys.argv[5]),
          iterations_between_reports = int(sys.argv[6]),
          function_output = 'libfann.'+sys.argv[7],
          function_hidden = 'libfann.'+sys.argv[8],
          training_algorithm = 'libfann.'+sys.argv[9],
          file_out= sys.argv[10]):
    ''' Learning network'''
    title = open(os.path.join('export', '')+file_in, 'r').readline()
    sets, num_input, num_output = title.split(' ')
    num_input = int(num_input)
    num_output = int(num_output)
    print num_input, num_output
    # create training data, and ann object
    train_data = libfann.training_data()
    train_data.read_train_from_file(os.path.join('export', '')+file_in)
    ann = libfann.neural_net()
    ann.create_sparse_array(connection_rate, (num_input,
                                              num_neurons_hidden,
                                              num_neurons_hidden+1,
                                              num_neurons_hidden+2,
                                              num_output))
    ## start training the network
    ann.set_activation_function_output(eval(function_output))
    ann.set_activation_function_hidden(eval(function_hidden))
    ann.set_bit_fail_limit(0.1)
    ann.set_rprop_increase_factor(1.225)
    ann.set_rprop_decrease_factor(0.325)
    ann.set_rprop_delta_min(-10.0)
    ann.set_rprop_delta_max(10.0)
    ann.set_training_algorithm(eval(training_algorithm))
    ann.set_train_stop_function(libfann.STOPFUNC_MSE)
    ann.train_on_data(train_data, max_iterations,
                      iterations_between_reports, desired_error)
    # save network to disk
    ann.save(os.path.join('net','')+file_out)

if __name__ == "__main__":
    start()
<|endoftext|>


<commit_before>from __future__ import print_function

import time
import logging
from threading import Thread

import numpy as np

from bokeh.models.widgets import Button
from bokeh.plotting import (
    cursession, figure, show,
    output_server, curdoc, vplot,
    hplot
)

logger = logging.getLogger(__name__)

N = 80

x = np.linspace(0, 4*np.pi, N)
y = np.sin(x)

output_server("line_animate_widget")

p = figure()

p.line(x, y, color="#3333ee", name="sin")
p.line([0,4*np.pi], [-1, 1], color="#ee3333")

def play_handler():
    print("button_handler: start click")
    global play
    play = True

def stop_handler():
    print("button_handler: stop click")
    global play
    play = False

play = True

button_start = Button(label="Start", type="success")
button_start.on_click(play_handler)

button_stop = Button(label="Stop", type="danger")
button_stop.on_click(stop_handler)

controls = hplot(button_start, button_stop)
layout = vplot(controls, p)

show(layout)

renderer = p.select(dict(name="sin"))
ds = renderer[0].data_source

def should_play():
    """Return true if we should play animation, otherwise block"""
    global play
    while True:
        if play:
            return True
        else:
            time.sleep(0.05)

def background_thread(ds):
    """Plot animation, update data if play is True, otherwise stop"""
    try:
        while True:
            for i in np.hstack((np.linspace(1, -1, 100), np.linspace(-1, 1, 100))):
                if should_play():
                    ds.data["y"] = y * i
                    cursession().store_objects(ds)
                time.sleep(0.05)
    except:
        logger.exception("An error occurred")
        raise

# spin up a background thread with animation
Thread(target=background_thread, args=(ds,)).start()

# endlessly poll to check widgets
cursession().poll_document(curdoc(), 0.5)

<commit_msg>Update time span in poll_document<commit_after>from __future__ import print_function

import time
import logging
from threading import Thread

import numpy as np

from bokeh.models.widgets import Button
from bokeh.plotting import (
    cursession, figure, show,
    output_server, curdoc, vplot,
    hplot
)

logger = logging.getLogger(__name__)

N = 80

x = np.linspace(0, 4*np.pi, N)
y = np.sin(x)

output_server("line_animate_widget")

p = figure()

p.line(x, y, color="#3333ee", name="sin")
p.line([0,4*np.pi], [-1, 1], color="#ee3333")

def play_handler():
    print("button_handler: start click")
    global play
    play = True

def stop_handler():
    print("button_handler: stop click")
    global play
    play = False

play = True

button_start = Button(label="Start", type="success")
button_start.on_click(play_handler)

button_stop = Button(label="Stop", type="danger")
button_stop.on_click(stop_handler)

controls = hplot(button_start, button_stop)
layout = vplot(controls, p)

show(layout)

renderer = p.select(dict(name="sin"))
ds = renderer[0].data_source

def should_play():
    """Return true if we should play animation, otherwise block"""
    global play
    while True:
        if play:
            return True
        else:
            time.sleep(0.05)

def background_thread(ds):
    """Plot animation, update data if play is True, otherwise stop"""
    try:
        while True:
            for i in np.hstack((np.linspace(1, -1, 100), np.linspace(-1, 1, 100))):
                if should_play():
                    ds.data["y"] = y * i
                    cursession().store_objects(ds)
                time.sleep(0.05)
    except:
        logger.exception("An error occurred")
        raise

# spin up a background thread with animation
Thread(target=background_thread, args=(ds,)).start()

# endlessly poll to check widgets
cursession().poll_document(curdoc(), 0.05)

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    ocr_pdf.py
    ~~~~~~~~~~~~~~

    A brief description goes here.
"""
import subprocess
import os
import shutil

def call(cmd, check=True, stdout=None, stderr=None):
    """
        Args:
            check: check return code or not
    """
    if check:
        return subprocess.check_call(cmd, stdout=stdout, stderr=stderr, shell=True)
    else:
        return subprocess.call(cmd, stdout=stdout, stderr=stderr, shell=True)

def unzip(zip_file, func=call):
    cmd = "unzip -o %s" % zip_file
    try:
        return func(cmd)
    except subprocess.CalledProcessError as e:
        if e.returncode != 2:
            raise e

def k2pdfopt(pdf_file, output_file, func=call):
    """convert multi-column PDF into single column

    K2pdfopt (Kindle 2 PDF Optimizer) is a stand-alone program which optimizes the format of PDF (or DJVU) files for viewing on small (e.g. 6-inch) mobile reader and smartphone screens such as the Kindle's.
    The output from k2pdfopt is a new (optimized) PDF file.
    http://www.willus.com/k2pdfopt/

    Args:
        output_file: this is a required parameter, because k2pdfopt always return 0

    Returns:
        0: WARNING, k2pdfopt will always return 0, judge its succeed by looking at the output_file
    """
    try:
        os.remove(output_file)
    except OSError as e:
        if e.errno != 2:
            raise e
    cmd = "./k2pdfopt -ui- -x -w 2160 -h 3840 -odpi 300 %s -o %s" % (pdf_file, output_file)
    return func(cmd)

def pdf_to_png(pdf_file, tmp_folder=None, func=call):
    if tmp_folder:
        cmd = "./codes/convert/cde-exec 'gs' -dBATCH -dNOPAUSE -sDEVICE=png16m -dGraphicsAlphaBits=4 -dTextAlphaBits=4 -r300 -sOutputFile=%s/page-%%d.png %s"\
            % (tmp_folder, pdf_file)
    else:
        cmd = "./codes/convert/cde-exec 'gs' -dBATCH -dNOPAUSE -sDEVICE=png16m -dGraphicsAlphaBits=4 -dTextAlphaBits=4 -r300 -sOutputFile=page-%%d.png %s" % pdf_file
    return func(cmd)

def pdf_to_bmp(pdf_file, tmp_folder=None, func=call):
    if tmp_folder:
        cmd = "./codes/convert/cde-exec 'gs' -SDEVICE=bmpmono -r300x300 -sOutputFile=%s/cuneiform-page-%%04d.bmp -dNOPAUSE -dBATCH -- %s"\
                % (tmp_folder, pdf_file)
    else:
        cmd = "./codes/convert/cde-exec 'gs' -SDEVICE=bmpmono -r300x300 -sOutputFile=cuneiform-page-%%04d.bmp -dNOPAUSE -dBATCH -- %s" % pdf_file
    return func(cmd)

def tesseract(png_folder_path, output_folder_path=None, func=call):
    png_folder_path = os.path.abspath(png_folder_path)
    if not output_folder_path:
        output_folder_path = png_folder_path
    for i in os.listdir(png_folder_path):
        if i.endswith('.png'):
            png_path = os.path.join(png_folder_path, i)
            ppm_filename = "%s.ppm" % png_path
            hocr_filename = os.path.join(output_folder_path, "%s.hocr" % i)
            cmd = "./codes/convert/cde-exec 'convert' -density 750 %s %s" % (png_path, ppm_filename)
            func(cmd)
            cmd = "./codes/tesseract/cde-exec 'tesseract' %s %s hocr" % (ppm_filename, hocr_filename)
            func(cmd)
            cmd = "rm -f %s" % (ppm_filename)
            func(cmd)

def cuneiform(bmp_folder_path, output_folder_path=None, func=call):
    bmp_folder_path = os.path.abspath(bmp_folder_path)
    if not output_folder_path:
        output_folder_path = bmp_folder_path
    for i in os.listdir(bmp_folder_path):
        if i.endswith('.bmp'):
            cmd = "./cde-package/cde-exec '/scratch.1/pdf2xml/cuneiform/bin/cuneiform' -f hocr -o %s.html %s"\
                % (os.path.join(output_folder_path, i), os.path.join(bmp_folder_path, i))
            func(cmd)

class OcrPdf(object):
    def __init__(self, pdf_path, stdout_filepath, stderr_filepath, output_folder_path=None):
        try:
            self.stdout = open(stdout_filepath, 'a')
            self.stderr = open(stderr_filepath, 'a')
            self.pdf_path = pdf_path
            self.output_folder_path = output_folder_path
        except IOError as e:
            print "ERROR\tInvalid filepath %s, %s" % (stdout_filepath, stderr_filepath)
            if self.stdout:
                self.stdout.close()
            if self.stderr:
                self.stderr.close()
            raise e

        shutil.rmtree('tmp', True)
        try:
            os.mkdir('tmp')
        except OSError as e:
            print "ERROR\tCreate tmp folder"
            raise e

        if self.output_folder_path and not os.path.isdir(self.output_folder_path):
            try:
                os.mkdir(self.output_folder_path)
            except OSError as e:
                print "ERROR\tCreate output folder"
                raise e

    def __del__(self):
        shutil.rmtree('tmp', True)

    def call(self, cmd, check=True):
        return call(cmd, check=check, stdout=self.stdout, stderr=self.stderr)

    def do(self):
        output_file = "k2_pdf_%s" % self.pdf_path
        unzip("ocr2.zip", func=self.call)
        unzip("cuneiform.zip", func=self.call)
        print k2pdfopt(self.pdf_path, output_file, func=self.call)
        print pdf_to_png(output_file, tmp_folder='tmp', func=self.call)
        print pdf_to_bmp(output_file, tmp_folder='tmp', func=self.call)
        print tesseract('tmp', self.output_folder_path, self.call)
        print cuneiform('tmp', self.output_folder_path, self.call)

def main(argv):
    o = OcrPdf(argv[1], 'out.txt', 'out.txt', 'out')
    o.do()

if __name__ == '__main__':
    import sys
    main(sys.argv)
<commit_msg>let tesseract and cuneiform return 0<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    ocr_pdf.py
    ~~~~~~~~~~~~~~

    A brief description goes here.
"""
import subprocess
import os
import shutil

def call(cmd, check=True, stdout=None, stderr=None):
    """
    Args:
        check: check return code or not
    """
    if check:
        return subprocess.check_call(cmd, stdout=stdout, stderr=stderr, shell=True)
    else:
        return subprocess.call(cmd, stdout=stdout, stderr=stderr, shell=True)

def unzip(zip_file, func=call):
    cmd = "unzip -o %s" % zip_file
    try:
        return func(cmd)
    except subprocess.CalledProcessError as e:
        if e.returncode != 2:
            raise e

def k2pdfopt(pdf_file, output_file, func=call):
    """convert multi-column PDF into single column

    K2pdfopt (Kindle 2 PDF Optimizer) is a stand-alone program which optimizes the format of PDF (or DJVU) files for viewing on small (e.g. 6-inch) mobile reader and smartphone screens such as the Kindle's.
    The output from k2pdfopt is a new (optimized) PDF file.
    http://www.willus.com/k2pdfopt/

    Args:
        output_file: this is a required parameter, because k2pdfopt always return 0

    Returns:
        0: WARNING, k2pdfopt will always return 0, judge its succeed by looking at the output_file
    """
    try:
        os.remove(output_file)
    except OSError as e:
        if e.errno != 2:
            raise e
    cmd = "./k2pdfopt -ui- -x -w 2160 -h 3840 -odpi 300 %s -o %s" % (pdf_file, output_file)
    return func(cmd)

def pdf_to_png(pdf_file, tmp_folder=None, func=call):
    if tmp_folder:
        cmd = "./codes/convert/cde-exec 'gs' -dBATCH -dNOPAUSE -sDEVICE=png16m -dGraphicsAlphaBits=4 -dTextAlphaBits=4 -r300 -sOutputFile=%s/page-%%d.png %s"\
            % (tmp_folder, pdf_file)
    else:
        cmd = "./codes/convert/cde-exec 'gs' -dBATCH -dNOPAUSE -sDEVICE=png16m -dGraphicsAlphaBits=4 -dTextAlphaBits=4 -r300 -sOutputFile=page-%%d.png %s" % pdf_file
    return func(cmd)

def pdf_to_bmp(pdf_file, tmp_folder=None, func=call):
    if tmp_folder:
        cmd = "./codes/convert/cde-exec 'gs' -SDEVICE=bmpmono -r300x300 -sOutputFile=%s/cuneiform-page-%%04d.bmp -dNOPAUSE -dBATCH -- %s"\
                % (tmp_folder, pdf_file)
    else:
        cmd = "./codes/convert/cde-exec 'gs' -SDEVICE=bmpmono -r300x300 -sOutputFile=cuneiform-page-%%04d.bmp -dNOPAUSE -dBATCH -- %s" % pdf_file
    return func(cmd)

def tesseract(png_folder_path, output_folder_path=None, func=call):
    """
    Returns:
        0, always return 0
    """
    png_folder_path = os.path.abspath(png_folder_path)
    if not output_folder_path:
        output_folder_path = png_folder_path
    for i in os.listdir(png_folder_path):
        if i.endswith('.png'):
            png_path = os.path.join(png_folder_path, i)
            ppm_filename = "%s.ppm" % png_path
            hocr_filename = os.path.join(output_folder_path, "%s.hocr" % i)
            cmd = "./codes/convert/cde-exec 'convert' -density 750 %s %s" % (png_path, ppm_filename)
            func(cmd)
            cmd = "./codes/tesseract/cde-exec 'tesseract' %s %s hocr" % (ppm_filename, hocr_filename)
            func(cmd)
            cmd = "rm -f %s" % (ppm_filename)
            func(cmd)
    return 0

def cuneiform(bmp_folder_path, output_folder_path=None, func=call):
    """
    Returns:
        0, always return 0
    """
    bmp_folder_path = os.path.abspath(bmp_folder_path)
    if not output_folder_path:
        output_folder_path = bmp_folder_path
    for i in os.listdir(bmp_folder_path):
        if i.endswith('.bmp'):
            cmd = "./cde-package/cde-exec '/scratch.1/pdf2xml/cuneiform/bin/cuneiform' -f hocr -o %s.html %s"\
                % (os.path.join(output_folder_path, i), os.path.join(bmp_folder_path, i))
            func(cmd)
    return 0

class OcrPdf(object):
    def __init__(self, pdf_path, stdout_filepath, stderr_filepath, output_folder_path=None):
        try:
            self.stdout = open(stdout_filepath, 'a')
            self.stderr = open(stderr_filepath, 'a')
            self.pdf_path = pdf_path
            self.output_folder_path = output_folder_path
        except IOError as e:
            print "ERROR\tInvalid filepath %s, %s" % (stdout_filepath, stderr_filepath)
            if self.stdout:
                self.stdout.close()
            if self.stderr:
                self.stderr.close()
            raise e

        shutil.rmtree('tmp', True)
        try:
            os.mkdir('tmp')
        except OSError as e:
            print "ERROR\tCreate tmp folder"
            raise e

        if self.output_folder_path and not os.path.isdir(self.output_folder_path):
            try:
                os.mkdir(self.output_folder_path)
            except OSError as e:
                print "ERROR\tCreate output folder"
                raise e

    def __del__(self):
        shutil.rmtree('tmp', True)

    def call(self, cmd, check=True):
        return call(cmd, check=check, stdout=self.stdout, stderr=self.stderr)

    def do(self):
        output_file = "k2_pdf_%s" % self.pdf_path
        unzip("ocr2.zip", func=self.call)
        unzip("cuneiform.zip", func=self.call)
        print k2pdfopt(self.pdf_path, output_file, func=self.call)
        print pdf_to_png(output_file, tmp_folder='tmp', func=self.call)
        print pdf_to_bmp(output_file, tmp_folder='tmp', func=self.call)
        print tesseract('tmp', self.output_folder_path, self.call)
        print cuneiform('tmp', self.output_folder_path, self.call)

def main(argv):
    o = OcrPdf(argv[1], 'out.txt', 'out.txt', 'out')
    o.do()

if __name__ == '__main__':
    import sys
    main(sys.argv)
<|endoftext|>