<commit_before>from celery import chain
from django.conf import settings
from django.db.models import F

from changes.tasks import get_engage_inbound_and_reply
from eventstore.models import DeliveryFailure, Event, OptOut
from eventstore.tasks import (
    async_create_flow_start,
    async_handle_whatsapp_delivery_error,
    get_rapidpro_contact_by_msisdn,
    send_helpdesk_response_to_dhis2,
    update_rapidpro_contact,
)
from ndoh_hub.utils import normalise_msisdn

def handle_outbound(message):
    """
    Triggers all the actions that are required for this outbound message
    """
    if message.is_operator_message:
        handle_operator_message(message)

def handle_operator_message(message):
    """
    Triggers all the tasks that need to run for an outbound message from an operator
    """
    whatsapp_contact_id = message.data["_vnd"]["v1"]["chat"]["owner"]
    # This should be in "+27xxxxxxxxx" format, but just in case it isn't
    msisdn = normalise_msisdn(whatsapp_contact_id)
    # Submit to Jembi
    chain(
        get_engage_inbound_and_reply.s(),
        get_rapidpro_contact_by_msisdn.s("inbound_address"),
        send_helpdesk_response_to_dhis2.s(),
    ).delay(whatsapp_contact_id, message.id)

    # Clear the wait_for_helpdesk flag
    update_rapidpro_contact.delay(
        f"whatsapp:{msisdn.lstrip('+')}", {"wait_for_helpdesk": ""}
    )

def handle_inbound(message):
    """
    Triggers all the actions that are required for this inbound message
    """
    if not message.fallback_channel:
        update_rapidpro_preferred_channel(message)

    if message.has_label("EDD"):
        handle_edd_message(message)

def update_rapidpro_preferred_channel(message):
    update_rapidpro_contact.delay(
        urn=f"whatsapp:{message.contact_id}", fields={"preferred_channel": "WhatsApp"}
    )

def handle_edd_message(message):
    whatsapp_contact_id = message.data["_vnd"]["v1"]["chat"]["owner"]
    # This should be in "+27xxxxxxxxx" format, but just in case it isn't
    msisdn = normalise_msisdn(whatsapp_contact_id)
    async_create_flow_start.delay(
        extra={},
        flow=settings.RAPIDPRO_EDD_LABEL_FLOW,
        urns=[f"whatsapp:{msisdn.lstrip('+')}"],
    )

def handle_event(event):
    """
    Triggers all the actions that are required for this event
    """
    if event.is_message_expired_error or event.is_whatsapp_failed_delivery_event:
        handle_whatsapp_delivery_error(event)

    if event.is_hsm_error:
        handle_whatsapp_hsm_error(event)

    if event.fallback_channel is True:
        handle_fallback_event(event)

def handle_fallback_event(event):
    if event.status != Event.SENT:
        if event.status == Event.FAILED:
            try:
                df = DeliveryFailure.objects.get(contact_id=event.recipient_id)

                df.number_of_failures = F("number_of_failures") + 1
                df.save()
                df.refresh_from_db()

                if df.number_of_failures >= 5:
                    async_create_flow_start.delay(
                        extra={
                            "optout_reason": OptOut.SMS_FAILURE_REASON,
                            "timestamp": event.timestamp.timestamp(),
                            "babyloss_subscription": "FALSE",
                            "delete_info_for_babyloss": "FALSE",
                            "delete_info_consent": "FALSE",
                            "source": "System",
                        },
                        flow=settings.RAPIDPRO_OPTOUT_FLOW,
                        urns=[f"whatsapp:{event.recipient_id}"],
                    )
            except DeliveryFailure.DoesNotExist:
                df = DeliveryFailure(
                    contact_id=event.recipient_id, number_of_failures=1
                )
                df.save()
        else:
            df = DeliveryFailure.objects.update_or_create(
                contact_id=event.recipient_id, defaults={"number_of_failures": 0}
            )

def handle_whatsapp_delivery_error(event):
    async_handle_whatsapp_delivery_error.delay(f"whatsapp:{event.recipient_id}")

def handle_whatsapp_hsm_error(event):
    if settings.ENABLE_UNSENT_EVENT_ACTION:
        async_create_flow_start.delay(
            extra={
                "popi_ussd": settings.POPI_USSD_CODE,
                "optout_ussd": settings.OPTOUT_USSD_CODE,
                "timestamp": event.timestamp.timestamp(),
            },
            flow=settings.RAPIDPRO_UNSENT_EVENT_FLOW,
            urns=[f"whatsapp:{event.recipient_id}"],
        )
<commit_msg>fixed if else logic<commit_after>from celery import chain
from django.conf import settings
from django.db.models import F

from changes.tasks import get_engage_inbound_and_reply
from eventstore.models import DeliveryFailure, Event, OptOut
from eventstore.tasks import (
    async_create_flow_start,
    async_handle_whatsapp_delivery_error,
    get_rapidpro_contact_by_msisdn,
    send_helpdesk_response_to_dhis2,
    update_rapidpro_contact,
)
from ndoh_hub.utils import normalise_msisdn

def handle_outbound(message):
    """
    Triggers all the actions that are required for this outbound message
    """
    if message.is_operator_message:
        handle_operator_message(message)

def handle_operator_message(message):
    """
    Triggers all the tasks that need to run for an outbound message from an operator
    """
    whatsapp_contact_id = message.data["_vnd"]["v1"]["chat"]["owner"]
    # This should be in "+27xxxxxxxxx" format, but just in case it isn't
    msisdn = normalise_msisdn(whatsapp_contact_id)
    # Submit to Jembi
    chain(
        get_engage_inbound_and_reply.s(),
        get_rapidpro_contact_by_msisdn.s("inbound_address"),
        send_helpdesk_response_to_dhis2.s(),
    ).delay(whatsapp_contact_id, message.id)

    # Clear the wait_for_helpdesk flag
    update_rapidpro_contact.delay(
        f"whatsapp:{msisdn.lstrip('+')}", {"wait_for_helpdesk": ""}
    )

def handle_inbound(message):
    """
    Triggers all the actions that are required for this inbound message
    """
    if not message.fallback_channel:
        update_rapidpro_preferred_channel(message)

    if message.has_label("EDD"):
        handle_edd_message(message)

def update_rapidpro_preferred_channel(message):
    update_rapidpro_contact.delay(
        urn=f"whatsapp:{message.contact_id}", fields={"preferred_channel": "WhatsApp"}
    )

def handle_edd_message(message):
    whatsapp_contact_id = message.data["_vnd"]["v1"]["chat"]["owner"]
    # This should be in "+27xxxxxxxxx" format, but just in case it isn't
    msisdn = normalise_msisdn(whatsapp_contact_id)
    async_create_flow_start.delay(
        extra={},
        flow=settings.RAPIDPRO_EDD_LABEL_FLOW,
        urns=[f"whatsapp:{msisdn.lstrip('+')}"],
    )

def handle_event(event):
    """
    Triggers all the actions that are required for this event
    """
    if event.is_message_expired_error or event.is_whatsapp_failed_delivery_event:
        handle_whatsapp_delivery_error(event)

    if event.is_hsm_error:
        handle_whatsapp_hsm_error(event)

    if event.fallback_channel is True:
        handle_fallback_event(event)

def handle_fallback_event(event):
    if event.status == Event.FAILED:
        try:
            df = DeliveryFailure.objects.get(contact_id=event.recipient_id)

            df.number_of_failures = F("number_of_failures") + 1
            df.save()
            df.refresh_from_db()

            if df.number_of_failures >= 5:
                async_create_flow_start.delay(
                    extra={
                        "optout_reason": OptOut.SMS_FAILURE_REASON,
                        "timestamp": event.timestamp.timestamp(),
                        "babyloss_subscription": "FALSE",
                        "delete_info_for_babyloss": "FALSE",
                        "delete_info_consent": "FALSE",
                        "source": "System",
                    },
                    flow=settings.RAPIDPRO_OPTOUT_FLOW,
                    urns=[f"whatsapp:{event.recipient_id}"],
                )
        except DeliveryFailure.DoesNotExist:
            df = DeliveryFailure(contact_id=event.recipient_id, number_of_failures=1)
            df.save()
    elif event.status == Event.READ or event.status == Event.DELIVERED:
        df = DeliveryFailure.objects.update_or_create(
            contact_id=event.recipient_id, defaults={"number_of_failures": 0}
        )

def handle_whatsapp_delivery_error(event):
    async_handle_whatsapp_delivery_error.delay(f"whatsapp:{event.recipient_id}")

def handle_whatsapp_hsm_error(event):
    if settings.ENABLE_UNSENT_EVENT_ACTION:
        async_create_flow_start.delay(
            extra={
                "popi_ussd": settings.POPI_USSD_CODE,
                "optout_ussd": settings.OPTOUT_USSD_CODE,
                "timestamp": event.timestamp.timestamp(),
            },
            flow=settings.RAPIDPRO_UNSENT_EVENT_FLOW,
            urns=[f"whatsapp:{event.recipient_id}"],
        )
<|endoftext|>


<commit_before>from cStringIO import StringIO
from sorl.thumbnail.engines.base import EngineBase

try:
    from PIL import Image, ImageFile, ImageDraw
except ImportError:
    import Image, ImageFile, ImageDraw

class Engine(EngineBase):
    def get_image(self, source):
        buf = StringIO(source.read())
        return Image.open(buf)

    def get_image_size(self, image):
        return image.size

    def is_valid_image(self, raw_data):
        buf = StringIO(raw_data)
        try:
            trial_image = Image.open(buf)
            trial_image.verify()
        except Exception:
            return False
        return True

    def _orientation(self, image):
        try:
            exif = image._getexif()
        except AttributeError:
            exif = None
        if exif:
            orientation = exif.get(0x0112)
            if orientation == 2:
                image = image.transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 3:
                image = image.rotate(180)
            elif orientation == 4:
                image = image.transpose(Image.FLIP_TOP_BOTTOM)
            elif orientation == 5:
                image = image.rotate(-90).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 6:
                image = image.rotate(-90)
            elif orientation == 7:
                image = image.rotate(90).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 8:
                image = image.rotate(90)
        return image

    def _colorspace(self, image, colorspace):
        if colorspace == 'RGB':
            if image.mode == 'RGBA':
                return image # RGBA is just RGB + Alpha
            if image.mode == 'P' and 'transparency' in image.info:
                return image.convert('RGBA')
            return image.convert('RGB')
        if colorspace == 'GRAY':
            return image.convert('L')
        return image

    def _scale(self, image, width, height):
        return image.resize((width, height), resample=Image.ANTIALIAS)

    def _crop(self, image, width, height, x_offset, y_offset):
        return image.crop((x_offset, y_offset,
                           width + x_offset, height + y_offset))

    def _get_raw_data(self, image, format_, quality, progressive=False):
        ImageFile.MAXBLOCK = 1024 * 1024
        buf = StringIO()
        params = {
            'format': format_,
            'quality': quality,
            'optimize': 1,
        }
        if format_ == 'JPEG' and progressive:
            params['progressive'] = True
        try:
            image.save(buf, **params)
        except IOError:
            params.pop('optimize')
            image.save(buf, **params)
        raw_data = buf.getvalue()
        buf.close()
        return raw_data

<commit_msg>Catching exceptions when _getexif is called<commit_after>from cStringIO import StringIO
from sorl.thumbnail.engines.base import EngineBase

try:
    from PIL import Image, ImageFile, ImageDraw
except ImportError:
    import Image, ImageFile, ImageDraw

class Engine(EngineBase):
    def get_image(self, source):
        buf = StringIO(source.read())
        return Image.open(buf)

    def get_image_size(self, image):
        return image.size

    def is_valid_image(self, raw_data):
        buf = StringIO(raw_data)
        try:
            trial_image = Image.open(buf)
            trial_image.verify()
        except Exception:
            return False
        return True

    def _orientation(self, image):
        try:
            exif = image._getexif()
        except (AttributeError, KeyError, IndexError, IOError):
            exif = None
        if exif:
            orientation = exif.get(0x0112)
            if orientation == 2:
                image = image.transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 3:
                image = image.rotate(180)
            elif orientation == 4:
                image = image.transpose(Image.FLIP_TOP_BOTTOM)
            elif orientation == 5:
                image = image.rotate(-90).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 6:
                image = image.rotate(-90)
            elif orientation == 7:
                image = image.rotate(90).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 8:
                image = image.rotate(90)
        return image

    def _colorspace(self, image, colorspace):
        if colorspace == 'RGB':
            if image.mode == 'RGBA':
                return image # RGBA is just RGB + Alpha
            if image.mode == 'P' and 'transparency' in image.info:
                return image.convert('RGBA')
            return image.convert('RGB')
        if colorspace == 'GRAY':
            return image.convert('L')
        return image

    def _scale(self, image, width, height):
        return image.resize((width, height), resample=Image.ANTIALIAS)

    def _crop(self, image, width, height, x_offset, y_offset):
        return image.crop((x_offset, y_offset,
                           width + x_offset, height + y_offset))

    def _get_raw_data(self, image, format_, quality, progressive=False):
        ImageFile.MAXBLOCK = 1024 * 1024
        buf = StringIO()
        params = {
            'format': format_,
            'quality': quality,
            'optimize': 1,
        }
        if format_ == 'JPEG' and progressive:
            params['progressive'] = True
        try:
            image.save(buf, **params)
        except IOError:
            params.pop('optimize')
            image.save(buf, **params)
        raw_data = buf.getvalue()
        buf.close()
        return raw_data

<|endoftext|>


<commit_before># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import math

# Third-party
import numpy as np
from astropy.constants import G

from .core import Potential, CartesianPotential

__all__ = ["PointMassPotential", "IsochronePotential", "HarmonicOscillatorPotential"]

# ============================================================================
#    Harmonic oscillator
#
def harmonic_osc_funcs(units):
    def f(x, omega):
        omega2 = omega*omega
        return np.sum(0.5*np.atleast_2d(omega2)*x*x, axis=-1)

    def gradient(x, omega):
        omega2 = omega*omega
        return np.atleast_2d(omega2)*x

    def hessian(x, x0, m):
        raise NotImplementedError()  # TODO:

    return f, gradient, None

class HarmonicOscillatorPotential(CartesianPotential):
    r"""
    Represents an N-dimensional harmonic oscillator.

    .. math::

        \Phi = \frac{1}{2}\omega^2 x^2

    Parameters
    ----------
    omega : numeric
        Frequency.
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.
    """

    def __init__(self, omega, units=None):
        parameters = dict(omega=np.array(omega))
        func,gradient,hessian = harmonic_osc_funcs(units)
        super(HarmonicOscillatorPotential, self).__init__(func=func, gradient=gradient,
                                                          hessian=hessian,
                                                          parameters=parameters, units=units)

    def action_angle(self, x, v):
        """
        Transform the input cartesian position and velocity to action-angle
        coordinates the Harmonic Oscillator potential. This transformation
        is analytic and can be used as a "toy potential" in the
        Sanders & Binney 2014 formalism for computing action-angle coordinates
        in _any_ potential.

        Adapted from Jason Sanders' code
        `genfunc <https://github.com/jlsanders/genfunc>`_.

        Parameters
        ----------
        x : array_like
            Positions.
        v : array_like
            Velocities.
        """
        from ..dynamics.analyticactionangle import harmonic_oscillator_xv_to_aa
        return harmonic_oscillator_xv_to_aa(x, v, self)

    def phase_space(self, actions, angles):
        """
        Transform the input action-angle coordinates to cartesian position and velocity
        assuming a Harmonic Oscillator potential. This transformation
        is analytic and can be used as a "toy potential" in the
        Sanders & Binney 2014 formalism for computing action-angle coordinates
        in _any_ potential.

        Adapted from Jason Sanders' code
        `genfunc <https://github.com/jlsanders/genfunc>`_.

        Parameters
        ----------
        x : array_like
            Positions.
        v : array_like
            Velocities.
        """
        from ..dynamics.analyticactionangle import harmonic_oscillator_aa_to_xv
        return harmonic_oscillator_aa_to_xv(actions, angles, self)

# ============================================================================
#    Potential due to a point mass at a given position
#
def point_mass_funcs(units):
    # scale G to be in this unit system
    if units is None:
        _G = 1.
    else:
        _G = G.decompose(units).value

    def f(x, x0, m):
        xx = x-x0
        R = np.sqrt(np.sum(xx**2, axis=-1))
        return -_G * m / R

    def gradient(x, x0, m):
        xx = x-x0
        a = (np.sum(xx**2, axis=-1)**-1.5)[...,None]
        return _G * m * xx * a

    def hessian(x, x0, m):
        raise NotImplementedError()  # TODO:

    return f, gradient, None

class PointMassPotential(Potential):
    r"""
    Represents a point-mass potential at the given origin.

    .. math::

        \Phi = -\frac{Gm}{x-x0}

    Parameters
    ----------
    m : numeric
        Mass.
    x0 : array_like, numeric
        Position of the point mass relative to origin of coordinates
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.
    """

    def __init__(self, m, x0, units=None):
        parameters = dict(m=m, x0=x0)
        func,gradient,hessian = point_mass_funcs(units)
        super(PointMassPotential, self).__init__(func=func, gradient=gradient,
                                                 hessian=hessian,
                                                 parameters=parameters, units=units)

# ============================================================================
#    Isochrone potential
#

def isochrone_funcs(units):
    # scale G to be in this unit system
    if units is None:
        _G = 1.
    else:
        _G = G.decompose(units).value

    def func(x, m, b):
        r2 = np.sum(x**2, axis=-1)
        val = -_G * m / (np.sqrt(r2 + b*b) + b)
        return val

    def gradient(x, m, b):
        r2 = np.sum(x**2, axis=-1)
        fac = _G*m / (np.sqrt(r2 + b*b) + b)**2 / np.sqrt(r2 + b*b)
        return fac[...,None] * x

    def hessian(x, m, b):
        raise NotImplementedError()  # TODO:

    return func, gradient, None

class IsochronePotential(CartesianPotential):
    r"""
    The Isochrone potential.

    .. math::

        \Phi_{spher} = -\frac{GM}{\sqrt{r^2+b^2} + b}

    Parameters
    ----------
    m : numeric
        Mass.
    b : numeric
        Core concentration.
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.

    """
    def __init__(self, m, b, units):
        parameters = dict(m=m, b=b)
        func,gradient,hessian = isochrone_funcs(units)
        super(IsochronePotential, self).__init__(func=func, gradient=gradient,
                                                 hessian=hessian,
                                                 parameters=parameters, units=units)

    def action_angle(self, x, v):
        """
        Transform the input cartesian position and velocity to action-angle
        coordinates the Isochrone potential. See Section 3.5.2 in
        Binney & Tremaine (2008), and be aware of the errata entry for
        Eq. 3.225.

        This transformation is analytic and can be used as a "toy potential"
        in the Sanders & Binney 2014 formalism for computing action-angle
        coordinates in _any_ potential.

        Adapted from Jason Sanders' code
        `here <https://github.com/jlsanders/genfunc>`_.

        Parameters
        ----------
        x : array_like
            Positions.
        v : array_like
            Velocities.
        """
        from ..dynamics.analyticactionangle import isochrone_xv_to_aa
        return isochrone_xv_to_aa(x, v, self)

    def phase_space(self, actions, angles):
        """
        Transform the input actions and angles to ordinary phase space (position
        and velocity) in cartesian coordinates. See Section 3.5.2 in
        Binney & Tremaine (2008), and be aware of the errata entry for
        Eq. 3.225.

        Parameters
        ----------
        actions : array_like
        angles : array_like
        """
        from ..dynamics.analyticactionangle import isochrone_aa_to_xv
        return isochrone_aa_to_xv(actions, angles, self)
<commit_msg>added kuzmin potential<commit_after># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import math

# Third-party
import numpy as np
from astropy.constants import G

from .core import Potential, CartesianPotential

__all__ = ["PointMassPotential", "IsochronePotential", "HarmonicOscillatorPotential",
           "KuzminPotential"]

# ============================================================================
#    Harmonic oscillator
#
def harmonic_osc_funcs(units):
    def f(x, omega):
        omega2 = omega*omega
        return np.sum(0.5*np.atleast_2d(omega2)*x*x, axis=-1)

    def gradient(x, omega):
        omega2 = omega*omega
        return np.atleast_2d(omega2)*x

    def hessian(x, x0, m):
        raise NotImplementedError()  # TODO:

    return f, gradient, None

class HarmonicOscillatorPotential(CartesianPotential):
    r"""
    Represents an N-dimensional harmonic oscillator.

    .. math::

        \Phi = \frac{1}{2}\omega^2 x^2

    Parameters
    ----------
    omega : numeric
        Frequency.
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.
    """

    def __init__(self, omega, units=None):
        parameters = dict(omega=np.array(omega))
        func,gradient,hessian = harmonic_osc_funcs(units)
        super(HarmonicOscillatorPotential, self).__init__(func=func, gradient=gradient,
                                                          hessian=hessian,
                                                          parameters=parameters, units=units)

    def action_angle(self, x, v):
        """
        Transform the input cartesian position and velocity to action-angle
        coordinates the Harmonic Oscillator potential. This transformation
        is analytic and can be used as a "toy potential" in the
        Sanders & Binney 2014 formalism for computing action-angle coordinates
        in _any_ potential.

        Adapted from Jason Sanders' code
        `genfunc <https://github.com/jlsanders/genfunc>`_.

        Parameters
        ----------
        x : array_like
            Positions.
        v : array_like
            Velocities.
        """
        from ..dynamics.analyticactionangle import harmonic_oscillator_xv_to_aa
        return harmonic_oscillator_xv_to_aa(x, v, self)

    def phase_space(self, actions, angles):
        """
        Transform the input action-angle coordinates to cartesian position and velocity
        assuming a Harmonic Oscillator potential. This transformation
        is analytic and can be used as a "toy potential" in the
        Sanders & Binney 2014 formalism for computing action-angle coordinates
        in _any_ potential.

        Adapted from Jason Sanders' code
        `genfunc <https://github.com/jlsanders/genfunc>`_.

        Parameters
        ----------
        x : array_like
            Positions.
        v : array_like
            Velocities.
        """
        from ..dynamics.analyticactionangle import harmonic_oscillator_aa_to_xv
        return harmonic_oscillator_aa_to_xv(actions, angles, self)

# ============================================================================
#    Potential due to a point mass at a given position
#
def point_mass_funcs(units):
    # scale G to be in this unit system
    if units is None:
        _G = 1.
    else:
        _G = G.decompose(units).value

    def f(x, x0, m):
        xx = x-x0
        R = np.sqrt(np.sum(xx**2, axis=-1))
        return -_G * m / R

    def gradient(x, x0, m):
        xx = x-x0
        a = (np.sum(xx**2, axis=-1)**-1.5)[...,None]
        return _G * m * xx * a

    def hessian(x, x0, m):
        raise NotImplementedError()  # TODO:

    return f, gradient, None

class PointMassPotential(Potential):
    r"""
    Represents a point-mass potential at the given origin.

    .. math::

        \Phi = -\frac{Gm}{x-x0}

    Parameters
    ----------
    m : numeric
        Mass.
    x0 : array_like, numeric
        Position of the point mass relative to origin of coordinates
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.
    """

    def __init__(self, m, x0, units=None):
        parameters = dict(m=m, x0=x0)
        func,gradient,hessian = point_mass_funcs(units)
        super(PointMassPotential, self).__init__(func=func, gradient=gradient,
                                                 hessian=hessian,
                                                 parameters=parameters, units=units)

# ============================================================================
#    Isochrone potential
#

def isochrone_funcs(units):
    # scale G to be in this unit system
    if units is None:
        _G = 1.
    else:
        _G = G.decompose(units).value

    def func(x, m, b):
        r2 = np.sum(x**2, axis=-1)
        val = -_G * m / (np.sqrt(r2 + b*b) + b)
        return val

    def gradient(x, m, b):
        r2 = np.sum(x**2, axis=-1)
        fac = _G*m / (np.sqrt(r2 + b*b) + b)**2 / np.sqrt(r2 + b*b)
        return fac[...,None] * x

    def hessian(x, m, b):
        raise NotImplementedError()  # TODO:

    return func, gradient, None

class IsochronePotential(CartesianPotential):
    r"""
    The Isochrone potential.

    .. math::

        \Phi = -\frac{GM}{\sqrt{r^2+b^2} + b}

    Parameters
    ----------
    m : numeric
        Mass.
    b : numeric
        Core concentration.
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.

    """
    def __init__(self, m, b, units):
        parameters = dict(m=m, b=b)
        func,gradient,hessian = isochrone_funcs(units)
        super(IsochronePotential, self).__init__(func=func, gradient=gradient,
                                                 hessian=hessian,
                                                 parameters=parameters, units=units)

    def action_angle(self, x, v):
        """
        Transform the input cartesian position and velocity to action-angle
        coordinates the Isochrone potential. See Section 3.5.2 in
        Binney & Tremaine (2008), and be aware of the errata entry for
        Eq. 3.225.

        This transformation is analytic and can be used as a "toy potential"
        in the Sanders & Binney 2014 formalism for computing action-angle
        coordinates in _any_ potential.

        Adapted from Jason Sanders' code
        `here <https://github.com/jlsanders/genfunc>`_.

        Parameters
        ----------
        x : array_like
            Positions.
        v : array_like
            Velocities.
        """
        from ..dynamics.analyticactionangle import isochrone_xv_to_aa
        return isochrone_xv_to_aa(x, v, self)

    def phase_space(self, actions, angles):
        """
        Transform the input actions and angles to ordinary phase space (position
        and velocity) in cartesian coordinates. See Section 3.5.2 in
        Binney & Tremaine (2008), and be aware of the errata entry for
        Eq. 3.225.

        Parameters
        ----------
        actions : array_like
        angles : array_like
        """
        from ..dynamics.analyticactionangle import isochrone_aa_to_xv
        return isochrone_aa_to_xv(actions, angles, self)

# ============================================================================
#    Kuzmin potential
#

def kuzmin_funcs(units):
    # scale G to be in this unit system
    if units is None:
        _G = 1.
    else:
        _G = G.decompose(units).value

    def func(q, m, a):
        x,y,z = q.T
        val = -_G * m / np.sqrt(x**2 + y**2 + (a + np.abs(z))**2)
        return val

    def gradient(q, m, a):
        x,y,z = q.T
        fac = _G * m / (x**2 + y**2 + (a + np.abs(z))**2)**1.5
        return fac[...,None] * q

    def hessian(x, m, a):
        raise NotImplementedError()  # TODO:

    return func, gradient, None

class KuzminPotential(CartesianPotential):
    r"""
    The Kuzmin flattened disk potential.

    .. math::

        \Phi = -\frac{Gm}{\sqrt{x^2 + y^2 + (a + |z|)^2}}

    Parameters
    ----------
    m : numeric
        Mass.
    a : numeric
        Flattening parameter.
    units : iterable
        Unique list of non-reducable units that specify (at minimum) the
        length, mass, time, and angle units.

    """
    def __init__(self, m, a, units):
        parameters = dict(m=m, a=a)
        func,gradient,hessian = kuzmin_funcs(units)
        super(KuzminPotential, self).__init__(func=func, gradient=gradient,
                                              hessian=hessian,
                                              parameters=parameters, units=units)
<|endoftext|>


<commit_before>import sys
import random
import chess
import chess.pgn
import numpy as np

NUM_PIECES = len(chess.PIECE_TYPES)
NUM_COLORS = len(chess.COLORS)
NUM_SQUARES = len(chess.SQUARE_NAMES)
NUM_COLS = 8
NUM_ROWS = 8

GAMMA = 0.99

class Dataset:
    def __init__(self, filename):
        self.filename = filename

    def random_white_state(self):
        """
        Returns (state, action, reward) tuple from white's perspective
        - state: np.array [12 pieces x 64 squares]
            - piece order:  wp wn wb wr wq wk bp bn bb br bq bk
            - square order: a1 b1 c1 ... h8
        - action: index in array [7 pieces x 64 squares x 64 squares]
            - action_array[ind2sub(action)]: move piece at square j to square k and promote to piece type i
            - promotion piece order: None p n b r q k
        - result: {-1, 0, 1} (lose, draw, win)
        """
        with open(self.filename) as pgn:
            while True:
                game = chess.pgn.read_game(pgn)
                if game is None:
                    break

                num_moves = int(game.headers["PlyCount"])
                # Choose a random white-turn state
                idx_move = random.randint(1, int(num_moves / 2)) * 2

                # Play moves up to idx_move
                board = game.board()
                node = game.root()
                for i in range(idx_move):
                    board.push(node.variations[0].move)
                    node = node.variations[0]
                move = node.variations[0].move
                promotion = move.promotion
                if promotion is None:
                    promotion = 0
                action = promotion * NUM_SQUARES * NUM_SQUARES + move.from_square * NUM_SQUARES + move.to_square

                # headers["Result"]: {"0-1", "1-0", "1/2-1/2"}
                # result: {-1, 0, 1}
                # Parse result from header
                white_score = game.headers["Result"].split("-")[0].split("/")
                if len(white_score) == 1:
                    result = 2 * int(white_score[0]) - 1
                else:
                    result = 0

                state = np.zeros((NUM_COLORS*NUM_PIECES, NUM_SQUARES))
                for piece_type in chess.PIECE_TYPES:
                    for color in chess.COLORS:
                        pieces = bin(board.pieces(piece_type, color))
                        for i, piece in enumerate(reversed(pieces)):
                            if piece == 'b':
                                break
                            elif piece == '1':
                                state[(1-color)*NUM_PIECES + piece_type - 1, i] = 1

                yield state, action, result

    def random_black_state(self):
        """
        Returns (state, reward, moves_remaining) tuple at black's turn from white's perspective
        - state: np.array [12 pieces x 8 rows x 8 cols]
            - piece order:  wp wn wb wr wq wk bp bn bb br bq bk
            - row order: a b c ... h
            - col order: 1 2 3 ... 8
        - result: {-1, 0, 1} (lose, draw, win)
        - moves_remaining: number of moves left to the end of the game
        """
        with open(self.filename) as pgn:
            while True:
                game = chess.pgn.read_game(pgn)
                if game is None:
                    break

                num_moves = int(game.headers["PlyCount"])
                if num_moves < 2:
                    continue
                # Choose a random black-turn state
                idx_move = random.randint(1, int(num_moves / 2)) * 2 - 1
                moves_remaining = num_moves - idx_move

                # Play moves up to idx_move
                board = game.board()
                node = game.root()
                for i in range(idx_move):
                    board.push(node.variations[0].move)
                    node = node.variations[0]

                # headers["Result"]: {"0-1", "1-0", "1/2-1/2"}
                # result: {-1, 0, 1}
                # Parse result from header
                white_score = game.headers["Result"].split("-")[0].split("/")
                if len(white_score) == 1:
                    result = 2 * int(white_score[0]) - 1
                else:
                    result = 0

                state = np.zeros((1, NUM_COLORS * NUM_PIECES, NUM_ROWS, NUM_COLS))
                for piece_type in chess.PIECE_TYPES:
                    for color in chess.COLORS:
                        pieces = bin(board.pieces(piece_type, color))
                        for i, piece in enumerate(reversed(pieces)):
                            if piece == 'b':
                                break
                            elif piece == '1':
                                row = i // NUM_ROWS
                                col = i % NUM_ROWS
                                state[0, (1-color)*NUM_PIECES + piece_type - 1, row, col] = 1

                yield state, np.array([(GAMMA ** moves_remaining) * result])
<commit_msg>loop forever on data, deterministic validation<commit_after>import sys
import random
import chess
import chess.pgn
import numpy as np

NUM_PIECES = len(chess.PIECE_TYPES)
NUM_COLORS = len(chess.COLORS)
NUM_SQUARES = len(chess.SQUARE_NAMES)
NUM_COLS = 8
NUM_ROWS = 8

GAMMA = 0.99

def state_from_board(board):
    state = np.zeros((1, NUM_COLORS * NUM_PIECES, NUM_ROWS, NUM_COLS))
    for piece_type in chess.PIECE_TYPES:
        for color in chess.COLORS:
            pieces = bin(board.pieces(piece_type, color))
            for i, piece in enumerate(reversed(pieces)):
                if piece == 'b':
                    break
                elif piece == '1':
                    row = i // NUM_ROWS
                    col = i % NUM_ROWS
                    state[0, (1-color)*NUM_PIECES + piece_type - 1, row, col] = 1
    return state

class Dataset:
    def __init__(self, filename, test_set=False):
        self.filename = filename
        self.test_set = test_set
        self.idx_game = 0
        self.idx_moves = []
        self.num_games = 0

    def random_white_state(self):
        """
        Returns (state, action, reward) tuple from white's perspective
        - state: np.array [12 pieces x 64 squares]
            - piece order:  wp wn wb wr wq wk bp bn bb br bq bk
            - square order: a1 b1 c1 ... h8
        - action: index in array [7 pieces x 64 squares x 64 squares]
            - action_array[ind2sub(action)]: move piece at square j to square k and promote to piece type i
            - promotion piece order: None p n b r q k
        - result: {-1, 0, 1} (lose, draw, win)
        """
        with open(self.filename) as pgn:
            while True:
                game = chess.pgn.read_game(pgn)
                if game is None:
                    break

                num_moves = int(game.headers["PlyCount"])
                # Choose a random white-turn state
                idx_move = random.randint(1, int(num_moves / 2)) * 2

                # Play moves up to idx_move
                board = game.board()
                node = game.root()
                for i in range(idx_move):
                    board.push(node.variations[0].move)
                    node = node.variations[0]
                move = node.variations[0].move
                promotion = move.promotion
                if promotion is None:
                    promotion = 0
                action = promotion * NUM_SQUARES * NUM_SQUARES + move.from_square * NUM_SQUARES + move.to_square

                # headers["Result"]: {"0-1", "1-0", "1/2-1/2"}
                # result: {-1, 0, 1}
                # Parse result from header
                white_score = game.headers["Result"].split("-")[0].split("/")
                if len(white_score) == 1:
                    result = 2 * int(white_score[0]) - 1
                else:
                    result = 0

                state = np.zeros((NUM_COLORS*NUM_PIECES, NUM_SQUARES))
                for piece_type in chess.PIECE_TYPES:
                    for color in chess.COLORS:
                        pieces = bin(board.pieces(piece_type, color))
                        for i, piece in enumerate(reversed(pieces)):
                            if piece == 'b':
                                break
                            elif piece == '1':
                                state[(1-color)*NUM_PIECES + piece_type - 1, i] = 1

                yield state, action, result

    def random_black_state(self):
        """
        Returns (state, reward) tuple at black's turn from white's perspective
        - state: np.array [12 pieces x 8 rows x 8 cols]
            - piece order:  wp wn wb wr wq wk bp bn bb br bq bk
            - row order: a b c ... h
            - col order: 1 2 3 ... 8
        - reward: GAMMA^moves_remaining * {-1, 0, 1} (lose, draw, win)
        """
        with open(self.filename) as pgn:
            while True:
                game = chess.pgn.read_game(pgn)
                if game is None:
                    pgn.seek(0)
                    self.num_games = self.idx_game
                    self.idx_game = 0
                    continue

                num_moves = int(game.headers["PlyCount"])
                if num_moves < 2:
                    continue

                # Choose a random black-turn state
                if self.test_set:
                    if self.num_games:
                        idx_move = self.idx_moves[self.idx_game]
                    else:
                        idx_move = random.randint(1, num_moves // 2) * 2 - 1
                        self.idx_moves.append(idx_move)
                else:
                    idx_move = random.randint(1, num_moves // 2) * 2 - 1
                moves_remaining = num_moves - idx_move

                # Play moves up to idx_move
                board = game.board()
                node = game.root()
                for i in range(idx_move):
                    board.push(node.variations[0].move)
                    node = node.variations[0]

                # headers["Result"]: {"0-1", "1-0", "1/2-1/2"}
                # result: {-1, 0, 1}
                # Parse result from header
                white_score = game.headers["Result"].split("-")[0].split("/")
                if len(white_score) == 1:
                    result = 2 * int(white_score[0]) - 1
                else:
                    result = 0

                state = state_from_board(board)
                reward = np.array([(GAMMA ** moves_remaining) * result])

                self.idx_game += 1
                print(idx_move)
                yield state, reward

    # def load_games(self):
    #     with open(self.filename) as pgn:
    #         self.games = []
    #         while True:
    #             game = chess.pgn.read_game(pgn)
    #             if game is None:
    #                 break

    #             num_moves = int(game.headers["PlyCount"])
    #             if num_moves < 2:
    #                     continue

    #             self.games.append(game)
    #         self.num_games = len(self.games)
    #         self.idx_game = 0

    # def pickle_games(self, filename):
    #     with open("filename", "w") as f:
    #         p = pickle.Pickler(f)
    #         p.dump(self.games)

    # def unpickle_games(self, filename):
    #     with open("filename", "w") as f:
    #         up = pickle.Unpickler(f)
    #         self.games = up.load()

    # def set_batch_size(self, batch_size):
    #     self.batch_size = batch_size
    #     self.num_batches = self.num_games // self.batch_size

    # def random_black_state_batch(self):
    #     for idx_batch in range(self.num_batches):
    #         X = []
    #         y = []
    #         for i in np.random.permutation(self.batch_size):
    #             game = self.games[self.batch_size * idx_batch + i]
    #             state, reward = self.fetch_black_state(game)
    #             X.append(state)
    #             y.append(reward)
    #         X = np.array(X)
    #         y = np.array(y)
    #         yield X, y

    # def fetch_black_state(self, game):
    #     num_moves = int(game.headers["PlyCount"])
    #     if num_moves < 2:
    #         return
    #     # Choose a random black-turn state
    #     idx_move = random.randint(1, int(num_moves / 2)) * 2 - 1
    #     moves_remaining = num_moves - idx_move

    #     # Play moves up to idx_move
    #     board = game.board()
    #     node = game.root()
    #     for i in range(idx_move):
    #         board.push(node.variations[0].move)
    #         node = node.variations[0]

    #     # headers["Result"]: {"0-1", "1-0", "1/2-1/2"}
    #     # result: {-1, 0, 1}
    #     # Parse result from header
    #     white_score = game.headers["Result"].split("-")[0].split("/")
    #     if len(white_score) == 1:
    #         result = 2 * int(white_score[0]) - 1
    #     else:
    #         result = 0

    #     state = np.zeros((1, NUM_COLORS * NUM_PIECES, NUM_ROWS, NUM_COLS))
    #     for piece_type in chess.PIECE_TYPES:
    #         for color in chess.COLORS:
    #             pieces = bin(board.pieces(piece_type, color))
    #             for i, piece in enumerate(reversed(pieces)):
    #                 if piece == 'b':
    #                     break
    #                 elif piece == '1':
    #                     row = i // NUM_ROWS
    #                     col = i % NUM_ROWS
    #                     state[0, (1-color)*NUM_PIECES + piece_type - 1, row, col] = 1

    #     return state, np.array([(GAMMA ** moves_remaining) * result])

    # def random_black_state(self):
    #     """
    #     Returns (state, reward, moves_remaining) tuple at black's turn from white's perspective
    #     - state: np.array [12 pieces x 8 rows x 8 cols]
    #         - piece order:  wp wn wb wr wq wk bp bn bb br bq bk
    #         - row order: a b c ... h
    #         - col order: 1 2 3 ... 8
    #     - result: {-1, 0, 1} (lose, draw, win)
    #     - moves_remaining: number of moves left to the end of the game
    #     """
    #     with open(self.filename) as pgn:
    #         while True:
    #             if self.games:
    #                 if self.idx_game >= self.num_games:
    #                     break

    #                 game = self.games[self.idx_game]
    #                 self.idx_game += 1
    #             else:
    #                 game = chess.pgn.read_game(pgn)
    #                 if game is None:
    #                     break

    #             yield self.fetch_black_state(game)
<|endoftext|>


<commit_before>  #!/usr/bin/python
#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import eventlet
from eventlet.green import threading, time, socket

# Global timeout for sockets in case something leaks
socket.setdefaulttimeout(900)

class Data():
    def __init__(self,bitHopper):
        self.users = {}
        self.bitHopper = bitHopper
        self.pool = self.bitHopper.pool
        self.db = self.bitHopper.db
        self.speed = self.bitHopper.speed
        self.difficulty = self.bitHopper.difficulty
        self.lock = threading.RLock()
        try:
            self.user_drop_time = self.bitHopper.config.get('main', 'user_drop_time')
        except:
            self.user_drop_time = 3600
        with self.lock:
            users = self.db.get_users()

            for user in users:
                self.users[user] = {'shares':users[user]['shares'],'rejects':users[user]['rejects'], 'last':0, 'shares_time': [], 'hash_rate': 0}
        eventlet.spawn_n(self.prune)

    def prune(self):
        while True:
            with self.lock:
                for user in self.users:
                    for share_time in self.users[user]['shares_time']:
                        if time.time() - share_time > 60 * 5:
                            self.users[user]['shares_time'].remove(share_time)
                    self.users[user]['hash_rate'] = (len(self.users[user]['shares_time']) * 2**32) / (60 * 5 * 1000000)
            eventlet.sleep(30)

    def get_users(self):
        with self.lock:
            users = {}
            for item in self.users:
                if self.users[item]['shares'] > 0:
                    shares_time = self.users[item]['shares_time']
                    if len(shares_time) > 0 and time.time()-max(shares_time) < int(self.user_drop_time):
                        users[item] = self.users[item]
            return users

    def user_share_add(self,user,password,shares,server):
        with self.lock:
            if user not in self.users:
                self.users[user] = {'shares':0,'rejects':0, 'last':0, 'shares_time': [], 'hash_rate': 0}
            self.users[user]['last'] = int(time.time())
            self.users[user]['shares'] += shares
            self.users[user]['shares_time'].append(int(time.time()))
            self.users[user]['hash_rate'] = (len(self.users[user]['shares_time']) * 2**32) / (60 * 5 * 1000000)

    def user_reject_add(self,user,password,rejects,server):
        with self.lock:
            if user not in self.users:
                self.users[user] = {'shares':0,'rejects':0, 'last':0, 'shares_time': [], 'hash_rate': 0}
            self.users[user]['rejects'] += rejects

    def reject_callback(self,server,data,user,password):
        try:
            self.db.update_rejects(server,1, user, password)
            self.pool.get_servers()[server]['rejects'] += 1
            self.user_reject_add(user, password, 1, server)
        except Exception, e:
            self.bitHopper.log_dbg('reject_callback_error')
            self.bitHopper.log_dbg(str(e))
            return

    def data_callback(self,server,data, user, password):
        try:
            if data != []:
                self.speed.add_shares(1)
                self.db.update_shares(server, 1, user, password)
                self.pool.get_servers()[server]['user_shares'] += 1
                self.pool.get_servers()[server]['expected_payout'] += 1.0 / self.difficulty['btc'] * 50.0
                self.user_share_add(user, password, 1, server)

        except Exception, e:
            self.bitHopper.log_dbg('data_callback_error')
            self.bitHopper.log_dbg(str(e))

<commit_msg>Fix to make user_drop_time work almost perfectly<commit_after>  #!/usr/bin/python
#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import eventlet
from eventlet.green import threading, time, socket

# Global timeout for sockets in case something leaks
socket.setdefaulttimeout(900)

class Data():
    def __init__(self,bitHopper):
        self.users = {}
        self.bitHopper = bitHopper
        self.pool = self.bitHopper.pool
        self.db = self.bitHopper.db
        self.speed = self.bitHopper.speed
        self.difficulty = self.bitHopper.difficulty
        self.lock = threading.RLock()
        try:
            self.user_drop_time = self.bitHopper.config.get('main', 'user_drop_time')
        except:
            self.user_drop_time = 3600
        with self.lock:
            users = self.db.get_users()

            for user in users:
                self.users[user] = {'shares':users[user]['shares'],'rejects':users[user]['rejects'], 'last':0, 'shares_time': [], 'hash_rate': 0}
        eventlet.spawn_n(self.prune)

    def prune(self):
        while True:
            with self.lock:
                for user in self.users:
                    for share_time in self.users[user]['shares_time']:
                        if time.time() - share_time > 60 * 15:
                            if len(self.users[user]['shares_time']) > 1:
                                self.users[user]['shares_time'].remove(share_time)
                    self.users[user]['hash_rate'] = (len(self.users[user]['shares_time']) * 2**32) / (60 * 15 * 1000000)
            eventlet.sleep(30)

    def get_users(self):
        with self.lock:
            users = {}
            for item in self.users:
                if self.users[item]['shares'] > 0:
                    shares_time = self.users[item]['shares_time']
                    if len(shares_time) > 0 and time.time()-max(shares_time) < int(self.user_drop_time):
                        users[item] = self.users[item]
            return users

    def user_share_add(self,user,password,shares,server):
        with self.lock:
            if user not in self.users:
                self.users[user] = {'shares':0,'rejects':0, 'last':0, 'shares_time': [], 'hash_rate': 0}
            self.users[user]['last'] = int(time.time())
            self.users[user]['shares'] += shares
            self.users[user]['shares_time'].append(int(time.time()))
            self.users[user]['hash_rate'] = (len(self.users[user]['shares_time']) * 2**32) / (60 * 15 * 1000000)

    def user_reject_add(self,user,password,rejects,server):
        with self.lock:
            if user not in self.users:
                self.users[user] = {'shares':0,'rejects':0, 'last':0, 'shares_time': [], 'hash_rate': 0}
            self.users[user]['rejects'] += rejects

    def reject_callback(self,server,data,user,password):
        try:
            self.db.update_rejects(server,1, user, password)
            self.pool.get_servers()[server]['rejects'] += 1
            self.user_reject_add(user, password, 1, server)
        except Exception, e:
            self.bitHopper.log_dbg('reject_callback_error')
            self.bitHopper.log_dbg(str(e))
            return

    def data_callback(self,server,data, user, password):
        try:
            if data != []:
                self.speed.add_shares(1)
                self.db.update_shares(server, 1, user, password)
                self.pool.get_servers()[server]['user_shares'] += 1
                self.pool.get_servers()[server]['expected_payout'] += 1.0 / self.difficulty['btc'] * 50.0
                self.user_share_add(user, password, 1, server)

        except Exception, e:
            self.bitHopper.log_dbg('data_callback_error')
            self.bitHopper.log_dbg(str(e))

<|endoftext|>


<commit_before>
import argparse
import binascii
import hashlib
import json
import logging
import requests
import requests.exceptions
import sys
import time

from os import listdir
from os.path import isfile, join

from strongfellowbtc.logging import configure_logging
import strongfellowbtc.block_io as block_io
import strongfellowbtc.constants as constants
import strongfellowbtc.hex

def md5(fname):
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

def post_blocks_from_blocks_dir(args=None):
  s = requests.Session()
  configure_logging()
  if args is None:
    args = sys.argv[1:]

  parser = argparse.ArgumentParser()
  parser.add_argument('--blocks-dir', required=False, default='/data/bitcoin/.bitcoin/blocks')
  parser.add_argument('--url', required=False, default='http://localhost:8080/internal/blocks')
  parser.add_argument('--network', required=False, default='main')

  args = parser.parse_args(args)

  url = args.url
  blocks_dir = args.blocks_dir

  cache = {}
  try:
    with open('cache.json') as input:
      cache = json.load(input)
  except IOError:
    cache = {}

  expected_magic = binascii.unhexlify(constants.NETWORKS[args.network].magic)

  while True:
    total_success_count = 0
    logging.info('begin iteration')
    for f in sorted(listdir(blocks_dir)):
      if isfile(join(blocks_dir, f)) and f.startswith('blk') and f.endswith('.dat'):
        logging.info('begin processing %s', f)
        md = md5(join(blocks_dir, f))
        logging.info('considering %s', f)
        if f in cache and cache[f] == md:
          logging.info('skipping %s', f)
        else:
          logging.info('processing %s', f)
          success_count = 0
          with open(join(blocks_dir, f), 'rb') as input:
            for (magic, block, h) in block_io.generate_blocks(input):
              block_hash = strongfellowbtc.hex.big_endian_hex(h)
              logging.debug('posting %s', block_hash)
              if magic != expected_magic:
                raise Exception('unexpected magic: %s; expected %s' % (strongfellowbtc.hex.little_endian_hex(magic), strongfellowbtc.hex.little_endian_hex(expected_magic)))
              if success_count % 1000 == 0:
                  logging.info('so far in file %s with %d successes', f, success_count)

              backoff = 1
              while True:
                retry = True
                try:
                  response = s.post(url, data=block,
                                    headers={ 'Content-Type': 'strongfellow/block'})
                  status = response.status_code
                  if status == 200:
                    success_count += 1
                    retry = False
                  else:
                    logging.error('non-200 response for block %s -- HTTP %d', block_hash, status)
                except requests.exceptions.RequestException as e:
                  logging.exception('problem posting %s', block_hash)
                if not retry:
                  break
                elif backoff < 60:
                  logging.info('backing off for %d seconds', backoff)
                  time.sleep(backoff)
                  backoff *= 2
                else:
                  logging.info('failed to post %s', block_hash)
                  logging.info('we backed off too many times; failing')
                  raise
          logging.info('finished file %s with %d successes', f, success_count)
          total_success_count += success_count
          logging.info('total success count: %d', total_success_count)
          cache[f] = md
          with open('cache.json', 'w') as output:
            json.dump(cache, output)
        logging.info('finished processing %s', f)
    delay_seconds = 600 if total_success_count == 0 else 10
    logging.info('and now, we sleep for %d seconds', delay_seconds)
    time.sleep(delay_seconds)
    logging.info('finished sleeping')
    logging.info('finished iteration')

if __name__ == '__main__':
  post_blocks_from_blocks_dir()
<commit_msg>change cache file location<commit_after>
import argparse
import binascii
import hashlib
import json
import logging
import requests
import requests.exceptions
import sys
import time

from os import listdir
from os.path import isfile, join

from strongfellowbtc.logging import configure_logging
import strongfellowbtc.block_io as block_io
import strongfellowbtc.constants as constants
import strongfellowbtc.hex

def md5(fname):
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

def post_blocks_from_blocks_dir(args=None):
  s = requests.Session()
  configure_logging()
  if args is None:
    args = sys.argv[1:]

  parser = argparse.ArgumentParser()
  parser.add_argument('--blocks-dir', required=False, default='/data/bitcoin/.bitcoin/blocks')
  parser.add_argument('--url', required=False, default='http://localhost:8080/internal/blocks')
  parser.add_argument('--network', required=False, default='main')

  args = parser.parse_args(args)

  url = args.url
  blocks_dir = args.blocks_dir

  cache = {}
  cache_file = '/tmp/cache.json'
  try:
    with open(cache_file) as input:
      cache = json.load(input)
  except IOError:
    cache = {}

  expected_magic = binascii.unhexlify(constants.NETWORKS[args.network].magic)

  while True:
    total_success_count = 0
    logging.info('begin iteration')
    for f in sorted(listdir(blocks_dir)):
      if isfile(join(blocks_dir, f)) and f.startswith('blk') and f.endswith('.dat'):
        logging.info('begin processing %s', f)
        md = md5(join(blocks_dir, f))
        logging.info('considering %s', f)
        if f in cache and cache[f] == md:
          logging.info('skipping %s', f)
        else:
          logging.info('processing %s', f)
          success_count = 0
          with open(join(blocks_dir, f), 'rb') as input:
            for (magic, block, h) in block_io.generate_blocks(input):
              block_hash = strongfellowbtc.hex.big_endian_hex(h)
              logging.debug('posting %s', block_hash)
              if magic != expected_magic:
                raise Exception('unexpected magic: %s; expected %s' % (strongfellowbtc.hex.little_endian_hex(magic), strongfellowbtc.hex.little_endian_hex(expected_magic)))
              if success_count % 1000 == 0:
                  logging.info('so far in file %s with %d successes', f, success_count)

              backoff = 1
              while True:
                retry = True
                try:
                  response = s.post(url, data=block,
                                    headers={ 'Content-Type': 'strongfellow/block'})
                  status = response.status_code
                  if status == 200:
                    success_count += 1
                    retry = False
                  else:
                    logging.error('non-200 response for block %s -- HTTP %d', block_hash, status)
                except requests.exceptions.RequestException as e:
                  logging.exception('problem posting %s', block_hash)
                if not retry:
                  break
                elif backoff < 60:
                  logging.info('backing off for %d seconds', backoff)
                  time.sleep(backoff)
                  backoff *= 2
                else:
                  logging.info('failed to post %s', block_hash)
                  logging.info('we backed off too many times; failing')
                  raise
          logging.info('finished file %s with %d successes', f, success_count)
          total_success_count += success_count
          logging.info('total success count: %d', total_success_count)
          cache[f] = md
          with open(cache_file, 'w') as output:
            json.dump(cache, output)
        logging.info('finished processing %s', f)
    delay_seconds = 600 if total_success_count == 0 else 10
    logging.info('and now, we sleep for %d seconds', delay_seconds)
    time.sleep(delay_seconds)
    logging.info('finished sleeping')
    logging.info('finished iteration')

if __name__ == '__main__':
  post_blocks_from_blocks_dir()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys, re, json, string

import nltk
from codecs import open
from guess_language import guessLanguage

from util import (get_file_names,
                  extract_title,
                  get_document_content_paf,
                  get_document_content,
                  is_monocase,
                  normalize_title)

from cap_transform import (make_capitalized_title,
                           make_lowercase_title,
                           make_uppercase_title)

from word_shape_util import (without_alpha,
                             contains_lowercase,
                             contains_uppercase)

from feature_extractor import FeatureExtractor

def print_filenames_and_titles():
    """print title each per one line from the corpus"""
    paths = ['/group/home/puls/Shared/capitalization-recovery/%2d' %i
             for i in xrange(9, 32)]
    titles = []
    for fname in get_file_names(paths):
        title = extract_title(fname)

        if not title: # no title
            continue

        title = normalize_title(title)

        if not is_monocase(nltk.word_tokenize(title)) and guessLanguage(title) == "en": #is not monocase and is English
            print json.dumps([fname, unicode(title).encode("utf8")])

def get_label(word, **kwargs):
    """
    Possible labels

    IC: initial capital, note, there should be at least one alphabetic letter, and the alphabetic ones are in lower case
    AU: all uppercase
    AL: all lowercase
    MX: mixed case
    AN: all no case

    >>> get_label(u'Click')
    'IC'
    >>> get_label(u'F11')
    'AU'
    >>> get_label(u'OK')
    'AU'
    >>> get_label(u'/hone/Doc')
    'MX'
    >>> get_label(u'HealthCare')
    'MX'
    >>> get_label(u'ealthCare')
    'MX'
    >>> get_label(u'lower')
    'AL'
    >>> get_label(u'$123')
    'AN'
    >>> get_label(u'.')
    'AN'
    >>> get_label(u'Fb11')
    'IC'
    >>> get_label(u'11.5a')
    'AL'
    >>> get_label(u'11.5A')
    'AU'
    >>> get_label(u'11.5Aa')
    'MX'
    """
    if without_alpha(word):
        return "AN"
    else:
        if word[0].isalpha(): # starts with alpha
            # a word
            if word.lower() == word:
                return "AL"
            elif word[0].upper() == word[0] and contains_lowercase(word) and word[1:].lower() == word[1:]:
                return "IC"
            elif word.upper() == word:
                return "AU"
            else:
                return "MX"
        else:
            lower_flag = contains_lowercase(word)
            upper_flag = contains_uppercase(word)
            if lower_flag and upper_flag:
                return "MX"
            elif lower_flag and not upper_flag:
                return "AL"
            else:
                return "AU"

    if word.upper() == word:
        return "AU"
    elif word[0].isalpha() and word[0] == word[0].upper():
        return "C"
    elif word[0].isalpha():
        return "L"
    else:
        return "I"
        # raise ValueError("Invalid value `%s`" %(word))

def convert_to_trainable_format(raw_title, title_transform_func, feature_extractor, doc = None):
    """
    Given some title(before capitalization), return the trainable(for CRF-suite) format

    >>> from cap_transform import make_capitalized_title
    >>> from feature_extractor import FeatureExtractor
    >>> extractor = FeatureExtractor()
    >>> sent = convert_to_trainable_format(u"CIS FMs hold summit in Belarus on Oct 10", make_capitalized_title, extractor)
    >>> sent[2]["word"]
    u'Hold'
    >>> sent[5]["lower-in-dict"]
    False
    >>> sent[1]["y"]
    'MX'
    """
    words = nltk.word_tokenize(raw_title)

    transformed_words = title_transform_func(title_words = words)

    words_with_features = feature_extractor.extract(transformed_words)

    #add the labels
    for word_str, word_info in zip(words, words_with_features):
        word_info["y"] = get_label(word_str)

    return words_with_features

def print_trainable_data(path,
                         extractor, feature_names,
                         start, end = None,
                         title_transform_func = make_capitalized_title,
                         content_extractor = None):
    """
    >>> extractor = FeatureExtractor()
    >>> print_trainable_data(path = "data/title_test.txt", extractor = extractor, feature_names = extractor.feature_names, start = 0, end = 1, title_transform_func = make_capitalized_title) #doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    One True True True True True False True False CD IC
    ...
    PaaS False False False False False False True False NNP MX
    ...
    IBM False False True False True True True False NNP AU
    ...
    """
    feature_names += ['y'] #add the label feature name
    i = 0
    with open(path, "r", "utf8") as f:
        for line in f:
            fname, title = json.loads(line)
            if i < start:
                i += 1
                continue

            if i % 1000 == 0:
                sys.stderr.write("Finished %d\n" %i)

            words = convert_to_trainable_format(title,
                                                title_transform_func,
                                                extractor,
                                                (content_extractor(fname)
                                                 if content_extractor and callable(content_extractor)
                                                 else None)
                                            )

            for word in words:
                word_feature_str = ' '.join([unicode(word[feature_name]) for feature_name in feature_names])
                print unicode(word_feature_str).encode('utf8')
            print

            i += 1

            if end is not None and i >= end:
                sys.stderr.write("Reached %d.\nTerminate.\n" %(end))
                break

def load_labeled_data(path):
    """

    >>> d = load_labeled_data(path = "fnames_and_titles.txt")
    >>> d.next()[:8]
    [(u'The', 'IC'), (u'Sun', 'IC'), (u'Life', 'IC'), (u'Building', 'IC'), (u'receives', 'AL'), (u'LEED', 'AU'), (u'Silver', 'IC'), (u'Certification', 'IC')]
    """
    with open(path, "r", "utf8") as f:
        for line in f:
            _, title = json.loads(line)
            words = nltk.word_tokenize(title)
            yield [(w, get_label(w)) for w in words]

if __name__ == "__main__":
    import sys

    path = sys.argv[1]
    op = sys.argv[2]
    start = int(sys.argv[3])

    try:
        end = int(sys.argv[4])
    except IndexError:
        end = None

    op_table = {
        "upper": make_uppercase_title,
        "lower": make_lowercase_title,
        "cap": make_capitalized_title
    }
    assert op in op_table.keys()

    extractor = FeatureExtractor()

    print_trainable_data(path = sys.argv[1],
                         extractor = extractor,
                         feature_names = extractor.feature_names,
                         start = start, end = end,
                         # content_extractor = get_document_content,
                         title_transform_func = op_table[op])
    # path = "test-titles.txt",
<commit_msg>updated data.py for make_data.sh<commit_after># -*- coding: utf-8 -*-

import sys, re, json, string

import nltk
from codecs import open
from guess_language import guessLanguage

from util import (get_file_names,
                  extract_title,
                  get_document_content_paf,
                  get_document_content,
                  is_monocase,
                  normalize_title)

from cap_transform import (make_capitalized_title,
                           make_lowercase_title,
                           make_uppercase_title)

from word_shape_util import (without_alpha,
                             contains_lowercase,
                             contains_uppercase)

from feature_extractor import FeatureExtractor

def print_filenames_and_titles():
    """print title each per one line from the corpus"""
    paths = ['/group/home/puls/Shared/capitalization-recovery/%2d' %i
             for i in xrange(9, 32)]
    titles = []
    for fname in get_file_names(paths):
        title = extract_title(fname)

        if not title: # no title
            continue

        title = normalize_title(title)

        if not is_monocase(nltk.word_tokenize(title)) and guessLanguage(title) == "en": #is not monocase and is English
            print json.dumps([fname, unicode(title).encode("utf8")])

def get_label(word, **kwargs):
    """
    Possible labels

    IC: initial capital, note, there should be at least one alphabetic letter, and the alphabetic ones are in lower case
    AU: all uppercase
    AL: all lowercase
    MX: mixed case
    AN: all no case

    >>> get_label(u'Click')
    'IC'
    >>> get_label(u'F11')
    'AU'
    >>> get_label(u'OK')
    'AU'
    >>> get_label(u'/hone/Doc')
    'MX'
    >>> get_label(u'HealthCare')
    'MX'
    >>> get_label(u'ealthCare')
    'MX'
    >>> get_label(u'lower')
    'AL'
    >>> get_label(u'$123')
    'AN'
    >>> get_label(u'.')
    'AN'
    >>> get_label(u'Fb11')
    'IC'
    >>> get_label(u'11.5a')
    'AL'
    >>> get_label(u'11.5A')
    'AU'
    >>> get_label(u'11.5Aa')
    'MX'
    """
    if without_alpha(word):
        return "AN"
    else:
        if word[0].isalpha(): # starts with alpha
            # a word
            if word.lower() == word:
                return "AL"
            elif word[0].upper() == word[0] and contains_lowercase(word) and word[1:].lower() == word[1:]:
                return "IC"
            elif word.upper() == word:
                return "AU"
            else:
                return "MX"
        else:
            lower_flag = contains_lowercase(word)
            upper_flag = contains_uppercase(word)
            if lower_flag and upper_flag:
                return "MX"
            elif lower_flag and not upper_flag:
                return "AL"
            else:
                return "AU"

    if word.upper() == word:
        return "AU"
    elif word[0].isalpha() and word[0] == word[0].upper():
        return "C"
    elif word[0].isalpha():
        return "L"
    else:
        return "I"
        # raise ValueError("Invalid value `%s`" %(word))

def convert_to_trainable_format(raw_title, title_transform_func, feature_extractor, docpath = None):
    """
    Given some title(before capitalization), return the trainable(for CRF-suite) format

    >>> from cap_transform import make_capitalized_title
    >>> from feature_extractor import FeatureExtractor
    >>> extractor = FeatureExtractor()
    >>> sent = convert_to_trainable_format(u"CIS FMs hold summit in Belarus on Oct 10", make_capitalized_title, extractor)
    >>> sent[2]["word"]
    u'Hold'
    >>> sent[5]["lower-in-dict"]
    False
    >>> sent[1]["y"]
    'MX'
    """
    words = nltk.word_tokenize(raw_title)

    transformed_words = title_transform_func(title_words = words)

    kwargs={}

    if docpath:
        kwargs['docpath']=docpath

    words_with_features = feature_extractor.extract(transformed_words, **kwargs)

    #add the labels
    for word_str, word_info in zip(words, words_with_features):
        word_info["y"] = get_label(word_str)

    return words_with_features

def print_trainable_data(path,
                         extractor, feature_names,
                         start, end = None,
                         title_transform_func = make_capitalized_title):
    """
    >>> extractor = FeatureExtractor()
    >>> print_trainable_data(path = "data/title_test.txt", extractor = extractor, feature_names = extractor.feature_names, start = 0, end = 1, title_transform_func = make_capitalized_title) #doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    One True True True True True False True False CD IC
    ...
    PaaS False False False False False False True False NNP MX
    ...
    IBM False False True False True True True False NNP AU
    ...
    """
    feature_names += ['y'] #add the label feature name
    i = 0
    with open(path, "r", "utf8") as f:
        for line in f:
            fname, title = json.loads(line)
            if i < start:
                i += 1
                continue

            if i % 1000 == 0:
                sys.stderr.write("Finished %d\n" %i)

            words = convert_to_trainable_format(title,
                                                title_transform_func,
                                                extractor,
                                                docpath=fname
                                            )

            for word in words:
                word_feature_str = ' '.join([unicode(word[feature_name]) for feature_name in feature_names])
                print unicode(word_feature_str).encode('utf8')
            print

            i += 1

            if end is not None and i >= end:
                sys.stderr.write("Reached %d.\nTerminate.\n" %(end))
                break

def load_labeled_data(path):
    """

    >>> d = load_labeled_data(path = "fnames_and_titles.txt")
    >>> d.next()[:8]
    [(u'The', 'IC'), (u'Sun', 'IC'), (u'Life', 'IC'), (u'Building', 'IC'), (u'receives', 'AL'), (u'LEED', 'AU'), (u'Silver', 'IC'), (u'Certification', 'IC')]
    """
    with open(path, "r", "utf8") as f:
        for line in f:
            _, title = json.loads(line)
            words = nltk.word_tokenize(title)
            yield [(w, get_label(w)) for w in words]

if __name__ == "__main__":
    import sys

    path = sys.argv[1]
    op = sys.argv[2]
    start = int(sys.argv[3])

    try:
        end = int(sys.argv[4])
    except IndexError:
        end = None

    op_table = {
        "upper": make_uppercase_title,
        "lower": make_lowercase_title,
        "cap": make_capitalized_title
    }
    assert op in op_table.keys()

    extractor = FeatureExtractor()

    print_trainable_data(path = sys.argv[1],
                         extractor = extractor,
                         feature_names = extractor.feature_names,
                         start = start, end = end,
                         # content_extractor = get_document_content,
                         title_transform_func = op_table[op])
    # path = "test-titles.txt",
<|endoftext|>


<commit_before># coding: utf8

from __future__ import print_function, unicode_literals
import codecs, subprocess
from collections import Counter
from itertools import islice
from nltk.tag import untag
from sklearn.cross_validation import train_test_split
from hazm import *
from hazm.Chunker import tree2brackets

def create_words_file(dic_file='resources/persian.dic', output='hazm/data/words.dat'):
	""" prepares list of persian word words from [Virastyar](https://sourceforge.net/projects/virastyar/) dic file.
	"""

	dic_words = sorted([line.split('\t')[0] for line in codecs.open(dic_file, encoding='utf8')])
	print(*dic_words, sep='\n', file=codecs.open(output, 'w', 'utf8'))
	print(output, 'created')

def evaluate_lemmatizer(conll_file='resources/train.conll', peykare_root='corpora/peykare'):
	lemmatizer = Lemmatizer()

	errors = []
	with codecs.open('resources/lemmatizer_errors.txt', 'w', 'utf8') as output:
		dadegan = DadeganReader(conll_file)
		for tree in dadegan.trees():
			for node in tree.nodelist[1:]:
				word, lemma, pos = node['word'], node['lemma'], node['mtag']
				if lemmatizer.lemmatize(word, pos) != lemma:
					errors.append((word, lemma, pos, lemmatizer.lemmatize(word, pos)))
		print(len(errors), 'errors', file=output)
		counter = Counter(errors)
		for item, count in sorted(counter.items(), key=lambda t: t[1], reverse=True):
			print(count, *item, file=output)

	missed = []
	with codecs.open('resources/lemmatizer_missed.txt', 'w', 'utf8') as output:
		peykare = PeykareReader(peykare_root)
		for sentence in peykare.sents():
			for word in sentence:
				if word[1] == 'V':
					if word[0] == lemmatizer.lemmatize(word[0]):
						missed.append(word[0])
		print(len(missed), 'missed', file=output)
		counter = Counter(missed)
		for item, count in sorted(counter.items(), key=lambda t: t[1], reverse=True):
			print(count, item, file=output)

def evaluate_chunker(treebank_root='corpora/treebank'):
	treebank = TreebankReader(treebank_root, join_clitics=True, join_verb_parts=True)
	chunker = Chunker()
	chunked_trees = list(treebank.chunked_trees())

	print(chunker.evaluate(chunked_trees))

	output = codecs.open('resources/chunker_errors.txt', 'w', 'utf8')
	for sentence, gold in zip(treebank.sents(), chunked_trees):
		chunked = chunker.parse(sentence)
		if chunked != gold:
			print(tree2brackets(chunked), file=output)
			print(tree2brackets(gold), file=output)
			print(file=output)

def train_postagger(peykare_root='corpora/peykare', model_file='resources/postagger.model', test_size=.1, sents_limit=None):

	tagger = SequencePOSTagger(type='crf', algo='rprop', compact=True, patterns=[
		'*',

		'u:wll=%X[-2,0]',
		'u:wl=%X[-1,0]',
		'u:w=%X[0,0]',
		'u:wr=%X[1,0]',
		'u:wrr=%X[2,0]',

		'u:w2l=%X[-1,0]/%X[0,0]',
		'u:w2r=%X[0,0]/%X[1,0]',

		# 'u:w3l=%X[-2,0]/%X[-1,0]/%X[0,0]',
		# 'u:w3r=%X[0,0]/%X[1,0]/%X[2,0]',

		'*:p1=%m[0,0,"^.?"]',
		'*:p2=%m[0,0,"^.?.?"]',
		'*:p3=%m[0,0,"^.?.?.?"]',
		'*:p4=%m[0,0,"^.?.?.?.?"]',

		'*:s1=%m[0,0,".?$"]',
		'*:s2=%m[0,0,".?.?$"]',
		'*:s3=%m[0,0,".?.?.?$"]',
		'*:s4=%m[0,0,".?.?.?.?$"]',

		'*:p?l=%t[-1,0,"\p"]',
		'*:p?=%t[0,0,"\p"]',
		'*:p?r=%t[1,0,"\p"]',
		'*:p?a=%t[0,0,"^\p*$"]',

		'*:n?l=%t[-1,0,"\d"]',
		'*:n?=%t[0,0,"\d"]',
		'*:n?r=%t[1,0,"\d"]',
		'*:n?a=%t[0,0,"^\d*$"]',
	])

	peykare = PeykareReader(peykare_root)
	train_sents, test_sents = train_test_split(list(islice(peykare.sents(), sents_limit)), test_size=test_size, random_state=0)

	tagger.train(train_sents)
	tagger.save_model(model_file)

	print(tagger.evaluate(test_sents))

def train_chunker(train_file='corpora/train.conll', validation_file='corpora/validation.conll', test_file='corpora/test.conll', model_file='resources/chunker.model'):

	tagger = SequencePOSTagger(model='resources/postagger.model')
	chunker = Chunker(type='crf', algo='l-bfgs', compact=True, patterns=[
		'*',

		'u:wll=%X[-2,0]',
		'u:wl=%X[-1,0]',
		'u:w=%X[0,0]',
		'u:wr=%X[1,0]',
		'u:wrr=%X[2,0]',

		'*:tll=%X[-2,1]',
		'*:tl=%X[-1,1]',
		'*:t=%X[0,1]',
		'*:tr=%X[1,1]',
		'*:trr=%X[2,1]',
	])

	train, validation, test = DadeganReader(train_file), DadeganReader(validation_file), DadeganReader(test_file)
	train_sents = list(train.sents()) + list(validation.sents())
	train_trees = list(train.chunked_trees()) + list(validation.chunked_trees())

	tagged_sents = tagger.tag_sents([untag(sent) for sent in train_sents])
	for tree, sentence in zip(train_trees, tagged_sents):
		for (n, word) in zip(tree.treepositions('leaves'), sentence):
			tree[n] = word

	chunker.train(train_trees)
	chunker.save_model(model_file)

	print(chunker.evaluate(test.chunked_trees()))

def train_maltparser(train_file='corpora/train.conll', validation_file='corpora/validation.conll', test_file='corpora/test.conll', model_file='langModel.mco', path_to_jar='resources/malt.jar', options_file='resources/malt-options.xml', features_file='resources/malt-features.xml', memory_min='-Xms7g', memory_max='-Xmx8g'):

	lemmatizer, tagger = Lemmatizer(), POSTagger()
	train, validation, test = DadeganReader(train_file), DadeganReader(validation_file), DadeganReader(test_file)
	train_sents = list(train.sents()) + list(validation.sents())
	train_trees = list(train.trees()) + list(validation.trees())

	train_data = train_file +'.data'
	with codecs.open(train_data, 'w', 'utf8') as output:
		for tree, sentence in zip(train_trees, tagger.tag_sents(train_sents)):
			for i, (node, word) in enumerate(zip(tree.nodelist[1:], sentence), start=1):
				node['tag'] = word[1]
				node['lemma'] = lemmatizer.lemmatize(node['word'].replace('_', ' '), node['tag'])
				print(i, node['word'].replace(' ', '_'), node['lemma'].replace(' ', '_'), node['tag'], node['tag'], '_', node['head'], node['rel'], '_', '_', sep='\t', file=output)
			print(file=output)

	subprocess.Popen(['java', memory_min, memory_max, '-jar', path_to_jar, '-w', 'resources', '-c', model_file, '-i', train_data, '-f', options_file, '-F', features_file, '-m', 'learn']).wait()

	# evaluation
	print('\nEvaluating trained model on test data:')
	parser = DependencyParser(tagger=tagger, model_file=model_file)

	tagged = tagger.tag_sents(test.sents())
	parsed = parser.tagged_parse_sents(tagged)

	test_data, test_results = test_file +'.data', test_file +'.results'
	print('\n'.join([sentence.to_conll(10).replace('/', '') for sentence in test.trees()]).strip(), file=codecs.open(test_data, 'w', 'utf8'))
	print('\n'.join([sentence.to_conll(10) for sentence in parsed]).strip(), file=codecs.open(test_results, 'w', 'utf8'))

	subprocess.Popen(['java', '-jar', 'resources/MaltEval.jar', '-g', test_data, '-s', test_results]).wait()

def train_stanford_postagger(peykare_root='corpora/peykare', path_to_model='resources/persian.tagger', path_to_jar='resources/stanford-postagger.jar', properties_file='resources/stanford-postagger.props', memory_min='-Xms1g', memory_max='-Xmx6g', test_size=.1):
	peykare = PeykareReader(peykare_root)
	train_file = 'resources/tagger_train_data.txt'
	train, test = train_test_split(list(peykare.sents()), test_size=test_size, random_state=0)
	print('Peykare loaded.')

	output = codecs.open(train_file, 'w', 'utf8')
	for sentence in train:
		print(*(map(lambda w: '/'.join(w).replace(' ', '_'), sentence)), file=output)
	subprocess.Popen(['java', memory_min, memory_max, '-classpath', path_to_jar, 'edu.stanford.nlp.tagger.maxent.MaxentTagger', '-prop', properties_file, '-model', path_to_model,  '-trainFile', train_file, '-tagSeparator', '/', '-search', 'owlqn2']).wait()

	tagger = POSTagger()
	print('Tagger Accuracy on Test Split:')
	print(tagger.evaluate(test))
<commit_msg>remove POSTagger unnecessary features<commit_after># coding: utf8

from __future__ import print_function, unicode_literals
import codecs, subprocess
from collections import Counter
from itertools import islice
from nltk.tag import untag
from sklearn.cross_validation import train_test_split
from hazm import *
from hazm.Chunker import tree2brackets

def create_words_file(dic_file='resources/persian.dic', output='hazm/data/words.dat'):
	""" prepares list of persian word words from [Virastyar](https://sourceforge.net/projects/virastyar/) dic file.
	"""

	dic_words = sorted([line.split('\t')[0] for line in codecs.open(dic_file, encoding='utf8')])
	print(*dic_words, sep='\n', file=codecs.open(output, 'w', 'utf8'))
	print(output, 'created')

def evaluate_lemmatizer(conll_file='resources/train.conll', peykare_root='corpora/peykare'):
	lemmatizer = Lemmatizer()

	errors = []
	with codecs.open('resources/lemmatizer_errors.txt', 'w', 'utf8') as output:
		dadegan = DadeganReader(conll_file)
		for tree in dadegan.trees():
			for node in tree.nodelist[1:]:
				word, lemma, pos = node['word'], node['lemma'], node['mtag']
				if lemmatizer.lemmatize(word, pos) != lemma:
					errors.append((word, lemma, pos, lemmatizer.lemmatize(word, pos)))
		print(len(errors), 'errors', file=output)
		counter = Counter(errors)
		for item, count in sorted(counter.items(), key=lambda t: t[1], reverse=True):
			print(count, *item, file=output)

	missed = []
	with codecs.open('resources/lemmatizer_missed.txt', 'w', 'utf8') as output:
		peykare = PeykareReader(peykare_root)
		for sentence in peykare.sents():
			for word in sentence:
				if word[1] == 'V':
					if word[0] == lemmatizer.lemmatize(word[0]):
						missed.append(word[0])
		print(len(missed), 'missed', file=output)
		counter = Counter(missed)
		for item, count in sorted(counter.items(), key=lambda t: t[1], reverse=True):
			print(count, item, file=output)

def evaluate_chunker(treebank_root='corpora/treebank'):
	treebank = TreebankReader(treebank_root, join_clitics=True, join_verb_parts=True)
	chunker = Chunker()
	chunked_trees = list(treebank.chunked_trees())

	print(chunker.evaluate(chunked_trees))

	output = codecs.open('resources/chunker_errors.txt', 'w', 'utf8')
	for sentence, gold in zip(treebank.sents(), chunked_trees):
		chunked = chunker.parse(sentence)
		if chunked != gold:
			print(tree2brackets(chunked), file=output)
			print(tree2brackets(gold), file=output)
			print(file=output)

def train_postagger(peykare_root='corpora/peykare', model_file='resources/postagger.model', test_size=.1, sents_limit=None):

	tagger = SequencePOSTagger(type='crf', algo='rprop', compact=True, patterns=[
		'*',

		'u:wll=%x[-2,0]',
		'u:wl=%x[-1,0]',
		'u:w=%x[0,0]',
		'u:wr=%x[1,0]',
		'u:wrr=%x[2,0]',

		# 'u:w2l=%x[-1,0]/%x[0,0]',
		# 'u:w2r=%x[0,0]/%x[1,0]',

		'*:p1=%m[0,0,"^.?"]',
		'*:p2=%m[0,0,"^.?.?"]',
		'*:p3=%m[0,0,"^.?.?.?"]',

		'*:s1=%m[0,0,".?$"]',
		'*:s2=%m[0,0,".?.?$"]',
		'*:s3=%m[0,0,".?.?.?$"]',

		'*:p?l=%t[-1,0,"\p"]',
		'*:p?=%t[0,0,"\p"]',
		'*:p?r=%t[1,0,"\p"]',
		'*:p?a=%t[0,0,"^\p*$"]',

		'*:n?l=%t[-1,0,"\d"]',
		'*:n?=%t[0,0,"\d"]',
		'*:n?r=%t[1,0,"\d"]',
		'*:n?a=%t[0,0,"^\d*$"]',
	])

	peykare = PeykareReader(peykare_root)
	train_sents, test_sents = train_test_split(list(islice(peykare.sents(), sents_limit)), test_size=test_size, random_state=0)

	tagger.train(train_sents)
	tagger.save_model(model_file)

	print(tagger.evaluate(test_sents))

def train_chunker(train_file='corpora/train.conll', validation_file='corpora/validation.conll', test_file='corpora/test.conll', model_file='resources/chunker.model'):

	tagger = SequencePOSTagger(model='resources/postagger.model')
	chunker = Chunker(type='crf', algo='l-bfgs', compact=True, patterns=[
		'*',

		'u:wll=%x[-2,0]',
		'u:wl=%x[-1,0]',
		'u:w=%x[0,0]',
		'u:wr=%x[1,0]',
		'u:wrr=%x[2,0]',

		'*:tll=%x[-2,1]',
		'*:tl=%x[-1,1]',
		'*:t=%x[0,1]',
		'*:tr=%x[1,1]',
		'*:trr=%x[2,1]',
	])

	train, validation, test = DadeganReader(train_file), DadeganReader(validation_file), DadeganReader(test_file)
	train_sents = list(train.sents()) + list(validation.sents())
	train_trees = list(train.chunked_trees()) + list(validation.chunked_trees())

	tagged_sents = tagger.tag_sents([untag(sent) for sent in train_sents])
	for tree, sentence in zip(train_trees, tagged_sents):
		for (n, word) in zip(tree.treepositions('leaves'), sentence):
			tree[n] = word

	chunker.train(train_trees)
	chunker.save_model(model_file)

	print(chunker.evaluate(test.chunked_trees()))

def train_maltparser(train_file='corpora/train.conll', validation_file='corpora/validation.conll', test_file='corpora/test.conll', model_file='langModel.mco', path_to_jar='resources/malt.jar', options_file='resources/malt-options.xml', features_file='resources/malt-features.xml', memory_min='-Xms7g', memory_max='-Xmx8g'):

	lemmatizer, tagger = Lemmatizer(), POSTagger()
	train, validation, test = DadeganReader(train_file), DadeganReader(validation_file), DadeganReader(test_file)
	train_sents = list(train.sents()) + list(validation.sents())
	train_trees = list(train.trees()) + list(validation.trees())

	train_data = train_file +'.data'
	with codecs.open(train_data, 'w', 'utf8') as output:
		for tree, sentence in zip(train_trees, tagger.tag_sents(train_sents)):
			for i, (node, word) in enumerate(zip(tree.nodelist[1:], sentence), start=1):
				node['tag'] = word[1]
				node['lemma'] = lemmatizer.lemmatize(node['word'].replace('_', ' '), node['tag'])
				print(i, node['word'].replace(' ', '_'), node['lemma'].replace(' ', '_'), node['tag'], node['tag'], '_', node['head'], node['rel'], '_', '_', sep='\t', file=output)
			print(file=output)

	subprocess.Popen(['java', memory_min, memory_max, '-jar', path_to_jar, '-w', 'resources', '-c', model_file, '-i', train_data, '-f', options_file, '-F', features_file, '-m', 'learn']).wait()

	# evaluation
	print('\nEvaluating trained model on test data:')
	parser = DependencyParser(tagger=tagger, model_file=model_file)

	tagged = tagger.tag_sents(test.sents())
	parsed = parser.tagged_parse_sents(tagged)

	test_data, test_results = test_file +'.data', test_file +'.results'
	print('\n'.join([sentence.to_conll(10).replace('/', '') for sentence in test.trees()]).strip(), file=codecs.open(test_data, 'w', 'utf8'))
	print('\n'.join([sentence.to_conll(10) for sentence in parsed]).strip(), file=codecs.open(test_results, 'w', 'utf8'))

	subprocess.Popen(['java', '-jar', 'resources/MaltEval.jar', '-g', test_data, '-s', test_results]).wait()

def train_stanford_postagger(peykare_root='corpora/peykare', path_to_model='resources/persian.tagger', path_to_jar='resources/stanford-postagger.jar', properties_file='resources/stanford-postagger.props', memory_min='-Xms1g', memory_max='-Xmx6g', test_size=.1):
	peykare = PeykareReader(peykare_root)
	train_file = 'resources/tagger_train_data.txt'
	train, test = train_test_split(list(peykare.sents()), test_size=test_size, random_state=0)
	print('Peykare loaded.')

	output = codecs.open(train_file, 'w', 'utf8')
	for sentence in train:
		print(*(map(lambda w: '/'.join(w).replace(' ', '_'), sentence)), file=output)
	subprocess.Popen(['java', memory_min, memory_max, '-classpath', path_to_jar, 'edu.stanford.nlp.tagger.maxent.MaxentTagger', '-prop', properties_file, '-model', path_to_model,  '-trainFile', train_file, '-tagSeparator', '/', '-search', 'owlqn2']).wait()

	tagger = POSTagger()
	print('Tagger Accuracy on Test Split:')
	print(tagger.evaluate(test))
<|endoftext|>