<commit_before>import warnings

from geopy.compat import urlencode
from geopy.exc import (
    ConfigurationError,
    GeocoderInsufficientPrivileges,
    GeocoderQueryError,
    GeocoderServiceError,
)
from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.timezone import ensure_pytz_is_installed, from_timezone_name
from geopy.util import logger

__all__ = ("GeoNames", )

class GeoNames(Geocoder):
    """GeoNames geocoder.

    Documentation at:
        http://www.geonames.org/export/geonames-search.html

    Reverse geocoding documentation at:
        http://www.geonames.org/export/web-services.html#findNearbyPlaceName
    """

    geocode_path = '/searchJSON'
    reverse_path = '/findNearbyPlaceNameJSON'
    reverse_nearby_path = '/findNearbyJSON'
    timezone_path = '/timezoneJSON'

    def __init__(
            self,
            country_bias=None,
            username=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            format_string=None,
            ssl_context=DEFAULT_SENTINEL,
    ):
        """
        :param str country_bias:

        :param str username: GeoNames username, required. Sign up here:
            http://www.geonames.org/login

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

            .. versionadded:: 1.12.0

        :param str format_string:
            See :attr:`geopy.geocoders.options.default_format_string`.

            .. versionadded:: 1.14.0

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

            .. versionadded:: 1.14.0
        """
        super(GeoNames, self).__init__(
            format_string=format_string,
            scheme='http',
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
        )
        if username is None:
            raise ConfigurationError(
                'No username given, required for api access.  If you do not '
                'have a GeoNames username, sign up here: '
                'http://www.geonames.org/login'
            )
        self.username = username
        self.country_bias = country_bias
        domain = 'api.geonames.org'
        self.api = (
            "%s://%s%s" % (self.scheme, domain, self.geocode_path)
        )
        self.api_reverse = (
            "%s://%s%s" % (self.scheme, domain, self.reverse_path)
        )
        self.api_timezone = (
            "%s://%s%s" % (self.scheme, domain, self.timezone_path)
        )
        self.api_reverse_nearby = (
            "%s://%s%s" % (self.scheme, domain, self.reverse_nearby_path)
        )

    def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL):
        """
        Return a location point by address.

        :param str query: The address or query you wish to geocode.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        params = {
            'q': self.format_string % query,
            'username': self.username
        }
        if self.country_bias:
            params['countryBias'] = self.country_bias
        if exactly_one:
            params['maxRows'] = 1
        url = "?".join((self.api, urlencode(params)))
        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        return self._parse_json(
            self._call_geocoder(url, timeout=timeout),
            exactly_one,
        )

    def reverse(
            self,
            query,
            exactly_one=DEFAULT_SENTINEL,
            timeout=DEFAULT_SENTINEL,
            feature_code=None,
            lang=None,
            find_nearby_type='findNearbyPlaceNameJSON',
    ):
        """
        Return an address by location point.

            .. versionadded:: 1.2.0

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

            .. versionchanged:: 1.14.0
               Default value for ``exactly_one`` was ``False``, which differs
               from the conventional default across geopy. Please always pass
               this argument explicitly, otherwise you would get a warning.
               In geopy 2.0 the default value will become ``True``.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param str feature_code: A GeoNames feature code
            .. versionadded:: 1.18.0

        :param str lang: language of returned ``name`` element (the pseudo
            language code 'local' will return it in local language)
            Full list of supported languages ISO639-2 can be found here:
            https://www.geonames.org/countries/

        :param str find_nearby_type: A flag to switch between different
            endpoints. The default value is ``findNearbyPlaceNameJSON`` which
            was the only option before geopy 1.18 and returns the closest
            populated place. Another currently implemented option is
            ``findNearbyJSON`` return the closest toponym for the lat/lng query

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """
        if exactly_one is DEFAULT_SENTINEL:
            warnings.warn('%s.reverse: default value for `exactly_one` '
                          'argument will become True in geopy 2.0. '
                          'Specify `exactly_one=False` as the argument '
                          'explicitly to get rid of this warning.' % type(self).__name__,
                          DeprecationWarning, stacklevel=2)
            exactly_one = False

        try:
            lat, lng = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")
        params = {
            'lat': lat,
            'lng': lng,
            'username': self.username
        }
        if feature_code:
            params['featureCode'] = feature_code

        if lang:
            params['lang'] = lang
        if find_nearby_type == 'findNearbyJSON':
            if lang:
                raise ValueError("Not supported argument for this api")
            url = "?".join((self.api_reverse_nearby, urlencode(params)))
        elif find_nearby_type == 'findNearbyPlaceNameJSON':
            if feature_code:
                raise ValueError("Not supported argument for this api")
            url = "?".join((self.api_reverse, urlencode(params)))
        else:
            raise GeocoderQueryError(
                '%s type is not supported by geopy yet' % find_nearby_type
            )

        return self._parse_json(
            self._call_geocoder(url, timeout=timeout),
            exactly_one
        )

    def reverse_timezone(self, query, timeout=DEFAULT_SENTINEL):
        """
        Find the timezone a point in `query`.

        .. versionadded:: 1.18.0

        :param query: The coordinates for which you want a timezone.
        :type query: :class:`geopy.point.Point`, list or tuple of (latitude,
            longitude), or string as "%(latitude)s, %(longitude)s"

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: :class:`geopy.timezone.Timezone`
        """
        ensure_pytz_is_installed()
        try:
            lat, lng = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")

        params = {
            "lat": lat,
            "lng": lng,
            "username": self.username,
        }

        url = "?".join((self.api_timezone, urlencode(params)))

        logger.debug("%s.timezone: %s", self.__class__.__name__, url)
        response = self._call_geocoder(url, timeout=timeout)

        return self._parse_json_timezone(response)

    def _parse_json_timezone(self, response):
        return from_timezone_name(response["timezoneId"], raw=response)

    def _parse_json(self, doc, exactly_one):
        """
        Parse JSON response body.
        """
        places = doc.get('geonames', [])
        err = doc.get('status', None)
        if err and 'message' in err:
            if err['message'].startswith("user account not enabled to use"):
                raise GeocoderInsufficientPrivileges(err['message'])
            else:
                raise GeocoderServiceError(err['message'])
        if not len(places):
            return None

        def parse_code(place):
            """
            Parse each record.
            """
            latitude = place.get('lat', None)
            longitude = place.get('lng', None)
            if latitude and longitude:
                latitude = float(latitude)
                longitude = float(longitude)
            else:
                return None

            placename = place.get('name')
            state = place.get('adminCode1', None)
            country = place.get('countryCode', None)

            location = ', '.join(
                [x for x in [placename, state, country] if x]
            )

            return Location(location, (latitude, longitude), place)

        if exactly_one:
            return parse_code(places[0])
        else:
            return [parse_code(place) for place in places]
<commit_msg>Geonames: use full names instead of codes for `Location.address`<commit_after>import warnings

from geopy.compat import urlencode
from geopy.exc import (
    ConfigurationError,
    GeocoderInsufficientPrivileges,
    GeocoderQueryError,
    GeocoderServiceError,
)
from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.timezone import ensure_pytz_is_installed, from_timezone_name
from geopy.util import logger

__all__ = ("GeoNames", )

class GeoNames(Geocoder):
    """GeoNames geocoder.

    Documentation at:
        http://www.geonames.org/export/geonames-search.html

    Reverse geocoding documentation at:
        http://www.geonames.org/export/web-services.html#findNearbyPlaceName
    """

    geocode_path = '/searchJSON'
    reverse_path = '/findNearbyPlaceNameJSON'
    reverse_nearby_path = '/findNearbyJSON'
    timezone_path = '/timezoneJSON'

    def __init__(
            self,
            country_bias=None,
            username=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            format_string=None,
            ssl_context=DEFAULT_SENTINEL,
    ):
        """
        :param str country_bias:

        :param str username: GeoNames username, required. Sign up here:
            http://www.geonames.org/login

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

            .. versionadded:: 1.12.0

        :param str format_string:
            See :attr:`geopy.geocoders.options.default_format_string`.

            .. versionadded:: 1.14.0

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

            .. versionadded:: 1.14.0
        """
        super(GeoNames, self).__init__(
            format_string=format_string,
            scheme='http',
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
        )
        if username is None:
            raise ConfigurationError(
                'No username given, required for api access.  If you do not '
                'have a GeoNames username, sign up here: '
                'http://www.geonames.org/login'
            )
        self.username = username
        self.country_bias = country_bias
        domain = 'api.geonames.org'
        self.api = (
            "%s://%s%s" % (self.scheme, domain, self.geocode_path)
        )
        self.api_reverse = (
            "%s://%s%s" % (self.scheme, domain, self.reverse_path)
        )
        self.api_timezone = (
            "%s://%s%s" % (self.scheme, domain, self.timezone_path)
        )
        self.api_reverse_nearby = (
            "%s://%s%s" % (self.scheme, domain, self.reverse_nearby_path)
        )

    def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL):
        """
        Return a location point by address.

        :param str query: The address or query you wish to geocode.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        params = {
            'q': self.format_string % query,
            'username': self.username
        }
        if self.country_bias:
            params['countryBias'] = self.country_bias
        if exactly_one:
            params['maxRows'] = 1
        url = "?".join((self.api, urlencode(params)))
        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        return self._parse_json(
            self._call_geocoder(url, timeout=timeout),
            exactly_one,
        )

    def reverse(
            self,
            query,
            exactly_one=DEFAULT_SENTINEL,
            timeout=DEFAULT_SENTINEL,
            feature_code=None,
            lang=None,
            find_nearby_type='findNearbyPlaceNameJSON',
    ):
        """
        Return an address by location point.

            .. versionadded:: 1.2.0

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

            .. versionchanged:: 1.14.0
               Default value for ``exactly_one`` was ``False``, which differs
               from the conventional default across geopy. Please always pass
               this argument explicitly, otherwise you would get a warning.
               In geopy 2.0 the default value will become ``True``.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param str feature_code: A GeoNames feature code
            .. versionadded:: 1.18.0

        :param str lang: language of returned ``name`` element (the pseudo
            language code 'local' will return it in local language)
            Full list of supported languages ISO639-2 can be found here:
            https://www.geonames.org/countries/

        :param str find_nearby_type: A flag to switch between different
            endpoints. The default value is ``findNearbyPlaceNameJSON`` which
            was the only option before geopy 1.18 and returns the closest
            populated place. Another currently implemented option is
            ``findNearbyJSON`` return the closest toponym for the lat/lng query

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """
        if exactly_one is DEFAULT_SENTINEL:
            warnings.warn('%s.reverse: default value for `exactly_one` '
                          'argument will become True in geopy 2.0. '
                          'Specify `exactly_one=False` as the argument '
                          'explicitly to get rid of this warning.' % type(self).__name__,
                          DeprecationWarning, stacklevel=2)
            exactly_one = False

        try:
            lat, lng = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")
        params = {
            'lat': lat,
            'lng': lng,
            'username': self.username
        }
        if feature_code:
            params['featureCode'] = feature_code

        if lang:
            params['lang'] = lang
        if find_nearby_type == 'findNearbyJSON':
            if lang:
                raise ValueError("Not supported argument for this api")
            url = "?".join((self.api_reverse_nearby, urlencode(params)))
        elif find_nearby_type == 'findNearbyPlaceNameJSON':
            if feature_code:
                raise ValueError("Not supported argument for this api")
            url = "?".join((self.api_reverse, urlencode(params)))
        else:
            raise GeocoderQueryError(
                '%s type is not supported by geopy yet' % find_nearby_type
            )

        return self._parse_json(
            self._call_geocoder(url, timeout=timeout),
            exactly_one
        )

    def reverse_timezone(self, query, timeout=DEFAULT_SENTINEL):
        """
        Find the timezone a point in `query`.

        .. versionadded:: 1.18.0

        :param query: The coordinates for which you want a timezone.
        :type query: :class:`geopy.point.Point`, list or tuple of (latitude,
            longitude), or string as "%(latitude)s, %(longitude)s"

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: :class:`geopy.timezone.Timezone`
        """
        ensure_pytz_is_installed()
        try:
            lat, lng = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")

        params = {
            "lat": lat,
            "lng": lng,
            "username": self.username,
        }

        url = "?".join((self.api_timezone, urlencode(params)))

        logger.debug("%s.timezone: %s", self.__class__.__name__, url)
        response = self._call_geocoder(url, timeout=timeout)

        return self._parse_json_timezone(response)

    def _parse_json_timezone(self, response):
        return from_timezone_name(response["timezoneId"], raw=response)

    def _parse_json(self, doc, exactly_one):
        """
        Parse JSON response body.
        """
        places = doc.get('geonames', [])
        err = doc.get('status', None)
        if err and 'message' in err:
            if err['message'].startswith("user account not enabled to use"):
                raise GeocoderInsufficientPrivileges(err['message'])
            else:
                raise GeocoderServiceError(err['message'])
        if not len(places):
            return None

        def parse_code(place):
            """
            Parse each record.
            """
            latitude = place.get('lat', None)
            longitude = place.get('lng', None)
            if latitude and longitude:
                latitude = float(latitude)
                longitude = float(longitude)
            else:
                return None

            placename = place.get('name')
            state = place.get('adminName1', None)
            country = place.get('countryName', None)

            location = ', '.join(
                [x for x in [placename, state, country] if x]
            )

            return Location(location, (latitude, longitude), place)

        if exactly_one:
            return parse_code(places[0])
        else:
            return [parse_code(place) for place in places]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'Observation'
        db.create_table(u'observations_observation', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('observer', self.gf('django.db.models.fields.related.ForeignKey')(related_name=u'observations', to=orm['observers.Observer'])),
            ('star', self.gf('django.db.models.fields.related.ForeignKey')(related_name=u'observations', to=orm['stars.Star'])),
            ('jd', self.gf('django.db.models.fields.FloatField')()),
            ('magnitude', self.gf('django.db.models.fields.FloatField')()),
            ('fainter_than', self.gf('django.db.models.fields.BooleanField')(default=False)),
            ('comp1', self.gf('django.db.models.fields.CharField')(default=u'', max_length=5, blank=True)),
            ('comp2', self.gf('django.db.models.fields.CharField')(default=u'', max_length=5, blank=True)),
            ('comment_code', self.gf('django.db.models.fields.CharField')(default=u'', max_length=10, blank=True)),
            ('chart', self.gf('django.db.models.fields.CharField')(default=u'', max_length=20, blank=True)),
            ('notes', self.gf('django.db.models.fields.CharField')(default=u'', max_length=100, blank=True)),
        ))
        db.send_create_signal(u'observations', ['Observation'])

    def backwards(self, orm):
        # Deleting model 'Observation'
        db.delete_table(u'observations_observation')

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        u'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'observations.observation': {
            'Meta': {'ordering': "(u'-jd',)", 'object_name': 'Observation'},
            'chart': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '20', 'blank': 'True'}),
            'comment_code': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '10', 'blank': 'True'}),
            'comp1': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '5', 'blank': 'True'}),
            'comp2': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '5', 'blank': 'True'}),
            'fainter_than': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'jd': ('django.db.models.fields.FloatField', [], {}),
            'magnitude': ('django.db.models.fields.FloatField', [], {}),
            'notes': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '100', 'blank': 'True'}),
            'observer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'observations'", 'to': u"orm['observers.Observer']"}),
            'star': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'observations'", 'to': u"orm['stars.Star']"})
        },
        u'observers.observer': {
            'Meta': {'ordering': "(u'-created',)", 'object_name': 'Observer'},
            'aavso_code': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '10', 'blank': 'True'}),
            'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'limiting_magnitude': ('django.db.models.fields.FloatField', [], {'default': '6.0', 'null': 'True', 'blank': 'True'}),
            'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "u'observer'", 'unique': 'True', 'to': u"orm['auth.User']"})
        },
        u'stars.star': {
            'Meta': {'ordering': "(u'constellation', u'name')", 'object_name': 'Star'},
            'constellation': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
            'dec': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '15'}),
            'epoch': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'max_magnitude': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'min_magnitude': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '20', 'db_index': 'True'}),
            'period': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'ra': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '15'}),
            'variability_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stars.VariabilityType']", 'null': 'True', 'blank': 'True'})
        },
        u'stars.variabilitytype': {
            'Meta': {'ordering': "(u'code',)", 'object_name': 'VariabilityType'},
            'code': ('django.db.models.fields.CharField', [], {'max_length': '12', 'db_index': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'long_description': ('django.db.models.fields.TextField', [], {'default': "u''"}),
            'short_description': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '100', 'blank': 'True'})
        }
    }

    complete_apps = ['observations']<commit_msg>Fixed migration dependency order.<commit_after># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    depends_on = (
        ('observers', '0001_initial'),
        ('stars', '0001_initial'),
    )

    def forwards(self, orm):
        # Adding model 'Observation'
        db.create_table(u'observations_observation', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('observer', self.gf('django.db.models.fields.related.ForeignKey')(related_name=u'observations', to=orm['observers.Observer'])),
            ('star', self.gf('django.db.models.fields.related.ForeignKey')(related_name=u'observations', to=orm['stars.Star'])),
            ('jd', self.gf('django.db.models.fields.FloatField')()),
            ('magnitude', self.gf('django.db.models.fields.FloatField')()),
            ('fainter_than', self.gf('django.db.models.fields.BooleanField')(default=False)),
            ('comp1', self.gf('django.db.models.fields.CharField')(default=u'', max_length=5, blank=True)),
            ('comp2', self.gf('django.db.models.fields.CharField')(default=u'', max_length=5, blank=True)),
            ('comment_code', self.gf('django.db.models.fields.CharField')(default=u'', max_length=10, blank=True)),
            ('chart', self.gf('django.db.models.fields.CharField')(default=u'', max_length=20, blank=True)),
            ('notes', self.gf('django.db.models.fields.CharField')(default=u'', max_length=100, blank=True)),
        ))
        db.send_create_signal(u'observations', ['Observation'])

    def backwards(self, orm):
        # Deleting model 'Observation'
        db.delete_table(u'observations_observation')

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        u'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'observations.observation': {
            'Meta': {'ordering': "(u'-jd',)", 'object_name': 'Observation'},
            'chart': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '20', 'blank': 'True'}),
            'comment_code': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '10', 'blank': 'True'}),
            'comp1': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '5', 'blank': 'True'}),
            'comp2': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '5', 'blank': 'True'}),
            'fainter_than': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'jd': ('django.db.models.fields.FloatField', [], {}),
            'magnitude': ('django.db.models.fields.FloatField', [], {}),
            'notes': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '100', 'blank': 'True'}),
            'observer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'observations'", 'to': u"orm['observers.Observer']"}),
            'star': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'observations'", 'to': u"orm['stars.Star']"})
        },
        u'observers.observer': {
            'Meta': {'ordering': "(u'-created',)", 'object_name': 'Observer'},
            'aavso_code': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '10', 'blank': 'True'}),
            'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'limiting_magnitude': ('django.db.models.fields.FloatField', [], {'default': '6.0', 'null': 'True', 'blank': 'True'}),
            'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "u'observer'", 'unique': 'True', 'to': u"orm['auth.User']"})
        },
        u'stars.star': {
            'Meta': {'ordering': "(u'constellation', u'name')", 'object_name': 'Star'},
            'constellation': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}),
            'dec': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '15'}),
            'epoch': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'max_magnitude': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'min_magnitude': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '20', 'db_index': 'True'}),
            'period': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'ra': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '15'}),
            'variability_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stars.VariabilityType']", 'null': 'True', 'blank': 'True'})
        },
        u'stars.variabilitytype': {
            'Meta': {'ordering': "(u'code',)", 'object_name': 'VariabilityType'},
            'code': ('django.db.models.fields.CharField', [], {'max_length': '12', 'db_index': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'long_description': ('django.db.models.fields.TextField', [], {'default': "u''"}),
            'short_description': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '100', 'blank': 'True'})
        }
    }

    complete_apps = ['observations']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Add LIWC words as entities in FoLiA XML file.
Usage: python add_liwc_entities.py <file in>
"""
from lxml import etree
from datetime import datetime
import argparse
import codecs
import os

def add_entity(sentence, cls, words, text_content_tag):
    entity_tag = '{http://ilk.uvt.nl/folia}entities'
    if sentence.find(entity_tag) is not None:
        entities = sentence.find(entity_tag)
    else:
        entities = etree.SubElement(sentence, 'entities')

    entity = etree.SubElement(entities, 'entity', {'class': cls})
    for w in words:
        wref_attrs = {
            'id': w.attrib.get('{http://www.w3.org/XML/1998/namespace}id'),
            't': w.find(text_content_tag).text
        }
        etree.SubElement(entity, 'wref', wref_attrs)

def write_folia_file(context, folia_in, ext):
    head, tail = os.path.split(folia_in)
    p = tail.split('.')
    file_out = '{h}{s}{n}-{e}.xml'.format(n=p[0], h=head, s=os.sep, e=ext)
    with open(file_out, 'w') as f:
        f.write(etree.tostring(context.root,
                               encoding='utf8',
                               xml_declaration=True,
                               pretty_print=True))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('file_in', help='the name of the FoLiA XML file add '
                        'LIWC entities to')
    args = parser.parse_args()

    file_name = args.file_in

    # Load liwc dict
    with codecs.open('historic_Dutch_LIWC.dic', 'rb', 'utf8') as f:
        lines = f.readlines()

    liwc_categories = {}
    liwc_dict = {}

    for line in lines:
        # LIWC category
        if line[0].isdigit():
            entry = line.split()
            # remove 0 from strings like 01
            c = str(int(entry[0]))
            liwc_categories[c] = entry[1]
        # word
        elif line[0].isalpha():
            entry = line.split()
            term = entry[0]
            categories = entry[1:]
            liwc_dict[term] = categories

    # Load document
    context = etree.iterparse(file_name, events=('end',))
    annotations_tag = '{http://ilk.uvt.nl/folia}annotations'
    sentence_tag = '{http://ilk.uvt.nl/folia}s'
    word_tag = '{http://ilk.uvt.nl/folia}w'
    text_content_tag = '{http://ilk.uvt.nl/folia}t'

    for event, elem in context:
        if elem.tag == annotations_tag:
            # add entity-annotation for liwc
            annotation_attrs = {
                'annotator': 'liwc',
                'annotatortype': 'auto',
                'datetime': datetime.now().isoformat(),
                'set': 'liwc-set'
            }
            etree.SubElement(elem, 'entity-annotation', annotation_attrs)

        if elem.tag == sentence_tag:
            words = elem.findall(word_tag)
            for word in words:
                w = word.find(text_content_tag).text
                if w in liwc_dict.keys():
                    for cat in liwc_dict[w]:
                        cat_label = 'liwc-{}'.format(liwc_categories[cat])
                        add_entity(elem, cat_label, [word], text_content_tag)

    write_folia_file(context, file_name, 'liwc')
<commit_msg>Fix pretty printing output XML<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Add LIWC words as entities in FoLiA XML file.
Usage: python add_liwc_entities.py <file in>
"""
from lxml import etree
from datetime import datetime
import argparse
import codecs
import os

def add_entity(sentence, cls, words, text_content_tag):
    entity_tag = '{http://ilk.uvt.nl/folia}entities'
    if sentence.find(entity_tag) is not None:
        entities = sentence.find(entity_tag)
    else:
        entities = etree.SubElement(sentence, 'entities')

    entity = etree.SubElement(entities, 'entity', {'class': cls})
    for w in words:
        wref_attrs = {
            'id': w.attrib.get('{http://www.w3.org/XML/1998/namespace}id'),
            't': w.find(text_content_tag).text
        }
        etree.SubElement(entity, 'wref', wref_attrs)

def write_folia_file(context, folia_in, ext):
    head, tail = os.path.split(folia_in)
    p = tail.split('.')
    file_out = '{h}{s}{n}-{e}.xml'.format(n=p[0], h=head, s=os.sep, e=ext)
    with open(file_out, 'w') as f:
        f.write(etree.tostring(context.root,
                               encoding='utf8',
                               xml_declaration=True,
                               pretty_print=True))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('file_in', help='the name of the FoLiA XML file add '
                        'LIWC entities to')
    args = parser.parse_args()

    file_name = args.file_in

    # Load liwc dict
    with codecs.open('historic_Dutch_LIWC.dic', 'rb', 'utf8') as f:
        lines = f.readlines()

    liwc_categories = {}
    liwc_dict = {}

    for line in lines:
        # LIWC category
        if line[0].isdigit():
            entry = line.split()
            # remove 0 from strings like 01
            c = str(int(entry[0]))
            liwc_categories[c] = entry[1]
        # word
        elif line[0].isalpha():
            entry = line.split()
            term = entry[0]
            categories = entry[1:]
            liwc_dict[term] = categories

    # Load document
    context = etree.iterparse(file_name,
                              events=('end',),
                              remove_blank_text=True)
    annotations_tag = '{http://ilk.uvt.nl/folia}annotations'
    sentence_tag = '{http://ilk.uvt.nl/folia}s'
    word_tag = '{http://ilk.uvt.nl/folia}w'
    text_content_tag = '{http://ilk.uvt.nl/folia}t'

    for event, elem in context:
        if elem.tag == annotations_tag:
            # add entity-annotation for liwc
            annotation_attrs = {
                'annotator': 'liwc',
                'annotatortype': 'auto',
                'datetime': datetime.now().isoformat(),
                'set': 'liwc-set'
            }
            etree.SubElement(elem, 'entity-annotation', annotation_attrs)

        if elem.tag == sentence_tag:
            words = elem.findall(word_tag)
            for word in words:
                w = word.find(text_content_tag).text
                if w in liwc_dict.keys():
                    for cat in liwc_dict[w]:
                        cat_label = 'liwc-{}'.format(liwc_categories[cat])
                        add_entity(elem, cat_label, [word], text_content_tag)

    write_folia_file(context, file_name, 'liwc')
<|endoftext|>


<commit_before>import os

from django.core.urlresolvers import reverse
from django.conf import settings
from main.tests.test_base import MainTestCase
from odk_logger.models import Attachment
from odk_viewer.views import attachment_url

class TestAttachmentUrl(MainTestCase):

    def setUp(self):
        self.attachment_count = 0
        MainTestCase.setUp(self)
        self._create_user_and_login()
        self._publish_transportation_form()
        self._submit_transport_instance_w_attachment()
        self.url = reverse(
            attachment_url, kwargs={'size': 'original'})

    def test_attachment_url(self):
        self.assertEqual(
            Attachment.objects.count(), self.attachment_count + 1)
        response = self.client.get(
            self.url, {"media_file": self.attachment_media_file})
        print response
        print response.content
        print self.url
        print self.attachment_media_file
        self.assertEqual(response.status_code, 302) #redirects to amazon

    def test_attachment_not_found(self):
        response = self.client.get(
            self.url, {"media_file": "non_existent_attachment.jpg"})
        self.assertEqual(response.status_code, 404)

    def test_attachment_has_mimetype(self):
        attachment = Attachment.objects.all().reverse()[0]
        self.assertEqual(attachment.mimetype, 'image/jpeg')

    def tearDown(self):
        path = os.path.join(settings.MEDIA_ROOT, self.user.username)
        for root, dirs, files in os.walk(path, topdown=False):
            for name in files:
                os.remove(os.path.join(root, name))
            for name in dirs:
                os.rmdir(os.path.join(root, name))

<commit_msg>expecting 200 http status code<commit_after>import os

from django.core.urlresolvers import reverse
from django.conf import settings
from main.tests.test_base import MainTestCase
from odk_logger.models import Attachment
from odk_viewer.views import attachment_url

class TestAttachmentUrl(MainTestCase):

    def setUp(self):
        self.attachment_count = 0
        MainTestCase.setUp(self)
        self._create_user_and_login()
        self._publish_transportation_form()
        self._submit_transport_instance_w_attachment()
        self.url = reverse(
            attachment_url, kwargs={'size': 'original'})

    def test_attachment_url(self):
        self.assertEqual(
            Attachment.objects.count(), self.attachment_count + 1)
        response = self.client.get(
            self.url, {"media_file": self.attachment_media_file})
        print response
        print response.content
        print self.url
        print self.attachment_media_file
        self.assertEqual(response.status_code, 200) #redirects to amazon

    def test_attachment_not_found(self):
        response = self.client.get(
            self.url, {"media_file": "non_existent_attachment.jpg"})
        self.assertEqual(response.status_code, 404)

    def test_attachment_has_mimetype(self):
        attachment = Attachment.objects.all().reverse()[0]
        self.assertEqual(attachment.mimetype, 'image/jpeg')

    def tearDown(self):
        path = os.path.join(settings.MEDIA_ROOT, self.user.username)
        for root, dirs, files in os.walk(path, topdown=False):
            for name in files:
                os.remove(os.path.join(root, name))
            for name in dirs:
                os.rmdir(os.path.join(root, name))

<|endoftext|>


<commit_before>import numpy as np
from scipy.misc import imresize

from PyQt5.QtCore import QCoreApplication
from PyQt5.QtWidgets import QMainWindow, QTabWidget, QAction, QStatusBar
from PyQt5.QtGui import QIcon

from qtgui.panels import ActivationsPanel
from util import ArgumentError

class DeepVisMainWindow(QMainWindow):
    '''The main window of the deep visualization toolbox. The window is
    intended to hold different panels that allow for different kinds
    of visualizations.

    This class also provides central entry points to programmatically
    set certain aspects, like loading a network or input data,
    switching between different panels, etc.

    Attributes
    ----------
    _title   :   str
                Window title

    '''

    def __init__(self, network=None, data=None, title='QtPyVis'):
        super().__init__()

        self._title = title

        self.initUI()

    def initUI(self):
        '''Initialize the graphical components of this user interface.'''
        self.setWindowTitle(self._title)

        menubar = self.menuBar()
        ########################################################################
        #                            Add exit menu                             #
        ########################################################################
        exitAction = QAction(QIcon('exit.png'), '&Exit', self)
        exitAction.setShortcuts(['Ctrl+W', 'Ctrl+Q'])
        exitAction.setStatusTip('Exit application')
        exitAction.triggered.connect(QCoreApplication.quit)

        fileMenu = menubar.addMenu('&File')
        fileMenu.addAction(exitAction)

        ##
        # Initialize the panels ##
        self._activations = ActivationsPanel()
        self.initActivationPanel()

        self._tabs = QTabWidget(self)
        self._tabs.addTab(self._activations, 'Main')

        self.setCentralWidget(self._tabs)

        ##
        # Initialize the status bar
        ##
        self._statusBar = QStatusBar()
        self.setStatusBar(self._statusBar)

    def initActivationPanel(self):
        '''Initialiase the actiations panel.
        This will connect the Input button and the Layer buttons
        '''
        self._activations.inputselector.selected.connect(self.setInputData)
        self._activations.inputSelected.connect(self.setInputData)

    def setNetwork(self, network):
        '''Set the model to visualise
        Parameters
        ----------
        network :   network.network.Network
                    The network to use
        '''
        self._network = network
        self._activations.addNetwork(network)

    def setInputDataArray(self, data: np.ndarray):
        '''Set the input data to be used.

        Parameters
        ----------
        data    :   np.ndarray
                    Input data array
        '''
        self._activations.setInputDataArray(data)

    def setInputDataFile(self, filename: str):
        '''Set the input data to be used (read from file)

        Parameters
        ----------
        filename    :   str
                        Input filename

        '''
        self._activations.setInputDataFile(filename)

    def setInputDataDirectory(self, dirname: str):
        '''Set input data directory.

        Parameters
        ----------
        dirname     :   str
                        Directory to read from

        '''
        self._activations.setInputDataDirectory(dirname)

    def setInputDataSet(self, name: str):
        '''Set the input data to be used.'''
        self._activations.setInputDataSet(name)

    def showStatusMessage(self, message):
        self._statusBar.showMessage(message, 2000)

    def setInputData(self, data: np.ndarray=None, description: str=None):
        '''Provide one data vector as input for the network.'''
        # FIXME[hack]: there seems to be a bug in PyQt forbidding to emit
        # signals with None parameter. See code in 'widget/inputselector.py'

        if not data.ndim:
            raise ArgumentError('Data cannot be none.')

        raw_data = data
        if self._network is not None and data is not None:
            network_shape = self._network.get_input_shape(include_batch=False)
            if data.ndim > 4 or data.ndim < 2:
                raise ArgumentError('Data must have between 2 and 4 dimensions.')

            if data.ndim == 4:
                if data.shape[0] == 1:
                    # first dimension has size of 1 -> remove
                    data = data.squeeze(0)
                else:
                    raise ArgumentError('Cannot visualize batch of images')

            if data.ndim == 2:
                data = data[..., np.newaxis].repeat(3, axis=2).copy()

            if data is not None and data.shape[0:2] != network_shape[0:2]:
                data = imresize(data, network_shape[0:2])

            if data is not None and data.shape[2] != network_shape[2]:
                # FIXME[hack]: find better way to doe RGB <-> grayscale
                # conversion
                if network_shape[2] == 1:
                    data = data[:, :, 0:1]
                elif network_shape[2] == 3 and data.shape[2] == 1:
                    data = data.repeat(3, axis=2)
                else:
                    data = None

        self._activations.setInputData(raw_data, data, description)
<commit_msg>Comment setInputData<commit_after>import numpy as np
from scipy.misc import imresize

from PyQt5.QtCore import QCoreApplication
from PyQt5.QtWidgets import QMainWindow, QTabWidget, QAction, QStatusBar
from PyQt5.QtGui import QIcon

from qtgui.panels import ActivationsPanel
from util import ArgumentError

class DeepVisMainWindow(QMainWindow):
    '''The main window of the deep visualization toolbox. The window is
    intended to hold different panels that allow for different kinds
    of visualizations.

    This class also provides central entry points to programmatically
    set certain aspects, like loading a network or input data,
    switching between different panels, etc.

    Attributes
    ----------
    _title   :   str
                Window title

    '''

    def __init__(self, network=None, data=None, title='QtPyVis'):
        super().__init__()

        self._title = title

        self.initUI()

    def initUI(self):
        '''Initialize the graphical components of this user interface.'''
        self.setWindowTitle(self._title)

        menubar = self.menuBar()
        ########################################################################
        #                            Add exit menu                             #
        ########################################################################
        exitAction = QAction(QIcon('exit.png'), '&Exit', self)
        exitAction.setShortcuts(['Ctrl+W', 'Ctrl+Q'])
        exitAction.setStatusTip('Exit application')
        exitAction.triggered.connect(QCoreApplication.quit)

        fileMenu = menubar.addMenu('&File')
        fileMenu.addAction(exitAction)

        ##
        # Initialize the panels ##
        self._activations = ActivationsPanel()
        self.initActivationPanel()

        self._tabs = QTabWidget(self)
        self._tabs.addTab(self._activations, 'Main')

        self.setCentralWidget(self._tabs)

        ##
        # Initialize the status bar
        ##
        self._statusBar = QStatusBar()
        self.setStatusBar(self._statusBar)

    def initActivationPanel(self):
        '''Initialiase the actiations panel.
        This will connect the Input button and the Layer buttons
        '''
        self._activations.inputselector.selected.connect(self.setInputData)
        self._activations.inputSelected.connect(self.setInputData)

    def setNetwork(self, network):
        '''Set the model to visualise
        Parameters
        ----------
        network :   network.network.Network
                    The network to use
        '''
        self._network = network
        self._activations.addNetwork(network)

    def setInputDataArray(self, data: np.ndarray):
        '''Set the input data to be used.

        Parameters
        ----------
        data    :   np.ndarray
                    Input data array
        '''
        self._activations.setInputDataArray(data)

    def setInputDataFile(self, filename: str):
        '''Set the input data to be used (read from file)

        Parameters
        ----------
        filename    :   str
                        Input filename

        '''
        self._activations.setInputDataFile(filename)

    def setInputDataDirectory(self, dirname: str):
        '''Set input data directory.

        Parameters
        ----------
        dirname     :   str
                        Directory to read from

        '''
        self._activations.setInputDataDirectory(dirname)

    def setInputDataSet(self, name: str):
        '''Set the input data to be used.'''
        self._activations.setInputDataSet(name)

    def showStatusMessage(self, message):
        self._statusBar.showMessage(message, 2000)

    def setInputData(self, data: np.ndarray=None, description: str=None):
        '''Provide one data vector as input for the network.
        The input data must have 2, 3, or 4 dimensions.
            - 2 dimensions means a single gray value image
            - 3 dimensions means a single three-channel image. The channels will
                be repeated thrice to form a three-dimensional input
            - 4 dimensions are only supported insofar as the shape must be
                (1, A, B, C), meaning the fist dimension is singular and can be
                dropped. Actual batches are not supported
        The input image shape will be adapted for the chosen network by resizing
        the images

        Parameters
        ----------
        data    :   np.ndarray
                    The data array
        description :   str
                        Data description
        '''
        # FIXME[hack]: there seems to be a bug in PyQt forbidding to emit
        # signals with None parameter. See code in 'widget/inputselector.py'

        if not data.ndim:
            raise ArgumentError('Data cannot be none.')

        raw_data = data
        if self._network is not None and data is not None:
            network_shape = self._network.get_input_shape(include_batch=False)
            if data.ndim > 4 or data.ndim < 2:
                raise ArgumentError('Data must have between 2 '
                                    'and 4 dimensions.')

            if data.ndim == 4:
                if data.shape[0] == 1:
                    # first dimension has size of 1 -> remove
                    data = data.squeeze(0)
                else:
                    raise ArgumentError('Cannot visualize batch of images')

            if data.ndim == 2:
                # Blow up to three dimensions by repeating the channel
                data = data[..., np.newaxis].repeat(3, axis=2)

            if data.shape[0:2] != network_shape[0:2]:
                # Image does not fit into network -> resize
                data = imresize(data, network_shape[0:2])

            if data.shape[2] != network_shape[2]:
                # different number of channels
                # FIXME[hack]: find better way to do RGB <-> grayscale
                # conversion
                if network_shape[2] == 1:
                    data = np.mean(data, axis=2)
                elif network_shape[2] == 3 and data.shape[2] == 1:
                    data = data.repeat(3, axis=2)
                else:
                    raise ArgumentError('Incompatible network input shape.')

        self._activations.setInputData(raw_data, data, description)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import six
from airflow.contrib.utils.weekday import WeekDay
from airflow.sensors.base_sensor_operator import BaseSensorOperator
from airflow.utils import timezone
from airflow.utils.decorators import apply_defaults

class DayOfWeekSensor(BaseSensorOperator):
    """
    Waits until the first specified day of the week. For example, if the execution
    day of the task is '2018-12-22' (Saturday) and you pass 'FRIDAY', the task will wait
    until next Friday.

    **Example** (with single day): ::

        weekend_check = DayOfWeekSensor(
            task_id='weekend_check',
            week_day='Saturday',
            use_task_execution_day=True,
            dag=dag)

    **Example** (with multiple day using set): ::

        weekend_check = DayOfWeekSensor(
            task_id='weekend_check',
            week_day={'Saturday', 'Sunday'},
            use_task_execution_day=True,
            dag=dag)

    **Example** (with :class:`~airflow.contrib.utils.weekday.WeekDay` enum): ::

        # import WeekDay Enum
        from airflow.contrib.utils.weekday import WeekDay

        weekend_check = DayOfWeekSensor(
            task_id='weekend_check',
            week_day={WeekDay.Saturday, WeekDay.Sunday},
            use_task_execution_day=True,
            dag=dag)

    :param week_day: Day of the week to check (full name). Optionally, a set
        of days can also be provided using a set.
        Example values:
            * ``"MONDAY"``,
            * ``{"Saturday", "Sunday"}``
            * ``{WeekDay.TUESDAY}``
            * ``{WeekDay.Saturday, WeekDay.Sunday}``
    :type week_day: set or str or WeekDay
    :param use_task_execution_day: If ``True``, uses task's execution day to compare
        with week_day. Execution Date is Useful for backfilling.
        If ``False``, uses system's day of the week. Useful when you
        don't want to run anything on weekdays on the system.
    :type use_task_execution_day: bool
    """

    @apply_defaults
    def __init__(self, week_day,
                 use_task_execution_day=False,
                 *args, **kwargs):
        super(DayOfWeekSensor, self).__init__(*args, **kwargs)
        self.week_day = week_day
        self.use_task_execution_day = use_task_execution_day
        if isinstance(self.week_day, six.string_types):
            self._week_day_num = {WeekDay.get_weekday_number(week_day_str=self.week_day)}
        elif isinstance(self.week_day, WeekDay):
            self._week_day_num = {self.week_day}
        elif isinstance(self.week_day, set):
            if all(isinstance(day, six.string_types) for day in self.week_day):
                self._week_day_num = {WeekDay.get_weekday_number(day) for day in week_day}
            elif all(isinstance(day, WeekDay) for day in self.week_day):
                self._week_day_num = self.week_day
        else:
            raise TypeError(
                'Unsupported Type for week_day parameter: {}. It should be one of str'
                ', set or Weekday enum type'.format(type(week_day)))

    def poke(self, context):
        self.log.info('Poking until weekday is in %s, Today is %s',
                      self.week_day,
                      WeekDay(timezone.utcnow().isoweekday()).name)
        if self.use_task_execution_day:
            return context['execution_date'].isoweekday() in self._week_day_num
        else:
            return timezone.utcnow().isoweekday() in self._week_day_num
<commit_msg>[AIRFLOW-XXX] Fix WeekDay Sensor Example (#4431)<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import six
from airflow.contrib.utils.weekday import WeekDay
from airflow.sensors.base_sensor_operator import BaseSensorOperator
from airflow.utils import timezone
from airflow.utils.decorators import apply_defaults

class DayOfWeekSensor(BaseSensorOperator):
    """
    Waits until the first specified day of the week. For example, if the execution
    day of the task is '2018-12-22' (Saturday) and you pass 'FRIDAY', the task will wait
    until next Friday.

    **Example** (with single day): ::

        weekend_check = DayOfWeekSensor(
            task_id='weekend_check',
            week_day='Saturday',
            use_task_execution_day=True,
            dag=dag)

    **Example** (with multiple day using set): ::

        weekend_check = DayOfWeekSensor(
            task_id='weekend_check',
            week_day={'Saturday', 'Sunday'},
            use_task_execution_day=True,
            dag=dag)

    **Example** (with :class:`~airflow.contrib.utils.weekday.WeekDay` enum): ::

        # import WeekDay Enum
        from airflow.contrib.utils.weekday import WeekDay

        weekend_check = DayOfWeekSensor(
            task_id='weekend_check',
            week_day={WeekDay.SATURDAY, WeekDay.SUNDAY},
            use_task_execution_day=True,
            dag=dag)

    :param week_day: Day of the week to check (full name). Optionally, a set
        of days can also be provided using a set.
        Example values:
            * ``"MONDAY"``,
            * ``{"Saturday", "Sunday"}``
            * ``{WeekDay.TUESDAY}``
            * ``{WeekDay.SATURDAY, WeekDay.SUNDAY}``
    :type week_day: set or str or WeekDay
    :param use_task_execution_day: If ``True``, uses task's execution day to compare
        with week_day. Execution Date is Useful for backfilling.
        If ``False``, uses system's day of the week. Useful when you
        don't want to run anything on weekdays on the system.
    :type use_task_execution_day: bool
    """

    @apply_defaults
    def __init__(self, week_day,
                 use_task_execution_day=False,
                 *args, **kwargs):
        super(DayOfWeekSensor, self).__init__(*args, **kwargs)
        self.week_day = week_day
        self.use_task_execution_day = use_task_execution_day
        if isinstance(self.week_day, six.string_types):
            self._week_day_num = {WeekDay.get_weekday_number(week_day_str=self.week_day)}
        elif isinstance(self.week_day, WeekDay):
            self._week_day_num = {self.week_day}
        elif isinstance(self.week_day, set):
            if all(isinstance(day, six.string_types) for day in self.week_day):
                self._week_day_num = {WeekDay.get_weekday_number(day) for day in week_day}
            elif all(isinstance(day, WeekDay) for day in self.week_day):
                self._week_day_num = self.week_day
        else:
            raise TypeError(
                'Unsupported Type for week_day parameter: {}. It should be one of str'
                ', set or Weekday enum type'.format(type(week_day)))

    def poke(self, context):
        self.log.info('Poking until weekday is in %s, Today is %s',
                      self.week_day,
                      WeekDay(timezone.utcnow().isoweekday()).name)
        if self.use_task_execution_day:
            return context['execution_date'].isoweekday() in self._week_day_num
        else:
            return timezone.utcnow().isoweekday() in self._week_day_num
<|endoftext|>


<commit_before>import os.path
import pandas
import numpy as np
from sklearn.cluster import KMeans
from database_service import DatabaseService
import yaml

class Clustering:

    def __init__(self, secret_config):
        self.__databaseService = DatabaseService(secret_config['mysql'])

    def __fetch_data(self, data):
        headers = [data._info_axis._data[1:]][0] #fetch headers
        repos = [row[0] for row in data._values] #fetch repo names
        features = [row[1:len(row)] for row in data._values] #fetch features.
        return headers, repos, features

    def k_means_clustering(self, file_name, k=5):
        data = pandas.read_csv(file_name, sep=';') #read the csv file
        headers, repos, features = self.__fetch_data(data)
        kmeans = KMeans(n_clusters=k, random_state=0, n_init=200).fit(features) #apply kmeans algorithm

        #form clusters
        clusters = []
        for i in range(0, k): # k cluster
            repo_list = []
            for j in range (0, len(kmeans.labels_)): # a label for each repo.
                if i == kmeans.labels_[j]: ## if repo label is equal to Cluster number
                    repo_list.append(repos[j]) # add repo to cluster i's list.
            clusters.append(repo_list)

        self.__exportResults(kmeans, headers, clusters, file_name[:-4]) #avoid ".csv"

    def k_means_clustering_repostats(self, k=5):
        repo_stats = self.__databaseService.getRepoStats()
        repo_stats_df = pandas.DataFrame.from_dict(repo_stats)
        headers, repos, features = self.__fetch_data(repo_stats_df)

        kmeans = KMeans(n_clusters=k, random_state=0, n_init=200).fit(features)  # apply kmeans algorithm

        # form clusters
        clusters = []
        for i in range(0, k):  # k cluster
            repo_list = []
            for j in range(0, len(kmeans.labels_)):  # a label for each repo.
                if i == kmeans.labels_[j]:  ## if repo label is equal to Cluster number
                    repo_list.append(repos[j])  # add repo to cluster i's list.
            clusters.append(repo_list)

        self.__exportResults(kmeans, headers, clusters, "repo_stats")

    def __exportResults(self, kmeans, headers, clusters, file_name):

        #export clusters and cluster members.
        with open("clusters_" + file_name + ".txt", "w") as out_file:
            for i in range(0, kmeans.n_clusters):
                out_file.write("Cluster " + str(i) + "\n")
                for repo in clusters[i]:
                    repo_lang = self.__databaseService.getLanguageByRepoFullName(repo)
                    out_file.write(str(repo) + " - " + str(repo_lang) + "\n")
                out_file.write("\n")

        #export cluster centers matrix.

        indexes = []
        for i in range(0, kmeans.n_clusters):
            indexes.append("Cluster " + str(i))

        cluster_centers_df = pandas.DataFrame(kmeans.cluster_centers_, index=indexes, columns=headers)
        cluster_centers_df.to_csv("cluster_centers_" + file_name + ".csv", sep=';')

        return

with open(os.path.join(os.path.dirname(__file__), os.pardir, 'config_secret.yaml'), 'r') as ymlfile:
    secret_config = yaml.load(ymlfile)

clustering = Clustering(secret_config)
clustering.k_means_clustering("file_metrics.csv", k=5)
clustering.k_means_clustering_repostats(k=10)<commit_msg>Added printing cluster sizes.<commit_after>import os.path
import pandas
import numpy as np
from sklearn.cluster import KMeans
from database_service import DatabaseService
import yaml

class Clustering:

    def __init__(self, secret_config):
        self.__databaseService = DatabaseService(secret_config['mysql'])

    def __fetch_data(self, data):
        headers = [data._info_axis._data[1:]][0] #fetch headers
        repos = [row[0] for row in data._values] #fetch repo names
        features = [row[1:len(row)] for row in data._values] #fetch features.
        return headers, repos, features

    def k_means_clustering(self, file_name, k=5):
        data = pandas.read_csv(file_name, sep=';') #read the csv file
        headers, repos, features = self.__fetch_data(data)
        kmeans = KMeans(n_clusters=k, random_state=0, n_init=200).fit(features) #apply kmeans algorithm

        #form clusters
        clusters = []
        for i in range(0, k): # k cluster
            repo_list = []
            for j in range (0, len(kmeans.labels_)): # a label for each repo.
                if i == kmeans.labels_[j]: ## if repo label is equal to Cluster number
                    repo_list.append(repos[j]) # add repo to cluster i's list.
            clusters.append(repo_list)

        self.__exportResults(kmeans, headers, clusters, file_name[:-4]) #avoid ".csv"

    def k_means_clustering_repostats(self, k=5):
        repo_stats = self.__databaseService.getRepoStats()
        repo_stats_df = pandas.DataFrame.from_dict(repo_stats)
        headers, repos, features = self.__fetch_data(repo_stats_df)

        kmeans = KMeans(n_clusters=k, random_state=0, n_init=200).fit(features)  # apply kmeans algorithm

        # form clusters
        clusters = []
        for i in range(0, k):  # k cluster
            repo_list = []
            for j in range(0, len(kmeans.labels_)):  # a label for each repo.
                if i == kmeans.labels_[j]:  ## if repo label is equal to Cluster number
                    repo_list.append(repos[j])  # add repo to cluster i's list.
            clusters.append(repo_list)

        self.__exportResults(kmeans, headers, clusters, "repo_stats")

    def __exportResults(self, kmeans, headers, clusters, file_name):

        #export clusters and cluster members.
        with open("clusters_" + file_name + ".txt", "w") as out_file:
            for i in range(0, kmeans.n_clusters):
                out_file.write("Cluster " + str(i) + " size: " + str(len(clusters[i])) + "\n" )
                for repo in clusters[i]:
                    repo_lang = self.__databaseService.getLanguageByRepoFullName(repo)
                    out_file.write(str(repo) + " - " + str(repo_lang) + "\n")
                out_file.write("\n")

        #export cluster centers matrix.

        indexes = []
        for i in range(0, kmeans.n_clusters):
            indexes.append("Cluster " + str(i))

        cluster_centers_df = pandas.DataFrame(kmeans.cluster_centers_, index=indexes, columns=headers)
        cluster_centers_df.to_csv("cluster_centers_" + file_name + ".csv", sep=';')

        return

with open(os.path.join(os.path.dirname(__file__), os.pardir, 'config_secret.yaml'), 'r') as ymlfile:
    secret_config = yaml.load(ymlfile)

clustering = Clustering(secret_config)
clustering.k_means_clustering("file_metrics.csv", k=5)
clustering.k_means_clustering_repostats(k=10)<|endoftext|>


<commit_before>"""jobs.py

Jobs for the Harvest System.
"""

import os
import requests
import datetime

import sys
sys.path.append('/d1lod')
from d1lod.sesame import Store, Repository

from glharvest import registry, void, util

SESAME_HOST = os.getenv('GRAPHDB_PORT_8080_TCP_ADDR', 'localhost')
SESAME_PORT = os.getenv('GRAPHDB_PORT_8080_TCP_PORT', '8080')
SESAME_REPOSITORY = 'test'

def main_job():
    """The main worker method for the Harvest System.

    1. Parse the registry file.
    2. For each provider in the registry,
        2a. Try to find their VoID file
        2b. Parse that VoID file
        2c. Determine if work needs to be done
            2c1. If none, go to the next provider
            2c2. If the modified time is different
                2c2a. Grab the data dump
                2c2b. Import the triples
                2c2c. Update the registry
    """

    # Parse registry file
    print "Parsing registry file."
    regfile = registry.parse_registry_file('/glharvest/registry.yml')

    if regfile is None:
        print "Failed to parse registry file. Exiting."
        return

    print "  %d provider(s) found." % len(regfile)

    for provider in regfile:
        print "Processing provider: `%s`." % provider

        # Void dump location
        if 'void' not in regfile[provider]:
            print "Location of VoID dataset not found for this provider in the registry. Skipping."
            continue

        voidfile = regfile[provider]['void']

        # Modified value
        if 'modified' not in regfile[provider]:
            print "Location of VoID dataset not found for this provider in the registry."
            provider_modified = None
        else:
            provider_modified = regfile[provider]['modified']

        # Get and parse the VoID file
        r = None

        try:
            r = requests.get(voidfile)
        except:
            print "Failed to get voidfile located at `%s`." % voidfile

        if r is None:
              continue

        model = util.load_string_into_model(r.text, format="turtle")
        void_model = void.parse_void_model(model)

        print void_model

        # Process the graph
        for graph in void_model:
            if 'modified' not in void_model[graph]:
                print "Graph found in VoID dump did not have dcterms:modified value. Skipping."

            modified = void_model[graph]['modified']

            # Parse datestring parts as ints
            try:
                year, month, day = [int(part) for part in modified.split("-")]
            except:
                print "Failed to parse datetime parts from string: `%s`. Skipping." % modified
                continue

            modified_dt = datetime.date(year, month, day)
            print "Registry says modified should be %s and the VoID dump says %s." % (provider_modified, modified_dt)

            if modified_dt > provider_modified:
                print "datadump has been updated since the last visit. We should fetch and import it."

                try:
                    void_location = void_model[graph]['dataDump']
                    r = requests.get(void_location)
                except:
                    print "Failed to fetch the void file at `%s`. Skipping." % void_location
                    # continue

                str = Store(SESAME_HOST, SESAME_PORT)
                rep = Repository(str, SESAME_REPOSITORY)
                rep.import_from_text(r.text, context=provider)

                # Update registry file on disk
                regfile[provider]['modified'] = datetime.date.today()
                registry.save_registry("registry.yml", regfile)
            else:
                print "datadump has not been updated since the last visit. Doing nothing."

def export_job():
    """Export the repository as Turtle."""

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    with open('/www/export', 'wb') as f:
        exported_text = r.export()
        f.write(exported_text.encode('utf-8'))

def status_job():
    """Gets the status of the system and prints to stdout."""
    print "Getting the status of the system."

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    print "Repository size is %d" % r.size()
<commit_msg>Clean up job names<commit_after>"""jobs.py

Jobs for the Harvest System.
"""

import os
import requests
import datetime

import sys
sys.path.append('/d1lod')
from d1lod.sesame import Store, Repository

from glharvest import registry, void, util

SESAME_HOST = os.getenv('GRAPHDB_PORT_8080_TCP_ADDR', 'localhost')
SESAME_PORT = os.getenv('GRAPHDB_PORT_8080_TCP_PORT', '8080')
SESAME_REPOSITORY = 'test'

def update():
    """Update the contents of the Harvester.

    Reads the registry file and does work depending on what it finds.
    """

    # Parse registry file
    print "Parsing registry file."
    regfile = registry.parse_registry_file('/glharvest/registry.yml')

    if regfile is None:
        print "Failed to parse registry file. Exiting."
        return

    print "  %d provider(s) found." % len(regfile)

    for provider in regfile:
        print "Processing provider: `%s`." % provider

        # Void dump location
        if 'void' not in regfile[provider]:
            print "Location of VoID dataset not found for this provider in the registry. Skipping."
            continue

        voidfile = regfile[provider]['void']

        # Modified value
        if 'modified' not in regfile[provider]:
            print "Location of VoID dataset not found for this provider in the registry."
            provider_modified = None
        else:
            provider_modified = regfile[provider]['modified']

        # Get and parse the VoID file
        r = None

        try:
            r = requests.get(voidfile)
        except:
            print "Failed to get voidfile located at `%s`." % voidfile

        if r is None:
              continue

        model = util.load_string_into_model(r.text, format="turtle")
        void_model = void.parse_void_model(model)

        print void_model

        # Process the graph
        for graph in void_model:
            if 'modified' not in void_model[graph]:
                print "Graph found in VoID dump did not have dcterms:modified value. Skipping."

            modified = void_model[graph]['modified']

            # Parse datestring parts as ints
            try:
                year, month, day = [int(part) for part in modified.split("-")]
            except:
                print "Failed to parse datetime parts from string: `%s`. Skipping." % modified
                continue

            modified_dt = datetime.date(year, month, day)
            print "Registry says modified should be %s and the VoID dump says %s." % (provider_modified, modified_dt)

            if modified_dt > provider_modified:
                print "datadump has been updated since the last visit. We should fetch and import it."

                try:
                    void_location = void_model[graph]['dataDump']
                    r = requests.get(void_location)
                except:
                    print "Failed to fetch the void file at `%s`. Skipping." % void_location
                    # continue

                str = Store(SESAME_HOST, SESAME_PORT)
                rep = Repository(str, SESAME_REPOSITORY)
                rep.import_from_text(r.text, context=provider)

                # Update registry file on disk
                regfile[provider]['modified'] = datetime.date.today()
                registry.save_registry("registry.yml", regfile)
            else:
                print "datadump has not been updated since the last visit. Doing nothing."

def export():
    """Export the repository as Turtle."""

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    with open('/www/export', 'wb') as f:
        exported_text = r.export()
        f.write(exported_text.encode('utf-8'))

def status():
    """Gets the status of the system and prints to stdout."""
    print "Getting the status of the system."

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    print "Repository size is %d" % r.size()
<|endoftext|>