<commit_before>from django.contrib.admin.widgets import AdminSplitDateTime
from django.contrib.admin.checks import ModelAdminChecks
from django.contrib import admin
from django import forms
from django.contrib.admin.templatetags.admin_static import static
from django.http import HttpResponseRedirect
from django.conf.urls import patterns, url

    #necessary right now because of the error about exclude not being a tuple since we are using @property to dynamicall
    #change it
class VAdminChecks(ModelAdminChecks):
    def _check_exclude(self, cls, model):
        return []

class DateTimeForm(forms.Form):
    def __init__(self, request, *args, **kwargs):
        field_name = kwargs.pop('field_name')
        super(DateTimeForm, self).__init__(*args, **kwargs)
        self.request = request
        self.fields['%s_as_of' % field_name] = forms.SplitDateTimeField(
            label='',
            input_time_formats=['%I:%M %p', '%H:%M:%S'],
            widget=AdminSplitDateTime(
                attrs={'placeholder': ('as of date and time')}
            ),
            localize=True,
            required=True
        )

    @property
    def media(self):
        try:
            if getattr(self.request, 'daterange_filter_media_included'):
                return forms.Media()
        except AttributeError:
            setattr(self.request, 'daterange_filter_media_included', True)

            js = ["calendar.js", "admin/DateTimeShortcuts.js"]
            css = ['widgets.css']

            return forms.Media(
                js=[static("admin/js/%s" % path) for path in js],
                css={'all': [static("admin/css/%s" % path) for path in css]}
            )

class DateTimeFilter(admin.FieldListFilter):
    template = 'datetimefilter.html'

    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg_as_ofdate = '%s_as_of_0' % field_path
        self.lookup_kwarg_as_oftime = '%s_as_of_1' % field_path
        super(DateTimeFilter, self).__init__(field, request, params, model, model_admin, field_path)
        self.form = self.get_form(request)

    def choices(self, cl):
        return []

    def expected_parameters(self):
        return [self.lookup_kwarg_as_ofdate,self.lookup_kwarg_as_oftime]

    def get_form(self, request):
        return DateTimeForm(request, data=self.used_parameters,
                             field_name=self.field_path)

    def queryset(self, request, queryset):
        if self.form.is_valid() and self.form.cleaned_data.values()[0] is not None:
            filter_params = self.form.cleaned_data.values()[0]
            return queryset.as_of(filter_params)
        else:
            return queryset

class VersionedAdmin(admin.ModelAdmin):
    #actions = ['delete_model']
    #these are so that the subclasses can overwrite these attributes
    # to have the identity, end date,or start date column not show
    list_display_show_identity = True
    list_display_show_end_date = True
    list_display_show_start_date = True
    ordering = []
    #new attribute for working with self.exclude method so that the subclass can specify more fields to exclude
    _exclude = None
    checks_class = VAdminChecks
    change_form_template = 'admin/versions/changeform.html'

    def get_readonly_fields(self, request, obj=None):
        """this method is needed so that if a subclass of VersionedAdmin has readonly_fields the
                the ones written above won't be undone"""
        if obj:
            return self.readonly_fields + ('id','identity','is_current')
        return self.readonly_fields

    def get_ordering(self, request):
        return ['identity', '-version_start_date'] + self.ordering

    def get_list_display(self, request):
        """this method determines which fields go in the changelist"""

        list_display = super(VersionedAdmin,self).get_list_display(request)
        #force cast to list as super get_list_display could return a tuple
        list_display = list(list_display)
        if self.list_display_show_identity:
            list_display = ['identity_shortener'] + list_display

        if self.list_display_show_start_date:
            list_display += ['version_start_date']

        if self.list_display_show_end_date:
            list_display += ['version_end_date']

        return list_display + ['is_current']

    def get_list_filter(self, request):
        list_filter = super(VersionedAdmin,self).get_list_filter(request)
        return list_filter + (('version_start_date',DateTimeFilter),)

    def will_not_clone(self, request, *args, **kwargs):
        """needed for save but not clone capability, this is a view"""
        paths = request.path_info.split('/')

        object_id = paths[3]
        self.change_view(request,object_id)
        #this gets the adminsite for the app, and the model name and joins together with /
        path = '/'+'/'.join(paths[1:3])
        return HttpResponseRedirect(path)

    @property
    def exclude(self):
        """need a getter for exclude since there is no get_exclude method to be overridden"""
        VERSIONED_EXCLUDE = ['id', 'identity', 'version_end_date', 'version_start_date', 'version_birth_date']
        #creating _exclude so that self.exclude doesn't need to be called prompting recursion, and subclasses
        #have a way to change what is excluded
        if self._exclude is None:
            return VERSIONED_EXCLUDE
        else:
            return list(self._exclude) + VERSIONED_EXCLUDE

    def get_object(self, request, object_id, from_field=None):
        obj = super(VersionedAdmin, self).get_object(request, object_id) #from_field breaks in 1.7.8
        #the things tested for in the if are for Updating an object; get_object is called three times: changeview, delete, and history
        if request.method == "POST" and obj and obj.is_latest and not 'will_not_clone' in request.path and not 'delete' in request.path:
            obj = obj.clone()

        return obj

    def get_urls(self):
        not_clone_url = [url(r'^(.+)/will_not_clone/$',admin.site.admin_view(self.will_not_clone),name="willNotClone")]
        return not_clone_url + super(VersionedAdmin,self).get_urls()

    def is_current(self, obj):
        return obj.is_current

    is_current.boolean = True
    is_current.short_description = "Current"

    def identity_shortener(self,obj):
        return "..."+obj.identity[-12:]

    identity_shortener.boolean = False
    identity_shortener.short_description = "Short Identity"

<commit_msg>Using super.exlude in @property exclude instead of new attribute _exclude<commit_after>from django.contrib.admin.widgets import AdminSplitDateTime
from django.contrib.admin.checks import ModelAdminChecks
from django.contrib import admin
from django import forms
from django.contrib.admin.templatetags.admin_static import static
from django.http import HttpResponseRedirect
from django.conf.urls import patterns, url

    #necessary right now because of the error about exclude not being a tuple since we are using @property to dynamicall
    #change it
class VAdminChecks(ModelAdminChecks):
    def _check_exclude(self, cls, model):
        return []

class DateTimeForm(forms.Form):
    def __init__(self, request, *args, **kwargs):
        field_name = kwargs.pop('field_name')
        super(DateTimeForm, self).__init__(*args, **kwargs)
        self.request = request
        self.fields['%s_as_of' % field_name] = forms.SplitDateTimeField(
            label='',
            input_time_formats=['%I:%M %p', '%H:%M:%S'],
            widget=AdminSplitDateTime(
                attrs={'placeholder': ('as of date and time')}
            ),
            localize=True,
            required=True
        )

    @property
    def media(self):
        try:
            if getattr(self.request, 'daterange_filter_media_included'):
                return forms.Media()
        except AttributeError:
            setattr(self.request, 'daterange_filter_media_included', True)

            js = ["calendar.js", "admin/DateTimeShortcuts.js"]
            css = ['widgets.css']

            return forms.Media(
                js=[static("admin/js/%s" % path) for path in js],
                css={'all': [static("admin/css/%s" % path) for path in css]}
            )

class DateTimeFilter(admin.FieldListFilter):
    template = 'datetimefilter.html'

    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg_as_ofdate = '%s_as_of_0' % field_path
        self.lookup_kwarg_as_oftime = '%s_as_of_1' % field_path
        super(DateTimeFilter, self).__init__(field, request, params, model, model_admin, field_path)
        self.form = self.get_form(request)

    def choices(self, cl):
        return []

    def expected_parameters(self):
        return [self.lookup_kwarg_as_ofdate,self.lookup_kwarg_as_oftime]

    def get_form(self, request):
        return DateTimeForm(request, data=self.used_parameters,
                             field_name=self.field_path)

    def queryset(self, request, queryset):
        if self.form.is_valid() and self.form.cleaned_data.values()[0] is not None:
            filter_params = self.form.cleaned_data.values()[0]
            return queryset.as_of(filter_params)
        else:
            return queryset

class VersionedAdmin(admin.ModelAdmin):
    #actions = ['delete_model']
    #these are so that the subclasses can overwrite these attributes
    # to have the identity, end date,or start date column not show
    list_display_show_identity = True
    list_display_show_end_date = True
    list_display_show_start_date = True
    ordering = []

    checks_class = VAdminChecks
    change_form_template = 'admin/versions/changeform.html'

    def get_readonly_fields(self, request, obj=None):
        """this method is needed so that if a subclass of VersionedAdmin has readonly_fields the
                the ones written above won't be undone"""
        if obj:
            return self.readonly_fields + ('id','identity','is_current')
        return self.readonly_fields

    def get_ordering(self, request):
        return ['identity', '-version_start_date'] + self.ordering

    def get_list_display(self, request):
        """this method determines which fields go in the changelist"""

        list_display = super(VersionedAdmin,self).get_list_display(request)
        #force cast to list as super get_list_display could return a tuple
        list_display = list(list_display)
        if self.list_display_show_identity:
            list_display = ['identity_shortener'] + list_display

        if self.list_display_show_start_date:
            list_display += ['version_start_date']

        if self.list_display_show_end_date:
            list_display += ['version_end_date']

        return list_display + ['is_current']

    def get_list_filter(self, request):
        list_filter = super(VersionedAdmin,self).get_list_filter(request)
        return list_filter + (('version_start_date',DateTimeFilter),)

    def will_not_clone(self, request, *args, **kwargs):
        """needed for save but not clone capability, this is a view"""
        paths = request.path_info.split('/')

        object_id = paths[3]
        self.change_view(request,object_id)
        #this gets the adminsite for the app, and the model name and joins together with /
        path = '/'+'/'.join(paths[1:3])
        return HttpResponseRedirect(path)

    @property
    def exclude(self):
        """need a getter for exclude since there is no get_exclude method to be overridden"""
        VERSIONED_EXCLUDE = ['id', 'identity', 'version_end_date', 'version_start_date', 'version_birth_date']

        if super(VersionedAdmin, self).exclude is None:
            return VERSIONED_EXCLUDE
        else:
            return list(super(VersionedAdmin, self).exclude) + VERSIONED_EXCLUDE

    def get_object(self, request, object_id, from_field=None):
        obj = super(VersionedAdmin, self).get_object(request, object_id) #from_field breaks in 1.7.8
        #the things tested for in the if are for Updating an object; get_object is called three times: changeview, delete, and history
        if request.method == "POST" and obj and obj.is_latest and not 'will_not_clone' in request.path and not 'delete' in request.path:
            obj = obj.clone()

        return obj

    def get_urls(self):
        not_clone_url = [url(r'^(.+)/will_not_clone/$',admin.site.admin_view(self.will_not_clone),name="willNotClone")]
        return not_clone_url + super(VersionedAdmin,self).get_urls()

    def is_current(self, obj):
        return obj.is_current

    is_current.boolean = True
    is_current.short_description = "Current"

    def identity_shortener(self,obj):
        return "..."+obj.identity[-12:]

    identity_shortener.boolean = False
    identity_shortener.short_description = "Short Identity"

<|endoftext|>


<commit_before>"""
Neuroimaging tools for Python (NIPY).

The aim of NIPY is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.1.2'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__    = "$LastChangedDate$"
__url__     = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

from neuroimaging import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms',
  'neuroimaging.algorithms.tests',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.modalities.fmri',
  'neuroimaging.modalities.fmri.tests',
  'neuroimaging.modalities.fmri.fmristat',
  'neuroimaging.modalities.fmri.fmristat.tests',
  'neuroimaging.utils',
  'neuroimaging.utils.tests',
  'neuroimaging.utils.tests.data')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr):
            return False
    return True

def haslength(obj):
    return hasattr(obj,"__len__")

def flatten(arr, dim=0):
    if len(arr.shape) < 2:
        return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

def reorder(seq, order):
    return [seq[i] for i in order]

def reverse(seq):
    return reorder(seq, range(len(seq)-1, -1, -1))

def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try:
        return getattr(module, objectname)
    except AttributeError:
        return None

<commit_msg>Added core and modalities to package list to fix __init__.py copying<commit_after>"""
Neuroimaging tools for Python (NIPY).

The aim of NIPY is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.1.2'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__    = "$LastChangedDate$"
__url__     = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

from neuroimaging import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms',
  'neuroimaging.algorithms.tests',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.modalities',
  'neuroimaging.modalities.fmri',
  'neuroimaging.modalities.fmri.tests',
  'neuroimaging.modalities.fmri.fmristat',
  'neuroimaging.modalities.fmri.fmristat.tests',
  'neuroimaging.utils',
  'neuroimaging.utils.tests',
  'neuroimaging.utils.tests.data')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr):
            return False
    return True

def haslength(obj):
    return hasattr(obj,"__len__")

def flatten(arr, dim=0):
    if len(arr.shape) < 2:
        return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

def reorder(seq, order):
    return [seq[i] for i in order]

def reverse(seq):
    return reorder(seq, range(len(seq)-1, -1, -1))

def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try:
        return getattr(module, objectname)
    except AttributeError:
        return None

<|endoftext|>


<commit_before>"""
This is the HRL submodule for coordinating photometers. Photometers must
implement the Photometer abstract base class, which simply defines a common
function for reading luminance.
"""

import abc

### Classes ###

class Photometer(object):
    """
    The Photometer abstract base class. New photometers must instantiate
    this class. The only method is 'readLuminance', which returns a luminance
    reading from the device.
    """
    __metaclass__ = abc.ABCMeta

    # Abstract Methods #

    def readLuminance(self,n,slp):
        """
        Reads a value from the photometer, returning a real value in lumens.

        (Some decisions need to be made about this interface still wrt arguments.)
        """
        return
<commit_msg>Fixed a typo<commit_after>"""
This is the HRL submodule for coordinating photometers. Photometers must
implement the Photometer abstract base class, which simply defines a common
function for reading luminance.
"""

import abc

### Classes ###

class Photometer(object):
    """
    The Photometer abstract base class. New photometers must instantiate
    this class. The only method is 'readLuminance', which returns a luminance
    reading from the device.
    """
    __metaclass__ = abc.ABCMeta

    # Abstract Methods #

    def readLuminance(self,n,slp):
        """
        Reads a value from the photometer, returning a real value in candela/m^2.

        (Some decisions need to be made about this interface still wrt arguments.)
        """
        return
<|endoftext|>


<commit_before>#!coding:utf-8

import unittest
import flagjudge
import os

class MyException(Exception):
    pass

class flagjudgeTest(unittest.TestCase):

    def testcapture(self):
        cap = flagjudge.flagcapture()
        if cap.capture() is True:
            print(cap.judge())

    def testjudge1(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone.jpg'
        print(cap.imagename)
        direction, count = cap.judge()
        self.assertEqual(direction, 'right')
        self.assertEqual(count, '4923')
        self.assertNotEqual(count, 4923)

    def testjudge1green(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone.jpg'
        print(cap.imagename)
        direction, count = cap.judge(selectcolor='green')
        self.assertEqual(direction, 'center')
        self.assertEqual(count, '149')
        self.assertNotEqual(count, 149)

    def testjudge1blue(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone.jpg'
        print(cap.imagename)
        direction, count = cap.judge(selectcolor='blue')
        self.assertEqual(direction, 'center')
        self.assertEqual(count, '93')
        self.assertNotEqual(count, 93)

    def testjudge2(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone2.jpg'
        print(cap.imagename)
        direction, count = cap.judge()
        self.assertEqual(direction, 'center')
        self.assertEqual(count, '3328')
        self.assertNotEqual(count, 3328)

if __name__ == '__main__':
    unittest.main()
<commit_msg>refactoring<commit_after>#!coding:utf-8

import unittest
import flagjudge
import os

class MyException(Exception):
    pass

class flagjudgeTest(unittest.TestCase):

    def testcapture(self):
        cap = flagjudge.flagcapture()
        if cap.capture() is True:
            print(cap.judge())

    def testjudge1(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone.jpg'
        print(cap.imagename)
        direction, count = cap.judge()
        self.assertEqual(direction, 'right')
        # self.assertEqual(count, '4923')
        # self.assertNotEqual(count, 4923)

    def testjudge1green(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone.jpg'
        print(cap.imagename)
        direction, count = cap.judge(selectcolor='green')
        self.assertEqual(direction, 'center')
        self.assertEqual(count, '149')
        self.assertNotEqual(count, 149)

    def testjudge1blue(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone.jpg'
        print(cap.imagename)
        direction, count = cap.judge(selectcolor='blue')
        self.assertEqual(direction, 'center')
        self.assertEqual(count, '93')
        self.assertNotEqual(count, 93)

    def testjudge2(self):
        cap = flagjudge.flagcapture()
        cap.imagename = os.path.dirname(__file__)+'/testimage/colorcone2.jpg'
        print(cap.imagename)
        direction, count = cap.judge()
        self.assertEqual(direction, 'center')
        self.assertEqual(count, '3328')
        self.assertNotEqual(count, 3328)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started.')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

    print('Transaction ended.')

def checkout(url, auth, branch, transactionId):
    print "Checking out "+branch+" branch..."
    params = {'output_format': 'JSON', 'branch': branch, 'transactionId':transactionId}
    request = make_request(url=url+'checkout.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Checkout for branch "+branch+" failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    #taskID = response['task']['id']
    #print response

    newBranch = response['response']['NewTarget']
    print "Checked out "+newBranch+' branch.'
    return newBranch

def getTaskStatus(url, auth, taskID, printStatus):
    #print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    #print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "RUNNING":
            #print response
            #taskAmount = response['task']['amount']
            taskAmount = "#" #Amount value isn't showing up in response.
            print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        elif taskStatus == "FAILED":
            errorMessage = response['task']['error']['message']
            print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
        elif taskStatus == "FINISHED":
            print response
            print "++Task "+str(taskID)+" is finished."
        else:
            print "++Task "+str(taskID)+" is "+taskStatus+"."

    return taskStatus;

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+"."

    return taskStatus;

def waitOnTask(url, auth, taskID):
    maxTime = 30
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None
    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime:
        taskStatus = getTaskStatus(url, auth, taskID, True)
        if not (taskStatus in ['WAITING','RUNNING']):
            break
        #print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID)
        maxTime = 30
        timeSlept = 0
        sleepCycle = 1
        while timeSlept < maxTime:
            taskStatus = cancelTask(url, auth, taskID, True)
            if not (taskStatus in ['WAITING','RUNNING']):
                break
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId, update, mapping, bbox):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': update, 'mapping': mapping, 'bbox': bbox, 'transactionId':transactionId}
    print params
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def getRepoID(geoserver, auth, workspace, datastore):
    params = {}
    url = geoserver+"rest/workspaces/"+workspace+"/datastores/"+datastore+".json"
    request = make_request(url=url, params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    repoID = None
    for entry in response['dataStore']['connectionParameters']['entry']:
        if entry['@key'] == 'geogig_repository':
            repoID = entry['$']
            break

    return repoID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def parse_bbox(extent):
    file_extent = "/opt/cybergis-osm-mappings.git/extents/"+extent.replace(":","/")+".txt"
    bbox = None
    with open (file_extent, "r") as f:
        bbox = f.read().replace('\n', '').replace(' ',',')
    return bbox

def parse_mapping(ns_mapping):
    file_mapping = "/opt/cybergis-osm-mappings.git/mappings/"+ns_mapping.replace(":","/")+".json"
    return file_mapping

def run(args):
    #==#
    geoserver = parse_url(args.geoserver)
    repo = args.repo
    datastore = args.datastore
    workspace = args.workspace
    #==#
    authorname = args.authorname
    authoremail = args.authoremail
    #==#
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))
    #==#
    update = args.update in ["1","y","t","true"]
    bbox = parse_bbox(args.extent)
    mapping = parse_mapping(args.mapping)
    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    if repo:
        pass
    elif geoserver and workspace and datastore:
        repo = getRepoID(geoserver, auth, workspace, datastore)
    else:
        print "You need to include the repo id or the datastore name to sync"
    #==#
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'
    #==#
    if update:
        pass
    elif bbox and mapping:
       pass
    else:
        return "Update is false and no new data will be brought in because the extent and mapping aren't specified"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
            taskID = downloadFromOSM(url_repo, auth, transID, update, mapping, bbox)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
        except Exception:
            pass

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')

parser.add_argument("update", help="true/false.  Update existing features only or download new features.  If false, extent and mapping are required.")

#
parser.add_argument("--workspace", help="The workspace of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--datastore", help="The name of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")

parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")
parser.add_argument("--extent", help="The extent of the OpenStreetMap extract. For example, basic:buildings_and_roads.")
parser.add_argument("--mapping", help="The mapping of the OpenStreetMap extract.  For example, dominican_republic:santo_domingo.")

args = parser.parse_args()
run(args)
<commit_msg>Update geogig_sync_osm.py<commit_after>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started.')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

    print('Transaction ended.')

def checkout(url, auth, branch, transactionId):
    print "Checking out "+branch+" branch..."
    params = {'output_format': 'JSON', 'branch': branch, 'transactionId':transactionId}
    request = make_request(url=url+'checkout.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Checkout for branch "+branch+" failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    #taskID = response['task']['id']
    #print response

    newBranch = response['response']['NewTarget']
    print "Checked out "+newBranch+' branch.'
    return newBranch

def pollTask(url, auth, taskID, printStatus):
    #print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    #print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    status = response['task']['status']
    result = response['task']['result']

    if printStatus:
        if status == "RUNNING":
            #print response
            #taskAmount = response['task']['amount']
            taskAmount = "#" #Amount value isn't showing up in response.
            print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        elif status == "FAILED":
            errorMessage = response['task']['error']['message']
            print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
        elif status == "FINISHED":
            print response
            print "++Task "+str(taskID)+" is finished."
        else:
            print "++Task "+str(taskID)+" is "+status+"."

    return status, result

def printTaskStatus(taskID, taskStatus, taskResult):

    if status == "RUNNING":
        print "----"
        #print response
        #taskAmount = response['task']['amount']
        taskAmount = "#" #Amount value isn't showing up in response.
        print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        print "Entities Processed: "+taskResult['OSMReport']['processedEntities']
    elif status == "FAILED":
        print "----"
        errorMessage = response['task']['error']['message']
        print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
    elif status == "FINISHED":
        print "----"
        print response
        print "++Task "+str(taskID)+" is finished."
        print "Entities Processed: "+taskResult['OSMReport']['processedEntities']
    elif status == "CANCELLED":
        print "----"
        print response
        print "++Task "+str(taskID)+" was cancelled."
        print "Entities Processed: "+taskResult['OSMReport']['processedEntities']
    else:
        print "----"
        print "++Task "+str(taskID)+" is "+status+"."

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+"."

    return taskStatus;

def waitOnTask(url, auth, taskID):
    maxTime = 30
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None
    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime:
        taskStatus, taskResult = getTaskStatus(url, auth, taskID)
        printTaskStatus(taskId, taskStatus, taskResult)
        if not (taskStatus in ['WAITING','RUNNING']):
            break
        #print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID)
        maxTime = 30
        timeSlept = 0
        sleepCycle = 1
        while timeSlept < maxTime:
            taskStatus = cancelTask(url, auth, taskID, True)
            if not (taskStatus in ['WAITING','RUNNING']):
                break
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId, update, mapping, bbox):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': update, 'mapping': mapping, 'bbox': bbox, 'transactionId':transactionId}
    print params
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def getRepoID(geoserver, auth, workspace, datastore):
    params = {}
    url = geoserver+"rest/workspaces/"+workspace+"/datastores/"+datastore+".json"
    request = make_request(url=url, params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    repoID = None
    for entry in response['dataStore']['connectionParameters']['entry']:
        if entry['@key'] == 'geogig_repository':
            repoID = entry['$']
            break

    return repoID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def parse_bbox(extent):
    file_extent = "/opt/cybergis-osm-mappings.git/extents/"+extent.replace(":","/")+".txt"
    bbox = None
    with open (file_extent, "r") as f:
        bbox = f.read().replace('\n', '').replace(' ',',')
    return bbox

def parse_mapping(ns_mapping):
    file_mapping = "/opt/cybergis-osm-mappings.git/mappings/"+ns_mapping.replace(":","/")+".json"
    return file_mapping

def run(args):
    #==#
    geoserver = parse_url(args.geoserver)
    repo = args.repo
    datastore = args.datastore
    workspace = args.workspace
    #==#
    authorname = args.authorname
    authoremail = args.authoremail
    #==#
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))
    #==#
    update = args.update in ["1","y","t","true"]
    bbox = parse_bbox(args.extent)
    mapping = parse_mapping(args.mapping)
    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    if repo:
        pass
    elif geoserver and workspace and datastore:
        repo = getRepoID(geoserver, auth, workspace, datastore)
    else:
        print "You need to include the repo id or the datastore name to sync"
    #==#
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'
    #==#
    if update:
        pass
    elif bbox and mapping:
       pass
    else:
        return "Update is false and no new data will be brought in because the extent and mapping aren't specified"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
            taskID = downloadFromOSM(url_repo, auth, transID, update, mapping, bbox)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
        except Exception:
            pass

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')

parser.add_argument("update", help="true/false.  Update existing features only or download new features.  If false, extent and mapping are required.")

#
parser.add_argument("--workspace", help="The workspace of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--datastore", help="The name of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")

parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")
parser.add_argument("--extent", help="The extent of the OpenStreetMap extract. For example, basic:buildings_and_roads.")
parser.add_argument("--mapping", help="The mapping of the OpenStreetMap extract.  For example, dominican_republic:santo_domingo.")

args = parser.parse_args()
run(args)
<|endoftext|>


<commit_before>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

def getTaskStatus(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "RUNNING":
            #print response
            #taskAmount = response['task']['amount']
            taskAmount = "#" #Amount value isn't showing up in response.
            print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        elif taskStatus == "FAILED":
            errorMessage = response['task']['error']['message']
            print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
        else:
            print "++Task "+str(taskID)+" is "+taskStatus+"."

    return taskStatus;

def waitOnTask(url, auth, taskID):
    print "Waiting for task "+str(taskID)+"..."

    maxTime = 60
    timeSlept = 0
    sleepCycle = 2

    while timeSlept < maxTime and getTaskStatus(url, auth, taskID, True) in ['WAITING','RUNNING']:
        #print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': 'true'}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def run(args):
    #url = parse_url(args)
    #if url is None:
    #    print "Please specify a url"
    #    return 1

    geoserver = parse_url(args.geoserver)
    repo = args.repo
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'

    authorname = args.authorname
    authoremail = args.authoremail
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))

    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        try:
            taskID = downloadFromOSM(url_repo, auth, transID)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')
parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")

args = parser.parse_args()
run(args)
<commit_msg>Update geogig_sync_osm.py<commit_after>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

def getTaskStatus(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "RUNNING":
            #print response
            #taskAmount = response['task']['amount']
            taskAmount = "#" #Amount value isn't showing up in response.
            print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        elif taskStatus == "FAILED":
            errorMessage = response['task']['error']['message']
            print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
        else:
            print "++Task "+str(taskID)+" is "+taskStatus+"."

    return taskStatus;

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+"."

    return taskStatus;

def waitOnTask(url, auth, taskID):
    maxTime = 30
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None
    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime and (taskStatus = getTaskStatus(url, auth, taskID, True) in ['WAITING','RUNNING']):
        #print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID).
        maxTime = 30
        timeSlept = 0
        sleepCycle = 1
        while timeSlept < maxTime and (taskStatus = cancelTask(url, auth, taskID, True) in ['WAITING','RUNNING']):
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': 'true'}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def run(args):
    #url = parse_url(args)
    #if url is None:
    #    print "Please specify a url"
    #    return 1

    geoserver = parse_url(args.geoserver)
    repo = args.repo
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'

    authorname = args.authorname
    authoremail = args.authoremail
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))

    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        try:
            taskID = downloadFromOSM(url_repo, auth, transID)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')
parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")

args = parser.parse_args()
run(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Compute pattern discovery evaluation metrics.

Usage:
    ./pattern_eval.py REFERENCE.TXT ESTIMATION.TXT

Example:
    ./pattern_eval.py ../tests/data/pattern/reference-mono.txt \
                      ../tests/data/pattern/estimate-mono.txt

Written by Oriol Nieto (oriol@nyu.edu), 2014
"""

import argparse
import os
from collections import OrderedDict

import mir_eval

def evaluate(ref_file, est_file):
    """Load data and perform the evaluation.

    :param ref_file: Path to the reference file.
    :type ref_file: str
    :param est_file: Path to the estimation file.
    :type est_file: str
    :returns:
        - M: dict
            Results contained in an ordered dictionary.
    """
    # load the data
    ref_patterns = mir_eval.io.load_patterns(ref_file)
    est_patterns = mir_eval.io.load_patterns(est_file)

    # Now compute all the metrics
    M = OrderedDict()

    # Standard scores
    M['F'], M['P'], M['R'] = \
        mir_eval.pattern.standard_FPR(ref_patterns, est_patterns)

    # Establishment scores
    M['F_est'], M['P_est'], M['R_est'] = \
        mir_eval.pattern.establishment_FPR(ref_patterns, est_patterns)

    # Occurrence scores
    M['F_occ.5'], M['P_occ.5'], M['R_occ.5'] = \
        mir_eval.pattern.occurrence_FPR(ref_patterns, est_patterns, thres=.5)
    M['F_occ.75'], M['P_occ.75'], M['R_occ.75'] = \
        mir_eval.pattern.occurrence_FPR(ref_patterns, est_patterns, thres=.75)

    # Three-layer scores
    M['F_3'], M['P_3'], M['R_3'] = \
        mir_eval.pattern.three_layer_FPR(ref_patterns, est_patterns)

    # First Five Patterns scores
    M['FFP'] = mir_eval.pattern.first_n_three_layer_P(ref_patterns,
                                                      est_patterns, n=5)
    M['FFTP_est'] = mir_eval.pattern.first_n_target_proportion_R(ref_patterns,
                                                                 est_patterns,
                                                                 n=5)

    return M

def print_evaluation(estimation_file, M):
    # And print them
    print os.path.basename(estimation_file)
    for key, value in M.iteritems():
        print '\t%12s:\t%0.3f' % (key, value)

def main():
    """Main function to evaluate the pattern discovery task."""
    parser = argparse.ArgumentParser(description="mir_eval pattern discovery "
                                                 "evaluation",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("ref_file",
                        action="store",
                        help="Path to the reference file.")
    parser.add_argument("est_file",
                        action="store",
                        help="Path to the estimation file.")
    args = parser.parse_args()

    # Run the evaluations
    scores = evaluate(args.ref_file, args.est_file)

    # Print results
    print_evaluation(args.ref_file, scores)

if __name__ == '__main__':
    main()
<commit_msg>pattern_eval for #6<commit_after>#!/usr/bin/env python
"""
Compute pattern discovery evaluation metrics.

Usage:
    ./pattern_eval.py REFERENCE.TXT ESTIMATION.TXT

Example:
    ./pattern_eval.py ../tests/data/pattern/reference-mono.txt \
                      ../tests/data/pattern/estimate-mono.txt

Written by Oriol Nieto (oriol@nyu.edu), 2014
"""

import argparse
import os
from collections import OrderedDict
import json
import sys

import mir_eval

def evaluate(ref_file, est_file):
    """Load data and perform the evaluation.

    :param ref_file: Path to the reference file.
    :type ref_file: str
    :param est_file: Path to the estimation file.
    :type est_file: str
    :returns:
        - M: dict
            Results contained in an ordered dictionary.
    """
    # load the data
    ref_patterns = mir_eval.io.load_patterns(ref_file)
    est_patterns = mir_eval.io.load_patterns(est_file)

    # Now compute all the metrics
    M = OrderedDict()

    # Standard scores
    M['F'], M['P'], M['R'] = \
        mir_eval.pattern.standard_FPR(ref_patterns, est_patterns)

    # Establishment scores
    M['F_est'], M['P_est'], M['R_est'] = \
        mir_eval.pattern.establishment_FPR(ref_patterns, est_patterns)

    # Occurrence scores
    M['F_occ.5'], M['P_occ.5'], M['R_occ.5'] = \
        mir_eval.pattern.occurrence_FPR(ref_patterns, est_patterns, thres=.5)
    M['F_occ.75'], M['P_occ.75'], M['R_occ.75'] = \
        mir_eval.pattern.occurrence_FPR(ref_patterns, est_patterns, thres=.75)

    # Three-layer scores
    M['F_3'], M['P_3'], M['R_3'] = \
        mir_eval.pattern.three_layer_FPR(ref_patterns, est_patterns)

    # First Five Patterns scores
    M['FFP'] = mir_eval.pattern.first_n_three_layer_P(ref_patterns,
                                                      est_patterns, n=5)
    M['FFTP_est'] = mir_eval.pattern.first_n_target_proportion_R(ref_patterns,
                                                                 est_patterns,
                                                                 n=5)

    return M

def save_results(results, output_file):
    '''Save a results dict into a json file'''
    with open(output_file, 'w') as f:
        json.dump(results, f)

def print_evaluation(estimation_file, M):
    # And print them
    print os.path.basename(estimation_file)
    for key, value in M.iteritems():
        print '\t%12s:\t%0.3f' % (key, value)

def main():
    """Main function to evaluate the pattern discovery task."""
    parser = argparse.ArgumentParser(description="mir_eval pattern discovery "
                                                 "evaluation",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-o',
                        dest='output_file',
                        default=None,
                        type=str,
                        action='store',
                        help='Store results in json format')
    parser.add_argument("ref_file",
                        action="store",
                        help="Path to the reference file.")
    parser.add_argument("est_file",
                        action="store",
                        help="Path to the estimation file.")
    parameters = vars(parser.parse_args(sys.argv[1:]))

    # Compute all the scores
    scores = evaluate(ref_file=parameters['ref_file'],
                      est_file=parameters['est_file'])
    print_evaluation(parameters['est_file'], scores)

    if parameters['output_file']:
        print 'Saving results to: ', parameters['output_file']
        save_results(scores, parameters['output_file'])

if __name__ == '__main__':
    main()
<|endoftext|>