<commit_before># -*- coding: utf-8 -*-

"""Sign up management handling.

   .. note::
      Views have to be registered at the end of this module because of circular dependencies.

   .. warning::
      Some code analyzers may flag view imports as unused, because they are only imported for their side effects.
"""

import os

from flask import Flask, session, g
from flask.ext.assets import Environment
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import LoginManager, current_user
from flask.ext.mail import Mail
from flask.ext.cache import Cache

from jinja2 import Markup

from spz import assets

class CustomFlask(Flask):
    """Internal customizations to the Flask class.

       This is mostly for Jinja2's whitespace and newline control, and improved template performance.
    """
    jinja_options = dict(Flask.jinja_options, trim_blocks=True, lstrip_blocks=True, auto_reload=False)

app = CustomFlask(__name__, instance_relative_config=True)

# set up login system
login_manager = LoginManager()
login_manager.init_app(app)

@login_manager.user_loader
def login_by_id(id):
    # local imports to avoid import before config
    from spz.models import User
    return User.query.filter(User.id == id).first()

@login_manager.token_loader
def login_by_token(tokenstring):
    # local imports to avoid import before config
    from spz.models import User
    return User.get_by_token(tokenstring)

# add `include_raw` Jinja helper
app.jinja_env.globals['include_raw'] = lambda filename : Markup(app.jinja_loader.get_source(app.jinja_env, filename)[0])

# Assets handling; keep the spz.assets module in sync with the static directory
assets_env = Environment(app)

bundles = assets.get_bundles()

for name, bundle in bundles.iteritems():
    assets_env.register(name, bundle)

# Configuration loading
if 'SPZ_CFG_FILE' in os.environ:
    app.config.from_pyfile(os.environ['SPZ_CFG_FILE'])
else:
    app.config.from_pyfile('development.cfg')
    # app.config.from_pyfile('production.cfg')

# Set up logging before anything else, in order to catch early errors
if not app.debug:
    from logging import FileHandler
    file_handler = FileHandler(app.config['LOGFILE'])
    app.logger.addHandler(file_handler)

# Database handling
db = SQLAlchemy(app)

# Mail sending
mail = Mail(app)

# Cache setup
cache = Cache(app, config=app.config['CACHE_CONFIG'])

# Register all views here
from spz import views, errorhandlers, pdf

# Permission handling
@app.before_request
def detect_permission_level():
    if not current_user.is_anonymous:
        mail = current_user.id
        acl = app.config['ACCESS_CONTROL']

        if mail in acl['unrestricted']:
            g.access = 'unrestricted'
        elif mail in acl['restricted']:
            g.access = 'restricted'
        else:
            g.access = None

        g.user = mail
    else:
        g.access = None
        g.user = None

routes = [('/', views.index, ['GET', 'POST']),
          ('/licenses', views.licenses, ['GET']),

          ('/internal', views.internal, ['GET']),

          ('/internal/import', views.importer, ['GET']),
          ('/internal/import/registrations', views.registrations, ['GET', 'POST']),
          ('/internal/import/approvals', views.approvals, ['GET', 'POST']),

          ('/internal/export_course/<int:course_id>', views.export_course, ['GET']),
          ('/internal/print_course/<int:course_id>', pdf.print_course, ['GET']),
          ('/internal/print_course_presence/<int:course_id>', pdf.print_course_presence, ['GET']),
          ('/internal/export_language/<int:language_id>', views.export_language, ['GET']),
          ('/internal/print_language/<int:language_id>', pdf.print_language, ['GET']),
          ('/internal/print_language_presence/<int:language_id>', pdf.print_language_presence, ['GET']),

          ('/internal/notifications', views.notifications, ['GET', 'POST']),

          ('/internal/lists', views.lists, ['GET']),
          ('/internal/applicant/<int:id>', views.applicant, ['GET', 'POST']),
          ('/internal/language/<int:id>', views.language, ['GET']),
          ('/internal/course/<int:id>', views.course, ['GET']),

          ('/internal/add_attendance/<int:applicant_id>/<int:course_id>', views.add_attendance, ['GET']),
          ('/internal/remove_attendance/<int:applicant_id>/<int:course_id>', views.remove_attendance, ['GET']),

          ('/internal/applicants/search_applicant', views.search_applicant, ['GET', 'POST']),
          ('/internal/applicants/applicant_attendances/<int:id>', views.applicant_attendances, ['GET']),

          ('/internal/payments', views.payments, ['GET', 'POST']),
          ('/internal/outstanding', views.outstanding, ['GET', 'POST']),
          ('/internal/status/<int:applicant_id>/<int:course_id>', views.status, ['GET', 'POST']),
          ('/internal/print_bill/<int:applicant_id>/<int:course_id>', pdf.print_bill, ['GET']),

          # First-come-first-serve selection
          ('/internal/restock_fcfs', views.restock_fcfs, ['GET', 'POST']),
          # Weighted random selection
          ('/internal/restock_rnd', views.restock_rnd, ['GET', 'POST']),

          ('/internal/unique', views.unique, ['GET', 'POST']),

          ('/internal/preterm', views.preterm, ['GET', 'POST']),

          ('/internal/statistics', views.statistics, ['GET']),
          ('/internal/statistics/free_courses', views.free_courses, ['GET']),
          ('/internal/statistics/origins_breakdown', views.origins_breakdown, ['GET']),
          ('/internal/statistics/task_queue', views.task_queue, ['GET']),

          ('/internal/duplicates', views.duplicates, ['GET']),

          ('/internal/login', views.login, ['GET', 'POST']),
          ('/internal/logout', views.logout, ['GET', 'POST']),
]

for rule, view_func, methods in routes:
    app.add_url_rule(rule, view_func=view_func, methods=methods)

handlers = [
    (401, errorhandlers.unauthorized),
    (404, errorhandlers.page_not_found),
    (403, errorhandlers.page_forbidden),
    (410, errorhandlers.page_gone),
    (500, errorhandlers.not_found),
]

for errno, handler in handlers:
    app.register_error_handler(errno, handler)
<commit_msg>backend: stronger session protection<commit_after># -*- coding: utf-8 -*-

"""Sign up management handling.

   .. note::
      Views have to be registered at the end of this module because of circular dependencies.

   .. warning::
      Some code analyzers may flag view imports as unused, because they are only imported for their side effects.
"""

import os

from flask import Flask, session, g
from flask.ext.assets import Environment
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import LoginManager, current_user
from flask.ext.mail import Mail
from flask.ext.cache import Cache

from jinja2 import Markup

from spz import assets

class CustomFlask(Flask):
    """Internal customizations to the Flask class.

       This is mostly for Jinja2's whitespace and newline control, and improved template performance.
    """
    jinja_options = dict(Flask.jinja_options, trim_blocks=True, lstrip_blocks=True, auto_reload=False)

app = CustomFlask(__name__, instance_relative_config=True)

# set up login system
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.session_protection = 'strong'

@login_manager.user_loader
def login_by_id(id):
    # local imports to avoid import before config
    from spz.models import User
    return User.query.filter(User.id == id).first()

@login_manager.token_loader
def login_by_token(tokenstring):
    # local imports to avoid import before config
    from spz.models import User
    return User.get_by_token(tokenstring)

# add `include_raw` Jinja helper
app.jinja_env.globals['include_raw'] = lambda filename : Markup(app.jinja_loader.get_source(app.jinja_env, filename)[0])

# Assets handling; keep the spz.assets module in sync with the static directory
assets_env = Environment(app)

bundles = assets.get_bundles()

for name, bundle in bundles.iteritems():
    assets_env.register(name, bundle)

# Configuration loading
if 'SPZ_CFG_FILE' in os.environ:
    app.config.from_pyfile(os.environ['SPZ_CFG_FILE'])
else:
    app.config.from_pyfile('development.cfg')
    # app.config.from_pyfile('production.cfg')

# Set up logging before anything else, in order to catch early errors
if not app.debug:
    from logging import FileHandler
    file_handler = FileHandler(app.config['LOGFILE'])
    app.logger.addHandler(file_handler)

# Database handling
db = SQLAlchemy(app)

# Mail sending
mail = Mail(app)

# Cache setup
cache = Cache(app, config=app.config['CACHE_CONFIG'])

# Register all views here
from spz import views, errorhandlers, pdf

# Permission handling
@app.before_request
def detect_permission_level():
    if not current_user.is_anonymous:
        mail = current_user.id
        acl = app.config['ACCESS_CONTROL']

        if mail in acl['unrestricted']:
            g.access = 'unrestricted'
        elif mail in acl['restricted']:
            g.access = 'restricted'
        else:
            g.access = None

        g.user = mail
    else:
        g.access = None
        g.user = None

routes = [('/', views.index, ['GET', 'POST']),
          ('/licenses', views.licenses, ['GET']),

          ('/internal', views.internal, ['GET']),

          ('/internal/import', views.importer, ['GET']),
          ('/internal/import/registrations', views.registrations, ['GET', 'POST']),
          ('/internal/import/approvals', views.approvals, ['GET', 'POST']),

          ('/internal/export_course/<int:course_id>', views.export_course, ['GET']),
          ('/internal/print_course/<int:course_id>', pdf.print_course, ['GET']),
          ('/internal/print_course_presence/<int:course_id>', pdf.print_course_presence, ['GET']),
          ('/internal/export_language/<int:language_id>', views.export_language, ['GET']),
          ('/internal/print_language/<int:language_id>', pdf.print_language, ['GET']),
          ('/internal/print_language_presence/<int:language_id>', pdf.print_language_presence, ['GET']),

          ('/internal/notifications', views.notifications, ['GET', 'POST']),

          ('/internal/lists', views.lists, ['GET']),
          ('/internal/applicant/<int:id>', views.applicant, ['GET', 'POST']),
          ('/internal/language/<int:id>', views.language, ['GET']),
          ('/internal/course/<int:id>', views.course, ['GET']),

          ('/internal/add_attendance/<int:applicant_id>/<int:course_id>', views.add_attendance, ['GET']),
          ('/internal/remove_attendance/<int:applicant_id>/<int:course_id>', views.remove_attendance, ['GET']),

          ('/internal/applicants/search_applicant', views.search_applicant, ['GET', 'POST']),
          ('/internal/applicants/applicant_attendances/<int:id>', views.applicant_attendances, ['GET']),

          ('/internal/payments', views.payments, ['GET', 'POST']),
          ('/internal/outstanding', views.outstanding, ['GET', 'POST']),
          ('/internal/status/<int:applicant_id>/<int:course_id>', views.status, ['GET', 'POST']),
          ('/internal/print_bill/<int:applicant_id>/<int:course_id>', pdf.print_bill, ['GET']),

          # First-come-first-serve selection
          ('/internal/restock_fcfs', views.restock_fcfs, ['GET', 'POST']),
          # Weighted random selection
          ('/internal/restock_rnd', views.restock_rnd, ['GET', 'POST']),

          ('/internal/unique', views.unique, ['GET', 'POST']),

          ('/internal/preterm', views.preterm, ['GET', 'POST']),

          ('/internal/statistics', views.statistics, ['GET']),
          ('/internal/statistics/free_courses', views.free_courses, ['GET']),
          ('/internal/statistics/origins_breakdown', views.origins_breakdown, ['GET']),
          ('/internal/statistics/task_queue', views.task_queue, ['GET']),

          ('/internal/duplicates', views.duplicates, ['GET']),

          ('/internal/login', views.login, ['GET', 'POST']),
          ('/internal/logout', views.logout, ['GET', 'POST']),
]

for rule, view_func, methods in routes:
    app.add_url_rule(rule, view_func=view_func, methods=methods)

handlers = [
    (401, errorhandlers.unauthorized),
    (404, errorhandlers.page_not_found),
    (403, errorhandlers.page_forbidden),
    (410, errorhandlers.page_gone),
    (500, errorhandlers.not_found),
]

for errno, handler in handlers:
    app.register_error_handler(errno, handler)
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.http import Http404
from django.shortcuts import render_to_response
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from tracking.models import Visitor, UntrackedUserAgent, BannedIP
from tracking import utils
from datetime import datetime, timedelta
import random
import time
import re
import urllib2

title_re = re.compile('<title>(.*?)</title>')

class VisitorTrackingMiddleware:
    """
    Keeps track of your active users.  Anytime a visitor accesses a valid URL,
    their unique record will be updated with the page they're on and the last
    time they requested a page.

    Records are considered to be unique when the session key and IP address
    are unique together.  Sometimes the same user used to have two different
    records, so I added a check to see if the session key had changed for the
    same IP and user agent in the last 5 minutes
    """

    def process_request(self, request):
        # don't process AJAX requests
        if request.is_ajax():
            return

        # create some useful variables
        session_key = request.session.session_key
        ip_address = request.META.get('REMOTE_ADDR', '')
        user_agent = request.META.get('HTTP_USER_AGENT', '')

        # see if the user agent is not supposed to be tracked
        for ua in UntrackedUserAgent.objects.all():
            # if the keyword is found in the user agent, stop tracking
            if str(user_agent).find(ua.keyword) != -1:
                return

        prefixes = utils.get_untracked_prefixes()
        # don't track media files
        prefixes.append(settings.MEDIA_URL)
        prefixes.append(settings.ADMIN_MEDIA_PREFIX)

        # ensure that the request.path does not begin with any of the prefixes
        validURL = True
        for prefix in prefixes:
            if request.path.startswith(prefix):
                validURL = False
                break

        # if the URL needs to be tracked, track it!
        if validURL:
            # determine what time it is
            now = datetime.now()

            attrs = {
                        'session_key': session_key,
                        'ip_address': ip_address
                    }

            # for some reason, Visitor.objects.get_or_create was not working here
            try:
                visitor = Visitor.objects.get(**attrs)
            except Visitor.DoesNotExist:
                try:
                    # see if there's a visitor with the same IP and user agent
                    # within the last 5 minutes
                    cutoff = now - timedelta(minutes=5)
                    visitor = Visitor.objects.get(
                                    ip_address=ip_address,
                                    user_agent=user_agent,
                                    last_update__gte=cutoff
                                )
                    visitor.session_key = session_key
                except Visitor.DoesNotExist:
                    # it's probably safe to assume that the visitor is brand new
                    visitor = Visitor(**attrs)

            # determine whether or not the user is logged in
            user = request.user
            if isinstance(user, AnonymousUser):
                user = None

            # update the tracking information
            visitor.user = user
            visitor.user_agent = user_agent

            # if the visitor record is new, or the visitor hasn't been here for
            # at least an hour, update their referrer URL
            one_hour_ago = now + timedelta(hours=-1)
            if not visitor.last_update or \
                visitor.last_update <= one_hour_ago:
                visitor.referrer = request.META.get('HTTP_REFERER', 'unknown')

                # reset the number of pages they've been to
                visitor.page_views = 0
                visitor.session_start = now

            visitor.url = request.path
            visitor.page_views += 1
            visitor.last_update = now
            visitor.save()

class VisitorCleanUpMiddleware:
    """
    Clean up old visitor tracking records in the database
    """
    def process_request(self, request):
        timeout = datetime.now() - timedelta(hours=utils.get_cleanup_timeout())
        Visitor.objects.filter(last_update__lte=timeout).delete()

class BannedIPMiddleware:
    """
    Raises an Http404 error for any page request from a banned IP.  IP addresses
    may be added to the list of banned IPs via the Django admin.

    The banned users do not actually receive the 404 error--instead they get
    an "Internal Server Error", effectively eliminating any access to the site.
    """
    def process_request(self, request):
        # compile a list of all banned IP addresses
        ips = [b.ip_address for b in BannedIP.objects.all()]

        # check to see if the current user's IP address is in that list
        if request.META.get('REMOTE_ADDR', '') in ips:
            raise Http404

class GoogleAnalyticsMiddleware:
    """
    This is a server-side version of the Google Analytics tracking.  It should
    be able to track things like requests to RSS feeds and whatnot, but it does
    tend to lose some information, such as where the request is coming from.
    """
    def process_response(self, request, response):
        # get the title from the response if possible
        try:
            title = title_re.search(response.content).group(1)
        except:
            title = ''

        # setup a dictionary of values for use in the query string
        info = {
            'id': settings.GOOGLE_ANALYTICS_ID,
            'host': request.META.get('HTTP_HOST', ''),
            'path': request.META.get('PATH_INFO', '/'),
            'referer': request.META.get('HTTP_REFERER', ''),
            'uservar': '%s; %s' % (
                                    request.META.get('REMOTE_ADDR', ''),
                                    request.META.get('HTTP_USER_AGENT', 'unknown'),
                                ),
            'rand_request': random.randint(1000000000, 9999999999),
            'rand_cookie': random.randint(10000000, 99999999),
            'rand_number': random.randint(1000000000, 2147483647),
            'today': int(time.mktime(datetime.now().timetuple())),
            'resolution': '-',
            'color_depth': '-',
            'language': '-',
            'java': '-',
            'flash': '-',
            'title': title,
        }

        # put all of the info values where they belong
        data = 'utmwv=4.3&utmn=%(rand_request)s&utmsr=%(resolution)s&utmsc=%(color_depth)s&utmul=%(language)s&utmje=%(java)s&utmfl=%(flash)s&utmdt=%(title)s&utmhn=%(host)s&utmr=%(referer)s&utmp=%(path)s&utmac=%(id)s&utmcc=__utma%%3D%(rand_cookie)s.%(rand_number)s.%(today)s.%(today)s.%(today)s.2%%3B%%2B__utmb%%3D%(rand_cookie)s%%3B%%2B__utmc%%3D%(rand_cookie)s%%3B%%2B__utmz%%3D%(rand_cookie)s.%(today)s.2.2.utmccn%%3D(direct)%%7Cutmcsr%%3D(direct)%%7Cutmcmd%%3D(none)%%3B%%2B__utmv%%3D%(rand_cookie)s.%(uservar)s%%3B' % info

        # talk to Google Analytics
        conn = urllib2.urlopen('http://www.google-analytics.com/__utm.gif', data)

        # send the response back to the client
        return response
<commit_msg>Worked a little more on the server-side Google Analytics<commit_after>from django.core.urlresolvers import reverse
from django.http import Http404
from django.shortcuts import render_to_response
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from tracking.models import Visitor, UntrackedUserAgent, BannedIP
from tracking import utils
from datetime import datetime, timedelta
import random
import time
import re
import urllib2

title_re = re.compile('<title>(.*?)</title>')

class VisitorTrackingMiddleware:
    """
    Keeps track of your active users.  Anytime a visitor accesses a valid URL,
    their unique record will be updated with the page they're on and the last
    time they requested a page.

    Records are considered to be unique when the session key and IP address
    are unique together.  Sometimes the same user used to have two different
    records, so I added a check to see if the session key had changed for the
    same IP and user agent in the last 5 minutes
    """

    def process_request(self, request):
        # don't process AJAX requests
        if request.is_ajax():
            return

        # create some useful variables
        session_key = request.session.session_key
        ip_address = request.META.get('REMOTE_ADDR', '')
        user_agent = request.META.get('HTTP_USER_AGENT', '')

        # see if the user agent is not supposed to be tracked
        for ua in UntrackedUserAgent.objects.all():
            # if the keyword is found in the user agent, stop tracking
            if str(user_agent).find(ua.keyword) != -1:
                return

        prefixes = utils.get_untracked_prefixes()
        # don't track media files
        prefixes.append(settings.MEDIA_URL)
        prefixes.append(settings.ADMIN_MEDIA_PREFIX)

        # ensure that the request.path does not begin with any of the prefixes
        validURL = True
        for prefix in prefixes:
            if request.path.startswith(prefix):
                validURL = False
                break

        # if the URL needs to be tracked, track it!
        if validURL:
            # determine what time it is
            now = datetime.now()

            attrs = {
                        'session_key': session_key,
                        'ip_address': ip_address
                    }

            # for some reason, Visitor.objects.get_or_create was not working here
            try:
                visitor = Visitor.objects.get(**attrs)
            except Visitor.DoesNotExist:
                try:
                    # see if there's a visitor with the same IP and user agent
                    # within the last 5 minutes
                    cutoff = now - timedelta(minutes=5)
                    visitor = Visitor.objects.get(
                                    ip_address=ip_address,
                                    user_agent=user_agent,
                                    last_update__gte=cutoff
                                )
                    visitor.session_key = session_key
                except Visitor.DoesNotExist:
                    # it's probably safe to assume that the visitor is brand new
                    visitor = Visitor(**attrs)

            # determine whether or not the user is logged in
            user = request.user
            if isinstance(user, AnonymousUser):
                user = None

            # update the tracking information
            visitor.user = user
            visitor.user_agent = user_agent

            # if the visitor record is new, or the visitor hasn't been here for
            # at least an hour, update their referrer URL
            one_hour_ago = now + timedelta(hours=-1)
            if not visitor.last_update or \
                visitor.last_update <= one_hour_ago:
                visitor.referrer = request.META.get('HTTP_REFERER', 'unknown')

                # reset the number of pages they've been to
                visitor.page_views = 0
                visitor.session_start = now

            visitor.url = request.path
            visitor.page_views += 1
            visitor.last_update = now
            visitor.save()

class VisitorCleanUpMiddleware:
    """
    Clean up old visitor tracking records in the database
    """
    def process_request(self, request):
        timeout = datetime.now() - timedelta(hours=utils.get_cleanup_timeout())
        Visitor.objects.filter(last_update__lte=timeout).delete()

class BannedIPMiddleware:
    """
    Raises an Http404 error for any page request from a banned IP.  IP addresses
    may be added to the list of banned IPs via the Django admin.

    The banned users do not actually receive the 404 error--instead they get
    an "Internal Server Error", effectively eliminating any access to the site.
    """
    def process_request(self, request):
        # compile a list of all banned IP addresses
        ips = [b.ip_address for b in BannedIP.objects.all()]

        # check to see if the current user's IP address is in that list
        if request.META.get('REMOTE_ADDR', '') in ips:
            raise Http404

class GoogleAnalyticsMiddleware:
    """
    This is a server-side version of the Google Analytics tracking.  It should
    be able to track things like requests to RSS feeds and whatnot, but it does
    tend to lose some information, such as where the request is coming from.
    """
    def process_response(self, request, response):
        # get the title from the response if possible
        try:
            title = title_re.search(response.content).group(1)
        except:
            title = ''

        # setup a dictionary of values for use in the query string
        info = {
            'id': settings.GOOGLE_ANALYTICS_ID,
            'host': request.META.get('HTTP_HOST', ''),
            'path': request.META.get('PATH_INFO', '/'),
            'referer': request.META.get('HTTP_REFERER', ''),
            'uservar': '%s; %s' % (
                                    request.META.get('REMOTE_ADDR', ''),
                                    request.META.get('HTTP_USER_AGENT', 'unknown'),
                                ),
            'rand_request': random.randint(1000000000, 9999999999),
            'rand_cookie': random.randint(10000000, 99999999),
            'rand_number': random.randint(1000000000, 2147483647),
            'today': int(time.mktime(datetime.now().timetuple())),
            'resolution': '-',
            'color_depth': '-',
            'language': '-',
            'java': '-',
            'flash': '-',
            'title': title,
        }

        # put all of the info values where they belong
        data = 'utmwv=4.3&utmn=%(rand_request)s&utmsr=%(resolution)s&utmsc=%(color_depth)s&utmul=%(language)s&utmje=%(java)s&utmfl=%(flash)s&utmdt=%(title)s&utmhn=%(host)s&utmr=%(referer)s&utmp=%(path)s&utmac=%(id)s&utmcc=__utma%%3D%(rand_cookie)s.%(rand_number)s.%(today)s.%(today)s.%(today)s.2%%3B%%2B__utmb%%3D%(rand_cookie)s%%3B%%2B__utmc%%3D%(rand_cookie)s%%3B%%2B__utmz%%3D%(rand_cookie)s.%(today)s.2.2.utmccn%%3D(direct)%%7Cutmcsr%%3D(direct)%%7Cutmcmd%%3D(none)%%3B%%2B__utmv%%3D%(rand_cookie)s.%(uservar)s%%3B' % info

        # talk to Google Analytics
        conn = urllib2.urlopen('http://www.google-analytics.com/__utm.gif', data)
        conn.read()

        # send the response back to the client
        return response
<|endoftext|>


<commit_before>#!/usr/bin/env python

def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('tract_querier', parent_package, top_path)
    config.add_subpackage('tractography')
    config.add_subpackage('code_util')
    config.add_subpackage('nipype')
    config.add_data_files(('queries', [
        'data/FreeSurfer.qry',
        'data/JHU_MNI_SS_WMPM_Type_II.qry',
        'data/freesurfer_queries.qry',
        'data/freesurfer_queries_new.qry',
        'data/mori_queries.qry',
    ]))
    return config

if __name__ == '__main__':
    from distutils.core import setup
    setup(**configuration(top_path='').todict())
<commit_msg>ENH: Cleaning tract definitions<commit_after>#!/usr/bin/env python

def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('tract_querier', parent_package, top_path)
    config.add_subpackage('tractography')
    config.add_subpackage('code_util')
    config.add_subpackage('nipype')
    config.add_data_files(('queries', [
        'data/FreeSurfer.qry',
        'data/JHU_MNI_SS_WMPM_Type_II.qry',
        'data/freesurfer_queries.qry',
        'data/mori_queries.qry',
    ]))
    return config

if __name__ == '__main__':
    from distutils.core import setup
    setup(**configuration(top_path='').todict())
<|endoftext|>


<commit_before>
from traffic import rootwrap
from traffic import utils
from traffic import db
from traffic.db import base
import os

class API(base.Base):

    def set_execute(self, execute):
        self._execute = execute

    def create(self, context, ip, class_id, instanceid, prio=1):
        ips = ip + '/32'
        cmd = ['tc filter add dev br100 parent 10: protocol ip prio ', prio ,' u32 match ip src ', ips, ' flowid ', class_id]
        cmd = map(str, cmd)
        #self._execute(cmd)

        handle = self.db.tfilter_get_last_handle(context)
        if not handle:
            handle = 799
        else:
            handle = handle[0]
        self.db.tfilter_create(context,
                               {'ip': ip,
                                'classid': class_id,
                                'flowid': class_id,
                                'instanceid': instanceid,
                                'handle': handle+1,
                                'prio': prio})
        os.system(''.join(cmd))

    def delete(self, context, instanceid):
        tfilter = self.db.tfilter_get_by_instance(context, instanceid)
        handle_r = '800::' + str(tfilter[6])
        cmd = ['tc filter del dev eth0 parent 10: prio', tfilter[9], 'handle', handle_r]
        os.system(''.join(cmd))
#        self._execute(cmd)
        self.db.tfilter_delete_by_instance(context, instanceid)

    def get(self, context, class_id):
        result = self.db.tfilter_get_by_classid(context, class_id)
        return result<commit_msg>change all to string <commit_after>
from traffic import rootwrap
from traffic import utils
from traffic import db
from traffic.db import base
import os

class API(base.Base):

    def set_execute(self, execute):
        self._execute = execute

    def create(self, context, ip, class_id, instanceid, prio=1):
        ips = ip + '/32'
        cmd = ['tc filter add dev br100 parent 10: protocol ip prio ', prio ,' u32 match ip src ', ips, ' flowid ', class_id]
        cmd = map(str, cmd)
        #self._execute(cmd)

        handle = self.db.tfilter_get_last_handle(context)
        if not handle:
            handle = 799
        else:
            handle = handle[0]
        self.db.tfilter_create(context,
                               {'ip': ip,
                                'classid': class_id,
                                'flowid': class_id,
                                'instanceid': instanceid,
                                'handle': handle+1,
                                'prio': prio})
        os.system(''.join(cmd))

    def delete(self, context, instanceid):
        tfilter = self.db.tfilter_get_by_instance(context, instanceid)
        handle_r = '800::' + str(tfilter[6])
        cmd = ['tc filter del dev eth0 parent 10: prio', tfilter[9], 'handle', handle_r]
        cmd = map(str, cmd)
        os.system(''.join(cmd))
#        self._execute(cmd)
        self.db.tfilter_delete_by_instance(context, instanceid)

    def get(self, context, class_id):
        result = self.db.tfilter_get_by_classid(context, class_id)
        return result<|endoftext|>


<commit_before>'''
Created on Jul 19, 2017

@author: kashefy
'''
import os
import numpy as np
import tensorflow as tf
#from tensorflow.python import debug as tf_debug
from abstract_runner import AbstractRunner, setup_optimizer
from nideep.datasets.mnist.mnist_tf import MNIST

class AERunner(AbstractRunner):
    '''
    classdocs
    '''
    def learn(self, sess):
        dir_train = os.path.join(self.run_dir, self.prefix,  'train')
        summary_writer_train = tf.summary.FileWriter(dir_train,
                                                     sess.graph)
        dir_val = os.path.join(self.run_dir, self.prefix, 'validation')
        summary_writer_val = tf.summary.FileWriter(dir_val)
        itr_exp = 0
        for dim in self.stack_dims:
            itr_depth = 0
            self.logger.debug('Stacking %d nodes.' % dim)
            self.model.stack(dim)
            loss = self.model.cost(name='train/loss_reconstruction')
            if self.lambda_l2 != 0:
                regularization = self._regularization(name=self.prefix + '/train/regularization_l2')
                cost = tf.add(loss, self.lambda_l2 * regularization,
                              name=self.prefix + '/train/cost')
            else:
                cost = loss
            vars_new = self.model.vars_new()
            self.logger.debug('Variables added: %s' % [v.name for v in vars_new])
            self._vars_added.append(vars_new)
            optimizer = setup_optimizer(cost, self.learning_rate, var_list=vars_new)
            vars_new = self.model.vars_new()
            self.init_vars(sess, vars_new)

            self.logger.debug('encoder-0: %s' % sess.run(self.model.sae[0].w['encoder-0/w'][10,5:10]))
            summaries_merged_train = self._merge_summaries_scalars([loss, cost])

            summaries = []
#            for value in [loss, cost]:
#                self.logger.debug("log scalar: %s" % value.op.name)
#                summaries.append(tf.summary.scalar(value.op.name, value))
            self.logger.debug("log weights (histograms, images): %s" % self.model.w.keys())
            self._append_summaries_hists(summaries, self.model.w.values())
            self._append_summaries_var_imgs(summaries, self.model.w.values())
            self._append_summaries_op_imgs(summaries, [self.model.p])
            summaries_merged_val = tf.summary.merge(summaries)

            self._init_saver()

            for epoch in xrange(self.training_epochs):
                self.logger.info("Start epoch %d, step %d" % (epoch, itr_exp))
                # Loop over all batches
                for itr_epoch in xrange(self.num_batches_train):
                    batch_xs, _ = self.data.train.next_batch(self.num_batches_train)
        #            batch_xs_as_img = tf.reshape(batch_xs, [-1, 28, 28, 1])
        #            rots_cur = np.random.choice(rotations, batch_size)
        #            batch_xs_as_img_rot = tf.contrib.image.rotate(batch_xs_as_img, rots_cur)
        #            # Run optimization op (backprop) and cost op (to get loss value)
        #
        #            batch_xs_as_img, batch_xs_as_img_rot = \
        #                sess.run([batch_xs_as_img, batch_xs_as_img_rot],
        #                         feed_dict={ae1.x: batch_xs})
        #            f, a = plt.subplots(2, 10, figsize=(10, 2))
        #            for i in xrange(examples_to_show):
        #                print (batch_xs_as_img[i].shape, np.rad2deg(rots_cur)[i])
        #                a[0][i].imshow(np.squeeze(batch_xs_as_img[i]))
        #                a[1][i].imshow(np.squeeze(batch_xs_as_img_rot[i]))
        #            f.show()
        #            plt.draw()
        #            plt.waitforbuttonpress()
                    _, c, sess_summary = sess.run([optimizer, cost, summaries_merged_train],
                                                  feed_dict={self.model.x: batch_xs})
                    summary_writer_train.add_summary(sess_summary, itr_exp)
                    itr_exp += 1
                    itr_depth += 1
                    # run metric op one more time, data in feed dict is dummy data, does not influence metric
                _, sess_summary = sess.run([cost, summaries_merged_val],
                                         feed_dict={self.model.x  : self.batch_viz_xs}
                                         )
                summary_writer_val.add_summary(sess_summary, itr_exp)
                fpath_save = os.path.join(dir_train, self._get_save_name())
                self.logger.debug("Save model at step %d to '%s'" % (itr_exp, fpath_save))
                self.saver.save(sess, fpath_save, global_step=itr_exp)
            self.logger.info("Optimization Finished!")
            self.logger.debug('encoder-0: %s' % sess.run(self.model.sae[0].w['encoder-0/w'][10,5:10]))
#        if dim == 128:
#            print('encoder_2',sess.run(sae.sae[1].w['encoder_2/w'][10,5:10]))
#            encode_decode = sess.run(
#                sae.p, feed_dict={sae.x: mnist.test.images[:args.examples_to_show]})
        # Compare original images with their reconstructions
#    fig, a = plt.subplots(2, 10, figsize=(10, 2))
#    for i in xrange(args.examples_to_show):
#        a[0][i].imshow(np.reshape(mnist.test.images[i], (28, 28)), clim=(0.0, 1.0))
#        a[1][i].imshow(np.reshape(encode_decode[i], (28, 28)), clim=(0.0, 1.0))
#    fig.savefig(os.path.join(args.log_dir, run_dir, 'train_layerwise_reconstruct.png'))
    def validate(self, sess):
        pass

    def __init__(self, params):
        '''
        Constructor
        '''
        super(AERunner, self).__init__(params)
        self.data = MNIST.read_data_sets("MNIST_data",
                                         validation_size=self.validation_size)
        self.num_batches_train = int(self.data.train.num_examples/self.batch_size_train)
        self.logger.debug("No. of batches per epoch: %d" % self.num_batches_train)
        self._check_validation_batch_size()
        self.stack_dims = params['stack_dims']
        self.logger.debug("Stack dims: %s" % self.stack_dims)
        self.prefix = 'reconstruction'
        self._vars_added = []
        self.batch_size_val = 3
        self.batch_viz_xs, self.batch_viz_ys = self.data.validation.next_batch(self.batch_size_val, shuffle=False)
        <commit_msg>remove hardcoded batch size<commit_after>'''
Created on Jul 19, 2017

@author: kashefy
'''
import os
import numpy as np
import tensorflow as tf
#from tensorflow.python import debug as tf_debug
from abstract_runner import AbstractRunner, setup_optimizer
from nideep.datasets.mnist.mnist_tf import MNIST

class AERunner(AbstractRunner):
    '''
    classdocs
    '''
    def learn(self, sess):
        dir_train = os.path.join(self.run_dir, self.prefix,  'train')
        summary_writer_train = tf.summary.FileWriter(dir_train,
                                                     sess.graph)
        dir_val = os.path.join(self.run_dir, self.prefix, 'validation')
        summary_writer_val = tf.summary.FileWriter(dir_val)
        itr_exp = 0
        for dim in self.stack_dims:
            itr_depth = 0
            self.logger.debug('Stacking %d nodes.' % dim)
            self.model.stack(dim)
            loss = self.model.cost(name='train/loss_reconstruction')
            if self.lambda_l2 != 0:
                regularization = self._regularization(name=self.prefix + '/train/regularization_l2')
                cost = tf.add(loss, self.lambda_l2 * regularization,
                              name=self.prefix + '/train/cost')
            else:
                cost = loss
            vars_new = self.model.vars_new()
            self.logger.debug('Variables added: %s' % [v.name for v in vars_new])
            self._vars_added.append(vars_new)
            optimizer = setup_optimizer(cost, self.learning_rate, var_list=vars_new)
            vars_new = self.model.vars_new()
            self.init_vars(sess, vars_new)

            self.logger.debug('encoder-0: %s' % sess.run(self.model.sae[0].w['encoder-0/w'][10,5:10]))
            summaries_merged_train = self._merge_summaries_scalars([loss, cost])

            summaries = []
#            for value in [loss, cost]:
#                self.logger.debug("log scalar: %s" % value.op.name)
#                summaries.append(tf.summary.scalar(value.op.name, value))
            self.logger.debug("log weights (histograms, images): %s" % self.model.w.keys())
            self._append_summaries_hists(summaries, self.model.w.values())
            self._append_summaries_var_imgs(summaries, self.model.w.values())
            self._append_summaries_op_imgs(summaries, [self.model.p])
            summaries_merged_val = tf.summary.merge(summaries)

            self._init_saver()

            for epoch in xrange(self.training_epochs):
                self.logger.info("Start epoch %d, step %d" % (epoch, itr_exp))
                # Loop over all batches
                for itr_epoch in xrange(self.num_batches_train):
                    batch_xs, _ = self.data.train.next_batch(self.num_batches_train)
        #            batch_xs_as_img = tf.reshape(batch_xs, [-1, 28, 28, 1])
        #            rots_cur = np.random.choice(rotations, batch_size)
        #            batch_xs_as_img_rot = tf.contrib.image.rotate(batch_xs_as_img, rots_cur)
        #            # Run optimization op (backprop) and cost op (to get loss value)
        #
        #            batch_xs_as_img, batch_xs_as_img_rot = \
        #                sess.run([batch_xs_as_img, batch_xs_as_img_rot],
        #                         feed_dict={ae1.x: batch_xs})
        #            f, a = plt.subplots(2, 10, figsize=(10, 2))
        #            for i in xrange(examples_to_show):
        #                print (batch_xs_as_img[i].shape, np.rad2deg(rots_cur)[i])
        #                a[0][i].imshow(np.squeeze(batch_xs_as_img[i]))
        #                a[1][i].imshow(np.squeeze(batch_xs_as_img_rot[i]))
        #            f.show()
        #            plt.draw()
        #            plt.waitforbuttonpress()
                    _, c, sess_summary = sess.run([optimizer, cost, summaries_merged_train],
                                                  feed_dict={self.model.x: batch_xs})
                    summary_writer_train.add_summary(sess_summary, itr_exp)
                    itr_exp += 1
                    itr_depth += 1
                    # run metric op one more time, data in feed dict is dummy data, does not influence metric
                _, sess_summary = sess.run([cost, summaries_merged_val],
                                         feed_dict={self.model.x  : self.batch_viz_xs}
                                         )
                summary_writer_val.add_summary(sess_summary, itr_exp)
                fpath_save = os.path.join(dir_train, self._get_save_name())
                self.logger.debug("Save model at step %d to '%s'" % (itr_exp, fpath_save))
                self.saver.save(sess, fpath_save, global_step=itr_exp)
            self.logger.info("Optimization Finished!")
            self.logger.debug('encoder-0: %s' % sess.run(self.model.sae[0].w['encoder-0/w'][10,5:10]))
#        if dim == 128:
#            print('encoder_2',sess.run(sae.sae[1].w['encoder_2/w'][10,5:10]))
#            encode_decode = sess.run(
#                sae.p, feed_dict={sae.x: mnist.test.images[:args.examples_to_show]})
        # Compare original images with their reconstructions
#    fig, a = plt.subplots(2, 10, figsize=(10, 2))
#    for i in xrange(args.examples_to_show):
#        a[0][i].imshow(np.reshape(mnist.test.images[i], (28, 28)), clim=(0.0, 1.0))
#        a[1][i].imshow(np.reshape(encode_decode[i], (28, 28)), clim=(0.0, 1.0))
#    fig.savefig(os.path.join(args.log_dir, run_dir, 'train_layerwise_reconstruct.png'))
    def validate(self, sess):
        pass

    def __init__(self, params):
        '''
        Constructor
        '''
        super(AERunner, self).__init__(params)
        self.data = MNIST.read_data_sets("MNIST_data",
                                         validation_size=self.validation_size)
        self.num_batches_train = int(self.data.train.num_examples/self.batch_size_train)
        self.logger.debug("No. of batches per epoch: %d" % self.num_batches_train)
        self._check_validation_batch_size()
        self.stack_dims = params['stack_dims']
        self.logger.debug("Stack dims: %s" % self.stack_dims)
        self.prefix = 'reconstruction'
        self._vars_added = []
        self.batch_viz_xs, self.batch_viz_ys = self.data.validation.next_batch(self.batch_size_val,
                                                                               shuffle=False)
        <|endoftext|>


<commit_before>"""
Prepare eo3 metadata for Sentinel-2 Level 1C data produced by Sinergise or esa.
"""

from pathlib import Path
from xml.dom import minidom
import json
import os
from os import listdir
from os.path import isfile, join
import uuid
import click
from typing import Tuple, Dict
from eodatasets3 import DatasetAssembler
from eodatasets3.ui import PathPath
import zipfile

HARDCODED = {
    "file_format": "JPEG2000",
    "instrument": "MSI",
    "product_family": "level1",
    "data_type": "Level-1C",
    "product_type": "S2MSI1C",
}

SENTINEL_MSI_BAND_ALIASES = {
    "01": "coastal_aerosol",
    "02": "blue",
    "03": "green",
    "04": "red",
    "05": "red_edge_1",
    "06": "red_edge_2",
    "07": "red_edge_3",
    "08": "nir_1",
    "8A": "nir_2",
    "09": "water_vapour",
    "10": "swir_1_cirrus",
    "11": "swir_2",
    "12": "swir_3",
}

def find_metadata_path(find_filename, dataset_dir):
    for root, dir, fn in os.walk(dataset_dir):
        if find_filename in fn:
            return os.path.join(root, fn[fn.index(find_filename)])

def process_product_info(product_path):
    with open(product_path) as fp:
        product = json.loads(fp.read())

        synergise_product_name = product["name"]
        synergise_product_id = product["id"]
        if len(product["tiles"]) > 1:
            raise NotImplementedError("No support for multi-tiled products yet")
        timestamp = product["tiles"][0]["timestamp"]
        utm_zone = product["tiles"][0]["utmZone"]
        latitude_band = product["tiles"][0]["latitudeBand"]
        grid_square = product["tiles"][0]["gridSquare"]
        region_code = "%s%s%s" % (utm_zone, latitude_band, grid_square)

        return {
            "synergise_product_name": synergise_product_name,
            "synergise_product_id": synergise_product_id,
            "timestamp": timestamp,
            "region_code": region_code,
        }

def process_metadata_xml(metadata_xml_path: str) -> Dict:
    xmldoc = minidom.parse(metadata_xml_path)

    cloud = float(
        xmldoc.getElementsByTagName("CLOUDY_PIXEL_PERCENTAGE")[0].firstChild.data
    )
    downlink_priority = xmldoc.getElementsByTagName("DOWNLINK_PRIORITY")[
        0
    ].firstChild.data
    datastrip_id = xmldoc.getElementsByTagName("DATASTRIP_ID")[0].firstChild.data
    solar_azimuth = float(
        xmldoc.getElementsByTagName("Mean_Sun_Angle")[0]
        .getElementsByTagName("ZENITH_ANGLE")[0]
        .firstChild.data
    )
    solar_zenith = float(
        xmldoc.getElementsByTagName("Mean_Sun_Angle")[0]
        .getElementsByTagName("AZIMUTH_ANGLE")[0]
        .firstChild.data
    )

    resolutions = xmldoc.getElementsByTagName("Size")
    r_list = []
    for i in resolutions:
        r_list.append(int(i.attributes["resolution"].value))
    resolution = min(r_list)

    return {
        "cloud": cloud,
        "downlink_priority": downlink_priority,
        "datastrip_id": datastrip_id,
        "solar_azimuth": solar_azimuth,
        "solar_zenith": solar_zenith,
        "resolution": resolution,
    }

def process_mtd_ds(mtd_ds_zip_path: str, zip_object: object) -> Dict:
    xmldoc = minidom.parseString(zip_object.read(mtd_ds_zip_path))

    reception_station = xmldoc.getElementsByTagName("RECEPTION_STATION")[
        0
    ].firstChild.data
    downlink_orbit_number = xmldoc.getElementsByTagName("DOWNLINK_ORBIT_NUMBER")[
        0
    ].firstChild.data
    processing_center = xmldoc.getElementsByTagName("PROCESSING_CENTER")[
        0
    ].firstChild.data

    resolutions = xmldoc.getElementsByTagName("RESOLUTION")
    r_list = []
    for i in resolutions:
        r_list.append(int(i.firstChild.data))
    resolution = min(r_list)
    return {
        "reception_station": reception_station,
        "downlink_orbit_number": downlink_orbit_number,
        "processing_center": processing_center,
        "resolution": resolution,
    }

def process_mtd_tl(mtd_tl_zip_path: str, zip_object: object) -> Dict:
    xmldoc = minidom.parseString(zip_object.read(mtd_tl_zip_path))

    sun_azimuth = xmldoc.getElementsByTagName("AZIMUTH_ANGLE")[0].firstChild.data
    sun_elevation = xmldoc.getElementsByTagName("ZENITH_ANGLE")[0].firstChild.data

    return {
        "sun_azimuth": sun_azimuth,
        "sun_elevation": sun_elevation,
    }

def process_mtd_msil1c(mtd_msil1c_zip_path: str, zip_object: object) -> Dict:
    xmldoc = minidom.parseString(zip_object.read(mtd_msil1c_zip_path))

    data_type = xmldoc.getElementsByTagName("PROCESSING_LEVEL")[0].firstChild.data
    datastrip_id = xmldoc.getElementsByTagName("PRODUCT_URI")[0].firstChild.data
    product_type = xmldoc.getElementsByTagName("PRODUCT_TYPE")[0].firstChild.data
    platform = xmldoc.getElementsByTagName("SPACECRAFT_NAME")[0].firstChild.data
    orbit = xmldoc.getElementsByTagName("SENSING_ORBIT_NUMBER")[0].firstChild.data
    orbit_direction = xmldoc.getElementsByTagName("SENSING_ORBIT_DIRECTION")[
        0
    ].firstChild.data
    datatake_type = xmldoc.getElementsByTagName("DATATAKE_TYPE")[0].firstChild.data
    processing_datetime = xmldoc.getElementsByTagName("GENERATION_TIME")[
        0
    ].firstChild.data
    processing_baseline = xmldoc.getElementsByTagName("PROCESSING_BASELINE")[
        0
    ].firstChild.data
    datetime = xmldoc.getElementsByTagName("PRODUCT_START_TIME")[0].firstChild.data
    cloud_cover = xmldoc.getElementsByTagName("Cloud_Coverage_Assessment")[
        0
    ].firstChild.data
    region_code = datastrip_id.split("_")[5][1:]

    return {
        "data_type": data_type,
        "datastrip_id": datastrip_id,
        "product_type": product_type,
        "platform": platform,
        "orbit": orbit,
        "orbit_direction": orbit_direction,
        "datatake_type": datatake_type,
        "processing_datetime": processing_datetime,
        "processing_baseline": processing_baseline,
        "datetime": datetime,
        "cloud_cover": cloud_cover,
        "region_code": region_code,
    }

def process_format_correctness(
    format_correctness_path: str,
    zip_object: object,
) -> Dict:
    if zip_object is not None:
        xmldoc = minidom.parseString(zip_object.read(format_correctness_path))

        source_system = xmldoc.getElementsByTagName("System")[0].firstChild.data
        software_version = xmldoc.getElementsByTagName("Creator_Version")[
            0
        ].firstChild.data

        return {
            "source_system": source_system,
            "software_version": software_version,
        }
    else:
        xmldoc = minidom.parse(format_correctness_path)

        source_system = xmldoc.getElementsByTagName("System")[0].firstChild.data
        creator_version = xmldoc.getElementsByTagName("Creator_Version")[
            0
        ].firstChild.data
        creation_date = xmldoc.getElementsByTagName("Creation_Date")[0].firstChild.data
        datastrip_metadata = xmldoc.getElementsByTagName("File_Name")[0].firstChild.data

        return {
            "source_system": source_system,
            "creator_version": creator_version,
            "creation_date": creation_date,
            "datastrip_metadata": datastrip_metadata,
        }

def prepare_and_write(
    dataset: Path,
    dataset_document: Path,
) -> Tuple[uuid.UUID, Path]:
    # Process esa dataset
    if "zip" in str(dataset):
        with zipfile.ZipFile(dataset, "r") as z:
            # Get file paths for esa metadata files
            mtd_ds_zip_path = [s for s in z.namelist() if "MTD_DS.xml" in s][0]
            mtd_tl_zip_path = [s for s in z.namelist() if "MTD_TL.xml" in s][0]
            mtd_msil1c_zip_path = [s for s in z.namelist() if "MTD_MSIL1C.xml" in s][0]
            format_correctness_zip_path = [
                s for s in z.namelist() if "FORMAT_CORRECTNESS.xml" in s
            ][0]

            # Crawl through metadata files and return a dict of useful information
            mtd_ds = process_mtd_ds(mtd_ds_zip_path, z)
            mtd_tl = process_mtd_tl(mtd_tl_zip_path, z)
            format_correctness = process_format_correctness(
                format_correctness_zip_path, z
            )
            mtd_msil1c = process_mtd_msil1c(mtd_msil1c_zip_path, z)

            with DatasetAssembler(
                metadata_path=dataset_document,
                dataset_location=dataset,
            ) as p:

                p.datetime = mtd_msil1c["datetime"]
                p.properties["eo:instrument"] = HARDCODED["instrument"]
                p.properties["eo:platform"] = mtd_msil1c["platform"]
                p.properties["odc:processing_datetime"] = mtd_msil1c[
                    "processing_datetime"
                ]
                p.properties["odc:dataset_version"] = f"1.0.{p.processed:%Y%m%d}"
                p.properties["odc:producer"] = "esa.int"
                p.properties["odc:product_family"] = HARDCODED["product_family"]
                p.properties["eo:sun_elevation"] = mtd_tl["sun_elevation"]
                p.properties["eo:sun_azimuth"] = mtd_tl["sun_azimuth"]
                p.properties["eo:gsd"] = mtd_ds["resolution"]
                p.properties["eo:cloud_cover"] = mtd_msil1c["cloud_cover"]
                p.properties["odc:file_format"] = HARDCODED["file_format"]
                p.properties["odc:region_code"] = mtd_msil1c["region_code"]
                p.properties["sentinel:data_type"] = mtd_msil1c["data_type"]
                p.properties["sentinel:product_type"] = mtd_msil1c["product_type"]
                p.properties["sentinel:software_version"] = format_correctness[
                    "software_version"
                ]
                p.properties["sentinel:source_system"] = format_correctness[
                    "source_system"
                ]
                p.properties["sentinel:datastrip_id"] = mtd_msil1c["datastrip_id"]
                p.properties["sentinel:downlink_orbit_number"] = mtd_ds[
                    "downlink_orbit_number"
                ]
                p.properties["sentinel:reception_station"] = mtd_ds["reception_station"]
                p.properties["sentinel:processing_center"] = mtd_ds["processing_center"]
                p.properties["sentinel:orbit"] = mtd_msil1c["orbit"]
                p.properties["sentinel:orbit_direction"] = mtd_msil1c["orbit_direction"]
                p.properties["sentinel:datatake_type"] = mtd_msil1c["datatake_type"]
                p.properties["sentinel:processing_baseline"] = mtd_msil1c[
                    "processing_baseline"
                ]

                for file in z.namelist():
                    # T55HFA_20201011T000249_B01.jp2
                    if ".jp2" in file and "TCI" not in file and "PVI" not in file:
                        band = (
                            file.split("_")[len(file.split("_")) - 1]
                            .replace(".jp2", "")
                            .replace("B", "")
                        )
                        name = SENTINEL_MSI_BAND_ALIASES[band]
                        # path = 'zip:%s!%s' % (str(dataset), str(file))
                        p.note_measurement(
                            path=file,
                            name=name,
                            relative_to_dataset_location=True
                            # path=path, name=name
                        )

                return p.done()

    # process sinergise dataset
    else:
        directory = [f for f in listdir(dataset) if isfile(join(dataset, f))]

        # Get file paths for sinergise metadata files
        product_info_path = find_metadata_path("productInfo.json", dataset)
        metadata_xml_path = find_metadata_path("metadata.xml", dataset)
        format_correctness_path = find_metadata_path("FORMAT_CORRECTNESS.xml", dataset)

        # Crawl through metadata files and return a dict of useful information
        product_info = process_product_info(product_info_path)
        metadata_xml = process_metadata_xml(metadata_xml_path)
        format_correctness = process_format_correctness(format_correctness_path, None)

        with DatasetAssembler(
            metadata_path=dataset_document,
            dataset_location=dataset,
        ) as p:
            p.datetime = product_info["timestamp"]
            p.properties["eo:instrument"] = HARDCODED["instrument"]
            p.properties["eo:platform"] = "sentinel-2a"
            p.properties["odc:processing_datetime"] = (
                format_correctness["creation_date"].split("=")[1].replace("T", " ")
            )
            p.properties["odc:dataset_version"] = f"1.0.{p.processed:%Y%m%d}"
            p.properties["odc:producer"] = "sinergise.com"
            p.properties["odc:product_family"] = HARDCODED["product_family"]
            p.properties["eo:sun_elevation"] = metadata_xml["solar_zenith"]
            p.properties["eo:sun_azimuth"] = metadata_xml["solar_azimuth"]
            p.properties["eo:gsd"] = metadata_xml["resolution"]
            p.properties["eo:cloud_cover"] = metadata_xml["cloud"]
            p.properties["sentinel:sinergise_product_name"] = product_info[
                "synergise_product_name"
            ]
            p.properties["sentinel:sinergise_product_id"] = product_info[
                "synergise_product_id"
            ]
            p.properties["odc:file_format"] = HARDCODED["file_format"]
            p.properties["odc:region_code"] = product_info["region_code"]
            p.properties["sentinel:data_type"] = HARDCODED["data_type"]
            p.properties["sentinel:product_type"] = HARDCODED["product_type"]
            p.properties["sentinel:software_version"] = format_correctness[
                "creator_version"
            ]
            p.properties["sentinel:source_system"] = format_correctness["source_system"]
            p.properties["sentinel:datastrip_metadata"] = format_correctness[
                "datastrip_metadata"
            ]
            p.properties["sentinel:downlink_priority"] = metadata_xml[
                "downlink_priority"
            ]
            p.properties["sentinel:datastrip_id"] = metadata_xml["datastrip_id"]

            for ds in directory:
                if ".jp2" in ds and "preview" not in ds and "TCI" not in ds:
                    band = ds.replace(".jp2", "").replace("B", "")
                    name = SENTINEL_MSI_BAND_ALIASES[band]
                    path = Path(dataset) / ds
                    p.note_measurement(path=path, name=name)

            return p.done()

@click.command(help=__doc__)
@click.option(
    "--dataset",
    type=PathPath(),
    required=True,
    help="Path to ESA zipped dataset or Sinergise dataset directory",
)
@click.option(
    "--dataset-document",
    type=PathPath(),
    required=True,
    help="Location to output the L1C dataset document (yaml)",
)
def main(
    dataset: Path,
    dataset_document: Path,
):

    uuid, path = prepare_and_write(
        dataset,
        dataset_document,
    )
    return path

if __name__ == "__main__":
    main()
<commit_msg>	modified:   sentinel_l1c_prepare.py 	deleted:    sentinel_L1C_prepare.py modified:   ../scripts/prepare.py<commit_after>"""
Prepare eo3 metadata for Sentinel-2 Level 1C data produced by Sinergise or esa.
"""

from pathlib import Path
from xml.dom import minidom
import json
import os
from os import listdir
from os.path import isfile, join
import uuid
import click
from typing import Tuple, Dict
from eodatasets3 import DatasetAssembler
from eodatasets3.ui import PathPath
import zipfile

HARDCODED = {
    "file_format": "JPEG2000",
    "instrument": "MSI",
    "product_family": "level1",
    "data_type": "Level-1C",
    "product_type": "S2MSI1C",
}

SENTINEL_MSI_BAND_ALIASES = {
    "01": "coastal_aerosol",
    "02": "blue",
    "03": "green",
    "04": "red",
    "05": "red_edge_1",
    "06": "red_edge_2",
    "07": "red_edge_3",
    "08": "nir_1",
    "8A": "nir_2",
    "09": "water_vapour",
    "10": "swir_1_cirrus",
    "11": "swir_2",
    "12": "swir_3",
}

def find_metadata_path(find_filename, dataset_dir):
    for root, dir, fn in os.walk(dataset_dir):
        if find_filename in fn:
            return os.path.join(root, fn[fn.index(find_filename)])

def process_product_info(product_path):
    with open(product_path) as fp:
        product = json.loads(fp.read())

        synergise_product_name = product["name"]
        synergise_product_id = product["id"]
        if len(product["tiles"]) > 1:
            raise NotImplementedError("No support for multi-tiled products yet")
        timestamp = product["tiles"][0]["timestamp"]
        utm_zone = product["tiles"][0]["utmZone"]
        latitude_band = product["tiles"][0]["latitudeBand"]
        grid_square = product["tiles"][0]["gridSquare"]
        region_code = "%s%s%s" % (utm_zone, latitude_band, grid_square)

        return {
            "synergise_product_name": synergise_product_name,
            "synergise_product_id": synergise_product_id,
            "timestamp": timestamp,
            "region_code": region_code,
        }

def process_metadata_xml(metadata_xml_path: str) -> Dict:
    xmldoc = minidom.parse(metadata_xml_path)

    cloud = float(
        xmldoc.getElementsByTagName("CLOUDY_PIXEL_PERCENTAGE")[0].firstChild.data
    )
    downlink_priority = xmldoc.getElementsByTagName("DOWNLINK_PRIORITY")[
        0
    ].firstChild.data
    datastrip_id = xmldoc.getElementsByTagName("DATASTRIP_ID")[0].firstChild.data
    solar_azimuth = float(
        xmldoc.getElementsByTagName("Mean_Sun_Angle")[0]
        .getElementsByTagName("ZENITH_ANGLE")[0]
        .firstChild.data
    )
    solar_zenith = float(
        xmldoc.getElementsByTagName("Mean_Sun_Angle")[0]
        .getElementsByTagName("AZIMUTH_ANGLE")[0]
        .firstChild.data
    )

    resolutions = xmldoc.getElementsByTagName("Size")
    r_list = []
    for i in resolutions:
        r_list.append(int(i.attributes["resolution"].value))
    resolution = min(r_list)

    return {
        "cloud": cloud,
        "downlink_priority": downlink_priority,
        "datastrip_id": datastrip_id,
        "solar_azimuth": solar_azimuth,
        "solar_zenith": solar_zenith,
        "resolution": resolution,
    }

def process_mtd_ds(mtd_ds_zip_path: str, zip_object: object) -> Dict:
    xmldoc = minidom.parseString(zip_object.read(mtd_ds_zip_path))

    reception_station = xmldoc.getElementsByTagName("RECEPTION_STATION")[
        0
    ].firstChild.data
    downlink_orbit_number = xmldoc.getElementsByTagName("DOWNLINK_ORBIT_NUMBER")[
        0
    ].firstChild.data
    processing_center = xmldoc.getElementsByTagName("PROCESSING_CENTER")[
        0
    ].firstChild.data

    resolutions = xmldoc.getElementsByTagName("RESOLUTION")
    r_list = []
    for i in resolutions:
        r_list.append(int(i.firstChild.data))
    resolution = min(r_list)
    return {
        "reception_station": reception_station,
        "downlink_orbit_number": downlink_orbit_number,
        "processing_center": processing_center,
        "resolution": resolution,
    }

def process_mtd_tl(mtd_tl_zip_path: str, zip_object: object) -> Dict:
    xmldoc = minidom.parseString(zip_object.read(mtd_tl_zip_path))

    sun_azimuth = xmldoc.getElementsByTagName("AZIMUTH_ANGLE")[0].firstChild.data
    sun_elevation = xmldoc.getElementsByTagName("ZENITH_ANGLE")[0].firstChild.data

    return {
        "sun_azimuth": sun_azimuth,
        "sun_elevation": sun_elevation,
    }

def process_mtd_msil1c(mtd_msil1c_zip_path: str, zip_object: object) -> Dict:
    xmldoc = minidom.parseString(zip_object.read(mtd_msil1c_zip_path))

    data_type = xmldoc.getElementsByTagName("PROCESSING_LEVEL")[0].firstChild.data
    datastrip_id = xmldoc.getElementsByTagName("PRODUCT_URI")[0].firstChild.data
    product_type = xmldoc.getElementsByTagName("PRODUCT_TYPE")[0].firstChild.data
    platform = xmldoc.getElementsByTagName("SPACECRAFT_NAME")[0].firstChild.data
    orbit = xmldoc.getElementsByTagName("SENSING_ORBIT_NUMBER")[0].firstChild.data
    orbit_direction = xmldoc.getElementsByTagName("SENSING_ORBIT_DIRECTION")[
        0
    ].firstChild.data
    datatake_type = xmldoc.getElementsByTagName("DATATAKE_TYPE")[0].firstChild.data
    processing_datetime = xmldoc.getElementsByTagName("GENERATION_TIME")[
        0
    ].firstChild.data
    processing_baseline = xmldoc.getElementsByTagName("PROCESSING_BASELINE")[
        0
    ].firstChild.data
    datetime = xmldoc.getElementsByTagName("PRODUCT_START_TIME")[0].firstChild.data
    cloud_cover = xmldoc.getElementsByTagName("Cloud_Coverage_Assessment")[
        0
    ].firstChild.data
    region_code = datastrip_id.split("_")[5][1:]

    return {
        "data_type": data_type,
        "datastrip_id": datastrip_id,
        "product_type": product_type,
        "platform": platform,
        "orbit": orbit,
        "orbit_direction": orbit_direction,
        "datatake_type": datatake_type,
        "processing_datetime": processing_datetime,
        "processing_baseline": processing_baseline,
        "datetime": datetime,
        "cloud_cover": cloud_cover,
        "region_code": region_code,
    }

def process_format_correctness(
    format_correctness_path: str,
    zip_object: object,
) -> Dict:
    if zip_object is not None:
        xmldoc = minidom.parseString(zip_object.read(format_correctness_path))

        source_system = xmldoc.getElementsByTagName("System")[0].firstChild.data
        software_version = xmldoc.getElementsByTagName("Creator_Version")[
            0
        ].firstChild.data

        return {
            "source_system": source_system,
            "software_version": software_version,
        }
    else:
        xmldoc = minidom.parse(format_correctness_path)

        source_system = xmldoc.getElementsByTagName("System")[0].firstChild.data
        creator_version = xmldoc.getElementsByTagName("Creator_Version")[
            0
        ].firstChild.data
        creation_date = xmldoc.getElementsByTagName("Creation_Date")[0].firstChild.data
        datastrip_metadata = xmldoc.getElementsByTagName("File_Name")[0].firstChild.data

        return {
            "source_system": source_system,
            "creator_version": creator_version,
            "creation_date": creation_date,
            "datastrip_metadata": datastrip_metadata,
        }

def prepare_and_write(
    dataset: Path,
    dataset_document: Path,
) -> Tuple[uuid.UUID, Path]:
    # Process esa dataset
    if "S2A_MSIL1C_" in str(dataset) or "S2B_MSIL1C_" in str(dataset):
        with zipfile.ZipFile(dataset, "r") as z:
            # Get file paths for esa metadata files
            mtd_ds_zip_path = [s for s in z.namelist() if "MTD_DS.xml" in s][0]
            mtd_tl_zip_path = [s for s in z.namelist() if "MTD_TL.xml" in s][0]
            mtd_msil1c_zip_path = [s for s in z.namelist() if "MTD_MSIL1C.xml" in s][0]
            format_correctness_zip_path = [
                s for s in z.namelist() if "FORMAT_CORRECTNESS.xml" in s
            ][0]

            # Crawl through metadata files and return a dict of useful information
            mtd_ds = process_mtd_ds(mtd_ds_zip_path, z)
            mtd_tl = process_mtd_tl(mtd_tl_zip_path, z)
            format_correctness = process_format_correctness(
                format_correctness_zip_path, z
            )
            mtd_msil1c = process_mtd_msil1c(mtd_msil1c_zip_path, z)

            with DatasetAssembler(
                metadata_path=dataset_document,
                dataset_location=dataset,
            ) as p:

                p.datetime = mtd_msil1c["datetime"]
                p.properties["eo:instrument"] = HARDCODED["instrument"]
                p.properties["eo:platform"] = mtd_msil1c["platform"]
                p.properties["odc:processing_datetime"] = mtd_msil1c[
                    "processing_datetime"
                ]
                p.properties["odc:dataset_version"] = f"1.0.{p.processed:%Y%m%d}"
                p.properties["odc:producer"] = "esa.int"
                p.properties["odc:product_family"] = HARDCODED["product_family"]
                p.properties["eo:sun_elevation"] = mtd_tl["sun_elevation"]
                p.properties["eo:sun_azimuth"] = mtd_tl["sun_azimuth"]
                p.properties["eo:gsd"] = mtd_ds["resolution"]
                p.properties["eo:cloud_cover"] = mtd_msil1c["cloud_cover"]
                p.properties["odc:file_format"] = HARDCODED["file_format"]
                p.properties["odc:region_code"] = mtd_msil1c["region_code"]
                p.properties["sentinel:data_type"] = mtd_msil1c["data_type"]
                p.properties["sentinel:product_type"] = mtd_msil1c["product_type"]
                p.properties["sentinel:software_version"] = format_correctness[
                    "software_version"
                ]
                p.properties["sentinel:source_system"] = format_correctness[
                    "source_system"
                ]
                p.properties["sentinel:datastrip_id"] = mtd_msil1c["datastrip_id"]
                p.properties["sentinel:downlink_orbit_number"] = mtd_ds[
                    "downlink_orbit_number"
                ]
                p.properties["sentinel:reception_station"] = mtd_ds["reception_station"]
                p.properties["sentinel:processing_center"] = mtd_ds["processing_center"]
                p.properties["sentinel:orbit"] = mtd_msil1c["orbit"]
                p.properties["sentinel:orbit_direction"] = mtd_msil1c["orbit_direction"]
                p.properties["sentinel:datatake_type"] = mtd_msil1c["datatake_type"]
                p.properties["sentinel:processing_baseline"] = mtd_msil1c[
                    "processing_baseline"
                ]

                for file in z.namelist():
                    # T55HFA_20201011T000249_B01.jp2
                    if ".jp2" in file and "TCI" not in file and "PVI" not in file:
                        band = (
                            file.split("_")[len(file.split("_")) - 1]
                            .replace(".jp2", "")
                            .replace("B", "")
                        )
                        name = SENTINEL_MSI_BAND_ALIASES[band]
                        # path = 'zip:%s!%s' % (str(dataset), str(file))
                        p.note_measurement(
                            path=file,
                            name=name,
                            relative_to_dataset_location=True
                            # path=path, name=name
                        )

                return p.done()

    # process sinergise dataset
    else:
        directory = [f for f in listdir(dataset) if isfile(join(dataset, f))]

        # Get file paths for sinergise metadata files
        product_info_path = find_metadata_path("productInfo.json", dataset)
        metadata_xml_path = find_metadata_path("metadata.xml", dataset)
        format_correctness_path = find_metadata_path("FORMAT_CORRECTNESS.xml", dataset)

        # Crawl through metadata files and return a dict of useful information
        product_info = process_product_info(product_info_path)
        metadata_xml = process_metadata_xml(metadata_xml_path)
        format_correctness = process_format_correctness(format_correctness_path, None)

        with DatasetAssembler(
            metadata_path=dataset_document,
            dataset_location=dataset,
        ) as p:
            p.datetime = product_info["timestamp"]
            p.properties["eo:instrument"] = HARDCODED["instrument"]
            p.properties["eo:platform"] = "sentinel-2a"
            p.properties["odc:processing_datetime"] = (
                format_correctness["creation_date"].split("=")[1].replace("T", " ")
            )
            p.properties["odc:dataset_version"] = f"1.0.{p.processed:%Y%m%d}"
            p.properties["odc:producer"] = "sinergise.com"
            p.properties["odc:product_family"] = HARDCODED["product_family"]
            p.properties["eo:sun_elevation"] = metadata_xml["solar_zenith"]
            p.properties["eo:sun_azimuth"] = metadata_xml["solar_azimuth"]
            p.properties["eo:gsd"] = metadata_xml["resolution"]
            p.properties["eo:cloud_cover"] = metadata_xml["cloud"]
            p.properties["sentinel:sinergise_product_name"] = product_info[
                "synergise_product_name"
            ]
            p.properties["sentinel:sinergise_product_id"] = product_info[
                "synergise_product_id"
            ]
            p.properties["odc:file_format"] = HARDCODED["file_format"]
            p.properties["odc:region_code"] = product_info["region_code"]
            p.properties["sentinel:data_type"] = HARDCODED["data_type"]
            p.properties["sentinel:product_type"] = HARDCODED["product_type"]
            p.properties["sentinel:software_version"] = format_correctness[
                "creator_version"
            ]
            p.properties["sentinel:source_system"] = format_correctness["source_system"]
            p.properties["sentinel:datastrip_metadata"] = format_correctness[
                "datastrip_metadata"
            ]
            p.properties["sentinel:downlink_priority"] = metadata_xml[
                "downlink_priority"
            ]
            p.properties["sentinel:datastrip_id"] = metadata_xml["datastrip_id"]

            for ds in directory:
                if ".jp2" in ds and "preview" not in ds and "TCI" not in ds:
                    band = ds.replace(".jp2", "").replace("B", "")
                    name = SENTINEL_MSI_BAND_ALIASES[band]
                    path = Path(dataset) / ds
                    p.note_measurement(path=path, name=name)

            return p.done()

@click.command(help=__doc__)
@click.option(
    "--dataset",
    type=PathPath(),
    required=True,
    help="Path to ESA zipped dataset or Sinergise dataset directory",
)
@click.option(
    "--dataset-document",
    type=PathPath(),
    required=True,
    help="Location to output the L1C dataset document (yaml)",
)
def main(
    dataset: Path,
    dataset_document: Path,
):

    uuid, path = prepare_and_write(
        dataset,
        dataset_document,
    )
    return path

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import keyring
import os
import json
import six
from keyring.errors import PasswordDeleteError
from ..cache import CACHE_ROOT_DIR as _DEFAULT_CACHE_ROOT_DIR
from ..utils import equal_paths

SYNAPSE_CACHED_SESSION_APLICATION_NAME = "SYNAPSE.ORG_CLIENT"
SESSION_CACHE_FILEPATH = os.path.expanduser("~/.synapseSession")

def get_api_key(username):
    """
    Retrieves the user's API key
    :param str username:
    :return: API key for the specified username
    :rtype: str
    """
    if username is not None:
            return keyring.get_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username)
    return None

def remove_api_key(username):
        try:
            keyring.delete_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username)
        except PasswordDeleteError:
            #The API key does not exist, but that is fine
            pass

def set_api_key(username, api_key):
    keyring.set_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username, api_key)

def get_most_recent_user():
    session_cache = _read_session_cache(SESSION_CACHE_FILEPATH)
    return session_cache.get("<mostRecent>")

def set_most_recent_user(username):
    cachedSessions = {"<mostRecent>": username}
    _write_session_cache(SESSION_CACHE_FILEPATH, cachedSessions)

def _read_session_cache(filepath):
    """Returns the JSON contents of CACHE_DIR/SESSION_FILENAME."""
    try:
        file = open(filepath, 'r')
        result = json.load(file)
        if isinstance(result, dict):
            return result
    except: pass
    return {}

def _write_session_cache(filepath, data):
    """Dumps the JSON data into CACHE_DIR/SESSION_FILENAME."""
    with open(filepath, 'w') as file:
        json.dump(data, file)
        file.write('\n') # For compatibility with R's JSON parser

def migrate_old_session_file_credentials_if_necessary(syn):
    old_session_file_path = os.path.join(syn.cache.cache_root_dir, '.session')

    # only migrate if the download cache is in the default location (i.e. user did set its location)
    # we don't want to migrate credentials if they were a part of a cache shared by multiple people
    if equal_paths(syn.cache.cache_root_dir, os.path.expanduser(_DEFAULT_CACHE_ROOT_DIR)):
        #iterate throught the old file and place in new credential storage
        old_session_dict = _read_session_cache(old_session_file_path)
        for key, value in six.iteritems(old_session_dict):
            if key == "<mostRecent>":
                set_most_recent_user(value)
            else:
                set_api_key(key, value)
    #always attempt to remove the old session file
    try:
        os.remove(old_session_file_path)
    except OSError:
        #file already removed.
        pass<commit_msg>fixed comment typo<commit_after>import keyring
import os
import json
import six
from keyring.errors import PasswordDeleteError
from ..cache import CACHE_ROOT_DIR as _DEFAULT_CACHE_ROOT_DIR
from ..utils import equal_paths

SYNAPSE_CACHED_SESSION_APLICATION_NAME = "SYNAPSE.ORG_CLIENT"
SESSION_CACHE_FILEPATH = os.path.expanduser("~/.synapseSession")

def get_api_key(username):
    """
    Retrieves the user's API key
    :param str username:
    :return: API key for the specified username
    :rtype: str
    """
    if username is not None:
            return keyring.get_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username)
    return None

def remove_api_key(username):
        try:
            keyring.delete_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username)
        except PasswordDeleteError:
            #The API key does not exist, but that is fine
            pass

def set_api_key(username, api_key):
    keyring.set_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username, api_key)

def get_most_recent_user():
    session_cache = _read_session_cache(SESSION_CACHE_FILEPATH)
    return session_cache.get("<mostRecent>")

def set_most_recent_user(username):
    cachedSessions = {"<mostRecent>": username}
    _write_session_cache(SESSION_CACHE_FILEPATH, cachedSessions)

def _read_session_cache(filepath):
    """Returns the JSON contents of CACHE_DIR/SESSION_FILENAME."""
    try:
        file = open(filepath, 'r')
        result = json.load(file)
        if isinstance(result, dict):
            return result
    except: pass
    return {}

def _write_session_cache(filepath, data):
    """Dumps the JSON data into CACHE_DIR/SESSION_FILENAME."""
    with open(filepath, 'w') as file:
        json.dump(data, file)
        file.write('\n') # For compatibility with R's JSON parser

def migrate_old_session_file_credentials_if_necessary(syn):
    old_session_file_path = os.path.join(syn.cache.cache_root_dir, '.session')

    # only migrate if the download cache is in the default location (i.e. user did not set its location)
    # we don't want to migrate credentials if they were a part of a cache shared by multiple people
    if equal_paths(syn.cache.cache_root_dir, os.path.expanduser(_DEFAULT_CACHE_ROOT_DIR)):
        #iterate throught the old file and place in new credential storage
        old_session_dict = _read_session_cache(old_session_file_path)
        for key, value in six.iteritems(old_session_dict):
            if key == "<mostRecent>":
                set_most_recent_user(value)
            else:
                set_api_key(key, value)
    #always attempt to remove the old session file
    try:
        os.remove(old_session_file_path)
    except OSError:
        #file already removed.
        pass<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#  Copyright 2011 Kévin Gomez <contact@kevingomez.fr>
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  The Software is provided "as is", without warranty of any kind, express or
#  implied, including but not limited to the warranties of merchantability,
#  fitness for a particular purpose and noninfringement. In no event shall the
#  authors or copyright holders be liable for any claim, damages or other
#  liability, whether in an action of contract, tort or otherwise, arising
#  from, out of or in connection with the software or the use or other dealing
#  in the Software.

from functools import wraps

def provider_required(func):
    """
        Decorator that forces the selection of a provider to execute the given
        function. If no provider is selected, a ValueError exception is raised.
    """

    @wraps(func)
    def wrapper(self, *args, **kwargs):
        if self.provider is None:
            raise ValueError('You must select a provider to do this')

        return func(self, *args, **kwargs)

    return wrapper

class GigaDl(object):
    """ Download manager class """

    def __init__(self, provider=None):
        self._providers = {}
        self._provider = provider

        self.register_provider(provider)

    @property
    def provider(self):
        """ Returns the current provider """

        return self._provider

    @provider.setter
    def provider(self, provider):
        """ Defines the provider to use (expects an object) """

        self.use(provider)

    @provider.deleter
    def provider(self):
        """ Deletes the current selected provider """

        del self._providers[repr(self._provider)]
        self._provider = None

    def register_provider(self, provider, name=None):
        """ Register the given lists of providers """

        if provider is not None:
            self._providers[name or repr(provider)] = provider

        return self

    def register_providers(self, *args, **kwargs):
        """ Register the given lists of providers """

        for provider in args:
            self.register_provider(provider)

        for name, provider in kwargs.items():
            self.register_provider(provider, name)

        return self

    def use(self, provider):
        """ Selects the provider to use """

        if type(provider) is not str and not provider in self:
            self.register_provider(provider)

        provider_name = provider if type(provider) is str else repr(provider)

        try:
            self._provider = self._providers[provider_name]
        except KeyError:
            raise KeyError('The provider "%s" is unknown' % provider_name)

        return self

    @provider_required
    def search(self, terms):
        """
          Use the selected provider to search items matching the given terms.
        """

        if not terms:
            return []

        return self.provider.search(terms)

    @provider_required
    def retrieve_data(self, url):
        """
            Use the selected provider to retrieve the data corresponding to
            the given URL
        """

        if not url:
            raise ValueError('You must pass an URL to retrieve')

        return self.provider.retrieve_data(url)

    def __iter__(self):
        """ Iterator for looping through the providers """

        return iter(self._providers)

    def __contains__(self, item):
        """ Test if the given item in registered as a provider """

        if type(item) is str:
            return item in self._providers

        return item in self._providers.values()

    def __len__(self):
        """ Counts the registered providers """

        return len(self._providers)
<commit_msg>Chg: added more comments for the GigaDl class<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#  Copyright 2011 Kévin Gomez <contact@kevingomez.fr>
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  The Software is provided "as is", without warranty of any kind, express or
#  implied, including but not limited to the warranties of merchantability,
#  fitness for a particular purpose and noninfringement. In no event shall the
#  authors or copyright holders be liable for any claim, damages or other
#  liability, whether in an action of contract, tort or otherwise, arising
#  from, out of or in connection with the software or the use or other dealing
#  in the Software.

from functools import wraps

def provider_required(func):
    """
        Decorator that forces the selection of a provider to execute the given
        function. If no provider is selected, a ValueError exception is raised.
    """

    @wraps(func)
    def wrapper(self, *args, **kwargs):
        if self.provider is None:
            raise ValueError('You must select a provider to do this')

        return func(self, *args, **kwargs)

    return wrapper

class GigaDl(object):
    """ Download manager class """

    def __init__(self, provider=None):
        self._providers = {}
        self._provider = provider

        self.register_provider(provider)

    @property
    def provider(self):
        """ Returns the current provider """

        return self._provider

    @provider.setter
    def provider(self, provider):
        """ Defines the provider to use (expects an object or a provider name) """

        self.use(provider)

    @provider.deleter
    def provider(self):
        """ Deletes the current selected provider and unregister it """

        del self._providers[repr(self._provider)]
        self._provider = None

    def register_provider(self, provider, name=None):
        """
            Register the given provider. If no name specified, repr(provider)
            will be used.
        """

        if provider is not None:
            self._providers[name or repr(provider)] = provider

        return self

    def register_providers(self, *args, **kwargs):
        """ Register the given providers """

        for provider in args:
            self.register_provider(provider)

        for name, provider in kwargs.items():
            self.register_provider(provider, name)

        return self

    def use(self, provider):
        """
            Selects the provider to use. If a string is given, we expect it to
            be an already registered provider. Otherwise we register the given
            provider if it's not already done and we select it.
        """

        if type(provider) is not str and not provider in self:
            self.register_provider(provider)

        provider_name = provider if type(provider) is str else repr(provider)

        try:
            self._provider = self._providers[provider_name]
        except KeyError:
            raise KeyError('The provider "%s" is unknown' % provider_name)

        return self

    @provider_required
    def search(self, terms):
        """
          Use the selected provider to search items matching the given terms.
        """

        if not terms:
            return []

        return self.provider.search(terms)

    @provider_required
    def retrieve_data(self, url):
        """
            Use the selected provider to retrieve the data corresponding to
            the given URL.
        """

        if not url:
            raise ValueError('You must pass an URL to retrieve')

        return self.provider.retrieve_data(url)

    def __iter__(self):
        """ Iterator for looping through the providers """

        return iter(self._providers)

    def __contains__(self, item):
        """ Test if the given item (name or provider) in registered as a provider """

        if type(item) is str:
            return item in self._providers

        return item in self._providers.values()

    def __len__(self):
        """ Counts the registered providers """

        return len(self._providers)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import subprocess

def do_fetch(repo, remotes):

    ORIGINAL_COLOR = "\033[0m" # mvtodo: would be better to try to detect the terminal's current standard color
    report = []

    print("\n* Fetching %s ..." % repo)
    try:
        out = subprocess.check_output(["git", "-C", repo, "fetch", "--multiple"] + remotes)
        out = "OK."
        color = "\033[32m" # green
    except OSError as oser:
        out = "Failed."
        color = "\033[31m" # red
    except subprocess.CalledProcessError as cper:
        out = "Failed."
        color = "\033[31m" # red

    report.append("%s%s: %s%s" % (color, repo, out, ORIGINAL_COLOR))

    return report

<commit_msg>removing unused import<commit_after>#!/usr/bin/env python

import subprocess

def do_fetch(repo, remotes):

    ORIGINAL_COLOR = "\033[0m" # mvtodo: would be better to try to detect the terminal's current standard color
    report = []

    print("\n* Fetching %s ..." % repo)
    try:
        out = subprocess.check_output(["git", "-C", repo, "fetch", "--multiple"] + remotes)
        out = "OK."
        color = "\033[32m" # green
    except OSError as oser:
        out = "Failed."
        color = "\033[31m" # red
    except subprocess.CalledProcessError as cper:
        out = "Failed."
        color = "\033[31m" # red

    report.append("%s%s: %s%s" % (color, repo, out, ORIGINAL_COLOR))

    return report

<|endoftext|>


<commit_before>
from __future__ import absolute_import

import datetime

import pymongo

import pmxbot
from . import storage
from . import logging
from pmxbot.core import on_join

class ParticipantLogger(storage.SelectableStorage):
	"Base class for logging participants"

	@classmethod
	def initialize(cls):
		cls.store = cls.from_URI(pmxbot.config.database)
		cls._finalizers.append(cls.finalize)

	@classmethod
	def finalize(cls):
		del cls.store

	def list_channels(self):
		return self._list_channels()

	def log_join(self, nick, channel):
		self.log(nick, channel, 'join')

	def log_leave(self, nick, channel):
		self.log(nick, channel, 'leave')

@on_join()
def log_join(nick, channel, **kwargs):
	if channel not in pmxbot.config.logged_channels:
		return
	ParticipantLogger.store.log_join(nick, channel)

class SQLiteLogger(ParticipantLogger, storage.SQLiteStorage):

	def init_tables(self):
		LOG_CREATE_SQL = '''
		CREATE TABLE IF NOT EXISTS rolls (
			id INTEGER NOT NULL,
			datetime DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
			channel VARCHAR NOT NULL,
			nick VARCHAR NOT NULL,
			change TEXT,
			PRIMARY KEY (id) )
		'''
		INDEX_DTC_CREATE_SQL = 'CREATE INDEX IF NOT EXISTS ix_rolls_datetime_channel ON rolls (datetime, channel)'
		INDEX_DT_CREATE_SQL = 'CREATE INDEX IF NOT EXISTS ix_rolls_datetime ON rolls (datetime desc)'
		self.db.execute(LOG_CREATE_SQL)
		self.db.execute(INDEX_DTC_CREATE_SQL)
		self.db.execute(INDEX_DT_CREATE_SQL)
		self.db.commit()

	def log(self, channel, nick, change):
		INSERT_LOG_SQL = 'INSERT INTO rolls (datetime, channel, nick, change) VALUES (?, ?, ?, ?)'
		now = datetime.datetime.utcnow()
		self.db.execute(INSERT_LOG_SQL, [now, channel, nick, change])
		self.db.commit()

class MongoDBLogger(ParticipantLogger, storage.MongoDBStorage):
	collection_name = 'rolls'

	def log(self, channel, nick, change):
		self.db.ensure_index([
			('datetime.d', pymongo.DESCENDING),
			('channel', pymongo.ASCENDING),
			])
		now = datetime.datetime.utcnow()
		self.db.insert(dict(channel=channel, nick=nick, change=change,
			datetime=logging.MongoDBLogger._fmt_date(now)))
<commit_msg>Use proper property<commit_after>
from __future__ import absolute_import

import datetime

import pymongo

import pmxbot
from . import storage
from . import logging
from pmxbot.core import on_join

class ParticipantLogger(storage.SelectableStorage):
	"Base class for logging participants"

	@classmethod
	def initialize(cls):
		cls.store = cls.from_URI(pmxbot.config.database)
		cls._finalizers.append(cls.finalize)

	@classmethod
	def finalize(cls):
		del cls.store

	def list_channels(self):
		return self._list_channels()

	def log_join(self, nick, channel):
		self.log(nick, channel, 'join')

	def log_leave(self, nick, channel):
		self.log(nick, channel, 'leave')

@on_join()
def log_join(nick, channel, **kwargs):
	if channel not in pmxbot.config.log_channels:
		return
	ParticipantLogger.store.log_join(nick, channel)

class SQLiteLogger(ParticipantLogger, storage.SQLiteStorage):

	def init_tables(self):
		LOG_CREATE_SQL = '''
		CREATE TABLE IF NOT EXISTS rolls (
			id INTEGER NOT NULL,
			datetime DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
			channel VARCHAR NOT NULL,
			nick VARCHAR NOT NULL,
			change TEXT,
			PRIMARY KEY (id) )
		'''
		INDEX_DTC_CREATE_SQL = 'CREATE INDEX IF NOT EXISTS ix_rolls_datetime_channel ON rolls (datetime, channel)'
		INDEX_DT_CREATE_SQL = 'CREATE INDEX IF NOT EXISTS ix_rolls_datetime ON rolls (datetime desc)'
		self.db.execute(LOG_CREATE_SQL)
		self.db.execute(INDEX_DTC_CREATE_SQL)
		self.db.execute(INDEX_DT_CREATE_SQL)
		self.db.commit()

	def log(self, channel, nick, change):
		INSERT_LOG_SQL = 'INSERT INTO rolls (datetime, channel, nick, change) VALUES (?, ?, ?, ?)'
		now = datetime.datetime.utcnow()
		self.db.execute(INSERT_LOG_SQL, [now, channel, nick, change])
		self.db.commit()

class MongoDBLogger(ParticipantLogger, storage.MongoDBStorage):
	collection_name = 'rolls'

	def log(self, channel, nick, change):
		self.db.ensure_index([
			('datetime.d', pymongo.DESCENDING),
			('channel', pymongo.ASCENDING),
			])
		now = datetime.datetime.utcnow()
		self.db.insert(dict(channel=channel, nick=nick, change=change,
			datetime=logging.MongoDBLogger._fmt_date(now)))
<|endoftext|>