<commit_before>#!/usr/bin/env python
"""
GAE app for gathering GitHub access tokens and Google emails.
Used for migrating issues from Google Code to GitHub.
"""

__author__ = "Eldar Abusalimov"

import json
import os
import urllib
import hashlib

import jinja2
import webapp2
from webapp2_extras import sessions
from google.appengine.ext import ndb

from authomatic import Authomatic
from authomatic import provider_id
from authomatic.adapters import Webapp2Adapter
from authomatic.providers import oauth2, BaseProvider
from authomatic.exceptions import BaseError
from authomatic.exceptions import HTTPError

import authomatic.core
import authomatic.providers
authomatic.providers.BaseProvider._logger = authomatic.core._logger  # XXX

import config

# Instantiate Authomatic.
auth = Authomatic(config={
        'google': {
            'id': authomatic.provider_id(),
            'class_': oauth2.Google,
            'consumer_key':    config.GOOGLE_AUTH['client_id'],
            'consumer_secret': config.GOOGLE_AUTH['client_secret'],
            'scope': ['email'],
        },

        'github': {
            'id': authomatic.provider_id(),
            'class_': oauth2.GitHub,
            'consumer_key':    config.GITHUB_AUTH['client_id'],
            'consumer_secret': config.GITHUB_AUTH['client_secret'],
            'scope': ['user:email', 'public_repo'],
        },
    },
    secret=config.SECRET)

def _monkey_patch_init_to_prompt_account(goog):
    orig_init = goog.__init__
    def init_with_prompting(self, *args, **kwargs):
        orig_init(self, *args, **kwargs)
        self.user_authorization_params['prompt'] = 'select_account'
    goog.__init__ = init_with_prompting
_monkey_patch_init_to_prompt_account(oauth2.Google)

DEFAULT_PROJECT_NAME = 'project'
USER_PROPS           = ['info', 'emails']
USER_INFO_PROPS      = ['id', 'username', 'name', 'link', 'picture']

def project_key(project_name=DEFAULT_PROJECT_NAME):
    """Constructs a Datastore key for a Project entity with project_name."""
    return ndb.Key('Project', project_name)

class User(ndb.Model):
    """Models an individual Project entry."""
    info   = ndb.JsonProperty(indexed=False)
    token  = ndb.StringProperty(indexed=False)
    emails = ndb.StringProperty(repeated=True)
    date   = ndb.DateTimeProperty(auto_now_add=True)

    @classmethod
    def query_all(cls, ancestor_key=project_key()):
        return cls.query(ancestor=ancestor_key).order(-cls.date)

templates_path = os.path.join(os.path.dirname(__file__), 'templates')
jinja2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))

class IndexHandler(webapp2.RequestHandler):
    INDEX_HTML = jinja2_env.get_template('index.html').render(config=config)
    def get(self):
        self.response.write(self.INDEX_HTML)

class GenericError(BaseError):
    def __init__(self, error):
        super(GenericError, self).__init__(str(error))

class BaseHandler(webapp2.RequestHandler):

    @webapp2.cached_property
    def session(self):
        # Returns a session using the default cookie key.
        return self.session_store.get_session(backend='memcache')

    def dispatch(self):
        # Get a session store for this request.
        self.session_store = sessions.get_store(request=self.request)

        try:
            # Dispatch the request.
            webapp2.RequestHandler.dispatch(self)
        finally:
            # Save all sessions.
            self.session_store.save_sessions(self.response)

    @webapp2.cached_property
    def user(self):
        user_key_id = self.session.get('user_key_id')
        if user_key_id:
            return User.get_by_id(user_key_id, parent=project_key())

    def put_user(self, user):
        if user:
            self.session['user_key_id'] = user.key.id()
        else:
            self.session.pop('user_key_id', None)

    def state_dict(self, **defaults):
        if self.user:
            defaults.update(user=self.user.to_dict(include=USER_PROPS))
        return defaults

    def get_user_by_email(self, email):
        users_query = User.query_all().filter(User.emails == email)
        users = list(users_query.fetch())
        if not users:
            return
        else:
            assert len(users) == 1
        return users[0]

    def add_user_email(self, user, email):
        old_user = self.get_user_by_email(email)
        if old_user:
            old_user.emails.remove(email)
            old_user.put()

        user.emails.append(email)

    @webapp2.cached_property
    def admin_user(self):
        user = self.get_user_by_email(config.ADMIN_EMAIL)
        if not user:
            self.abort(403)
        return user

    @webapp2.cached_property
    def admin_code(self):
        sha = hashlib.sha1()
        sha.update(config.ADMIN_EMAIL)
        sha.update(config.SECRET)
        sha.update(self.admin_user.token)
        return sha.hexdigest()

    @webapp2.cached_property
    def has_admin_rights(self):
        return (self.user == self.admin_user or
                self.request.get('code') == self.admin_code)

    def ensure_admin(self):
        if not self.has_admin_rights:
            self.abort(403)

class FormattedResultHandler(BaseHandler):

    def get(self, result_format):
        result = self.handle()
        self.respond(result, result_format)

    def handle(self):
        pass

    def respond(self, result, result_format):
        if result_format and result_format[0] == '.':
            result_format = result_format[1:]
        assert result_format in ('json', 'js') or not result_format, result_format

        def obj_to_dict(obj):
            if hasattr(obj, 'to_dict'):
                return obj.to_dict()

        result_json = json.dumps(result, default=obj_to_dict,
                                 indent=(4 if result_format == 'json' else None))

        if result_format == 'json':
            self.response.content_type = 'application/json'
            response_str = result_json

        elif result_format == 'js':
            self.response.content_type = 'application/javascript'
            response_str = ("handleState({result_json});"
                            .format(**locals()))

        else:
            response_str = """
            <!DOCTYPE html>

            """.format(**locals())

        self.response.write(response_str)

class StateHandler(FormattedResultHandler):
    def handle(self):
        return self.state_dict()

class LoginHandler(FormattedResultHandler):

    def any(self, provider_name='', result_format=None):
        # It all begins with login.
        def clean_session(result):  # XXX
            self.session.pop(auth.prefix, None)
        result = auth.login(Webapp2Adapter(self), provider_name,
                                  session=self.session,
                                  session_saver=self.session.on_update,
                                  callback=clean_session)
        if not result:
            # Do not write anything to the response if there is no result!
            return

        if result.user:
            if not result.error:
                try:
                    self.fetch_user_info(result)
                except BaseError as e:
                    result.error = e
                except Exception as e:
                    result.error = GenericError(e)

            if not result.error:
                self.login_successful(result)

        self.respond(self.state_dict(error=result.error), result_format)

    def fetch_user_info(self, result):
        response = result.user.update()
        response._data = result.user.data  # XXX
        self.ensure_response_http_ok(response)

        if result.provider.name == 'github':
            emails_url = oauth2.GitHub.user_info_url + '/emails'
            response = auth.access(result.user.credentials, emails_url)
            self.ensure_response_http_ok(response)
            result.user.github_emails = response.data or []

    def ensure_response_http_ok(self, response):
        if response.status >= 400:
            raise HTTPError(response.data
                            .get('error', {})
                            .get('message', response.data))

    def login_successful(self, result):
        if result.provider.name == 'github':
            user = User.get_or_insert(result.user.id, parent=project_key())
            user.token = result.user.credentials.token
            user.info = {prop: getattr(result.user, prop, None)
                         for prop in USER_INFO_PROPS}

            for entry in result.user.github_emails:
                if entry['verified']:
                    self.add_user_email(user, entry['email'])

            user.put()
            self.put_user(user)

        elif result.provider.name == 'google':
            if result.user.email and self.user:
                self.add_user_email(self.user, result.user.email.lower())
                self.user.put()

class LogoutHandler(FormattedResultHandler):
    def handle(self):
        self.session.clear()
        return self.state_dict()

class RemoveEmailHandler(FormattedResultHandler):
    def handle(self):
        if self.user:
            email = self.request.get('email').lower()
            if email in self.user.emails:
                self.user.emails.remove(email)
                self.user.put()

        return self.state_dict()

class AdminCodeHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()
        return dict(code=self.admin_code)

class UsersHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()

        users_query = User.query(ancestor=project_key()).order(-User.date)
        users = users_query.fetch()

        return [user.to_dict(include=USER_PROPS+['token']) for user in users]

class EmailsHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()

        users_query = User.query(ancestor=project_key()).order(-User.date)
        users = users_query.fetch()

        return {email: user.info['username'] for user in users
                for email in user.emails}

class ResetHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()

        users_query = User.query_all()
        for user in users_query.fetch():
            user.key.delete()

        self.session.clear()
        return self.state_dict()

# Instantiate the webapp2 WSGI application.
app = webapp2.WSGIApplication([
        webapp2.Route(r'/', IndexHandler),

        webapp2.Route(r'/app/state<result_format:(\.(json|js))?>', StateHandler),
        webapp2.Route(r'/app/login/', LoginHandler, handler_method='any'),
        webapp2.Route(r'/app/login/<provider_name:.*?><result_format:(\.(json|js))?>',
                      LoginHandler, handler_method='any'),
        webapp2.Route(r'/app/logout<result_format:(\.(json|js))?>', LogoutHandler),
        webapp2.Route(r'/app/remove<result_format:(\.(json|js))?>', RemoveEmailHandler),
        webapp2.Route(r'/app/admin<result_format:(\.(json|js))?>', AdminCodeHandler),
        webapp2.Route(r'/app/users<result_format:(\.(json|js))?>', UsersHandler),
        webapp2.Route(r'/app/emails<result_format:(\.(json|js))?>', EmailsHandler),
        webapp2.Route(r'/app/reset<result_format:(\.(json|js))?>', ResetHandler),
    ],
    config={
        'webapp2_extras.sessions': {
            'secret_key': config.SECRET,
            'backends': {
                'datastore':    'webapp2_extras.appengine.sessions_ndb.DatastoreSessionFactory',
                'memcache':     'webapp2_extras.appengine.sessions_memcache.MemcacheSessionFactory',
                'securecookie': 'webapp2_extras.sessions.SecureCookieSessionFactory',
            }
        },
    },
    debug=False)
<commit_msg>Use 'securecookie' session backend instead of 'memcache'<commit_after>#!/usr/bin/env python
"""
GAE app for gathering GitHub access tokens and Google emails.
Used for migrating issues from Google Code to GitHub.
"""

__author__ = "Eldar Abusalimov"

import json
import os
import urllib
import hashlib

import jinja2
import webapp2
from webapp2_extras import sessions
from google.appengine.ext import ndb

from authomatic import Authomatic
from authomatic import provider_id
from authomatic.adapters import Webapp2Adapter
from authomatic.providers import oauth2, BaseProvider
from authomatic.exceptions import BaseError
from authomatic.exceptions import HTTPError

import authomatic.core
import authomatic.providers
authomatic.providers.BaseProvider._logger = authomatic.core._logger  # XXX

import config

# Instantiate Authomatic.
auth = Authomatic(config={
        'google': {
            'id': authomatic.provider_id(),
            'class_': oauth2.Google,
            'consumer_key':    config.GOOGLE_AUTH['client_id'],
            'consumer_secret': config.GOOGLE_AUTH['client_secret'],
            'scope': ['email'],
        },

        'github': {
            'id': authomatic.provider_id(),
            'class_': oauth2.GitHub,
            'consumer_key':    config.GITHUB_AUTH['client_id'],
            'consumer_secret': config.GITHUB_AUTH['client_secret'],
            'scope': ['user:email', 'public_repo'],
        },
    },
    secret=config.SECRET)

def _monkey_patch_init_to_prompt_account(goog):
    orig_init = goog.__init__
    def init_with_prompting(self, *args, **kwargs):
        orig_init(self, *args, **kwargs)
        self.user_authorization_params['prompt'] = 'select_account'
    goog.__init__ = init_with_prompting
_monkey_patch_init_to_prompt_account(oauth2.Google)

DEFAULT_PROJECT_NAME = 'project'
USER_PROPS           = ['info', 'emails']
USER_INFO_PROPS      = ['id', 'username', 'name', 'link', 'picture']

def project_key(project_name=DEFAULT_PROJECT_NAME):
    """Constructs a Datastore key for a Project entity with project_name."""
    return ndb.Key('Project', project_name)

class User(ndb.Model):
    """Models an individual Project entry."""
    info   = ndb.JsonProperty(indexed=False)
    token  = ndb.StringProperty(indexed=False)
    emails = ndb.StringProperty(repeated=True)
    date   = ndb.DateTimeProperty(auto_now_add=True)

    @classmethod
    def query_all(cls, ancestor_key=project_key()):
        return cls.query(ancestor=ancestor_key).order(-cls.date)

templates_path = os.path.join(os.path.dirname(__file__), 'templates')
jinja2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))

class IndexHandler(webapp2.RequestHandler):
    INDEX_HTML = jinja2_env.get_template('index.html').render(config=config)
    def get(self):
        self.response.write(self.INDEX_HTML)

class GenericError(BaseError):
    def __init__(self, error):
        super(GenericError, self).__init__(str(error))

class BaseHandler(webapp2.RequestHandler):

    @webapp2.cached_property
    def session(self):
        # Returns a session using the default cookie key.
        return self.session_store.get_session(backend='securecookie')

    def dispatch(self):
        # Get a session store for this request.
        self.session_store = sessions.get_store(request=self.request)

        try:
            # Dispatch the request.
            webapp2.RequestHandler.dispatch(self)
        finally:
            # Save all sessions.
            self.session_store.save_sessions(self.response)

    @webapp2.cached_property
    def user(self):
        user_key_id = self.session.get('user_key_id')
        if user_key_id:
            return User.get_by_id(user_key_id, parent=project_key())

    def put_user(self, user):
        if user:
            self.session['user_key_id'] = user.key.id()
        else:
            self.session.pop('user_key_id', None)

    def state_dict(self, **defaults):
        if self.user:
            defaults.update(user=self.user.to_dict(include=USER_PROPS))
        return defaults

    def get_user_by_email(self, email):
        users_query = User.query_all().filter(User.emails == email)
        users = list(users_query.fetch())
        if not users:
            return
        else:
            assert len(users) == 1
        return users[0]

    def add_user_email(self, user, email):
        old_user = self.get_user_by_email(email)
        if old_user:
            old_user.emails.remove(email)
            old_user.put()

        user.emails.append(email)

    @webapp2.cached_property
    def admin_user(self):
        user = self.get_user_by_email(config.ADMIN_EMAIL)
        if not user:
            self.abort(403)
        return user

    @webapp2.cached_property
    def admin_code(self):
        sha = hashlib.sha1()
        sha.update(config.ADMIN_EMAIL)
        sha.update(config.SECRET)
        sha.update(self.admin_user.token)
        return sha.hexdigest()

    @webapp2.cached_property
    def has_admin_rights(self):
        return (self.user == self.admin_user or
                self.request.get('code') == self.admin_code)

    def ensure_admin(self):
        if not self.has_admin_rights:
            self.abort(403)

class FormattedResultHandler(BaseHandler):

    def get(self, result_format):
        result = self.handle()
        self.respond(result, result_format)

    def handle(self):
        pass

    def respond(self, result, result_format):
        if result_format and result_format[0] == '.':
            result_format = result_format[1:]
        assert result_format in ('json', 'js') or not result_format, result_format

        def obj_to_dict(obj):
            if hasattr(obj, 'to_dict'):
                return obj.to_dict()

        result_json = json.dumps(result, default=obj_to_dict,
                                 indent=(4 if result_format == 'json' else None))

        if result_format == 'json':
            self.response.content_type = 'application/json'
            response_str = result_json

        elif result_format == 'js':
            self.response.content_type = 'application/javascript'
            response_str = ("handleState({result_json});"
                            .format(**locals()))

        else:
            response_str = """
            <!DOCTYPE html>

            """.format(**locals())

        self.response.write(response_str)

class StateHandler(FormattedResultHandler):
    def handle(self):
        return self.state_dict()

class LoginHandler(FormattedResultHandler):

    def any(self, provider_name='', result_format=None):
        # It all begins with login.
        def clean_session(result):  # XXX
            self.session.pop(auth.prefix, None)
        result = auth.login(Webapp2Adapter(self), provider_name,
                                  session=self.session,
                                  session_saver=self.session.on_update,
                                  callback=clean_session)
        if not result:
            # Do not write anything to the response if there is no result!
            return

        if result.user:
            if not result.error:
                try:
                    self.fetch_user_info(result)
                except BaseError as e:
                    result.error = e
                except Exception as e:
                    result.error = GenericError(e)

            if not result.error:
                self.login_successful(result)

        self.respond(self.state_dict(error=result.error), result_format)

    def fetch_user_info(self, result):
        response = result.user.update()
        response._data = result.user.data  # XXX
        self.ensure_response_http_ok(response)

        if result.provider.name == 'github':
            emails_url = oauth2.GitHub.user_info_url + '/emails'
            response = auth.access(result.user.credentials, emails_url)
            self.ensure_response_http_ok(response)
            result.user.github_emails = response.data or []

    def ensure_response_http_ok(self, response):
        if response.status >= 400:
            raise HTTPError(response.data
                            .get('error', {})
                            .get('message', response.data))

    def login_successful(self, result):
        if result.provider.name == 'github':
            user = User.get_or_insert(result.user.id, parent=project_key())
            user.token = result.user.credentials.token
            user.info = {prop: getattr(result.user, prop, None)
                         for prop in USER_INFO_PROPS}

            for entry in result.user.github_emails:
                if entry['verified']:
                    self.add_user_email(user, entry['email'])

            user.put()
            self.put_user(user)

        elif result.provider.name == 'google':
            if result.user.email and self.user:
                self.add_user_email(self.user, result.user.email.lower())
                self.user.put()

class LogoutHandler(FormattedResultHandler):
    def handle(self):
        self.session.clear()
        return self.state_dict()

class RemoveEmailHandler(FormattedResultHandler):
    def handle(self):
        if self.user:
            email = self.request.get('email').lower()
            if email in self.user.emails:
                self.user.emails.remove(email)
                self.user.put()

        return self.state_dict()

class AdminCodeHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()
        return dict(code=self.admin_code)

class UsersHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()

        users_query = User.query(ancestor=project_key()).order(-User.date)
        users = users_query.fetch()

        return [user.to_dict(include=USER_PROPS+['token']) for user in users]

class EmailsHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()

        users_query = User.query(ancestor=project_key()).order(-User.date)
        users = users_query.fetch()

        return {email: user.info['username'] for user in users
                for email in user.emails}

class ResetHandler(FormattedResultHandler):
    def handle(self):
        self.ensure_admin()

        users_query = User.query_all()
        for user in users_query.fetch():
            user.key.delete()

        self.session.clear()
        return self.state_dict()

# Instantiate the webapp2 WSGI application.
app = webapp2.WSGIApplication([
        webapp2.Route(r'/', IndexHandler),

        webapp2.Route(r'/app/state<result_format:(\.(json|js))?>', StateHandler),
        webapp2.Route(r'/app/login/', LoginHandler, handler_method='any'),
        webapp2.Route(r'/app/login/<provider_name:.*?><result_format:(\.(json|js))?>',
                      LoginHandler, handler_method='any'),
        webapp2.Route(r'/app/logout<result_format:(\.(json|js))?>', LogoutHandler),
        webapp2.Route(r'/app/remove<result_format:(\.(json|js))?>', RemoveEmailHandler),
        webapp2.Route(r'/app/admin<result_format:(\.(json|js))?>', AdminCodeHandler),
        webapp2.Route(r'/app/users<result_format:(\.(json|js))?>', UsersHandler),
        webapp2.Route(r'/app/emails<result_format:(\.(json|js))?>', EmailsHandler),
        webapp2.Route(r'/app/reset<result_format:(\.(json|js))?>', ResetHandler),
    ],
    config={
        'webapp2_extras.sessions': {
            'secret_key': config.SECRET,
            'backends': {
                'securecookie': 'webapp2_extras.sessions.SecureCookieSessionFactory',
            }
        },
    },
    debug=False)
<|endoftext|>


<commit_before>"""
main.py
"""
from importlib import import_module
from pydoc import locate
from lxml import etree
from components.log_server import LogServer

# initalize base components
LOGSERVER = LogServer()

# read xml file
XSD_FILE = etree.XML(open('core/pipeline.xsd', 'r').read())
SCHEMA = etree.XMLSchema(XSD_FILE)
PARSER = etree.XMLParser(schema=SCHEMA)
PIPELINE = etree.parse('pipelines/dummy.xml', PARSER)
XML_COMPONENTS = PIPELINE.xpath('component')
COMPONENTS = []

# dynamically load components
for component in XML_COMPONENTS:
    className = component.xpath('class')[0]
    fileName = component.xpath('name')[0]
    # read properties
    xml_properties = component.xpath('property')
    properties = dict()
    for component_property in xml_properties:
        property_name = component_property.xpath('name')[0]
        property_value = component_property.xpath('value')[0]
        property_value_type = component_property.xpath('value/@type')
        # typecast components
        if len(property_value_type) > 0:
            property_type = locate(property_value_type[0])
            property_value = property_type(property_value.text)
        else:
            property_value = property_value.text
        properties[property_name.text] = property_value
    # get component class
    ComponentClass = getattr(import_module('components.' + fileName.text), className.text)
    # instantiate class
    instance = ComponentClass(LOGSERVER, properties)
    COMPONENTS.append(instance)

# glue components together
for index, component in enumerate(COMPONENTS):
    if index + 1 < len(COMPONENTS):
        component.output_notifier.add_observer(COMPONENTS[index + 1].input_observer)

# start at component
COMPONENTS[0].start()
<commit_msg>add main.py<commit_after>"""
main.py
"""
from importlib import import_module
from pydoc import locate
from lxml import etree
from core.log_server import LogServer

# initalize base components
LOGSERVER = LogServer()

# read xml file
XSD_FILE = etree.XML(open('core/pipeline.xsd', 'r').read())
SCHEMA = etree.XMLSchema(XSD_FILE)
PARSER = etree.XMLParser(schema=SCHEMA)
PIPELINE = etree.parse('pipelines/dummy.xml', PARSER)
XML_COMPONENTS = PIPELINE.xpath('component')
COMPONENTS = []

# dynamically load components
for component in XML_COMPONENTS:
    className = component.xpath('class')[0]
    fileName = component.xpath('name')[0]
    # read properties
    xml_properties = component.xpath('property')
    properties = dict()
    for component_property in xml_properties:
        property_name = component_property.xpath('name')[0]
        property_value = component_property.xpath('value')[0]
        property_value_type = component_property.xpath('value/@type')
        # typecast components
        if len(property_value_type) > 0:
            property_type = locate(property_value_type[0])
            property_value = property_type(property_value.text)
        else:
            property_value = property_value.text
        properties[property_name.text] = property_value
    # get component class
    ComponentClass = getattr(import_module('components.' + fileName.text), className.text)
    # instantiate class
    instance = ComponentClass(LOGSERVER, properties)
    print(instance.get_description())
    COMPONENTS.append(instance)

# glue components together
for index, component in enumerate(COMPONENTS):
    if index + 1 < len(COMPONENTS):
        component.output_notifier.add_observer(COMPONENTS[index + 1].input_observer)

# start at component
COMPONENTS[0].start()
<|endoftext|>


<commit_before>import scipy.optimize as so
import numpy as np
import matplotlib.pyplot as plt
import collections

#Try to see if we can
try:
    import matplotlib.pyplot as plt
    can_plot = True
except(Exception):
    can_plot = False
    print("Cannot import matplotlib")

from functions import *
from util import getDataFromFile, parseList, exportResults

################################################################################
# OPTIONS AND PARAMETERS
################################################################################

# Add your function in functions.py, then put the names here of all the ones
# you want to fit to your data. You can have as many as you want.
fitting_functions = [
    stepFunction,
    hingeFunction
    ]

# Put the name of your data file here
data_file = "DataforBastien.csv"

# Electrode/subject/condition list syntax (comma-separated):
#           x : isolated electrode
#           x..y : include isolated electrodes from x to y (included)
#           x+y : average several electrodes
# ex : "1, 2, 3, 4..10, 11+12, 13, 14"
electrodes = "1,96"
subjects = "1"
conditions = "1"

# Name of the file for the export
results_file = "results.txt"

# Number of points to try and put the "inflexion" point of the fitting function.
num_X0 = 100

# Choose wheter you want to plot the data + fitted functions
plot_results = True

################################################################################
# DATA EXTRACTION
################################################################################

# We get the data from a csv file
# The code is in util.py
d = getDataFromFile(data_file)

electrodes_numbers = parseList(electrodes)
conditions_numbers = parseList(conditions)
subjects_numbers = parseList(subjects)

################################################################################
# DATA STRUCTURES
################################################################################

# The array with the x coordinates for the data point (1,2,...,20)
x = np.arange(1,d.shape[3]+1)

# Will contain the value of the fitted function at the data coordinates
y = np.zeros(x.shape)

# Will contain the norm of the error of each fitted function
fitting_scores = np.zeros([len(electrodes_numbers),len(fitting_functions)])

# Will contain the optimal parameters for each fitted function, for each
# data set. First indice is for dataset, second for function.
popts_total = []

# Coordinates at which to try to place the "inflexion" point of the fitting
# functions
X0 = np.linspace(1.0,20.0,num_X0)

# Vectorise the fitting functions
# It's technical, it lets you define the function for a single point, and
# numpy transforms it to accept vectors.
fitting_functions_names = []
for index,f in enumerate(fitting_functions):
    fitting_functions_names.append(f.__name__)
    fitting_functions[index] = np.vectorize(f)

################################################################################
# FUNCTION FITTING
################################################################################

for condition in conditions_numbers:
    for subject in subjects_numbers:
        for electrode in electrodes_numbers:
            if not isinstance(electrode, collections.Iterable):
                electrode = [electrode]
            electrode_averaging = 0.0
            dataset = numpy.zeros(d.shape[3])
            for e in electrode:
                dataset += d[subject, condition, e, :]
                electrode_averaging += 1.0
            dataset = dataset / electrode_averaging

            # Compute the fit
            popts = []

            # Start the final plot
            if (plot_results and can_plot):
                plt.plot(x,dataset[:],'o')

            for index,f in enumerate(fitting_functions):
                fitting_scores[0][index] = 1e12
                popts.append([])

                for x0 in X0:
                    # Optimise the fitting of the curve
                    (popt, pcov) = so.curve_fit(f, x, dataset[:], [x0,-1.0,1.0])
                    y = f(x, *popt)

                    # Compute the norm of the error, if better than previous best, keep it.
                    norm = np.linalg.norm(y-dataset[:], 2)
                    if norm < fitting_scores[0][index]:
                        fitting_scores[0][index] = norm
                        popts[index] = popt

                # Compute the function for the final solution for display
                if (plot_results and can_plot):
                    y = f(X0,*(popts[index]))
                    plt.plot(X0,y)

            popts_total.append(popts)

            # Finally show the whole plot
            if (plot_results and can_plot):
                plt.show()

################################################################################
# RESULTS EXPORT
################################################################################

options = {"Data file" : data_file,
           "Number of tests" : num_X0
          }

exportResults(results_file, options, fitting_functions_names, fitting_functions,
              fitting_scores, popts_total)

<commit_msg>Fixed fitting error computation<commit_after>import scipy.optimize as so
import numpy as np
import matplotlib.pyplot as plt
import collections

#Try to see if we can
try:
    import matplotlib.pyplot as plt
    can_plot = True
except(Exception):
    can_plot = False
    print("Cannot import matplotlib")

from functions import *
from util import getDataFromFile, parseList, exportResults

################################################################################
# OPTIONS AND PARAMETERS
################################################################################

# Add your function in functions.py, then put the names here of all the ones
# you want to fit to your data. You can have as many as you want.
fitting_functions = [
    stepFunction,
    hingeFunction
    ]

# Put the name of your data file here
data_file = "DataforBastien.csv"

# Electrode/subject/condition list syntax (comma-separated):
#           x : isolated electrode
#           x..y : include isolated electrodes from x to y (included)
#           x+y : average several electrodes
# ex : "1, 2, 3, 4..10, 11+12, 13, 14"
electrodes = "1,96"
subjects = "1"
conditions = "1"

# Name of the file for the export
results_file = "results.txt"

# Number of points to try and put the "inflexion" point of the fitting function.
num_X0 = 100

# Choose wheter you want to plot the data + fitted functions
plot_results = True

################################################################################
# DATA EXTRACTION
################################################################################

# We get the data from a csv file
# The code is in util.py
d = getDataFromFile(data_file)

electrodes_numbers = parseList(electrodes)
conditions_numbers = parseList(conditions)
subjects_numbers = parseList(subjects)

################################################################################
# DATA STRUCTURES
################################################################################

# The array with the x coordinates for the data point (1,2,...,20)
x = np.arange(1,d.shape[3]+1)

# Will contain the value of the fitted function at the data coordinates
y = np.zeros(x.shape)

# Will contain the norm of the error of each fitted function
fitting_scores = np.zeros([len(electrodes_numbers),len(fitting_functions)])

# Will contain the optimal parameters for each fitted function, for each
# data set. First indice is for dataset, second for function.
popts_total = []

# Coordinates at which to try to place the "inflexion" point of the fitting
# functions
X0 = np.linspace(1.0,20.0,num_X0)

# Vectorise the fitting functions
# It's technical, it lets you define the function for a single point, and
# numpy transforms it to accept vectors.
fitting_functions_names = []
for index,f in enumerate(fitting_functions):
    fitting_functions_names.append(f.__name__)
    fitting_functions[index] = np.vectorize(f)

################################################################################
# FUNCTION FITTING
################################################################################

dataset_index = -1
for condition in conditions_numbers:
    for subject in subjects_numbers:
        for electrode in electrodes_numbers:
            dataset_index+=1
            if not isinstance(electrode, collections.Iterable):
                electrode = [electrode]
            electrode_averaging = 0.0
            dataset = numpy.zeros(d.shape[3])
            for e in electrode:
                dataset += d[subject, condition, e, :]
                electrode_averaging += 1.0
            dataset = dataset / electrode_averaging

            # Compute the fit
            popts = []

            # Start the final plot
            if (plot_results and can_plot):
                plt.plot(x,dataset[:],'o')

            for index,f in enumerate(fitting_functions):
                fitting_scores[dataset_index][index] = 1e12
                popts.append([])

                for x0 in X0:
                    # Optimise the fitting of the curve
                    (popt, pcov) = so.curve_fit(f, x, dataset[:], [x0,-1.0,1.0])
                    y = f(x, *popt)

                    # Compute the norm of the error, if better than previous best, keep it.
                    norm = np.linalg.norm(y-dataset[:], 2)
                    if norm < fitting_scores[dataset_index][index]:
                        fitting_scores[dataset_index][index] = norm
                        popts[index] = popt

                # Compute the function for the final solution for display
                if (plot_results and can_plot):
                    y = f(X0,*(popts[index]))
                    plt.plot(X0,y)

            popts_total.append(popts)

            # Finally show the whole plot
            if (plot_results and can_plot):
                plt.show()

################################################################################
# RESULTS EXPORT
################################################################################

options = {"Data file" : data_file,
           "Number of tests" : num_X0
          }

exportResults(results_file, options, fitting_functions_names, fitting_functions,
              fitting_scores, popts_total)

<|endoftext|>


<commit_before>#! /usr/bin/env python
import os
import argparse
import datetime
import torch
import torchtext.data as data
import torchtext.datasets as datasets
import model
import train
import mydatasets
import pdb
import vpdataset

parser = argparse.ArgumentParser(description='CNN text classificer')
# learning
parser.add_argument('-lr', type=float, default=0.001, help='initial learning rate [default: 0.001]')
parser.add_argument('-l2', type=float, default=1e-6, help='l2 regularization strength [default: 1e-6]')
parser.add_argument('-epochs', type=int, default=256, help='number of epochs for train [default: 256]')
parser.add_argument('-batch-size', type=int, default=64, help='batch size for training [default: 64]')
parser.add_argument('-log-interval',  type=int, default=1, help='how many steps to wait before logging training status [default: 1]')
parser.add_argument('-log-file', type=str, default='result.txt', help='the name of the file to store results')
parser.add_argument('-test-interval', type=int, default=100, help='how many steps to wait before testing [default: 100]')
parser.add_argument('-save-interval', type=int, default=500, help='how many steps to wait before saving [default:500]')
parser.add_argument('-save-dir', type=str, default='snapshot', help='where to save the snapshot')
# data
parser.add_argument('-shuffle', action='store_true', default=False, help='shuffle the data every epoch' )
# model
parser.add_argument('-dropout', type=float, default=0.5, help='the probability for dropout [default: 0.5]')
parser.add_argument('-max-norm', type=float, default=3.0, help='l2 constraint of parameters [default: 3.0]')
parser.add_argument('-char-embed-dim', type=int, default=128, help='number of char embedding dimension [default: 128]')
parser.add_argument('-word-embed-dim', type=int, default=300, help='number of word embedding dimension [default: 300]')
parser.add_argument('-kernel-num', type=int, default=100, help='number of each kind of kernel')
#parser.add_argument('-kernel-sizes', type=str, default='3,4,5', help='comma-separated kernel size to use for convolution')
parser.add_argument('-kernel-sizes', type=str, default='3', help='comma-separated kernel size to use for convolution')
parser.add_argument('-static', action='store_true', default=False, help='fix the embedding')
# device
parser.add_argument('-device', type=int, default=-1, help='device to use for iterate data, -1 mean cpu [default: -1]')
parser.add_argument('-no-cuda', action='store_true', default=False, help='disable the gpu' )
# option
parser.add_argument('-snapshot', type=str, default=None, help='filename of model snapshot [default: None]')
parser.add_argument('-predict', type=str, default=None, help='predict the sentence given')
parser.add_argument('-test', action='store_true', default=False, help='train or test')
parser.add_argument('-xfolds', type=int, default=10, help='number of folds for cross-validation')
parser.add_argument('-layer-num', type=int, default=2, help='the number of layers in the final MLP')
parser.add_argument('-word-vector', action='store_true', default=False, help="use of glove 6B vector")
args = parser.parse_args()

if args.word_vector:
    args.word_vector = 'glove.6B'

# load SST dataset
def sst(text_field, label_field,  **kargs):
    train_data, dev_data, test_data = datasets.SST.splits(text_field, label_field, fine_grained=True)
    text_field.build_vocab(train_data, dev_data, test_data)
    label_field.build_vocab(train_data, dev_data, test_data)
    train_iter, dev_iter, test_iter = data.BucketIterator.splits(
                                        (train_data, dev_data, test_data),
                                        batch_sizes=(args.batch_size,
                                                     len(dev_data),
                                                     len(test_data)),
                                        **kargs)
    return train_iter, dev_iter, test_iter

# load MR dataset
def mr(text_field, label_field, **kargs):
    train_data, dev_data = mydatasets.MR.splits(text_field, label_field)
    text_field.build_vocab(train_data, dev_data)
    label_field.build_vocab(train_data, dev_data)
    train_iter, dev_iter = data.Iterator.splits(
                                (train_data, dev_data),
                                batch_sizes=(args.batch_size, len(dev_data)),
                                **kargs)
    return train_iter, dev_iter

#load VP dataset
def vp(text_field, label_field, foldid, **kargs):
    train_data, dev_data, test_data = vpdataset.VP.splits(text_field, label_field, foldid=foldid)
    text_field.build_vocab(train_data, dev_data, test_data, **kargs )
    label_field.build_vocab(train_data, dev_data, test_data )
    train_iter, dev_iter, test_iter = data.Iterator.splits(
                                        (train_data, dev_data, test_data),
                                        batch_sizes=(args.batch_size,
                                                     len(dev_data),
                                                     len(test_data)),
                                        **kargs)
    return train_iter, dev_iter, test_iter

def char_tokenizer(mstring):
    return list(mstring)

print("Beginning {0}-fold cross-validation...".format(args.xfolds))
print("Logging the results in {}".format(args.log_file))
log_file = open(args.log_file, 'w')
char_fold_accuracies = []
word_fold_accuracies = []
ensemble_fold_accuracies = []
orig_save_dir = args.save_dir
update_args = True

max_kernel_length = max([int(x) for x in args.kernel_sizes.split(',')])

for xfold in range(args.xfolds):
    print("Fold {0}".format(xfold))
    # load data
    print("\nLoading data...")

    #text_field = data.Field(lower=True)
    #label_field = data.Field(sequential=False)
    #train_iter, dev_iter = mr(text_field, label_field, device=-1, repeat=False)
    #train_iter, dev_iter, test_iter = sst(text_field, label_field, device=-1, repeat=False)

    text_field = data.Field(lower=True, tokenize=char_tokenizer)
    word_field = data.Field(lower=True, fix_length=max_kernel_length )

    label_field = data.Field(sequential=False)
    train_iter, dev_iter, test_iter = vp(text_field, label_field, foldid=xfold, device=args.device, repeat=False, shuffle=False, sort=False)
    train_iter_word, dev_iter_word, test_iter_word = vp(word_field, label_field, foldid=xfold, device=args.device,
                                                        repeat=False, shuffle=False, sort=False, wv_type=args.word_vector,
                                                        wv_dim=args.word_embed_dim)

    args.embed_num = len(text_field.vocab)
    args.class_num = len(label_field.vocab) - 1
    args.cuda = args.no_cuda and torch.cuda.is_available()#; del args.no_cuda
    if update_args==True:
        args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
        args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'CHAR')
    else:
        args.save_dir = os.path.join(orig_save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'CHAR')

    print("\nParameters:", file=log_file)
    for attr, value in sorted(args.__dict__.items()):
        print("\t{}={}".format(attr.upper(), value), file=log_file)

    # char CNN training and dev
    if args.snapshot is None:
        char_cnn = model.CNN_Text(args, 'char')
    else :
        print('\nLoading model from [%s]...' % args.snapshot)
        try:
            char_cnn = torch.load(args.snapshot)
        except :
            print("Sorry, This snapshot doesn't exist."); exit()

    train.train(train_iter, dev_iter, char_cnn, args)
    result = train.eval(test_iter, char_cnn, args)
    char_fold_accuracies.append(result)
    print("Completed fold {0}. Accuracy: {1} for CHAR".format(xfold, result))
    print("Completed fold {0}. Accuracy: {1} for CHAR".format(xfold, result), file=log_file)

    # Word CNN training and dev
    args.embed_num = len(word_field.vocab)
    if update_args==True:
        # args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
        args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'WORD')
    else:
        args.save_dir = os.path.join(orig_save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'WORD')
    # print("\nParameters:")
    # for attr, value in sorted(args.__dict__.items()):
    #     print("\t{}={}".format(attr.upper(), value))

    if args.snapshot is None:
        word_cnn = model.CNN_Text(args, 'word', vectors =word_field.vocab.vectors)
    else :
        print('\nLoading model from [%s]...' % args.snapshot)
        try:
            word_cnn = torch.load(args.snapshot)
        except:
            print("Sorry, This snapshot doesn't exist."); exit()

    train.train(train_iter_word, dev_iter_word, word_cnn, args)
    result = train.eval(test_iter_word, word_cnn, args)
    word_fold_accuracies.append(result)
    print("Completed fold {0}. Accuracy: {1} for WORD".format(xfold, result))
    print("Completed fold {0}. Accuracy: {1} for WORD".format(xfold, result), file=log_file)

    # Ensemble training and dev
    if update_args==True:
        args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'LOGIT')
    else:
        args.save_dir = os.path.join(orig_save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'LOGIT')
    update_args = False

    if args.snapshot is None:
        final_logit = model.SimpleLogistic(args)
    else :
        print('\nLoading model from [%s]...' % args.snapshot)
        try:
            final_logit = torch.load(args.snapshot)
        except:
            print("Sorry, This snapshot doesn't exist."); exit()

    # train_iter, dev_iter, test_iter = vp(text_field, label_field, foldid=xfold, device=-1, repeat=False)
    # train_iter_word, dev_iter_word, test_iter_word = vp(word_field, label_field, foldid=xfold, device=-1, repeat=False)

    train.train_logistic(train_iter, dev_iter, train_iter_word, dev_iter_word, char_cnn, word_cnn, final_logit, args)
    result = train.eval_logistic(test_iter, test_iter_word, char_cnn, word_cnn, final_logit, args)
    ensemble_fold_accuracies.append(result)

    print("Completed fold {0}. Accuracy: {1} for LOGIT".format(xfold, result))
    print("Completed fold {0}. Accuracy: {1} for LOGIT".format(xfold, result), file=log_file)
    """
    # train or predict
    if args.predict is not None:
        label = train.predict(args.predict, cnn, text_field, label_field)
        print('\n[Text]  {}[Label] {}\n'.format(args.predict, label))
    elif args.test :
        try:
            train.eval(test_iter, cnn, args)
        except Exception as e:
            print("\nSorry. The test dataset doesn't  exist.\n")
    else :
        print()
        train.train(train_iter, dev_iter, cnn, args)
    """

average_xfold_accuracy = sum([res for res in ensemble_fold_accuracies]) / len(ensemble_fold_accuracies)
print("folds: {0}".format(len(ensemble_fold_accuracies)))
print("Average cross-fold accuracy: {0}".format(average_xfold_accuracy))
log_file.close()<commit_msg>fix wvtype<commit_after>#! /usr/bin/env python
import os
import argparse
import datetime
import torch
import torchtext.data as data
import torchtext.datasets as datasets
import model
import train
import mydatasets
import pdb
import vpdataset

parser = argparse.ArgumentParser(description='CNN text classificer')
# learning
parser.add_argument('-lr', type=float, default=0.001, help='initial learning rate [default: 0.001]')
parser.add_argument('-l2', type=float, default=1e-6, help='l2 regularization strength [default: 1e-6]')
parser.add_argument('-epochs', type=int, default=256, help='number of epochs for train [default: 256]')
parser.add_argument('-batch-size', type=int, default=64, help='batch size for training [default: 64]')
parser.add_argument('-log-interval',  type=int, default=1, help='how many steps to wait before logging training status [default: 1]')
parser.add_argument('-log-file', type=str, default='result.txt', help='the name of the file to store results')
parser.add_argument('-test-interval', type=int, default=100, help='how many steps to wait before testing [default: 100]')
parser.add_argument('-save-interval', type=int, default=500, help='how many steps to wait before saving [default:500]')
parser.add_argument('-save-dir', type=str, default='snapshot', help='where to save the snapshot')
# data
parser.add_argument('-shuffle', action='store_true', default=False, help='shuffle the data every epoch' )
# model
parser.add_argument('-dropout', type=float, default=0.5, help='the probability for dropout [default: 0.5]')
parser.add_argument('-max-norm', type=float, default=3.0, help='l2 constraint of parameters [default: 3.0]')
parser.add_argument('-char-embed-dim', type=int, default=128, help='number of char embedding dimension [default: 128]')
parser.add_argument('-word-embed-dim', type=int, default=300, help='number of word embedding dimension [default: 300]')
parser.add_argument('-kernel-num', type=int, default=100, help='number of each kind of kernel')
#parser.add_argument('-kernel-sizes', type=str, default='3,4,5', help='comma-separated kernel size to use for convolution')
parser.add_argument('-kernel-sizes', type=str, default='3', help='comma-separated kernel size to use for convolution')
parser.add_argument('-static', action='store_true', default=False, help='fix the embedding')
# device
parser.add_argument('-device', type=int, default=-1, help='device to use for iterate data, -1 mean cpu [default: -1]')
parser.add_argument('-no-cuda', action='store_true', default=False, help='disable the gpu' )
# option
parser.add_argument('-snapshot', type=str, default=None, help='filename of model snapshot [default: None]')
parser.add_argument('-predict', type=str, default=None, help='predict the sentence given')
parser.add_argument('-test', action='store_true', default=False, help='train or test')
parser.add_argument('-xfolds', type=int, default=10, help='number of folds for cross-validation')
parser.add_argument('-layer-num', type=int, default=2, help='the number of layers in the final MLP')
parser.add_argument('-word-vector', action='store_true', default=False, help="use of glove 6B vector")
args = parser.parse_args()

if args.word_vector:
    args.word_vector = 'glove.6B'

# load SST dataset
def sst(text_field, label_field,  **kargs):
    train_data, dev_data, test_data = datasets.SST.splits(text_field, label_field, fine_grained=True)
    text_field.build_vocab(train_data, dev_data, test_data)
    label_field.build_vocab(train_data, dev_data, test_data)
    train_iter, dev_iter, test_iter = data.BucketIterator.splits(
                                        (train_data, dev_data, test_data),
                                        batch_sizes=(args.batch_size,
                                                     len(dev_data),
                                                     len(test_data)),
                                        **kargs)
    return train_iter, dev_iter, test_iter

# load MR dataset
def mr(text_field, label_field, **kargs):
    train_data, dev_data = mydatasets.MR.splits(text_field, label_field)
    text_field.build_vocab(train_data, dev_data)
    label_field.build_vocab(train_data, dev_data)
    train_iter, dev_iter = data.Iterator.splits(
                                (train_data, dev_data),
                                batch_sizes=(args.batch_size, len(dev_data)),
                                **kargs)
    return train_iter, dev_iter

#load VP dataset
def vp(text_field, label_field, foldid, **kargs):
    train_data, dev_data, test_data = vpdataset.VP.splits(text_field, label_field, foldid=foldid)
    text_field.build_vocab(train_data, dev_data, test_data, wv_type=kargs["wv_type"], wv_dim=kargs["wv_dim"])
    label_field.build_vocab(train_data, dev_data, test_data )
    train_iter, dev_iter, test_iter = data.Iterator.splits(
                                        (train_data, dev_data, test_data),
                                        batch_sizes=(args.batch_size,
                                                     len(dev_data),
                                                     len(test_data)),
                                        **kargs)
    return train_iter, dev_iter, test_iter

def char_tokenizer(mstring):
    return list(mstring)

print("Beginning {0}-fold cross-validation...".format(args.xfolds))
print("Logging the results in {}".format(args.log_file))
log_file = open(args.log_file, 'w')
char_fold_accuracies = []
word_fold_accuracies = []
ensemble_fold_accuracies = []
orig_save_dir = args.save_dir
update_args = True

max_kernel_length = max([int(x) for x in args.kernel_sizes.split(',')])

for xfold in range(args.xfolds):
    print("Fold {0}".format(xfold))
    # load data
    print("\nLoading data...")

    #text_field = data.Field(lower=True)
    #label_field = data.Field(sequential=False)
    #train_iter, dev_iter = mr(text_field, label_field, device=-1, repeat=False)
    #train_iter, dev_iter, test_iter = sst(text_field, label_field, device=-1, repeat=False)

    text_field = data.Field(lower=True, tokenize=char_tokenizer)
    word_field = data.Field(lower=True, fix_length=max_kernel_length )

    label_field = data.Field(sequential=False)
    train_iter, dev_iter, test_iter = vp(text_field, label_field, foldid=xfold, device=args.device, repeat=False, shuffle=False, sort=False
                                         , wv_type=None, wv_dim=None)
    train_iter_word, dev_iter_word, test_iter_word = vp(word_field, label_field, foldid=xfold, device=args.device,
                                                        repeat=False, shuffle=False, sort=False, wv_type=args.word_vector,
                                                        wv_dim=args.word_embed_dim)

    args.embed_num = len(text_field.vocab)
    args.class_num = len(label_field.vocab) - 1
    args.cuda = args.no_cuda and torch.cuda.is_available()#; del args.no_cuda
    if update_args==True:
        args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
        args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'CHAR')
    else:
        args.save_dir = os.path.join(orig_save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'CHAR')

    print("\nParameters:", file=log_file)
    for attr, value in sorted(args.__dict__.items()):
        print("\t{}={}".format(attr.upper(), value), file=log_file)

    # char CNN training and dev
    if args.snapshot is None:
        char_cnn = model.CNN_Text(args, 'char')
    else :
        print('\nLoading model from [%s]...' % args.snapshot)
        try:
            char_cnn = torch.load(args.snapshot)
        except :
            print("Sorry, This snapshot doesn't exist."); exit()

    train.train(train_iter, dev_iter, char_cnn, args)
    result = train.eval(test_iter, char_cnn, args)
    char_fold_accuracies.append(result)
    print("Completed fold {0}. Accuracy: {1} for CHAR".format(xfold, result))
    print("Completed fold {0}. Accuracy: {1} for CHAR".format(xfold, result), file=log_file)

    # Word CNN training and dev
    args.embed_num = len(word_field.vocab)
    if update_args==True:
        # args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
        args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'WORD')
    else:
        args.save_dir = os.path.join(orig_save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'WORD')
    # print("\nParameters:")
    # for attr, value in sorted(args.__dict__.items()):
    #     print("\t{}={}".format(attr.upper(), value))

    if args.snapshot is None:
        word_cnn = model.CNN_Text(args, 'word', vectors =word_field.vocab.vectors)
    else :
        print('\nLoading model from [%s]...' % args.snapshot)
        try:
            word_cnn = torch.load(args.snapshot)
        except:
            print("Sorry, This snapshot doesn't exist."); exit()

    train.train(train_iter_word, dev_iter_word, word_cnn, args)
    result = train.eval(test_iter_word, word_cnn, args)
    word_fold_accuracies.append(result)
    print("Completed fold {0}. Accuracy: {1} for WORD".format(xfold, result))
    print("Completed fold {0}. Accuracy: {1} for WORD".format(xfold, result), file=log_file)

    # Ensemble training and dev
    if update_args==True:
        args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'LOGIT')
    else:
        args.save_dir = os.path.join(orig_save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), 'LOGIT')
    update_args = False

    if args.snapshot is None:
        final_logit = model.SimpleLogistic(args)
    else :
        print('\nLoading model from [%s]...' % args.snapshot)
        try:
            final_logit = torch.load(args.snapshot)
        except:
            print("Sorry, This snapshot doesn't exist."); exit()

    # train_iter, dev_iter, test_iter = vp(text_field, label_field, foldid=xfold, device=-1, repeat=False)
    # train_iter_word, dev_iter_word, test_iter_word = vp(word_field, label_field, foldid=xfold, device=-1, repeat=False)

    train.train_logistic(train_iter, dev_iter, train_iter_word, dev_iter_word, char_cnn, word_cnn, final_logit, args)
    result = train.eval_logistic(test_iter, test_iter_word, char_cnn, word_cnn, final_logit, args)
    ensemble_fold_accuracies.append(result)

    print("Completed fold {0}. Accuracy: {1} for LOGIT".format(xfold, result))
    print("Completed fold {0}. Accuracy: {1} for LOGIT".format(xfold, result), file=log_file)
    """
    # train or predict
    if args.predict is not None:
        label = train.predict(args.predict, cnn, text_field, label_field)
        print('\n[Text]  {}[Label] {}\n'.format(args.predict, label))
    elif args.test :
        try:
            train.eval(test_iter, cnn, args)
        except Exception as e:
            print("\nSorry. The test dataset doesn't  exist.\n")
    else :
        print()
        train.train(train_iter, dev_iter, cnn, args)
    """

average_xfold_accuracy = sum([res for res in ensemble_fold_accuracies]) / len(ensemble_fold_accuracies)
print("folds: {0}".format(len(ensemble_fold_accuracies)))
print("Average cross-fold accuracy: {0}".format(average_xfold_accuracy))
log_file.close()<|endoftext|>


<commit_before>import os
import json
import boto
import urllib2
from boto.s3.key import Key
import boto3
import click
import logging
from copy import copy
from collections import OrderedDict
from datetime import date, timedelta
from elasticsearch import Elasticsearch, RequestError, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
import requests

from reader import csv_reader

logger = logging.getLogger('landsat8.meta')
bucket_name = os.getenv('BUCKETNAME', 'landsat8-meta')
thumbs_bucket_name = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
s3 = boto3.resource('s3')
es_index = 'sat-api'
es_type = 'landsat8'

def get_url(url, j=False):
    req = urllib2.Request(url)
    r = urllib2.urlopen(req).read()
    if j:
        obj = json.loads(r)
    else:
        obj = r
    return obj

def get_role():
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
    obj = get_url(url)
    return obj

def get_credentials():
    role = get_role()
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/' + role
    obj = get_url(url, True)
    return obj

def connection_to_es(es_host, es_port, aws=False):

    args = {}

    if aws:
        cred = get_credentials()

        access_key = cred['AccessKeyId']
        secret_access = cred['SecretAccessKey']
        region = cred['us-east-1']
        awsauth = AWS4Auth(access_key, secret_access, region, 'es')

        args = {
            'http_auth': awsauth,
            'use_ssl': True,
            'verify_certs': True,
            'connection_class': RequestsHttpConnection
        }

    es = Elasticsearch(hosts=[{
        'host': es_host,
        'port': es_port
    }], **args)

    return es

def create_index(index_name, doc_type, es_host, es_port, **kwargs):

    body = {
        doc_type: {
            'properties': {
                'scene_id': {'type': 'string', 'index': 'not_analyzed'},
                'satellite_name': {'type': 'string'},
                'cloud_coverage': {'type': 'float'},
                'date': {'type': 'date'},
                'data_geometry': {
                    'type': 'geo_shape',
                    'tree': 'quadtree',
                    'precision': '5mi'}
            }
        }
    }
    es = connection_to_es(es_host, es_port, kwargs['aws'])

    es.indices.create(index=index_name, ignore=400)

    es.indices.put_mapping(
        doc_type=doc_type,
        body=body,
        index=index_name
    )

def meta_constructor(metadata):
    internal_meta = copy(metadata)

    data_geometry = {
        'type': 'Polygon',
        'crs': {
            'type': 'name',
            'properties': {
                'name': 'urn:ogc:def:crs:EPSG:8.9:4326'
            }
        },
        'coordinates': [[
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')],
            [metadata.get('upperLeftCornerLongitude'), metadata.get('upperLeftCornerLatitude')],
            [metadata.get('lowerLeftCornerLongitude'), metadata.get('lowerLeftCornerLatitude')],
            [metadata.get('lowerRightCornerLongitude'), metadata.get('lowerRightCornerLatitude')],
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')]
        ]]
    }

    body = OrderedDict([
        ('scene_id', metadata.get('sceneID')),
        ('satellite_name', 'landsat-8'),
        ('cloud_coverage', metadata.get('cloudCoverFull', 100)),
        ('date', metadata.get('acquisitionDate')),
        ('thumbnail', metadata.get('browseURL')),
        ('data_geometry', data_geometry)
    ])

    body.update(internal_meta)

    return body

def elasticsearch_updater(product_dir, metadata, **kwargs):

    try:
        es = connection_to_es(kwargs['es_host'], kwargs['es_port'], kwargs['aws'])

        body = meta_constructor(metadata)

        logger.info('Pushing to Elasticsearch')

        try:
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)
        except RequestError as e:
            body['data_geometry'] = None
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)

    except Exception as e:
        logger.error('Unhandled error occured while writing to elasticsearch')
        logger.error('Details: %s' % e.__str__())

def thumbnail_writer(product_dir, metadata, **kwargs):
    """
    Extra function to download images from USGS, then upload to S3 and call
    the ES metadata writer afterwards.
    """

    # Download original thumbnail
    output_file = metadata['sceneID'] + '.jpg'
    thumbnail = 'https://' + thumbs_bucket_name + '.s3.amazonaws.com/' + output_file

    new_thumb = requests.get(thumbnail)
    if new_thumb.status_code != 200:

        # Upload thumbnail to S3
        thumbs_bucket_name2 = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
        try:
            print('uploading %s' % output_file)
            c = boto.connect_s3()
            b = c.get_bucket(thumbs_bucket_name2)
            k = Key(b, name=output_file)
            k.set_metadata('Content-Type', 'image/jpeg')
            r = requests.get(metadata['browseURL'])
            k.set_contents_from_string(r.content, policy='public-read')
        except Exception as e:
            print(e)

    # Update metadata record
    metadata['thumbnail'] = thumbnail

    elasticsearch_updater(product_dir, metadata, **kwargs)
    return

def file_writer(product_dir, metadata):
    body = meta_constructor(metadata)

    if not os.path.exists(product_dir):
        os.makedirs(product_dir)

    f = open(os.path.join(product_dir, body['scene_id'] + '.json'), 'w')
    f.write(json.dumps(body))
    logger.info('saving to disk at %s' % product_dir)
    f.close()

def s3_writer(product_dir, metadata):
    # make sure product_dir doesn't start with slash (/) or dot (.)
    if product_dir.startswith('.'):
        product_dir = product_dir[1:]

    if product_dir.startswith('/'):
        product_dir = product_dir[1:]

    body = meta_constructor(metadata)

    key = os.path.join(product_dir, body['scene_id'] + '.json')
    s3.Object(bucket_name, key).put(Body=json.dumps(body), ACL='public-read', ContentType='application/json')

    logger.info('saving to s3 at %s')

def last_updated(today):
    """ Gets the latest time a product added to Elasticsearch """

    bucket = s3.Bucket(bucket_name)

    start_day = today.day
    start_month = today.month

    yr_counter = 0
    while True:
        m_counter = 0
        year = today.year - yr_counter
        if year < 2015:
            break
        while True:
            month = start_month - m_counter
            if month == 0:
                start_month = 12
                break
            d_counter = 0
            while True:
                day = start_day - d_counter
                if day == 0:
                    start_day = 31
                    break
                path = os.path.join(str(year), str(month), str(day))
                logger.info('checking %s' % path)
                objs = bucket.objects.filter(Prefix=path).limit(1)
                if list(objs):
                    return date(year, month, day)
                d_counter += 1
            m_counter += 1
        yr_counter += 1

    return None

@click.command()
@click.argument('ops', metavar='<operations: choices: s3 | es | disk | thumbs>', nargs=-1)
@click.option('--start', default=None, help='Start Date. Format: YYYY-MM-DD')
@click.option('--end', default=None, help='End Date. Format: YYYY-MM-DD')
@click.option('--es-host', default='localhost', help='Elasticsearch host address')
@click.option('--es-port', default=9200, type=int, help='Elasticsearch port number')
@click.option('--folder', default='.', help='Destination folder if is written to disk')
@click.option('--download', is_flag=True,
              help='Sets the updater to download the metadata file first instead of streaming it')
@click.option('--aws', is_flag=True, default=False,
              help='Uses AWS STS to obtain aws credentials for accessing the various services')
@click.option('--download-folder', default=None,
              help='The folder to save the downloaded metadata to. Defaults to a temp folder')
@click.option('-v', '--verbose', is_flag=True)
@click.option('--concurrency', default=20, type=int, help='Process concurrency. Default=20')
def main(ops, start, end, es_host, es_port, folder, download,
         aws, download_folder, verbose, concurrency):

    if not ops:
        raise click.UsageError('No Argument provided. Use --help if you need help')

    accepted_args = {
        'es': elasticsearch_updater,
        's3': s3_writer,
        'disk': file_writer,
        'thumbs': thumbnail_writer
    }

    writers = []
    for op in ops:
        if op in accepted_args.keys():
            writers.append(accepted_args[op])
        else:
            raise click.UsageError('Operation (%s) is not supported' % op)

    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()

    if verbose:
        ch.setLevel(logging.INFO)
    else:
        ch.setLevel(logging.ERROR)

    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if 'es' in ops or 'thumbs' in ops:
        create_index(es_index, es_type, es_host, es_port, aws=aws)

    if not start and not end:
        delta = timedelta(days=3)
        start = date.today() - delta
        start = '{0}-{1}-{2}'.format(start.year, start.month, start.day)

    csv_reader(folder, writers, start_date=start, end_date=end, download=download, download_path=download_folder,
               num_worker_threads=concurrency, es_host=es_host, es_port=es_port, aws=aws)

if __name__ == '__main__':
    main()
<commit_msg>fix region<commit_after>import os
import json
import boto
import urllib2
from boto.s3.key import Key
import boto3
import click
import logging
from copy import copy
from collections import OrderedDict
from datetime import date, timedelta
from elasticsearch import Elasticsearch, RequestError, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
import requests

from reader import csv_reader

logger = logging.getLogger('landsat8.meta')
bucket_name = os.getenv('BUCKETNAME', 'landsat8-meta')
thumbs_bucket_name = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
s3 = boto3.resource('s3')
es_index = 'sat-api'
es_type = 'landsat8'

def get_url(url, j=False):
    req = urllib2.Request(url)
    r = urllib2.urlopen(req).read()
    if j:
        obj = json.loads(r)
    else:
        obj = r
    return obj

def get_role():
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
    obj = get_url(url)
    return obj

def get_credentials():
    role = get_role()
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/' + role
    obj = get_url(url, True)
    return obj

def connection_to_es(es_host, es_port, aws=False):

    args = {}

    if aws:
        cred = get_credentials()

        access_key = cred['AccessKeyId']
        secret_access = cred['SecretAccessKey']
        region = os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
        awsauth = AWS4Auth(access_key, secret_access, region, 'es')

        args = {
            'http_auth': awsauth,
            'use_ssl': True,
            'verify_certs': True,
            'connection_class': RequestsHttpConnection
        }

    es = Elasticsearch(hosts=[{
        'host': es_host,
        'port': es_port
    }], **args)

    return es

def create_index(index_name, doc_type, es_host, es_port, **kwargs):

    body = {
        doc_type: {
            'properties': {
                'scene_id': {'type': 'string', 'index': 'not_analyzed'},
                'satellite_name': {'type': 'string'},
                'cloud_coverage': {'type': 'float'},
                'date': {'type': 'date'},
                'data_geometry': {
                    'type': 'geo_shape',
                    'tree': 'quadtree',
                    'precision': '5mi'}
            }
        }
    }
    es = connection_to_es(es_host, es_port, kwargs['aws'])

    es.indices.create(index=index_name, ignore=400)

    es.indices.put_mapping(
        doc_type=doc_type,
        body=body,
        index=index_name
    )

def meta_constructor(metadata):
    internal_meta = copy(metadata)

    data_geometry = {
        'type': 'Polygon',
        'crs': {
            'type': 'name',
            'properties': {
                'name': 'urn:ogc:def:crs:EPSG:8.9:4326'
            }
        },
        'coordinates': [[
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')],
            [metadata.get('upperLeftCornerLongitude'), metadata.get('upperLeftCornerLatitude')],
            [metadata.get('lowerLeftCornerLongitude'), metadata.get('lowerLeftCornerLatitude')],
            [metadata.get('lowerRightCornerLongitude'), metadata.get('lowerRightCornerLatitude')],
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')]
        ]]
    }

    body = OrderedDict([
        ('scene_id', metadata.get('sceneID')),
        ('satellite_name', 'landsat-8'),
        ('cloud_coverage', metadata.get('cloudCoverFull', 100)),
        ('date', metadata.get('acquisitionDate')),
        ('thumbnail', metadata.get('browseURL')),
        ('data_geometry', data_geometry)
    ])

    body.update(internal_meta)

    return body

def elasticsearch_updater(product_dir, metadata, **kwargs):

    try:
        es = connection_to_es(kwargs['es_host'], kwargs['es_port'], kwargs['aws'])

        body = meta_constructor(metadata)

        logger.info('Pushing to Elasticsearch')

        try:
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)
        except RequestError as e:
            body['data_geometry'] = None
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)

    except Exception as e:
        logger.error('Unhandled error occured while writing to elasticsearch')
        logger.error('Details: %s' % e.__str__())

def thumbnail_writer(product_dir, metadata, **kwargs):
    """
    Extra function to download images from USGS, then upload to S3 and call
    the ES metadata writer afterwards.
    """

    # Download original thumbnail
    output_file = metadata['sceneID'] + '.jpg'
    thumbnail = 'https://' + thumbs_bucket_name + '.s3.amazonaws.com/' + output_file

    new_thumb = requests.get(thumbnail)
    if new_thumb.status_code != 200:

        # Upload thumbnail to S3
        thumbs_bucket_name2 = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
        try:
            print('uploading %s' % output_file)
            c = boto.connect_s3()
            b = c.get_bucket(thumbs_bucket_name2)
            k = Key(b, name=output_file)
            k.set_metadata('Content-Type', 'image/jpeg')
            r = requests.get(metadata['browseURL'])
            k.set_contents_from_string(r.content, policy='public-read')
        except Exception as e:
            print(e)

    # Update metadata record
    metadata['thumbnail'] = thumbnail

    elasticsearch_updater(product_dir, metadata, **kwargs)
    return

def file_writer(product_dir, metadata):
    body = meta_constructor(metadata)

    if not os.path.exists(product_dir):
        os.makedirs(product_dir)

    f = open(os.path.join(product_dir, body['scene_id'] + '.json'), 'w')
    f.write(json.dumps(body))
    logger.info('saving to disk at %s' % product_dir)
    f.close()

def s3_writer(product_dir, metadata):
    # make sure product_dir doesn't start with slash (/) or dot (.)
    if product_dir.startswith('.'):
        product_dir = product_dir[1:]

    if product_dir.startswith('/'):
        product_dir = product_dir[1:]

    body = meta_constructor(metadata)

    key = os.path.join(product_dir, body['scene_id'] + '.json')
    s3.Object(bucket_name, key).put(Body=json.dumps(body), ACL='public-read', ContentType='application/json')

    logger.info('saving to s3 at %s')

def last_updated(today):
    """ Gets the latest time a product added to Elasticsearch """

    bucket = s3.Bucket(bucket_name)

    start_day = today.day
    start_month = today.month

    yr_counter = 0
    while True:
        m_counter = 0
        year = today.year - yr_counter
        if year < 2015:
            break
        while True:
            month = start_month - m_counter
            if month == 0:
                start_month = 12
                break
            d_counter = 0
            while True:
                day = start_day - d_counter
                if day == 0:
                    start_day = 31
                    break
                path = os.path.join(str(year), str(month), str(day))
                logger.info('checking %s' % path)
                objs = bucket.objects.filter(Prefix=path).limit(1)
                if list(objs):
                    return date(year, month, day)
                d_counter += 1
            m_counter += 1
        yr_counter += 1

    return None

@click.command()
@click.argument('ops', metavar='<operations: choices: s3 | es | disk | thumbs>', nargs=-1)
@click.option('--start', default=None, help='Start Date. Format: YYYY-MM-DD')
@click.option('--end', default=None, help='End Date. Format: YYYY-MM-DD')
@click.option('--es-host', default='localhost', help='Elasticsearch host address')
@click.option('--es-port', default=9200, type=int, help='Elasticsearch port number')
@click.option('--folder', default='.', help='Destination folder if is written to disk')
@click.option('--download', is_flag=True,
              help='Sets the updater to download the metadata file first instead of streaming it')
@click.option('--aws', is_flag=True, default=False,
              help='Uses AWS STS to obtain aws credentials for accessing the various services')
@click.option('--download-folder', default=None,
              help='The folder to save the downloaded metadata to. Defaults to a temp folder')
@click.option('-v', '--verbose', is_flag=True)
@click.option('--concurrency', default=20, type=int, help='Process concurrency. Default=20')
def main(ops, start, end, es_host, es_port, folder, download,
         aws, download_folder, verbose, concurrency):

    if not ops:
        raise click.UsageError('No Argument provided. Use --help if you need help')

    accepted_args = {
        'es': elasticsearch_updater,
        's3': s3_writer,
        'disk': file_writer,
        'thumbs': thumbnail_writer
    }

    writers = []
    for op in ops:
        if op in accepted_args.keys():
            writers.append(accepted_args[op])
        else:
            raise click.UsageError('Operation (%s) is not supported' % op)

    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()

    if verbose:
        ch.setLevel(logging.INFO)
    else:
        ch.setLevel(logging.ERROR)

    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if 'es' in ops or 'thumbs' in ops:
        create_index(es_index, es_type, es_host, es_port, aws=aws)

    if not start and not end:
        delta = timedelta(days=3)
        start = date.today() - delta
        start = '{0}-{1}-{2}'.format(start.year, start.month, start.day)

    csv_reader(folder, writers, start_date=start, end_date=end, download=download, download_path=download_folder,
               num_worker_threads=concurrency, es_host=es_host, es_port=es_port, aws=aws)

if __name__ == '__main__':
    main()
<|endoftext|>