<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
unittest class used in Selenium tests. Every test_cluster_* inherits methods
from ClusterTest.

@author: Ioannis Stenos, Nick Vrionis
'''

from selenium import webdriver
import sys
from os.path import join, dirname, abspath
sys.path.append(join(dirname(abspath(__file__)), '../..'))
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from ConfigParser import RawConfigParser, NoSectionError
import unittest, time, re

BASE_DIR = join(dirname(abspath(__file__)), "../..")

class ClusterTest(unittest.TestCase):
    '''
    setUp method is common for all test_cluster tests.
    Defines the path to okeanos token and the base url for
    the selenium test.
    '''
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(30)
        self.verificationErrors = []
        self.accept_next_alert = True
        parser = RawConfigParser()
        config_file = join(BASE_DIR, '.private/.config.txt')
        self.name = 'testcluster'
        parser.read(config_file)
        try:
            self.token = parser.get('cloud \"~okeanos\"', 'token')
            self.auth_url = parser.get('cloud \"~okeanos\"', 'url')
            self.base_url = parser.get('cloud \"~okeanos\"', 'app_url')
        except NoSectionError:
            self.token = 'INVALID_TOKEN'
            self.auth_url = "INVALID_AUTH_URL"
            self.base_url = "INVALID_ESCIENCE_APP_HOST_URL"
            print 'Current authentication details are kept off source control. ' \
                  '\nUpdate your .config.txt file in <projectroot>/.private/'

    def login(self):
        '''Method used for login by all test_cluster tests'''
        driver = self.driver
        driver.get(self.base_url + "#/homepage")
        driver.find_element_by_css_selector("button[type=\"submit\"]").click()
        driver.find_element_by_id("token").clear()
        driver.find_element_by_id("token").send_keys(self.token)
        driver.find_element_by_css_selector("button[type=\"login\"]").click()
        for i in range(60):
            try:
                if self.is_element_present(By.ID, "CreateHadoopCluster"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        driver.find_element_by_id("CreateHadoopCluster").click()
        for i in range(60):
            try:
                if self.is_element_present(By.XPATH, "//div[@id='sidebar']/p/select"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        return driver

    def is_element_present(self, how, what):
        try: self.driver.find_element(by=how, value=what)
        except NoSuchElementException, e: return False
        return True

    def is_alert_present(self):
        try: self.driver.switch_to_alert()
        except NoAlertPresentException, e: return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally: self.accept_next_alert = True

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)

    def calculate_cluster_resources(self, resource_list, available_resource):
        '''
        Method used by test_cluster_cpu, test_cluster_memory and
        test_cluster_disk to calculate the resources needed to be binded
        during each test in cluster/create screen. Those resources are
        the buttons pressed by selenium test and are not really binded
        in ~okeanos. The result is that much less real resources are binded
        in ~okeanos for the tests.
        '''
        avail = available_resource
        cluster_size = 0
        vms = []
        # Create a vms list with values the combinations of resource size
        # and number of vms we will bind in selenium with a particular,
        # resource flavor e.g for test_cluster_disk vms = [{100:2}, {80:1}]
        # means 2 vms with 100 disk size each and 1 vm with 80 disk size.
        for resource in reversed(resource_list):
            if (available_resource/resource) >= 1:
                vms.append({resource: available_resource/resource})
                if available_resource%resource ==0:
                    break
                available_resource = available_resource - resource*(available_resource/resource)
        # If the vms list has two or more elements
        if len(vms) >= 2:
            # Find the remaining resource that we will bind in ~okeanos for the test.
            remaining_resource = avail - vms[0].values()[0] * resource_list[len(resource_list)-1] - vms[1].values()[0] * resource_list[len(resource_list)-2]
            # Calculate the cluster_size we will use as input in selenium
            cluster_size = vms[0].values()[0] + vms[1].values()[0]
            for resource in resource_list:
                if resource == vms[0].keys()[0]:
                    # Select the buttons selenium will press
                    # in create_cluster screen
                    slave = str(resource_list.index(resource) + 1)
                    master = str(resource_list.index(resource))
                    break
        # If the vms list has zero elements
        elif len(vms) == 0:
            raise RuntimeError
        # If the vms list has only one element
        else:
            remaining_resource = 0
            cluster_size = cluster_size + vms[0].values()[0]
            for resource in resource_list:
                if resource == vms[0].keys()[0]:
                    slave = str(resource_list.index(resource) + 1)
                    master = str(resource_list.index(resource) + 1)
                    break
        return cluster_size, master, slave, remaining_resource
<commit_msg>ES-149 ClusterTest fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
unittest class used in Selenium tests. Every test_cluster_* inherits methods
from ClusterTest.

@author: Ioannis Stenos, Nick Vrionis
'''

from selenium import webdriver
import sys
from os.path import join, dirname, abspath
sys.path.append(join(dirname(abspath(__file__)), '../..'))
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from ConfigParser import RawConfigParser, NoSectionError
import unittest, time, re

BASE_DIR = join(dirname(abspath(__file__)), "../..")

class ClusterTest(unittest.TestCase):
    '''
    setUp method is common for all test_cluster tests.
    Defines the path to okeanos token and the base url for
    the selenium test.
    '''
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(30)
        self.verificationErrors = []
        self.accept_next_alert = True
        parser = RawConfigParser()
        config_file = join(BASE_DIR, '.private/.config.txt')
        self.name = 'testcluster'
        parser.read(config_file)
        try:
            self.token = parser.get('cloud \"~okeanos\"', 'token')
            self.auth_url = parser.get('cloud \"~okeanos\"', 'url')
            self.base_url = parser.get('cloud \"~okeanos\"', 'app_url')
        except NoSectionError:
            self.token = 'INVALID_TOKEN'
            self.auth_url = "INVALID_AUTH_URL"
            self.base_url = "INVALID_ESCIENCE_APP_HOST_URL"
            print 'Current authentication details are kept off source control. ' \
                  '\nUpdate your .config.txt file in <projectroot>/.private/'

    def login(self):
        '''Method used for login by all test_cluster tests'''
        driver = self.driver
        driver.get(self.base_url + "#/homepage")
        driver.find_element_by_css_selector("button[type=\"submit\"]").click()
        driver.find_element_by_id("token").clear()
        driver.find_element_by_id("token").send_keys(self.token)
        driver.find_element_by_css_selector("button[type=\"login\"]").click()
        for i in range(60):
            try:
                if self.is_element_present(By.ID, "CreateHadoopCluster"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        driver.find_element_by_id("CreateHadoopCluster").click()
        for i in range(60):
            try:
                if self.is_element_present(By.XPATH, "//div[@id='sidebar']/p/select"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        return driver

    def is_element_present(self, how, what):
        try: self.driver.find_element(by=how, value=what)
        except NoSuchElementException, e: return False
        return True

    def is_alert_present(self):
        try: self.driver.switch_to_alert()
        except NoAlertPresentException, e: return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally: self.accept_next_alert = True

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)

    def calculate_cluster_resources(self, resource_list, available_resource):
        '''
        Method used by test_cluster_cpu, test_cluster_memory and
        test_cluster_disk to calculate the resources needed to be binded
        during each test in cluster/create screen. Those resources are
        the buttons pressed by selenium test and are not really binded
        in ~okeanos. The result is that much less real resources are binded
        in ~okeanos for the tests.
        '''
        avail = available_resource
        cluster_size = 0
        vms = []
        # Create a vms list with values the combinations of resource size
        # and number of vms we will bind in selenium with a particular,
        # resource flavor e.g for test_cluster_disk vms = [{100:2}, {80:1}]
        # means 2 vms with 100 disk size each and 1 vm with 80 disk size.
        for resource in reversed(resource_list):
            if (available_resource/resource) >= 1:
                vms.append({resource: available_resource/resource})
                if available_resource%resource ==0:
                    break
                available_resource = available_resource - resource*(available_resource/resource)
        # If the vms list has two or more elements
        if len(vms) >= 2:
            # Find the remaining resource that we will bind in ~okeanos for the test.
            remaining_resource = avail - vms[0].values()[0] * resource_list[len(resource_list)-1] - vms[1].values()[0] * resource_list[len(resource_list)-2]
            # Calculate the cluster_size we will use as input in selenium
            cluster_size = vms[0].values()[0] + vms[1].values()[0]
            for resource in resource_list:
                if resource == vms[0].keys()[0]:
                    # Select the buttons selenium will press
                    # in create_cluster screen
                    slave = str(resource_list.index(vms[0].values()[0]) + 1)
                    master = str(resource_list.index(vms[1].values()[0]))
                    break
        # If the vms list has zero elements
        elif len(vms) == 0:
            raise RuntimeError
        # If the vms list has only one element
        else:
            remaining_resource = 0
            cluster_size = cluster_size + vms[0].values()[0]
            for resource in resource_list:
                if resource == vms[0].keys()[0]:
                    slave = str(resource_list.index(vms[0].keys()[0]) + 1)
                    master = str(resource_list.index(vms[0].keys()[0]) + 1)
                    break
        return cluster_size, master, slave, remaining_resource
<|endoftext|>


<commit_before>import json
import web

from config import DB

def load_machine(name):
    results = list(DB.select('machine', what='*',
                             where='name = $name',
                             vars=dict(name=name)))
    if not results:
        raise web.NotFound()
    return results[0]

class MachineLock:
    def GET(self, name):
        row = load_machine(name)
        row.locked_since = row.locked_since.isoformat()
        web.header('Content-type', 'text/json')
        return json.dumps(row)

    def DELETE(self, name):
        user = web.input('user')['user']
        machine = load_machine(name)
        if not machine.locked:
            raise web.BadRequest()
        if machine.locked_by != user:
            raise web.Forbidden()

        res = DB.update('machine',
                        where='locked = true AND name = $name AND locked_by = $user',
                        vars=dict(name=name, user=user),
                        locked=False, locked_by=None)
        assert res == 1, 'Failed to unlock machine {name}'.format(name=name)

    def POST(self, name):
        user = web.input('user')['user']
        machine = load_machine(name)
        if machine.locked:
            raise web.Forbidden()
        res = DB.update('machine', where='name = $name AND locked = false',
                        vars=dict(name=name),
                        locked=True,
                        locked_by=user,
                        locked_since=web.db.SQLLiteral('NOW()'))
        assert res == 1, 'Failed to lock machine {name}'.format(name=name)

    def PUT(self, name):
        desc = web.input(desc=None)['desc']
        status = web.input(status=None)['status']
        sshpubkey = web.input(sshpubkey=None)['sshpubkey']

        updated = {}
        if desc is not None:
            updated['description'] = desc
        if status is not None:
            updated['up'] = (status == 'up')
        if sshpubkey is not None:
            updated['sshpubkey'] = sshpubkey

        if not updated:
            raise web.BadRequest()
        DB.update('machine', where='name = $name',
                  vars=dict(name=name), **updated)

class Lock:
    def GET(self):
        rows = list(DB.select('machine', what='*'))
        if not rows:
            raise web.NotFound()
        for row in rows:
            row.locked_since = row.locked_since.isoformat()
        web.header('Content-type', 'text/json')
        return json.dumps(rows)

    def POST(self):
        user = web.input('user')['user']
        num = int(web.input('num')['num'])

        if num < 1:
            raise web.BadRequest()

        tries = 0
        while True:
            try:
                with DB.transaction():
                    results = list(DB.select('machine', what='name, sshpubkey',
                                             where='locked = false and up = true',
                                             limit=num))
                    if len(results) < num:
                        raise web.HTTPError(status='503 Service Unavailable')
                    name_keys = {}
                    for row in results:
                        name_keys[row.name] = row.sshpubkey
                    num_locked = DB.update('machine',
                                           where=web.db.sqlors('name = ',
                                                               name_keys.keys()),
                                           locked=True,
                                           locked_by=user,
                                           locked_since=web.db.SQLLiteral('NOW()'))
                    assert num_locked == num, 'Failed to lock machines'
            except:
                tries += 1
                if tries < 10:
                    continue
                raise
            else:
                break

        web.header('Content-type', 'text/json')
        return json.dumps(name_keys)
<commit_msg>locker: fix race in locking<commit_after>import json
import web

from config import DB

def load_machine(name):
    results = list(DB.select('machine', what='*',
                             where='name = $name',
                             vars=dict(name=name)))
    if not results:
        raise web.NotFound()
    return results[0]

class MachineLock:
    def GET(self, name):
        row = load_machine(name)
        row.locked_since = row.locked_since.isoformat()
        web.header('Content-type', 'text/json')
        return json.dumps(row)

    def DELETE(self, name):
        user = web.input('user')['user']
        machine = load_machine(name)
        if not machine.locked:
            raise web.BadRequest()
        if machine.locked_by != user:
            raise web.Forbidden()

        res = DB.update('machine',
                        where='locked = true AND name = $name AND locked_by = $user',
                        vars=dict(name=name, user=user),
                        locked=False, locked_by=None)
        assert res == 1, 'Failed to unlock machine {name}'.format(name=name)

    def POST(self, name):
        user = web.input('user')['user']
        machine = load_machine(name)
        if machine.locked:
            raise web.Forbidden()
        res = DB.update('machine', where='name = $name AND locked = false',
                        vars=dict(name=name),
                        locked=True,
                        locked_by=user,
                        locked_since=web.db.SQLLiteral('NOW()'))
        assert res == 1, 'Failed to lock machine {name}'.format(name=name)

    def PUT(self, name):
        desc = web.input(desc=None)['desc']
        status = web.input(status=None)['status']
        sshpubkey = web.input(sshpubkey=None)['sshpubkey']

        updated = {}
        if desc is not None:
            updated['description'] = desc
        if status is not None:
            updated['up'] = (status == 'up')
        if sshpubkey is not None:
            updated['sshpubkey'] = sshpubkey

        if not updated:
            raise web.BadRequest()
        DB.update('machine', where='name = $name',
                  vars=dict(name=name), **updated)

class Lock:
    def GET(self):
        rows = list(DB.select('machine', what='*'))
        if not rows:
            raise web.NotFound()
        for row in rows:
            row.locked_since = row.locked_since.isoformat()
        web.header('Content-type', 'text/json')
        return json.dumps(rows)

    def POST(self):
        user = web.input('user')['user']
        num = int(web.input('num')['num'])

        if num < 1:
            raise web.BadRequest()

        tries = 0
        while True:
            try:
                # transaction will be rolled back if an exception is raised
                with DB.transaction():
                    results = list(DB.select('machine', what='name, sshpubkey',
                                             where='locked = false AND up = true',
                                             limit=num))
                    if len(results) < num:
                        raise web.HTTPError(status='503 Service Unavailable')
                    name_keys = {}
                    for row in results:
                        name_keys[row.name] = row.sshpubkey
                    where_cond = web.db.sqlors('name = ', name_keys.keys()) \
                        + ' AND locked = false AND up = true'
                    num_locked = DB.update('machine',
                                           where=where_cond,
                                           locked=True,
                                           locked_by=user,
                                           locked_since=web.db.SQLLiteral('NOW()'))
                    assert num_locked == num, 'Failed to lock machines'
            except:
                tries += 1
                if tries < 10:
                    continue
                raise
            else:
                break

        web.header('Content-type', 'text/json')
        return json.dumps(name_keys)
<|endoftext|>


<commit_before>import logging
from get_database import get_section_db
from main import carbon, common, stats
from datetime import datetime, timedelta
from datetime import time as dttime
import time
from dao.user import User
import math
from get_database import get_uuid_db
from main import userclient

# sb375 is a weekly goal - we convert it to daily by dividing by 7
sb375DailyGoal = 40.142892/7

# BEGIN: Code to get and set client specific fields in the profile (currentScore and previousScore)
def getStoredScore(user):
    profile = user.getProfile()
    if profile is None:
        return (0, 0)
    currScore = profile.get('currentScore', 0)
    prevScore = profile.get('previousScore', 0)
    return (prevScore, currScore)

def setScores(user, prevScore, newScore):
    logging.debug("Changing score for user %s from %s to %s" % (user.uuid, prevScore, newScore))
    user.setClientSpecificProfileFields({'previousScore': prevScore,
                                         'currentScore': newScore})
    # TODO: Add a server side stat here so that we can know for sure how the score varies over time
# END: Code to get and set client specific fields in the profile (currentScore and previousScore)

# Returns the components on which the score is based. These will be combined in
# getScore later, but it is useful to see them to decide how to set up the
# weights

def getScoreComponents(user_uuid, start, end):
  # The score is based on the following components:
  # - Percentage of trips classified. We are not auto-classifying high
  # confidence trips, so don't need to handle those here
  user = User.fromUUID(user_uuid)

  pctClassified = common.getClassifiedRatio(user_uuid, start, end)

  (myModeShareCount, avgModeShareCount,
   myModeShareDistance, avgModeShareDistance,
   myModeCarbonFootprint, avgModeCarbonFootprint,
   myModeCarbonFootprintNoLongMotorized, avgModeCarbonFootprintNoLongMotorized,
   myOptimalCarbonFootprint, avgOptimalCarbonFootprint,
   myOptimalCarbonFootprintNoLongMotorized, avgOptimalCarbonFootprintNoLongMotorized) = carbon.getFootprintCompareForRange(user.uuid, start, end)

  carbon.delLongMotorizedModes(myModeShareDistance)
  myAllDrive = carbon.getAllDrive(user.uuid, myModeShareDistance)
  myCarbonFootprintSum = sum(myModeCarbonFootprintNoLongMotorized.values())
  myOptimalFootprintSum = sum(myOptimalCarbonFootprintNoLongMotorized.values())
  logging.debug("myCarbonFootprintSum = %s, myOptimalFootprintSum = %s, myAllDrive = %s" %
        (myCarbonFootprintSum, myOptimalFootprintSum, myAllDrive))
  handleZero = lambda x, y: 0 if y == 0 else float(x)/y
  components = [pctClassified,
                handleZero(myCarbonFootprintSum - myOptimalFootprintSum, myOptimalFootprintSum),
                handleZero(myAllDrive - myCarbonFootprintSum, myAllDrive),
                handleZero(sb375DailyGoal - myCarbonFootprintSum, sb375DailyGoal)]
  return components

def calcScore(componentArr):
  [pctClassified, mineMinusOptimal, allDriveMinusMine, sb375DailyGoal] = componentArr
  # We want the ratio between the three components to be 5 : 3 : 2
  # Let's just convert everything to percentages to keep the ratios consistent
  # Also, we subtract the mineMinusOptimal term, since being way above optimal
  # should lower your score
  return 50 * pctClassified + 30 * allDriveMinusMine - 20 * mineMinusOptimal + 10 * sb375DailyGoal

def getScore(user_uuid, start, end):
    components = getScoreComponents(user_uuid, start, end)
    [pctClassified, mineMinusOptimal, allDriveMinusMine, sb375DailyGoal] = components
    stats.storeResultEntry(user_uuid, stats.STAT_PCT_CLASSIFIED, time.time(), pctClassified)
    stats.storeResultEntry(user_uuid, stats.STAT_MINE_MINUS_OPTIMAL, time.time(), mineMinusOptimal)
    stats.storeResultEntry(user_uuid, stats.STAT_ALL_DRIVE_MINUS_MINE, time.time(), allDriveMinusMine)
    stats.storeResultEntry(user_uuid, stats.STAT_SB375_DAILY_GOAL, time.time(), sb375DailyGoal)
    return calcScore(components)

# Ok so this is a big tricky to get right.
# We want this to be in increments of a day, so that the SB375 target
# calculation makes sense.
# The high level plan is to run it around midnight everyday and update the
# score based on the day that just passed. But what exactly does "around
# midnight" mean?
# It can't be just before midnight, because then we will skip trips around midnight
# I guess we start right after midnight
# Couple of other challenges:
# - What to do about trips that span days?
# - What to do about people in other time zones who have a different midnight?
# In order to handle both of these, we should really have task scheduled from
# the app instead of a cronjob, and have it track the last time it was run. But
# let's do the cheap solution for now so that we know whether it works at all.
def updateScore(user_uuid):
    today = datetime.now().date()
    updateScoreForDay(user_uuid, today)

def updateScoreForDay(user_uuid, today):
    yesterday = today - timedelta(days = 1)
    dayBeforeYesterday = today - timedelta(days = 2)

    dayBeforeYesterdayStart = datetime.combine(dayBeforeYesterday, dttime.min)
    yesterdayStart = datetime.combine(yesterday, dttime.min)
    todayStart = datetime.combine(today, dttime.min)

    user = User.fromUUID(user_uuid)
    (discardedScore, prevScore) = getStoredScore(user)
    # Using score from dayBeforeYesterday instead of yesterday because there is
    # currently a significant lag in the time for e-mission to prompt for
    # entries, so people might not confirm yesterday's trips until sometime
    # today, which means that it won't be counted in their score
    newScore = prevScore + getScore(user_uuid, dayBeforeYesterdayStart, yesterdayStart)
    if newScore < 0:
        newScore = 0
    stats.storeResultEntry(user_uuid, stats.STAT_GAME_SCORE, time.time(), newScore)
    setScores(user, prevScore, newScore)

def getLevel(score):
  if score < 100:
    level = 1
    sublevel = math.floor(score / 20) + 1
  elif score < 1000:
    level = 2
    sublevel = math.floor(score / 200) + 1
  elif score < 10000:
    level = 3
    sublevel = math.floor(score / 2000) + 1
  else:
    # Off the charts, stay at the top image
    level = 3
    sublevel = 5
  return (level, sublevel)

def getFileName(level, sublevel):
    return "level_%s_%s.png" % (int(level), int(sublevel))

def getResult(user_uuid):
  # This is in here, as opposed to the top level as recommended by the PEP
  # because then we don't have to worry about loading bottle in the unit tests
  from bottle import template

   (prevScore, currScore) = getStoredScore(User.fromUUID(user_uuid))
      (level, sublevel) = getLevel(currScore)

   otherCurrScoreList = []
   for user_uuid_dict in get_uuid_db().find({}, {'uuid': 1, '_id': 0}):
    (currPrevScore, currCurrScore) = getStoredScore(User.fromUUID(user_uuid_dict['uuid']))
    otherCurrScoreList.append(currCurrScore)

   otherCurrScoreList.sort()
   renderedTemplate = template("clients/leaderboard/result_template.html",
                               level_picture_filename = getFileName(level, sublevel),
                               prevScore = prevScore,
                               currScore = currScore,
                               otherCurrScoreList = otherCurrScoreList)
   return renderedTemplate

# These are copy/pasted from our first client, the carshare study
def getSectionFilter(uuid):
  # We are not planning to do any filtering for this study. Bring on the worst!
  return []

def clientSpecificSetters(uuid, sectionId, predictedModeMap):
  return None

def getClientConfirmedModeField():
  return None

def runBackgroundTasks(uuid):
  today = datetime.now().date()
  runBackgroundTasksForDay(uuid, today)

def runBackgroundTasksForDay(uuid, today):
  updateScoreForDay(uuid, today)
<commit_msg>fixed indenting error<commit_after>import logging
from get_database import get_section_db
from main import carbon, common, stats
from datetime import datetime, timedelta
from datetime import time as dttime
import time
from dao.user import User
import math
from get_database import get_uuid_db
from main import userclient

# sb375 is a weekly goal - we convert it to daily by dividing by 7
sb375DailyGoal = 40.142892/7

# BEGIN: Code to get and set client specific fields in the profile (currentScore and previousScore)
def getStoredScore(user):
    profile = user.getProfile()
    if profile is None:
        return (0, 0)
    currScore = profile.get('currentScore', 0)
    prevScore = profile.get('previousScore', 0)
    return (prevScore, currScore)

def setScores(user, prevScore, newScore):
    logging.debug("Changing score for user %s from %s to %s" % (user.uuid, prevScore, newScore))
    user.setClientSpecificProfileFields({'previousScore': prevScore,
                                         'currentScore': newScore})
    # TODO: Add a server side stat here so that we can know for sure how the score varies over time
# END: Code to get and set client specific fields in the profile (currentScore and previousScore)

# Returns the components on which the score is based. These will be combined in
# getScore later, but it is useful to see them to decide how to set up the
# weights

def getScoreComponents(user_uuid, start, end):
  # The score is based on the following components:
  # - Percentage of trips classified. We are not auto-classifying high
  # confidence trips, so don't need to handle those here
  user = User.fromUUID(user_uuid)

  pctClassified = common.getClassifiedRatio(user_uuid, start, end)

  (myModeShareCount, avgModeShareCount,
   myModeShareDistance, avgModeShareDistance,
   myModeCarbonFootprint, avgModeCarbonFootprint,
   myModeCarbonFootprintNoLongMotorized, avgModeCarbonFootprintNoLongMotorized,
   myOptimalCarbonFootprint, avgOptimalCarbonFootprint,
   myOptimalCarbonFootprintNoLongMotorized, avgOptimalCarbonFootprintNoLongMotorized) = carbon.getFootprintCompareForRange(user.uuid, start, end)

  carbon.delLongMotorizedModes(myModeShareDistance)
  myAllDrive = carbon.getAllDrive(user.uuid, myModeShareDistance)
  myCarbonFootprintSum = sum(myModeCarbonFootprintNoLongMotorized.values())
  myOptimalFootprintSum = sum(myOptimalCarbonFootprintNoLongMotorized.values())
  logging.debug("myCarbonFootprintSum = %s, myOptimalFootprintSum = %s, myAllDrive = %s" %
        (myCarbonFootprintSum, myOptimalFootprintSum, myAllDrive))
  handleZero = lambda x, y: 0 if y == 0 else float(x)/y
  components = [pctClassified,
                handleZero(myCarbonFootprintSum - myOptimalFootprintSum, myOptimalFootprintSum),
                handleZero(myAllDrive - myCarbonFootprintSum, myAllDrive),
                handleZero(sb375DailyGoal - myCarbonFootprintSum, sb375DailyGoal)]
  return components

def calcScore(componentArr):
  [pctClassified, mineMinusOptimal, allDriveMinusMine, sb375DailyGoal] = componentArr
  # We want the ratio between the three components to be 5 : 3 : 2
  # Let's just convert everything to percentages to keep the ratios consistent
  # Also, we subtract the mineMinusOptimal term, since being way above optimal
  # should lower your score
  return 50 * pctClassified + 30 * allDriveMinusMine - 20 * mineMinusOptimal + 10 * sb375DailyGoal

def getScore(user_uuid, start, end):
    components = getScoreComponents(user_uuid, start, end)
    [pctClassified, mineMinusOptimal, allDriveMinusMine, sb375DailyGoal] = components
    stats.storeResultEntry(user_uuid, stats.STAT_PCT_CLASSIFIED, time.time(), pctClassified)
    stats.storeResultEntry(user_uuid, stats.STAT_MINE_MINUS_OPTIMAL, time.time(), mineMinusOptimal)
    stats.storeResultEntry(user_uuid, stats.STAT_ALL_DRIVE_MINUS_MINE, time.time(), allDriveMinusMine)
    stats.storeResultEntry(user_uuid, stats.STAT_SB375_DAILY_GOAL, time.time(), sb375DailyGoal)
    return calcScore(components)

# Ok so this is a big tricky to get right.
# We want this to be in increments of a day, so that the SB375 target
# calculation makes sense.
# The high level plan is to run it around midnight everyday and update the
# score based on the day that just passed. But what exactly does "around
# midnight" mean?
# It can't be just before midnight, because then we will skip trips around midnight
# I guess we start right after midnight
# Couple of other challenges:
# - What to do about trips that span days?
# - What to do about people in other time zones who have a different midnight?
# In order to handle both of these, we should really have task scheduled from
# the app instead of a cronjob, and have it track the last time it was run. But
# let's do the cheap solution for now so that we know whether it works at all.
def updateScore(user_uuid):
    today = datetime.now().date()
    updateScoreForDay(user_uuid, today)

def updateScoreForDay(user_uuid, today):
    yesterday = today - timedelta(days = 1)
    dayBeforeYesterday = today - timedelta(days = 2)

    dayBeforeYesterdayStart = datetime.combine(dayBeforeYesterday, dttime.min)
    yesterdayStart = datetime.combine(yesterday, dttime.min)
    todayStart = datetime.combine(today, dttime.min)

    user = User.fromUUID(user_uuid)
    (discardedScore, prevScore) = getStoredScore(user)
    # Using score from dayBeforeYesterday instead of yesterday because there is
    # currently a significant lag in the time for e-mission to prompt for
    # entries, so people might not confirm yesterday's trips until sometime
    # today, which means that it won't be counted in their score
    newScore = prevScore + getScore(user_uuid, dayBeforeYesterdayStart, yesterdayStart)
    if newScore < 0:
        newScore = 0
    stats.storeResultEntry(user_uuid, stats.STAT_GAME_SCORE, time.time(), newScore)
    setScores(user, prevScore, newScore)

def getLevel(score):
  if score < 100:
    level = 1
    sublevel = math.floor(score / 20) + 1
  elif score < 1000:
    level = 2
    sublevel = math.floor(score / 200) + 1
  elif score < 10000:
    level = 3
    sublevel = math.floor(score / 2000) + 1
  else:
    # Off the charts, stay at the top image
    level = 3
    sublevel = 5
  return (level, sublevel)

def getFileName(level, sublevel):
    return "level_%s_%s.png" % (int(level), int(sublevel))

def getResult(user_uuid):
  # This is in here, as opposed to the top level as recommended by the PEP
  # because then we don't have to worry about loading bottle in the unit tests
  from bottle import template

   (prevScore, currScore) = getStoredScore(User.fromUUID(user_uuid))
      (level, sublevel) = getLevel(currScore)

   otherCurrScoreList = []
   for user_uuid_dict in get_uuid_db().find({}, {'uuid': 1, '_id': 0}):
       (currPrevScore, currCurrScore) = getStoredScore(User.fromUUID(user_uuid_dict['uuid']))
       otherCurrScoreList.append(currCurrScore)

   otherCurrScoreList.sort()
   renderedTemplate = template("clients/leaderboard/result_template.html",
                               level_picture_filename = getFileName(level, sublevel),
                               prevScore = prevScore,
                               currScore = currScore,
                               otherCurrScoreList = otherCurrScoreList)
   return renderedTemplate

# These are copy/pasted from our first client, the carshare study
def getSectionFilter(uuid):
  # We are not planning to do any filtering for this study. Bring on the worst!
  return []

def clientSpecificSetters(uuid, sectionId, predictedModeMap):
  return None

def getClientConfirmedModeField():
  return None

def runBackgroundTasks(uuid):
  today = datetime.now().date()
  runBackgroundTasksForDay(uuid, today)

def runBackgroundTasksForDay(uuid, today):
  updateScoreForDay(uuid, today)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import unicode_literals

import time
import datetime

import requests

from lxml import etree

from dateutil.parser import *

from nameparser import HumanName

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = "clinicaltrials"

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None

def copy_to_unicode(element):

    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)

def consume(days_back=10):
    """ First, get a list of all recently updated study urls,
    then get the xml one by one and save it into a list
    of docs including other information """

    today = datetime.date.today()
    start_date = today - datetime.timedelta(days_back)

    month = today.strftime('%m')
    day = today.strftime('%d')
    year = today.strftime('%Y')

    y_month = start_date.strftime('%m')
    y_day = start_date.strftime('%d')
    y_year = start_date.strftime('%Y')

    base_url = 'http://clinicaltrials.gov/ct2/results?lup_s='
    url_end = '{}%2F{}%2F{}%2F&lup_e={}%2F{}%2F{}&displayxml=true'.\
                format(y_month, y_day, y_year, month, day, year)

    url = base_url + url_end

    # grab the total number of studies
    initial_request = requests.get(url)
    record_encoding = initial_request.encoding
    initial_request_xml = etree.XML(initial_request.content)
    count = int(initial_request_xml.xpath('//search_results/@count')[0])
    # TODO this only gets the most recent 1000 - fix this!
    xml_list = []
    if int(count) > 0:
        # get a new url with all results in it
        url = url + '&count=' + str(count)
        total_requests = requests.get(url)
        initial_doc = etree.XML(total_requests.content)

        # make a list of urls from that full list of studies
        study_urls = []
        for study in initial_doc.xpath('//clinical_study'):
            study_urls.append(study.xpath('url/node()')[0] + '?displayxml=true')

        # grab each of those urls for full content
        print("There are {} urls to consume - be patient...".format(len(study_urls)))
        count = 0
        official_count = 0
        for study_url in study_urls:
            try:
                content = requests.get(study_url)
            except requests.exceptions.ConnectionError as e:
                print('Connection error: {}, wait a bit...'.format(e))
                time.sleep(30)
                continue
            doc = etree.XML(content.content)
            record = etree.tostring(doc, encoding=record_encoding)
            doc_id = doc.xpath('//nct_id/node()')[0]
            xml_list.append(RawDocument({
                    'doc': record,
                    'source': NAME,
                    'docID': copy_to_unicode(doc_id),
                    'filetype': 'xml',
                }))
            official_count += 1
            count += 1
            if count%100 == 0:
                print("You've requested {} studies, keep going!".format(official_count))
                count = 0
            time.sleep(1)

    return xml_list

def get_contributors(xml_doc):
    contributor_list = []
    #TODO - fix this for weird contributor names like companies
    contributors = xml_doc.xpath('//overall_official/last_name/node()') or xml_doc.xpath('//lead_sponsor/agency/node()') or ['']
    for person in contributors:
        name = HumanName(person)
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
            'ORCID': ''
        }
        contributor_list.append(contributor)
    return contributor_list

def get_ids(raw_doc, xml_doc):
    url = (xml_doc.xpath('//required_header/url/node()') or [''])[0]
    ids = {'serviceID': raw_doc.get('docID'), 'doi': '', 'url': copy_to_unicode(url)}
    if url == '':
        raise Exception('Warning: No url provided!')
    return ids

def get_tags(xml_doc):
    keywords = [copy_to_unicode(tag.lower()) for  tag in xml_doc.xpath('//keyword/node()')]
    return keywords

def get_properties(xml_doc):
    lead_sponsor  = {
            'agency': (xml_doc.xpath('//lead_sponsor/agency/node()') or [''])[0],
            'agency_class': (xml_doc.xpath('//lead_sponsor/agency_class/node()') or [''])[0]
    }

    for key, value in lead_sponsor.iteritems():
        lead_sponsor[key] = copy_to_unicode(value)

    primary_outcome = {
            'measure': (xml_doc.xpath('//primary_outcome/measure/node()') or [''])[0],
            'time_frame': (xml_doc.xpath('//primary_outcome/time_frame/node()') or [''])[0],
            'safety_issue': (xml_doc.xpath('//primary_outcome/safety_issue/node()') or [''])[0]
    }

    for key, value in primary_outcome.iteritems():
        primary_outcome[key] = copy_to_unicode(value)

    # gives a list of dictionaries of all secondary outcomes
    secondary_outcome_elements = xml_doc.xpath('//secondary_outcome')
    secondary_outcomes = []
    for item in secondary_outcome_elements:
        secondary_outcome = {element.tag: element.text for element in item.iterdescendants()}
        secondary_outcomes.append(copy_to_unicode(secondary_outcome))

    # enrollment - can have different types
    enrollment_list = xml_doc.xpath('//enrollment')
    try:
        enrollment = {item.values()[0]: copy_to_unicode(item.text) for item in enrollment_list}
    except IndexError:
        enrollment = []

    # arm group
    arm_group_elements = xml_doc.xpath('//arm_group')
    arm_groups = []
    for item in arm_group_elements:
        arm_group = {element.tag: copy_to_unicode(element.text) for element in item.iterdescendants()}
        arm_groups.append(arm_group)

    # intervention
    intervention_elements = xml_doc.xpath('//intervention')
    interventions = []
    for item in intervention_elements:
        intervention = {element.tag: copy_to_unicode(element.text) for element in item.iterdescendants()}
        interventions.append(intervention)

    # eligibility
    eligibility_elements = xml_doc.xpath('//eligibility')
    eligibility = {}
    if len(eligibility_elements) > 1:
        for element in eligibility_elements[0].iterchildren():
            if element.text.strip() == '':
                for child in element.getchildren():
                    if child.text.strip() != '':
                        eligibility[element.tag] = copy_to_unicode(child.text)
            else:
                eligibility[element.tag] = copy_to_unicode(element.text)
    else:
        eligibility_elements = ''

    ## TODO: location - undone for now
    ## location has a facility name - and address with separate city state zip country
    ## {name: 'facility name', address: {city: 'city', state:'state', zip: 'zip', country:'country'}, status: 'status'}

    ## TODO: location sometimes has contact with name and email - what do?
    # location_elements = xml_doc.xpath('//location')
    # locations = []
    # upper_level = [item.getchildren() for item in location_elements]
    # location = {}
    # for item in upper_level:
    #     for element in item:
    #         for detail in element:
    #             location[element.tag] = {detail.tag: detail.text}

    # link
    link_elements = xml_doc.xpath('//link')
    links = []
    for item in link_elements:
        link = {element.tag: copy_to_unicode(element.text) for element in item.iterdescendants()}
        links.append(link)

    # responsible party
    # TODO: is there ever more than one responsible party?
    responsible_party_elements = xml_doc.xpath('//responsible_party')
    try:
        responsible_party = {elem.tag: copy_to_unicode(elem.text) for elem in responsible_party_elements[0].iterdescendants()}
    except IndexError:
        responsible_party = {}

    #TODO: a more robust search for more metadata in different examples

    ## extra properties ##
    properties = {
        'sponsors': lead_sponsor,
        'oversightAuthority': xml_doc.xpath('//oversigh_info/authority/node()'),
        'studyDesign': (xml_doc.xpath('//study_design/node') or [''])[0],
        'primaryOutcome': primary_outcome,
        'secondary_outcomes': secondary_outcomes,
        'numberOfArms' : (xml_doc.xpath('//number_of_arms/node()') or [''])[0],
        'enrollment': enrollment,
        'source': (xml_doc.xpath('//source/node()') or [''])[0],
        'condition': (xml_doc.xpath('//condition/node()') or [''])[0],
        'armGroup' : arm_groups,
        'intervention': interventions,
        'eligibility': eligibility,
        'link' : links,
        'verificationDate': (xml_doc.xpath('//verification_date/node()') or [''])[0],
        'lastChanged': (xml_doc.xpath('//lastchanged_date/node()') or [''])[0],
        'responsible_party' : responsible_party,
        'status': (xml_doc.xpath('//status/node()') or [''])[0],
        'locationCountries': xml_doc.xpath('//location_countries/country/node()'),
        'isFDARegulated' : (xml_doc.xpath('//is_fda_regulated/node()') or [''])[0],
        'isSection801': (xml_doc.xpath('//is_section_801/node()') or [''])[0],
        'hasExpandedAccess': (xml_doc.xpath('//has_expanded_access/node()') or [''])[0]
    }

    for key, value in properties.iteritems():
        if isinstance(value, etree._ElementStringResult) or isinstance(value, etree._ElementUnicodeResult):
            properties[key] = copy_to_unicode(value)
        elif isinstance(value, list):
            unicode_list = []
            for item in value:
                unicode_list.append(copy_to_unicode(item))
            properties[key] = unicode_list

    return properties

def get_date_created(xml_doc):
    date_created = (xml_doc.xpath('//firstreceived_date/node()') or [''])[0]
    date = parse(date_created).isoformat()
    return copy_to_unicode(date)

def get_date_updated(xml_doc):
    date_updated = (xml_doc.xpath('//lastchanged_date/node()') or [''])[0]
    date = parse(date_updated).isoformat()
    return copy_to_unicode(date)

def normalize(raw_doc):
    raw_doc_text = raw_doc.get('doc')
    xml_doc = etree.XML(raw_doc_text)

    # Title
    title = (xml_doc.xpath('//official_title/node()') or xml_doc.xpath('//brief_title/node()') or [''])[0]

    # abstract
    abstract = (xml_doc.xpath('//brief_summary/textblock/node()') or xml_doc.xpath('//brief_summary/textblock/node()') or [''])[0]

    normalized_dict = {
            'title': copy_to_unicode(title),
            'contributors': get_contributors(xml_doc),
            'properties': get_properties(xml_doc),
            'description': copy_to_unicode(abstract),
            'id': get_ids(raw_doc, xml_doc),
            'source': NAME,
            'tags': get_tags(xml_doc),
            'dateCreated': get_date_created(xml_doc),
            'dateUpdated': get_date_updated(xml_doc),
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(consume, normalize))
<commit_msg>pep8 fixes<commit_after>#!/usr/bin/env python
from __future__ import unicode_literals

import time
import datetime

import requests

from lxml import etree

from dateutil.parser import *

from nameparser import HumanName

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = "clinicaltrials"

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None

def copy_to_unicode(element):
    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)

def consume(days_back=10):
    """ First, get a list of all recently updated study urls,
    then get the xml one by one and save it into a list
    of docs including other information """

    today = datetime.date.today()
    start_date = today - datetime.timedelta(days_back)

    month = today.strftime('%m')
    day = today.strftime('%d')
    year = today.strftime('%Y')

    y_month = start_date.strftime('%m')
    y_day = start_date.strftime('%d')
    y_year = start_date.strftime('%Y')

    base_url = 'http://clinicaltrials.gov/ct2/results?lup_s='
    url_end = '{}%2F{}%2F{}%2F&lup_e={}%2F{}%2F{}&displayxml=true'.\
        format(y_month, y_day, y_year, month, day, year)

    url = base_url + url_end

    # grab the total number of studies
    initial_request = requests.get(url)
    record_encoding = initial_request.encoding
    initial_request_xml = etree.XML(initial_request.content)
    count = int(initial_request_xml.xpath('//search_results/@count')[0])
    xml_list = []
    if int(count) > 0:
        # get a new url with all results in it
        url = url + '&count=' + str(count)
        total_requests = requests.get(url)
        initial_doc = etree.XML(total_requests.content)

        # make a list of urls from that full list of studies
        study_urls = []
        for study in initial_doc.xpath('//clinical_study'):
            study_urls.append(study.xpath('url/node()')[0] + '?displayxml=true')

        # grab each of those urls for full content
        print("There are {} urls to consume - be patient...".format(len(study_urls)))
        count = 0
        official_count = 0
        for study_url in study_urls:
            try:
                content = requests.get(study_url)
            except requests.exceptions.ConnectionError as e:
                print('Connection error: {}, wait a bit...'.format(e))
                time.sleep(30)
                continue
            doc = etree.XML(content.content)
            record = etree.tostring(doc, encoding=record_encoding)
            doc_id = doc.xpath('//nct_id/node()')[0]
            xml_list.append(RawDocument({
                'doc': record,
                'source': NAME,
                'docID': copy_to_unicode(doc_id),
                'filetype': 'xml',
            }))
            official_count += 1
            count += 1
            if count % 100 == 0:
                print("You've requested {} studies, keep going!".format(official_count))
                count = 0
            time.sleep(1)

    return xml_list

def get_contributors(xml_doc):
    contributor_list = []
    #TODO - fix this for weird contributor names like companies
    contributors = xml_doc.xpath('//overall_official/last_name/node()') or xml_doc.xpath('//lead_sponsor/agency/node()') or ['']
    for person in contributors:
        name = HumanName(person)
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
            'ORCID': ''
        }
        contributor_list.append(contributor)
    return contributor_list

def get_ids(raw_doc, xml_doc):
    url = (xml_doc.xpath('//required_header/url/node()') or [''])[0]
    ids = {'serviceID': raw_doc.get('docID'), 'doi': '', 'url': copy_to_unicode(url)}
    if url == '':
        raise Exception('Warning: No url provided!')
    return ids

def get_tags(xml_doc):
    keywords = [copy_to_unicode(tag.lower()) for tag in xml_doc.xpath('//keyword/node()')]
    return keywords

def get_properties(xml_doc):
    lead_sponsor = {
        'agency': (xml_doc.xpath('//lead_sponsor/agency/node()') or [''])[0],
        'agency_class': (xml_doc.xpath('//lead_sponsor/agency_class/node()') or [''])[0]
    }

    for key, value in lead_sponsor.iteritems():
        lead_sponsor[key] = copy_to_unicode(value)

    primary_outcome = {
        'measure': (xml_doc.xpath('//primary_outcome/measure/node()') or [''])[0],
        'time_frame': (xml_doc.xpath('//primary_outcome/time_frame/node()') or [''])[0],
        'safety_issue': (xml_doc.xpath('//primary_outcome/safety_issue/node()') or [''])[0]
    }

    for key, value in primary_outcome.iteritems():
        primary_outcome[key] = copy_to_unicode(value)

    # gives a list of dictionaries of all secondary outcomes
    secondary_outcome_elements = xml_doc.xpath('//secondary_outcome')
    secondary_outcomes = []
    for item in secondary_outcome_elements:
        secondary_outcome = {element.tag: element.text for element in item.iterdescendants()}
        secondary_outcomes.append(copy_to_unicode(secondary_outcome))

    # enrollment - can have different types
    enrollment_list = xml_doc.xpath('//enrollment')
    try:
        enrollment = {item.values()[0]: copy_to_unicode(item.text) for item in enrollment_list}
    except IndexError:
        enrollment = []

    # arm group
    arm_group_elements = xml_doc.xpath('//arm_group')
    arm_groups = []
    for item in arm_group_elements:
        arm_group = {element.tag: copy_to_unicode(element.text) for element in item.iterdescendants()}
        arm_groups.append(arm_group)

    # intervention
    intervention_elements = xml_doc.xpath('//intervention')
    interventions = []
    for item in intervention_elements:
        intervention = {element.tag: copy_to_unicode(element.text) for element in item.iterdescendants()}
        interventions.append(intervention)

    # eligibility
    eligibility_elements = xml_doc.xpath('//eligibility')
    eligibility = {}
    if len(eligibility_elements) > 1:
        for element in eligibility_elements[0].iterchildren():
            if element.text.strip() == '':
                for child in element.getchildren():
                    if child.text.strip() != '':
                        eligibility[element.tag] = copy_to_unicode(child.text)
            else:
                eligibility[element.tag] = copy_to_unicode(element.text)
    else:
        eligibility_elements = ''

    ## TODO: location - undone for now
    ## location has a facility name - and address with separate city state zip country
    ## {name: 'facility name', address: {city: 'city', state:'state', zip: 'zip', country:'country'}, status: 'status'}

    ## TODO: location sometimes has contact with name and email - what do?
    # location_elements = xml_doc.xpath('//location')
    # locations = []
    # upper_level = [item.getchildren() for item in location_elements]
    # location = {}
    # for item in upper_level:
    #     for element in item:
    #         for detail in element:
    #             location[element.tag] = {detail.tag: detail.text}

    # link
    link_elements = xml_doc.xpath('//link')
    links = []
    for item in link_elements:
        link = {element.tag: copy_to_unicode(element.text) for element in item.iterdescendants()}
        links.append(link)

    # responsible party
    # TODO: is there ever more than one responsible party?
    responsible_party_elements = xml_doc.xpath('//responsible_party')
    try:
        responsible_party = {elem.tag: copy_to_unicode(elem.text) for elem in responsible_party_elements[0].iterdescendants()}
    except IndexError:
        responsible_party = {}

    #TODO: a more robust search for more metadata in different examples

    ## extra properties ##
    properties = {
        'sponsors': lead_sponsor,
        'oversightAuthority': xml_doc.xpath('//oversigh_info/authority/node()'),
        'studyDesign': (xml_doc.xpath('//study_design/node') or [''])[0],
        'primaryOutcome': primary_outcome,
        'secondary_outcomes': secondary_outcomes,
        'numberOfArms': (xml_doc.xpath('//number_of_arms/node()') or [''])[0],
        'enrollment': enrollment,
        'source': (xml_doc.xpath('//source/node()') or [''])[0],
        'condition': (xml_doc.xpath('//condition/node()') or [''])[0],
        'armGroup': arm_groups,
        'intervention': interventions,
        'eligibility': eligibility,
        'link': links,
        'verificationDate': (xml_doc.xpath('//verification_date/node()') or [''])[0],
        'lastChanged': (xml_doc.xpath('//lastchanged_date/node()') or [''])[0],
        'responsible_party': responsible_party,
        'status': (xml_doc.xpath('//status/node()') or [''])[0],
        'locationCountries': xml_doc.xpath('//location_countries/country/node()'),
        'isFDARegulated': (xml_doc.xpath('//is_fda_regulated/node()') or [''])[0],
        'isSection801': (xml_doc.xpath('//is_section_801/node()') or [''])[0],
        'hasExpandedAccess': (xml_doc.xpath('//has_expanded_access/node()') or [''])[0]
    }

    for key, value in properties.iteritems():
        if isinstance(value, etree._ElementStringResult) or isinstance(value, etree._ElementUnicodeResult):
            properties[key] = copy_to_unicode(value)
        elif isinstance(value, list):
            unicode_list = []
            for item in value:
                unicode_list.append(copy_to_unicode(item))
            properties[key] = unicode_list

    return properties

def get_date_created(xml_doc):
    date_created = (xml_doc.xpath('//firstreceived_date/node()') or [''])[0]
    date = parse(date_created).isoformat()
    return copy_to_unicode(date)

def get_date_updated(xml_doc):
    date_updated = (xml_doc.xpath('//lastchanged_date/node()') or [''])[0]
    date = parse(date_updated).isoformat()
    return copy_to_unicode(date)

def normalize(raw_doc):
    raw_doc_text = raw_doc.get('doc')
    xml_doc = etree.XML(raw_doc_text)

    # Title
    title = (xml_doc.xpath('//official_title/node()') or xml_doc.xpath('//brief_title/node()') or [''])[0]

    # abstract
    abstract = (xml_doc.xpath('//brief_summary/textblock/node()') or xml_doc.xpath('//brief_summary/textblock/node()') or [''])[0]

    normalized_dict = {
        'title': copy_to_unicode(title),
        'contributors': get_contributors(xml_doc),
        'properties': get_properties(xml_doc),
        'description': copy_to_unicode(abstract),
        'id': get_ids(raw_doc, xml_doc),
        'source': NAME,
        'tags': get_tags(xml_doc),
        'dateCreated': get_date_created(xml_doc),
        'dateUpdated': get_date_updated(xml_doc),
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(consume, normalize))
<|endoftext|>


<commit_before># -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
### BEGIN LICENSE
# Copyright (C) 2014 Frederic Branczyk fbranczyk@gmail.com
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
### END LICENSE

# This is your preferences window.
#
# Define your preferences in
# data/glib-2.0/schemas/net.launchpad.xf-indicator.gschema.xml
# See http://developer.gnome.org/gio/stable/GSettings.html for more info.

from gi.repository import Gio # pylint: disable=E0611

from locale import gettext as _

import logging
logger = logging.getLogger('xf_indicator')

from xf_indicator_lib.PreferencesWindow import PreferencesWindow
from NewBuildXfIndicatorWindow import NewBuildXfIndicatorWindow
from NewBuildServerXfIndicatorWindow import NewBuildServerXfIndicatorWindow
from gi.repository import Gtk, GObject, GdkPixbuf

class PreferencesXfIndicatorWindow(PreferencesWindow):
    __gtype_name__ = "PreferencesXfIndicatorWindow"
    _instance = None

    @classmethod
    def instance(self):
        if PreferencesXfIndicatorWindow._instance is None:
            PreferencesXfIndicatorWindow._instance = PreferencesXfIndicatorWindow()
        return PreferencesXfIndicatorWindow._instance

    def finish_initializing(self, builder): # pylint: disable=E1002
        """Set up the preferences window"""
        super(PreferencesXfIndicatorWindow, self).finish_initializing(builder)

        # Code for other initialization actions should be added here.

        self.buildTreeview = builder.get_object("buildStatusTreeview")
        builder.get_object("addBuildButton").connect("clicked", self.on_add_project)
        builder.get_object("removeBuildButton").connect("clicked", self.on_remove_project)

        self.buildServerTreeview = builder.get_object("buildServerTreeview")
        builder.get_object("addBuildServerButton").connect("clicked", self.on_add_build_server)
        builder.get_object("removeBuildServerButton").connect("clicked", self.on_remove_build_server)

        self.connect("delete-event", self.quit)
        self.set_position(Gtk.WindowPosition.CENTER)

        self.generalSettingsBox = builder.get_object("generalSettingsBox")

        self.show_all()

    def quit(self, window, event):
        self.return_from_preferences_callback(self.projects, self.build_servers)
        window.hide()
        return True

    def on_add_project(self, widget):
        new_build_window = NewBuildXfIndicatorWindow()
        new_build_window.set_build_servers(self.build_servers)
        new_build_window.set_add_callback(self.new_build_callback)
        new_build_window.present()

    def new_build_callback(self, project):
        self.projects.add(project)
        self.set_projects(self.projects)

    def on_remove_project(self, widget):
        (model, pathlist) = self.buildTreeview.get_selection().get_selected_rows()
        for path in pathlist:
            tree_iter = model.get_iter(path)
            project = model.get_value(tree_iter, 0)
            self.projects.remove(project)
            self.set_projects(self.projects)

    def set_projects(self, projects):
        self.projects = projects

        #cleanup
        old_columns = self.buildTreeview.get_columns()
        for column in old_columns:
            self.buildTreeview.remove_column(column)

        build_store = Gtk.ListStore(GObject.TYPE_PYOBJECT, GObject.TYPE_PYOBJECT)

        self.buildTreeview.set_model(model=build_store)

        #build server type column
        renderer = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn('Build Server', renderer, text=0)
        column.set_sort_column_id(0)
        def extract_project_type(column, cell, model, iter, user_data):
            build_server_name = model.get_value(iter, 0).build_server_name()
            cell.props.text = build_server_name
        column.set_cell_data_func(renderer, extract_project_type)
        self.buildTreeview.append_column(column)

        #build column
        renderer = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn('Name', renderer, text=1)
        column.set_sort_column_id(1)
        def extract_project_name(column, cell, model, iter, user_data):
            project = model.get_value(iter, 1)
            cell.props.text = str(project)
        column.set_cell_data_func(renderer, extract_project_name)
        self.buildTreeview.append_column(column)

        for project in projects:
            build_store.append([project, project])

    def on_add_build_server(self, widget):
        build_server_window = NewBuildServerXfIndicatorWindow()
        build_server_window.set_callback(self.new_build_server_callback)
        build_server_window.present()

    def new_build_server_callback(self, new_build_server):
        self.build_servers.add(new_build_server)
        self.set_build_servers(self.build_servers)

    def on_remove_build_server(self, widget):
        (model, pathlist) = self.buildServerTreeview.get_selection().get_selected_rows()
        for path in pathlist:
            tree_iter = model.get_iter(path)
            build_server = model.get_value(tree_iter, 0)
            self.build_servers.remove(build_server)
            self.set_build_servers(self.build_servers)

    def set_build_servers(self, build_servers):
        self.build_servers = build_servers

        #cleanup
        old_columns = self.buildServerTreeview.get_columns()
        for column in old_columns:
            self.buildServerTreeview.remove_column(column)

        build_server_store = Gtk.ListStore(GdkPixbuf.Pixbuf, GObject.TYPE_PYOBJECT)
        self.buildServerTreeview.set_model(model=build_server_store)

        #build server type column
        renderer = Gtk.CellRendererPixbuf()
        column = Gtk.TreeViewColumn('Type', renderer)
        column.add_attribute(renderer, "pixbuf", 0)
        self.buildServerTreeview.append_column(column)

        #build server name column
        renderer = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn('Name', renderer, text=0)
        column.set_sort_column_id(1)
        def extract_build_server_name(column, cell, model, iter, user_data):
            build_server_name = str(model.get_value(iter, 1))
            cell.props.text = build_server_name
        column.set_cell_data_func(renderer, extract_build_server_name)
        self.buildServerTreeview.append_column(column)

        build_server_images = {
            "Travis CI": "../data/media/travis.png",
            "Travis CI Pro": "../data/media/travis.png",
            "Travis CI Enterprise": "../data/media/travis.png",
            "Jenkins": "../data/media/jenkins.png"
        }

        for build_server in build_servers:
            image = GdkPixbuf.Pixbuf.new_from_file(build_server_images[build_server.type()])
            build_server_store.append([image, build_server])

    def set_callback(self, callback):
        self.return_from_preferences_callback = callback

    def on_autostart_switch_activate(self, widget, data=None):
        if self.autostart_switch.get_active():
            if not os.path.exists(autostart_dir):
                os.mkdir(autostart_dir)
            if os.path.isdir(autostart_dir):
                os.symlink(autostart_file, installed_file)
        else:
            os.unlink(installed_file)
<commit_msg>add autostart switch functionality<commit_after># -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
### BEGIN LICENSE
# Copyright (C) 2014 Frederic Branczyk fbranczyk@gmail.com
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
### END LICENSE

# This is your preferences window.
#
# Define your preferences in
# data/glib-2.0/schemas/net.launchpad.xf-indicator.gschema.xml
# See http://developer.gnome.org/gio/stable/GSettings.html for more info.

from gi.repository import Gio, GLib # pylint: disable=E0611

from locale import gettext as _

import logging
logger = logging.getLogger('xf_indicator')

import os

from xf_indicator_lib.PreferencesWindow import PreferencesWindow
from NewBuildXfIndicatorWindow import NewBuildXfIndicatorWindow
from NewBuildServerXfIndicatorWindow import NewBuildServerXfIndicatorWindow
from gi.repository import Gtk, GObject, GdkPixbuf
from xf_indicator_lib.helpers import get_media_file

autostart_dir = os.path.join(GLib.get_user_config_dir(),"autostart/")
autostart_template = "xf-indicator-autostart.desktop"
autostart_file = get_media_file(autostart_template)
autostart_file = autostart_file.replace("file:///", '')
installed_file = os.path.join(autostart_dir, autostart_template)

class PreferencesXfIndicatorWindow(PreferencesWindow):
    __gtype_name__ = "PreferencesXfIndicatorWindow"
    _instance = None

    @classmethod
    def instance(self):
        if PreferencesXfIndicatorWindow._instance is None:
            PreferencesXfIndicatorWindow._instance = PreferencesXfIndicatorWindow()
        return PreferencesXfIndicatorWindow._instance

    def finish_initializing(self, builder): # pylint: disable=E1002
        """Set up the preferences window"""
        super(PreferencesXfIndicatorWindow, self).finish_initializing(builder)

        # Code for other initialization actions should be added here.

        self.buildTreeview = builder.get_object("buildStatusTreeview")
        builder.get_object("addBuildButton").connect("clicked", self.on_add_project)
        builder.get_object("removeBuildButton").connect("clicked", self.on_remove_project)

        self.buildServerTreeview = builder.get_object("buildServerTreeview")
        builder.get_object("addBuildServerButton").connect("clicked", self.on_add_build_server)
        builder.get_object("removeBuildServerButton").connect("clicked", self.on_remove_build_server)

        self.connect("delete-event", self.quit)
        self.set_position(Gtk.WindowPosition.CENTER)

        self.generalSettingsBox = builder.get_object("generalSettingsBox")
        self.autostart_switch = builder.get_object("autostartSwitch")
        self.autostart_switch.connect("notify::active", self.on_autostart_switch_activate)
        if os.path.isfile(installed_file):
            self.autostart_switch.set_active(True)

        self.show_all()

    def quit(self, window, event):
        self.return_from_preferences_callback(self.projects, self.build_servers)
        window.hide()
        return True

    def on_add_project(self, widget):
        new_build_window = NewBuildXfIndicatorWindow()
        new_build_window.set_build_servers(self.build_servers)
        new_build_window.set_add_callback(self.new_build_callback)
        new_build_window.present()

    def new_build_callback(self, project):
        self.projects.add(project)
        self.set_projects(self.projects)

    def on_remove_project(self, widget):
        (model, pathlist) = self.buildTreeview.get_selection().get_selected_rows()
        for path in pathlist:
            tree_iter = model.get_iter(path)
            project = model.get_value(tree_iter, 0)
            self.projects.remove(project)
            self.set_projects(self.projects)

    def set_projects(self, projects):
        self.projects = projects

        #cleanup
        old_columns = self.buildTreeview.get_columns()
        for column in old_columns:
            self.buildTreeview.remove_column(column)

        build_store = Gtk.ListStore(GObject.TYPE_PYOBJECT, GObject.TYPE_PYOBJECT)

        self.buildTreeview.set_model(model=build_store)

        #build server type column
        renderer = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn('Build Server', renderer, text=0)
        column.set_sort_column_id(0)
        def extract_project_type(column, cell, model, iter, user_data):
            build_server_name = model.get_value(iter, 0).build_server_name()
            cell.props.text = build_server_name
        column.set_cell_data_func(renderer, extract_project_type)
        self.buildTreeview.append_column(column)

        #build column
        renderer = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn('Name', renderer, text=1)
        column.set_sort_column_id(1)
        def extract_project_name(column, cell, model, iter, user_data):
            project = model.get_value(iter, 1)
            cell.props.text = str(project)
        column.set_cell_data_func(renderer, extract_project_name)
        self.buildTreeview.append_column(column)

        for project in projects:
            build_store.append([project, project])

    def on_add_build_server(self, widget):
        build_server_window = NewBuildServerXfIndicatorWindow()
        build_server_window.set_callback(self.new_build_server_callback)
        build_server_window.present()

    def new_build_server_callback(self, new_build_server):
        self.build_servers.add(new_build_server)
        self.set_build_servers(self.build_servers)

    def on_remove_build_server(self, widget):
        (model, pathlist) = self.buildServerTreeview.get_selection().get_selected_rows()
        for path in pathlist:
            tree_iter = model.get_iter(path)
            build_server = model.get_value(tree_iter, 0)
            self.build_servers.remove(build_server)
            self.set_build_servers(self.build_servers)

    def set_build_servers(self, build_servers):
        self.build_servers = build_servers

        #cleanup
        old_columns = self.buildServerTreeview.get_columns()
        for column in old_columns:
            self.buildServerTreeview.remove_column(column)

        build_server_store = Gtk.ListStore(GdkPixbuf.Pixbuf, GObject.TYPE_PYOBJECT)
        self.buildServerTreeview.set_model(model=build_server_store)

        #build server type column
        renderer = Gtk.CellRendererPixbuf()
        column = Gtk.TreeViewColumn('Type', renderer)
        column.add_attribute(renderer, "pixbuf", 0)
        self.buildServerTreeview.append_column(column)

        #build server name column
        renderer = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn('Name', renderer, text=0)
        column.set_sort_column_id(1)
        def extract_build_server_name(column, cell, model, iter, user_data):
            build_server_name = str(model.get_value(iter, 1))
            cell.props.text = build_server_name
        column.set_cell_data_func(renderer, extract_build_server_name)
        self.buildServerTreeview.append_column(column)

        build_server_images = {
            "Travis CI": "../data/media/travis.png",
            "Travis CI Pro": "../data/media/travis.png",
            "Travis CI Enterprise": "../data/media/travis.png",
            "Jenkins": "../data/media/jenkins.png"
        }

        for build_server in build_servers:
            image = GdkPixbuf.Pixbuf.new_from_file(build_server_images[build_server.type()])
            build_server_store.append([image, build_server])

    def set_callback(self, callback):
        self.return_from_preferences_callback = callback

    def on_autostart_switch_activate(self, widget, data=None):
        if self.autostart_switch.get_active():
            if not os.path.exists(autostart_dir):
                os.mkdir(autostart_dir)
            if os.path.isdir(autostart_dir):
                os.symlink(autostart_file, installed_file)
        else:
            os.unlink(installed_file)
<|endoftext|>


<commit_before>#!/usr/bin/env python

''' Consumer and Normalizer for the eScholarship Repo
at the University of California
'''

from lxml import etree
from xml.etree import ElementTree
from datetime import date, timedelta
import requests
from scrapi_tools import lint
from scrapi_tools.document import RawDocument, NormalizedDocument

TODAY = date.today()
NAME = "eScholarship"

def consume(days_back=1):
    start_date = TODAY - timedelta(days_back)
    base_url = 'http://www.escholarship.org/uc/oai?verb=ListRecords&metadataPrefix=oai_dc&from='
    url = base_url + str(start_date)
    data = requests.get(url)
    doc =  etree.XML(data.content)

    namespaces = {'dc': 'http://purl.org/dc/elements/1.1/',
                'oai_dc': 'http://www.openarchives.org/OAI/2.0/',
                'ns0': 'http://www.openarchives.org/OAI/2.0/'}

    records = doc.xpath('//oai_dc:record', namespaces=namespaces)

    xml_list = []
    for record in records:
        doc_id = record.xpath('ns0:header/ns0:identifier',
                                namespaces=namespaces)[0].text
        record = ElementTree.tostring(record)
        record = '<?xml version="1.0" encoding="UTF-8"?>\n' + record
        xml_list.append(RawDocument({
                    'doc': record,
                    'source': NAME,
                    'doc_id': doc_id,
                    'filetype': 'xml'
                }))

    return xml_list

    ## TODO: fix if there are no records found... what would the XML look like?

def normalize(raw_doc, timestamp):
    raw_doc = raw_doc.get('doc')
    doc = etree.XML(raw_doc)

    namespaces = {'dc': 'http://purl.org/dc/elements/1.1/',
                'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
                'ns0': 'http://www.openarchives.org/OAI/2.0/'}

    contributors = doc.findall('ns0:metadata/oai_dc:dc/dc:creator', namespaces=namespaces)
    contributor_list = []
    for contributor in contributors:
        contributor_list.append({'full_name': contributor.text, 'email': ''})
    title = doc.findall('ns0:metadata/oai_dc:dc/dc:title', namespaces=namespaces)

    service_id = doc.xpath('ns0:header/ns0:identifier/node()',
                                    namespaces=namespaces)[0]
    identifiers = doc.xpath('//dc:identifier/node()', namespaces=namespaces)
    for item in identifiers:
        if 'escholarship.org' in item:
            url = item

    ids = {'url': url, 'service_id': service_id, 'doi': 'doi'}

    ## Using this for the abstract for now...
    ## TODO: make this an actual absttract maybe by going to the source...
    description = doc.xpath('ns0:metadata/oai_dc:dc/dc:description/node()', namespaces=namespaces) or ['']

    date_created = doc.xpath('//dc:date', namespaces=namespaces)[0].text

    tags = doc.xpath('//dc:subject/node()', namespaces=namespaces)

    normalized_dict = {
            'title': title[0].text,
            'contributors': contributor_list,
            'properties': {},
            'description': description[0],
            'meta': {},
            'id': ids,
            'source': NAME,
            'tags': tags,
            'date_created': date_created,
            'timestamp': str(timestamp)
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(consume, normalize))
<commit_msg>Add properties: type, format, published-in, coverage, relation, rights.<commit_after>#!/usr/bin/env python

''' Consumer and Normalizer for the eScholarship Repo
at the University of California
'''

from lxml import etree
from xml.etree import ElementTree
from datetime import date, timedelta
import requests
from scrapi_tools import lint
from scrapi_tools.document import RawDocument, NormalizedDocument
import json #delete

TODAY = date.today()
NAME = "eScholarship"

def consume(days_back=1):
    start_date = TODAY - timedelta(days_back)
    base_url = 'http://www.escholarship.org/uc/oai?verb=ListRecords&metadataPrefix=oai_dc&from='
    url = base_url + str(start_date)
    data = requests.get(url)
    doc =  etree.XML(data.content)

    namespaces = {'dc': 'http://purl.org/dc/elements/1.1/',
                'oai_dc': 'http://www.openarchives.org/OAI/2.0/',
                'ns0': 'http://www.openarchives.org/OAI/2.0/'}

    records = doc.xpath('//oai_dc:record', namespaces=namespaces)

    xml_list = []
    for record in records:
        doc_id = record.xpath('ns0:header/ns0:identifier',
                                namespaces=namespaces)[0].text
        record = ElementTree.tostring(record)
        record = '<?xml version="1.0" encoding="UTF-8"?>\n' + record
        xml_list.append(RawDocument({
                    'doc': record,
                    'source': NAME,
                    'doc_id': doc_id,
                    'filetype': 'xml'
                }))

    return xml_list

    ## TODO: fix if there are no records found... what would the XML look like?

def normalize(raw_doc, timestamp):
    raw_doc = raw_doc.get('doc')
    doc = etree.XML(raw_doc)

    namespaces = {'dc': 'http://purl.org/dc/elements/1.1/',
                'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
                'ns0': 'http://www.openarchives.org/OAI/2.0/'}

    contributors = doc.findall('ns0:metadata/oai_dc:dc/dc:creator', namespaces=namespaces)
    contributor_list = []
    for contributor in contributors:
        contributor_list.append({'full_name': contributor.text, 'email': ''})
    title = doc.findall('ns0:metadata/oai_dc:dc/dc:title', namespaces=namespaces)

    service_id = doc.xpath('ns0:header/ns0:identifier/node()',
                                    namespaces=namespaces)[0]
    identifiers = doc.xpath('//dc:identifier/node()', namespaces=namespaces)
    for item in identifiers:
        if 'escholarship.org' in item:
            url = item

    ids = {'url': url, 'service_id': service_id, 'doi': 'doi'}

    ## Using this for the abstract for now...
    ## TODO: make this an actual absttract maybe by going to the source...
    description = doc.xpath('ns0:metadata/oai_dc:dc/dc:description/node()', namespaces=namespaces) or ['']

    date_created = doc.xpath('//dc:date', namespaces=namespaces)[0].text

    tags = doc.xpath('//dc:subject/node()', namespaces=namespaces)

    citation = doc.xpath('//dc:source/node()', namespaces=namespaces) or ['']

    dc_type = doc.xpath('//dc:type/node()', namespaces=namespaces) or ['']

    format = doc.xpath('//dc:format/node()', namespaces=namespaces) or ['']

    coverage = doc.xpath('//dc:coverage/node()', namespaces=namespaces) or ['']

    relation = doc.xpath('//dc:relation/node()', namespaces=namespaces) or ['']

    rights = doc.xpath('//dc:rights/node()', namespaces=namespaces) or ['']

    normalized_dict = {
            'title': title[0].text,
            'contributors': contributor_list,
            'properties': {
                'type': dc_type[0],
                'format': format[0],
                'published-in': citation[0],
                'coverage': coverage[0],
                'relation': relation[0],
                'rights': rights[0]
            },
            'description': description[0],
            'meta': {},
            'id': ids,
            'source': NAME,
            'tags': tags,
            'date_created': date_created,
            'timestamp': str(timestamp)
    }
    print(json.dumps(normalized_dict, sort_keys=True, indent=4, separators=(',', ': '))) # delete

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(consume, normalize))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
'''
AutoRunner -- Determine the correct runner class (GDB, ASan, etc) for
              the given program, instantiate and return it.

@author:     Christian Holler (:decoder)

@license:

This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.

@contact:    choller@mozilla.com
'''

# Ensure print() compatibility with Python 3
from __future__ import print_function

import subprocess

from abc import ABCMeta
from FTB.Signatures.CrashInfo import CrashInfo
import os

class AutoRunner():
    '''
    Abstract base class that provides a method to instantiate the right sub class
    for running the given program and obtaining crash information.
    '''
    __metaclass__ = ABCMeta

    def __init__(self, binary, args=None, env=None, cwd=None):
        self.binary = binary
        self.cwd = cwd

        self.env = env
        if self.env is None:
            self.env = {}

        self.args = args
        if self.args is None:
            self.args = []

        assert isinstance(self.env, dict)
        assert isinstance(self.args, list)

        # The command that we will run for obtaining crash information
        self.cmdArgs = []

        # These will hold our results from running
        self.stdout = None
        self.stderr = None
        self.auxCrashData = None

    def getCrashInfo(self, configuration):
        if not self.auxCrashData:
            return None

        return CrashInfo.fromRawCrashData(self.stdout, self.stderr, configuration, self.auxCrashData)

    @staticmethod
    def fromBinaryArgs(binary, args=None, env=None, cwd=None):
        process = subprocess.Popen(["nm", "-g", binary],
                                   stdin = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   stderr = subprocess.PIPE,
                                   cwd=cwd, env=env
                                   )

        (stdout, stderr) = process.communicate()

        if stdout.find(" __asan_init") >= 0:
            return ASanRunner(binary, args, env, cwd)
        else:
            return GDBRunner(binary, args, env, cwd)

class GDBRunner(AutoRunner):
    def __init__(self, binary, args=None, env=None, cwd=None):
        AutoRunner.__init__(self, binary, args, env, cwd)

        classPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "GDB.py")
        self.gdbArgs = [
                        '--batch',
                        '-ex', 'source %s' % classPath,
                        '-ex', 'run',
                        '-ex', 'set pagination 0',
                        '-ex', 'set backtrace limit 128',
                        '-ex', 'bt',
                        '-ex', 'python printImportantRegisters()',
                        '-ex', 'x/2i $pc',
                        '-ex', 'quit',
                        '--args',
                        ]

        self.cmdArgs.append("gdb")
        self.cmdArgs.extend(self.gdbArgs)
        self.cmdArgs.append(self.binary)
        self.cmdArgs.extend(self.args)

    def run(self):
        process = subprocess.Popen(
                                   self.cmdArgs,
                                   stdin = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   stderr = subprocess.PIPE,
                                   cwd=self.cwd, env=self.env
                                   )

        (self.stdout, self.stderr) = process.communicate()

        # Detect where the GDB trace starts/ends
        traceStart = self.stdout.rfind("Program received signal")
        traceStop = self.stdout.rfind("A debugging session is active")

        if traceStart < 0:
            return False

        if traceStop < 0:
            traceStop = len(self.stdout)

        # Move the trace from stdout to auxCrashData
        self.auxCrashData = self.stdout[traceStart:traceStop]
        self.stdout = self.stdout[:traceStart] + self.stdout[traceStop:]

        return True

class ASanRunner(AutoRunner):
    def __init__(self, binary, args=None, env=None, cwd=None):
        AutoRunner.__init__(self, binary, args, env, cwd)

        self.cmdArgs.append(self.binary)
        self.cmdArgs.extend(self.args)

    def run(self):
        process = subprocess.Popen(
                                   self.cmdArgs,
                                   stdin = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   stderr = subprocess.PIPE,
                                   cwd=self.cwd, env=self.env
                                   )

        (self.stdout, stderr) = process.communicate()

        inTrace = False
        self.auxCrashData = []
        self.stderr = []
        for line in stderr.splitlines():
            if inTrace:
                self.auxCrashData.append(line)
                if line.find("==ABORTING") >= 0:
                    inTrace = False
            elif line.find("==ERROR: AddressSanitizer") >= 0:
                self.auxCrashData.append(line)
                inTrace = True
            else:
                self.stderr.append(line)

        # Move the trace from stdout to auxCrashData
        self.auxCrashData = os.linesep.join(self.auxCrashData)
        self.stderr = os.linesep.join(self.stderr)

        return True<commit_msg>Add llvm-symbolizer auto-detection<commit_after>#!/usr/bin/env python
# encoding: utf-8
'''
AutoRunner -- Determine the correct runner class (GDB, ASan, etc) for
              the given program, instantiate and return it.

@author:     Christian Holler (:decoder)

@license:

This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.

@contact:    choller@mozilla.com
'''

# Ensure print() compatibility with Python 3
from __future__ import print_function

import subprocess

from abc import ABCMeta
from distutils import spawn
from FTB.Signatures.CrashInfo import CrashInfo
import os

class AutoRunner():
    '''
    Abstract base class that provides a method to instantiate the right sub class
    for running the given program and obtaining crash information.
    '''
    __metaclass__ = ABCMeta

    def __init__(self, binary, args=None, env=None, cwd=None):
        self.binary = binary
        self.cwd = cwd

        self.env = env
        if self.env is None:
            self.env = {}

        self.args = args
        if self.args is None:
            self.args = []

        assert isinstance(self.env, dict)
        assert isinstance(self.args, list)

        # The command that we will run for obtaining crash information
        self.cmdArgs = []

        # These will hold our results from running
        self.stdout = None
        self.stderr = None
        self.auxCrashData = None

    def getCrashInfo(self, configuration):
        if not self.auxCrashData:
            return None

        return CrashInfo.fromRawCrashData(self.stdout, self.stderr, configuration, self.auxCrashData)

    @staticmethod
    def fromBinaryArgs(binary, args=None, env=None, cwd=None):
        process = subprocess.Popen(["nm", "-g", binary],
                                   stdin = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   stderr = subprocess.PIPE,
                                   cwd=cwd, env=env
                                   )

        (stdout, stderr) = process.communicate()

        if stdout.find(" __asan_init") >= 0:
            return ASanRunner(binary, args, env, cwd)
        else:
            return GDBRunner(binary, args, env, cwd)

class GDBRunner(AutoRunner):
    def __init__(self, binary, args=None, env=None, cwd=None):
        AutoRunner.__init__(self, binary, args, env, cwd)

        classPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "GDB.py")
        self.gdbArgs = [
                        '--batch',
                        '-ex', 'source %s' % classPath,
                        '-ex', 'run',
                        '-ex', 'set pagination 0',
                        '-ex', 'set backtrace limit 128',
                        '-ex', 'bt',
                        '-ex', 'python printImportantRegisters()',
                        '-ex', 'x/2i $pc',
                        '-ex', 'quit',
                        '--args',
                        ]

        self.cmdArgs.append("gdb")
        self.cmdArgs.extend(self.gdbArgs)
        self.cmdArgs.append(self.binary)
        self.cmdArgs.extend(self.args)

    def run(self):
        process = subprocess.Popen(
                                   self.cmdArgs,
                                   stdin = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   stderr = subprocess.PIPE,
                                   cwd=self.cwd, env=self.env
                                   )

        (self.stdout, self.stderr) = process.communicate()

        # Detect where the GDB trace starts/ends
        traceStart = self.stdout.rfind("Program received signal")
        traceStop = self.stdout.rfind("A debugging session is active")

        if traceStart < 0:
            return False

        if traceStop < 0:
            traceStop = len(self.stdout)

        # Move the trace from stdout to auxCrashData
        self.auxCrashData = self.stdout[traceStart:traceStop]
        self.stdout = self.stdout[:traceStart] + self.stdout[traceStop:]

        return True

class ASanRunner(AutoRunner):
    def __init__(self, binary, args=None, env=None, cwd=None):
        AutoRunner.__init__(self, binary, args, env, cwd)

        self.cmdArgs.append(self.binary)
        self.cmdArgs.extend(self.args)

        if "ASAN_SYMBOLIZER_PATH" in self.env:
            if not os.path.exists(self.env["ASAN_SYMBOLIZER_PATH"]):
                raise RuntimeError("Misconfigured ASAN_SYMBOLIZER_PATH: %s")
        else:
            llvmSymbolizer = spawn.find_executable("llvm-symbolizer")
            if llvmSymbolizer:
                self.env["ASAN_SYMBOLIZER_PATH"] = llvmSymbolizer
            else:
                raise RuntimeError("Unable to locate llvm-symbolizer for ASAN_SYMBOLIZER_PATH")

    def run(self):
        process = subprocess.Popen(
                                   self.cmdArgs,
                                   stdin = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   stderr = subprocess.PIPE,
                                   cwd=self.cwd, env=self.env
                                   )

        (self.stdout, stderr) = process.communicate()

        inTrace = False
        self.auxCrashData = []
        self.stderr = []
        for line in stderr.splitlines():
            if inTrace:
                self.auxCrashData.append(line)
                if line.find("==ABORTING") >= 0:
                    inTrace = False
            elif line.find("==ERROR: AddressSanitizer") >= 0:
                self.auxCrashData.append(line)
                inTrace = True
            else:
                self.stderr.append(line)

        # Move the trace from stdout to auxCrashData
        self.auxCrashData = os.linesep.join(self.auxCrashData)
        self.stderr = os.linesep.join(self.stderr)

        return True<|endoftext|>