<commit_before>import datetime
import logging
import urlparse

import requests

from framework.celery_tasks import app as celery_app

from website import settings

logger = logging.getLogger(__name__)

@celery_app.task(ignore_results=True)
def on_preprint_updated(preprint_id):
    # WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
    # transactions are implemented in View and Task application layers.
    from website.models import PreprintService
    preprint = PreprintService.load(preprint_id)

    if settings.SHARE_URL and settings.SHARE_API_TOKEN:
        resp = requests.post('{}api/v2/normalizeddata/'.format(settings.SHARE_URL), json={
            'created_at': datetime.datetime.utcnow().isoformat(),
            'normalized_data': {
                '@graph': format_preprint(preprint)
            },
        }, headers={'Authorization': 'Bearer {}'.format(settings.SHARE_API_TOKEN)})
        logger.debug(resp.content)
        resp.raise_for_status()

def format_institution(institution):
    return [{
        '@id': '_:{}'.format(institution._id),
        '@type': 'institution',
        'name': institution.title,
    }]

def format_user(user):
    return sum([[{
        '@id': '_:{}'.format(user._id),
        '@type': 'person',
        'suffix': user.suffix,
        'given_name': user.given_name,
        'family_name': user.family_name,
        'additional_name': user.middle_names,
    }, {
        '@id': '_:throughidentifier-{}'.format(user._id),
        '@type': 'throughidentifiers',
        'person': {
            '@id': '_:{}'.format(user._id),
            '@type': 'person',
        },
        'identifier': {
            '@id': '_:identifier-{}'.format(user._id),
            '@type': 'identifier',
        }
    }, {
        '@id': '_:identifier-{}'.format(user._id),
        '@type': 'identifier',
        'url': urlparse.urljoin(settings.DOMAIN, user.profile_url),
        'base_url': settings.DOMAIN
    }]] + [[{
        '@id': '_:{}-{}'.format(user._id, institution._id),
        '@type': 'affiliation',
        'entity': {
            '@id': '_:{}'.format(institution._id),
            '@type': 'institution',
        },
        'person': {
            '@id': '_:{}'.format(user._id),
            '@type': 'person',
        }
    }] + format_institution(institution) for institution in user.affiliated_institutions], [])

def format_contributor(preprint, user):
    return [{
        '@id': '_:{}-{}'.format(preprint._id, user._id),
        '@type': 'contributor',
        'person': {'@id': '_:{}'.format(user._id), '@type': 'person'},
        'creative_work': {'@id': '_:{}'.format(preprint._id), '@type': 'preprint'},
    }] + format_user(user)

def format_subjects(preprint):
    flat_subjs = []
    summed_subjs = sum(sum([[[{
        '@id': '_:{}'.format(subject['id']),
        '@type': 'subject',
        'name': subject['text']
    }, {
        '@id': '_:throughsubject-{}'.format(subject['id']),
        '@type': 'throughsubjects',
        'subject': {
            '@id': '_:{}'.format(subject['id']),
            '@type': 'subject',
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint'
        }
    }] for subject in subject_hier] for subject_hier in preprint.get_subjects()], []), [])
    for s in summed_subjs:
        if s not in flat_subjs:
            flat_subjs.append(s)
    return flat_subjs

def format_doi(preprint):
    """
    * All DOIs will be valid URIs
    * All DOIs will use http
    * All DOI paths will be uppercased
    """
    return {
        '@id': '_:doi-{}'.format(preprint._id),
        '@type': 'link',
        'url': 'http://dx.doi.org/{}'.format(preprint.article_doi.upper().strip('/')),
        'type': 'doi'
    }

def format_preprint(preprint):
    preprint_parts = [
        {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint',
            'title': preprint.node.title,
            'description': preprint.node.description or '',
            'is_deleted': not preprint.is_published or not preprint.node.is_public or preprint.node.is_preprint_orphan
        }, {
            '@id': '_:link-{}'.format(preprint._id),
            '@type': 'link',
            'url': urlparse.urljoin(settings.DOMAIN, preprint.url),
            'type': 'provider'
        }
    ]

    if preprint.article_doi:
        preprint_parts.append(format_doi(preprint))

    return sum([preprint_parts] + [
        format_contributor(preprint, user) for user in preprint.node.contributors
    ] + [
        format_subjects(preprint)
    ] + [[{
        '@id': '_:{}-{}'.format(preprint._id, institution._id),
        '@type': 'association',
        'entity': {
            '@id': '_:{}'.format(institution._id),
            '@type': 'institution',
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint',
        }
    }] + format_institution(institution) for institution in preprint.node.affiliated_institutions], [])
<commit_msg>Add throughlinks to SHARE payload<commit_after>import datetime
import logging
import urlparse

import requests

from framework.celery_tasks import app as celery_app

from website import settings

logger = logging.getLogger(__name__)

@celery_app.task(ignore_results=True)
def on_preprint_updated(preprint_id):
    # WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
    # transactions are implemented in View and Task application layers.
    from website.models import PreprintService
    preprint = PreprintService.load(preprint_id)

    if settings.SHARE_URL and settings.SHARE_API_TOKEN:
        resp = requests.post('{}api/v2/normalizeddata/'.format(settings.SHARE_URL), json={
            'created_at': datetime.datetime.utcnow().isoformat(),
            'normalized_data': {
                '@graph': format_preprint(preprint)
            },
        }, headers={'Authorization': 'Bearer {}'.format(settings.SHARE_API_TOKEN)})
        logger.debug(resp.content)
        resp.raise_for_status()

def format_institution(institution):
    return [{
        '@id': '_:{}'.format(institution._id),
        '@type': 'institution',
        'name': institution.title,
    }]

def format_user(user):
    return sum([[{
        '@id': '_:{}'.format(user._id),
        '@type': 'person',
        'suffix': user.suffix,
        'given_name': user.given_name,
        'family_name': user.family_name,
        'additional_name': user.middle_names,
    }, {
        '@id': '_:throughidentifier-{}'.format(user._id),
        '@type': 'throughidentifiers',
        'person': {
            '@id': '_:{}'.format(user._id),
            '@type': 'person',
        },
        'identifier': {
            '@id': '_:identifier-{}'.format(user._id),
            '@type': 'identifier',
        }
    }, {
        '@id': '_:identifier-{}'.format(user._id),
        '@type': 'identifier',
        'url': urlparse.urljoin(settings.DOMAIN, user.profile_url),
        'base_url': settings.DOMAIN
    }]] + [[{
        '@id': '_:{}-{}'.format(user._id, institution._id),
        '@type': 'affiliation',
        'entity': {
            '@id': '_:{}'.format(institution._id),
            '@type': 'institution',
        },
        'person': {
            '@id': '_:{}'.format(user._id),
            '@type': 'person',
        }
    }] + format_institution(institution) for institution in user.affiliated_institutions], [])

def format_contributor(preprint, user):
    return [{
        '@id': '_:{}-{}'.format(preprint._id, user._id),
        '@type': 'contributor',
        'person': {'@id': '_:{}'.format(user._id), '@type': 'person'},
        'cited_name': user.fullname,
        'creative_work': {'@id': '_:{}'.format(preprint._id), '@type': 'preprint'},
    }] + format_user(user)

def format_subjects(preprint):
    flat_subjs = []
    summed_subjs = sum(sum([[[{
        '@id': '_:{}'.format(subject['id']),
        '@type': 'subject',
        'name': subject['text']
    }, {
        '@id': '_:throughsubject-{}'.format(subject['id']),
        '@type': 'throughsubjects',
        'subject': {
            '@id': '_:{}'.format(subject['id']),
            '@type': 'subject',
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint'
        }
    }] for subject in subject_hier] for subject_hier in preprint.get_subjects()], []), [])
    for s in summed_subjs:
        if s not in flat_subjs:
            flat_subjs.append(s)
    return flat_subjs

def format_doi(preprint):
    """
    * All DOIs will be valid URIs
    * All DOIs will use http
    * All DOI paths will be uppercased
    """
    return [{
        '@id': '_:doi-{}'.format(preprint._id),
        '@type': 'link',
        'url': 'http://dx.doi.org/{}'.format(preprint.article_doi.upper().strip('/')),
        'type': 'doi'
    }, {
        '@id': '_:throughdoi-{}'.format(preprint._id),
        '@type': 'throughlinks',
        'link': {
            '@id': '_:doi-{}'.format(preprint._id),
            '@type': 'link'
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint'
        }
    }]

def format_preprint(preprint):
    preprint_parts = [
        {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint',
            'title': preprint.node.title,
            'description': preprint.node.description or '',
            'is_deleted': not preprint.is_published or not preprint.node.is_public or preprint.node.is_preprint_orphan
        }, {
            '@id': '_:link-{}'.format(preprint._id),
            '@type': 'link',
            'url': urlparse.urljoin(settings.DOMAIN, preprint.url),
            'type': 'provider'
        }, {
            '@id': '_:throughlink-{}'.format(preprint._id),
            '@type': 'throughlinks',
            'creative_work': {
                '@id': '_:{}'.format(preprint._id),
                '@type': 'preprint'
            },
            'link': {
                '@id': '_:link-{}'.format(preprint._id),
                '@type': 'link'
            }
        }
    ]

    if preprint.article_doi:
        for part in format_doi(preprint):
            preprint_parts.append(part)

    return sum([preprint_parts] + [
        format_contributor(preprint, user) for user in preprint.node.contributors
    ] + [
        format_subjects(preprint)
    ] + [[{
        '@id': '_:{}-{}'.format(preprint._id, institution._id),
        '@type': 'association',
        'entity': {
            '@id': '_:{}'.format(institution._id),
            '@type': 'institution',
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint',
        }
    }] + format_institution(institution) for institution in preprint.node.affiliated_institutions], [])
<|endoftext|>


<commit_before># Dogecoin Ticker Twitter Bot
# Written by Damen Knight
# Copyright 2014

import tweepy, time, sys
import json
import urllib2
import os

import ConfigParser

config = ConfigParser.ConfigParser()
config.read("/path/to/appconfig")

CONSUMER_KEY = config.get('appconfig','consumer_key')
CONSUMER_SECRET = config.get('appconfig','consumer_secret')
ACCESS_KEY = config.get('appconfig','access_key')
ACCESS_SECRET = config.get('appconfig','access_secret')

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
api = tweepy.API(auth)

messages = api.direct_messages()

for message in messages:
	if message.text.lower().find("cryptsy") != -1:
		filename=open('/path/to/cryptsyvalue.txt','r')
		f=filename.readline()
		filename.close()
		reply = f
		user = message.sender_id
		api.send_direct_message(user, text=reply)
	if message.text.lower().find("vircurex") != -1:
                filename=open('/path/to/vircurexvalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)
        if message.text.lower().find("bter") != -1:
                filename=open('/path/to/btervalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)
        if (message.text.lower().find("coins-e") != -1) or (message.text.lower().find("coinse") != -1):
                filename=open('/path/to/coinsevalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)
        if message.text.lower().find("mintpal") != -1:
                filename=open('/path/to/mintpalvalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)

#delete all existing DMs
for message in messages:
	api.destroy_direct_message(message.id)

<commit_msg>removed mintpal support<commit_after># Dogecoin Ticker Twitter Bot
# Written by Damen Knight
# Copyright 2014

import tweepy, time, sys
import json
import urllib2
import os

import ConfigParser

config = ConfigParser.ConfigParser()
config.read("/path/to/appconfig")

CONSUMER_KEY = config.get('appconfig','consumer_key')
CONSUMER_SECRET = config.get('appconfig','consumer_secret')
ACCESS_KEY = config.get('appconfig','access_key')
ACCESS_SECRET = config.get('appconfig','access_secret')

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
api = tweepy.API(auth)

messages = api.direct_messages()

for message in messages:
	if message.text.lower().find("cryptsy") != -1:
		filename=open('/path/to/cryptsyvalue.txt','r')
		f=filename.readline()
		filename.close()
		reply = f
		user = message.sender_id
		api.send_direct_message(user, text=reply)
	if message.text.lower().find("vircurex") != -1:
                filename=open('/path/to/vircurexvalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)
        if message.text.lower().find("bter") != -1:
                filename=open('/path/to/btervalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)
        if (message.text.lower().find("coins-e") != -1) or (message.text.lower().find("coinse") != -1):
                filename=open('/path/to/coinsevalue.txt','r')
                f=filename.readline()
                filename.close()
                reply = f
                user = message.sender_id
                api.send_direct_message(user, text=reply)

#delete all existing DMs
for message in messages:
	api.destroy_direct_message(message.id)

<|endoftext|>


<commit_before>from rest_framework.viewsets import ModelViewSet
from rest_framework.filters import SearchFilter, OrderingFilter
from django_filters.rest_framework import DjangoFilterBackend

from api.models import *
from api.permission import HasGroupPermission
from api.v330.astronaut.filters import AstronautsFilter
from api.v330.astronaut.serializers import AstronautDetailedSerializer, AstronautListSerializer, AstronautNormalSerializer

class AstronautViewSet(ModelViewSet):
    """
    API endpoint that allows Astronauts to be viewed.

    GET:
    Return a list of all the existing astronauts.

    MODE:
    Normal, List and Detailed
    /3.3.0/astronaut/?mode=detailed

    FILTERS:
    Parameters - 'name', 'status', 'nationality', 'agency__name', 'agency__abbrev', 'date_of_birth', 'date_of_death', 'status_ids'
    Example - /3.3.0/astronaut/?nationality=American

    SEARCH EXAMPLE:
    /3.3.0/astronaut/?search=armstrong
    Searches through name, nationality and agency name

    ORDERING:
    Fields - 'name', 'status'
    Example - /3.3.0/astronaut/?order=name

    """
    def get_queryset(self):
        ids = self.request.query_params.get('status_ids', None)
        if ids:
            ids = ids.split(',')
            return Astronauts.objects.filter(status_id__in=ids)
        else:
            return Astronauts.objects.all()
    queryset = Astronauts.objects.all()
    permission_classes = [HasGroupPermission]
    permission_groups = {
        'retrieve': ['_Public'], # retrieve can be accessed without credentials (GET 'site.com/api/foo/1')
        'list': ['_Public'] # list returns None and is therefore NOT accessible by anyone (GET 'site.com/api/foo')
    }

    def get_serializer_class(self):
        mode = self.request.query_params.get("mode", "normal")
        if mode == "detailed" or self.action == 'retrieve':
            return AstronautDetailedSerializer
        elif mode == "list":
            return AstronautListSerializer
        else:
            return AstronautNormalSerializer

    filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter)
    filter_class = AstronautsFilter
    search_fields = ('$name', 'nationality', 'agency__name')
    ordering_fields = ('name', 'status', )
<commit_msg>Add agency abbrev to search filter.<commit_after>from rest_framework.viewsets import ModelViewSet
from rest_framework.filters import SearchFilter, OrderingFilter
from django_filters.rest_framework import DjangoFilterBackend

from api.models import *
from api.permission import HasGroupPermission
from api.v330.astronaut.filters import AstronautsFilter
from api.v330.astronaut.serializers import AstronautDetailedSerializer, AstronautListSerializer, AstronautNormalSerializer

class AstronautViewSet(ModelViewSet):
    """
    API endpoint that allows Astronauts to be viewed.

    GET:
    Return a list of all the existing astronauts.

    MODE:
    Normal, List and Detailed
    /3.3.0/astronaut/?mode=detailed

    FILTERS:
    Parameters - 'name', 'status', 'nationality', 'agency__name', 'agency__abbrev', 'date_of_birth', 'date_of_death', 'status_ids'
    Example - /3.3.0/astronaut/?nationality=American

    SEARCH EXAMPLE:
    /3.3.0/astronaut/?search=armstrong
    Searches through name, nationality and agency name

    ORDERING:
    Fields - 'name', 'status'
    Example - /3.3.0/astronaut/?order=name

    """
    def get_queryset(self):
        ids = self.request.query_params.get('status_ids', None)
        if ids:
            ids = ids.split(',')
            return Astronauts.objects.filter(status_id__in=ids)
        else:
            return Astronauts.objects.all()
    queryset = Astronauts.objects.all()
    permission_classes = [HasGroupPermission]
    permission_groups = {
        'retrieve': ['_Public'], # retrieve can be accessed without credentials (GET 'site.com/api/foo/1')
        'list': ['_Public'] # list returns None and is therefore NOT accessible by anyone (GET 'site.com/api/foo')
    }

    def get_serializer_class(self):
        mode = self.request.query_params.get("mode", "normal")
        if mode == "detailed" or self.action == 'retrieve':
            return AstronautDetailedSerializer
        elif mode == "list":
            return AstronautListSerializer
        else:
            return AstronautNormalSerializer

    filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter)
    filter_class = AstronautsFilter
    search_fields = ('$name', 'nationality', 'agency__name', 'agency__abbrev')
    ordering_fields = ('name', 'status', )
<|endoftext|>


<commit_before>import sys
import re
import json
import os
import json
from StringIO import StringIO
from gcloud import storage
import logging
from os.path import basename
import pprint
from collections import defaultdict
import pandas as pd
import concurrent.futures
import time
import tempfile
from pandas import ExcelWriter
import sqlite3
from bigquery_etl.execution import process_manager
from bigquery_etl.extract.gcloud_wrapper import GcsConnector
from bigquery_etl.utils.logging_manager import configure_logging
import os.path
#------------------------------------
# parse mirna
#------------------------------------
def download_file(project_id, bucket_name, blobname, outfilename, dummy):
#    gcs.download_to_filename("/mnt/datadisk-3/isoform_files/" + Study + "/" + basename(blobname))
    print 'start'
    client = storage.Client(project_id)
    bucket = client.get_bucket(bucket_name)
    blob = bucket.get_blob(blobname)
    blob.download_to_filename(outfilename)
    print 'Donwload'
    return True

def submit_to_queue(queue_df, conn, table_name):
    """
     Submit to queue - sqllite database
    """
    queue_df.to_sql(con=conn, name=table_name, if_exists='replace', index=False)
    print 'Inserted ' + str(len(queue_df)) + ' records to the task_queue'

#-----------------------------------------------------------------------------
def main(config):

    log_filename = 'etl_download_isoform.log'
    log_name = 'etl_download_isoform'
    log = configure_logging(log_name, log_filename)
#    etl = util.DataETL("isb-cgc", "isb-cgc-open") # this starts a new connectioni
    project_id = config['project_id']
    bucket_name = config['buckets']['open']

    # connect to bucket to get files
    gcs = GcsConnector(project_id, bucket_name)
    isoform_file = re.compile("^.*.isoform.quantification.txt.json$")
    data_library = gcs.search_files(search_patterns=['.isoform.quantification.txt'], regex_search_pattern=isoform_file, prefixes=['tcga/intermediary/mirna/isoform/'])
    # we are eliminating bad files - size 0; could be hg18 etc
    data_library.loc[:, 'basefilename'] = data_library['filename'].map(lambda x: os.path.splitext(os.path.basename(x))[0].replace('.json', ''))
    data_library = data_library.query('size > 0')

    conn = sqlite3.connect('../etl-mirna-isoform.db')
    sql = 'SELECT * from {0}'.format('task_queue')
    all_files_df = pd.read_sql_query(sql, conn)
    conn.close()

    with open('downloadedfiles.txt') as f:
        lines = f.read().splitlines()

    all_files_df = all_files_df[ (all_files_df.DatafileName.isin(data_library.basefilename))]
    all_files_df = all_files_df[ ~ (all_files_df.DatafileName.isin(lines))]
    data_library = all_files_df
    print data_library

    conn = sqlite3.connect('etl-isoform-download.db')
    submit_to_queue(data_library, conn, 'task_queue')
    queue_df = data_library

     # restart ETL; this gets the diff; also takes care of errors
    try:
        sql = 'SELECT * from task_queue_status where errors="None"'
        queue_df2 = pd.read_sql_query(sql, conn)
        print 'completed: ', len(queue_df2)
        queue_df = queue_df[ ~(queue_df.DatafileNameKey.isin(queue_df2.DatafileNameKey))]
        print 'Not completed: ',  len(queue_df)
    except Exception, e:
        pass

    # -----------------------------------------------------
    # thread this with concurrent futures
    #------------------------------------------------------
    pm = process_manager.ProcessManager(max_workers=200, db='isoform_download', table='task_queue_status', log=log)
    for i, df in data_library.iterrows():
        row = df.to_dict()
        print  row['DatafileName']
        outfilename = "/mnt/datadisk-3/isoform_files/" + row['Platform'] + "/" + row['DatafileName']
        future = pm.submit(download_file, project_id, bucket_name, row['DatafileNameKey'], outfilename, '')
        time.sleep(0.2)

    pm.start()

if __name__ == "__main__":
    config = json.load(open(sys.argv[1]))
    main(config)

<commit_msg>added copyright; added config; added logging; general cleanup<commit_after>#!/usr/bin/env python

# Copyright 2015, Institute for Systems Biology.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# -*- coding: utf-8 -*-
import sys
import re
from gcloud import storage
import json
import pandas as pd
import time
import sqlite3
from bigquery_etl.execution import process_manager
from bigquery_etl.extract.gcloud_wrapper import GcsConnector
from bigquery_etl.utils.logging_manager import configure_logging
import os.path
#------------------------------------
# parse mirna
#------------------------------------
def download_file(project_id, bucket_name, blobname, outfilename, dummy, log):
#    gcs.download_to_filename("/mnt/datadisk-3/isoform_files/" + Study + "/" + basename(blobname))
    log.info('\tstart download of %s' % outfilename)
    client = storage.Client(project_id)
    bucket = client.get_bucket(bucket_name)
    blob = bucket.get_blob(blobname)
    blob.download_to_filename(outfilename)
    log.info('\tfinished download of %s' % outfilename)
    return True

def submit_to_queue(queue_df, conn, table_name, log):
    """
     Submit to queue - sqllite database
    """
    queue_df.to_sql(con=conn, name=table_name, if_exists='replace', index=False)
    log.info('\tInserted ' + str(len(queue_df)) + ' records to the task_queue')

#-----------------------------------------------------------------------------
def main(config):

    log_filename = 'etl_download_isoform.log'
    log_name = 'etl_download_isoform'
    log = configure_logging(log_name, log_filename)
    log.info('begin downloading isoform files')
#    etl = util.DataETL("isb-cgc", "isb-cgc-open") # this starts a new connection
    project_id = config['project_id']
    bucket_name = config['buckets']['open']

    # connect to bucket to get files
    gcs = GcsConnector(project_id, bucket_name)
    isoform_file = re.compile("^.*.isoform.quantification.txt.json$")
    data_library = gcs.search_files(search_patterns=['.isoform.quantification.txt'], regex_search_pattern=isoform_file, prefixes=['tcga/intermediary/mirna/isoform/'])
    # we are eliminating bad files - size 0; could be hg18 etc
    data_library.loc[:, 'basefilename'] = data_library['filename'].map(lambda x: os.path.splitext(os.path.basename(x))[0].replace('.json', ''))
    data_library = data_library.query('size > 0')

    log.info('\tbegin selecting isoform files from sql-lite isoform db')
    conn = sqlite3.connect(config['mirna_isoform_matrix']['isoform_file_db'])
    sql = 'SELECT * from {0}'.format('task_queue')
    all_files_df = pd.read_sql_query(sql, conn)
    conn.close()
    log.info('\tfinished selecting isoform files')

    log.info('\tbegin reading from down loaded files')
    with open('downloadedfiles.txt') as f:
        lines = f.read().splitlines()
    log.info('\tfinished reading from down loaded files')

    log.info('filter files.\n\tfiles in cloud storage: %s\n\tfiles previously marked to download: %s\n%s\n' % (len(data_library), len(all_files_df), data_library))
    all_files_df = all_files_df[ (all_files_df.DatafileName.isin(data_library.basefilename))]
    all_files_df = all_files_df[ ~ (all_files_df.DatafileName.isin(lines))]
    data_library = all_files_df
    log.info('finished filter files: %s\n%s\n' % (len(data_library), data_library))

    conn = sqlite3.connect(config['mirna_isoform_matrix']['isoform_file_db'])
    submit_to_queue(data_library, conn, 'task_queue', log)
    queue_df = data_library

    # restart ETL; this gets the diff; also takes care of errors
    try:
        sql = 'SELECT * from task_queue_status where errors="None"'
        queue_df2 = pd.read_sql_query(sql, conn)
        log.info('\tso far completed: ' % (len(queue_df2)))
        queue_df = queue_df[ ~(queue_df.DatafileNameKey.isin(queue_df2.DatafileNameKey))]
        log.info('\tso far not completed: ' % (len(queue_df)))
    except Exception:
        log.error('\n++++++++++++++++++++++\n\tproblem filtering completed jobs, ignoring\n++++++++++++++++++++++\n')

    # -----------------------------------------------------
    # thread this with concurrent futures
    #------------------------------------------------------
    log.info('\tsubmit jobs to process manager')
    pm = process_manager.ProcessManager(max_workers=200, db='isoform_download', table='task_queue_status', log=log)
    for _, df in data_library.iterrows():
        row = df.to_dict()
        print  row['DatafileName']
        outfilename = config['mirna_isoform_matrix']['isoform_download_dir'] + row['Platform'] + "/" + row['DatafileName']
        pm.submit(download_file, project_id, bucket_name, row['DatafileNameKey'], outfilename, '')
        time.sleep(0.2)

    log.info('\tstart process manager completion check')
    pm.start()

    log.info('finished downloading isoform files')

if __name__ == "__main__":
    config = json.load(open(sys.argv[1]))
    main(config)

<|endoftext|>


<commit_before>"""
Doc string here.

@author: mje
@email: mads [] cnru.dk
"""
import numpy as np
import mne
import os
import socket

from mne.minimum_norm import (apply_inverse_epochs, read_inverse_operator)
from mne.baseline import rescale

# Settings
n_jobs = 1

# Setup paths and prepare raw data
hostname = socket.gethostname()

if hostname == "Wintermute":
    basic_path = "/home/mje/mnt/Hyp_meg/scratch/"
else:
    basic_path = "/projects/MINDLAB2013_18-MEG-HypnosisAnarchicHand/" +\
                "scratch/"

subjects_dir = basic_path + "fs_subjects_dir/"
data_path = basic_path + "Tone_task_MNE_ver_2/"

epochs_fnrm = data_path + "subj_1-nrm-epo.fif"
epochs_fhyp = data_path + "subj_1-hyp-epo.fif"
inverse_fnrm = data_path + "subj_1-nrm-inv.fif"
inverse_fhyp = data_path + "subj_1-hyp-inv.fif"
# change dir to save files the rigth place
os.chdir(data_path)

reject = dict(grad=4000e-13,  # T / m (gradiometers)
              mag=4e-12,  # T (magnetometers)
              #  eog=250e-6  # uV (EOG channels)
              )

#
snr = 1.0  # Standard assumption for average data but using it for single trial
lambda2 = 1.0 / snr ** 2
method = "MNE"

# Load data
inverse_nrm = read_inverse_operator(inverse_fnrm)
inverse_hyp = read_inverse_operator(inverse_fhyp)

epochs_nrm = mne.read_epochs(epochs_fnrm)
epochs_hyp = mne.read_epochs(epochs_fhyp)

epochs_nrm = epochs_nrm["Tone"]
epochs_hyp = epochs_hyp["Tone"]

#
stcs_nrm = apply_inverse_epochs(epochs_nrm, inverse_nrm, lambda2,
                                method, pick_ori="normal",
                                return_generator=False)
stcs_hyp = apply_inverse_epochs(epochs_hyp, inverse_hyp, lambda2,
                                method, pick_ori="normal",
                                return_generator=False)

# resample
[stc.resample(300) for stc in stcs_nrm]
[stc.resample(300) for stc in stcs_hyp]

# Get labels from FreeSurfer cortical parcellation
labels = mne.read_labels_from_annot('subject_1', parc='PALS_B12_Brodmann',
                                    regexp="Brodmann",
                                    subjects_dir=subjects_dir)

# Average the source estimates within eachh label using sign-flips to reduce
# signal cancellations, also here we return a generator
src_nrm = inverse_nrm['src']
label_ts_nrm = mne.extract_label_time_course(stcs_nrm, labels,
                                             src_nrm,
                                             mode='pca_flip',
                                             return_generator=False)

src_hyp = inverse_hyp['src']
label_ts_hyp = mne.extract_label_time_course(stcs_hyp, labels, src_hyp,
                                             mode='pca_flip',
                                             return_generator=False)

# standardize TS's
label_ts_nrm_rescaled = []
for j in range(len(label_ts_nrm)):
    label_ts_nrm_rescaled += [rescale(label_ts_nrm[j], epochs_nrm.times,
                                      baseline=(None, -0.7), mode="zscore")]

label_ts_hyp_rescaled = []
for j in range(len(label_ts_hyp)):
    label_ts_hyp_rescaled += [rescale(label_ts_hyp[j], epochs_hyp.times,
                                      baseline=(None, -0.7), mode="zscore")]

fromTime = np.argmax(stcs_nrm[0].times == -0.5)
toTime = np.argmax(stcs_nrm[0].times == 0)

label_ts_nrm_rescaled_crop = []
for j in range(len(label_ts_nrm)):
    label_ts_nrm_rescaled_crop +=\
        [label_ts_nrm_rescaled[j][:, fromTime:toTime]]

label_ts_hyp_rescaled_crop = []
for j in range(len(label_ts_hyp)):
    label_ts_hyp_rescaled_crop +=\
       [label_ts_hyp_rescaled[j][:, fromTime:toTime]]

# np.save("labelTsHypToneMean-flipZscore_resample_crop.npy",
#         label_ts_hyp_rescaled_crop)
# np.save("labelTsNrmToneMean-flipZscore_resample_crop.npy",
#         label_ts_nrm_rescaled_crop)
<commit_msg>Fixed paths and changed from_time & to_time names<commit_after>"""
Doc string here.

@author: mje
@email: mads [] cnru.dk
"""
import numpy as np
import mne
import os
import socket

from mne.minimum_norm import (apply_inverse_epochs, read_inverse_operator)
from mne.baseline import rescale

# Settings
n_jobs = 1

# Setup paths and prepare raw data
hostname = socket.gethostname()

if hostname == "wintermute":
    data_path = "/home/mje/Projects/MEG_Hyopnosis/data/"
    subjects_dir = "/home/mje/Projects/MEG_Hyopnosis/data/fs_subjects_dir"
else:
    data_path = "/projects/" + \
                "MINDLAB2013_18-MEG-HypnosisAnarchicHand/" + \
                "scratch/Tone_task_MNE_2/"
    subjects_dir = "/projects/" + \
                   "MINDLAB2013_18-MEG-HypnosisAnarchicHand/" + \
                   "scratch/fs_subjects_dir/"

# CHANGE DIR TO SAVE FILES THE RIGTH PLACE
os.chdir(data_path)

epochs_fnrm = data_path + "subj_1_nrm-epo.fif"
epochs_fhyp = data_path + "subj_1_hyp-epo.fif"
inverse_fnrm = data_path + "subj_1-nrm-inv.fif"
inverse_fhyp = data_path + "subj_1-hyp-inv.fif"
# change dir to save files the rigth place
os.chdir(data_path)

reject = dict(grad=4000e-13,  # T / m (gradiometers)
              mag=4e-12,  # T (magnetometers)
              #  eog=250e-6  # uV (EOG channels)
              )

#
snr = 1.0  # Standard assumption for average data but using it for single trial
lambda2 = 1.0 / snr ** 2
method = "MNE"

# Load data
inverse_nrm = read_inverse_operator(inverse_fnrm)
inverse_hyp = read_inverse_operator(inverse_fhyp)

epochs_nrm = mne.read_epochs(epochs_fnrm)
epochs_hyp = mne.read_epochs(epochs_fhyp)

epochs_nrm = epochs_nrm["Tone"]
epochs_hyp = epochs_hyp["Tone"]

#
stcs_nrm = apply_inverse_epochs(epochs_nrm, inverse_nrm, lambda2,
                                method, pick_ori="normal",
                                return_generator=False)
stcs_hyp = apply_inverse_epochs(epochs_hyp, inverse_hyp, lambda2,
                                method, pick_ori="normal",
                                return_generator=False)

# resample
[stc.resample(300) for stc in stcs_nrm]
[stc.resample(300) for stc in stcs_hyp]

# Get labels from FreeSurfer cortical parcellation
labels = mne.read_labels_from_annot('subject_1', parc='PALS_B12_Brodmann',
                                    regexp="Brodmann",
                                    subjects_dir=subjects_dir)

# Average the source estimates within eachh label using sign-flips to reduce
# signal cancellations, also here we return a generator
src_nrm = inverse_nrm['src']
label_ts_nrm = mne.extract_label_time_course(stcs_nrm, labels,
                                             src_nrm,
                                             mode='pca_flip',
                                             return_generator=False)

src_hyp = inverse_hyp['src']
label_ts_hyp = mne.extract_label_time_course(stcs_hyp, labels, src_hyp,
                                             mode='pca_flip',
                                             return_generator=False)

# standardize TS's
label_ts_nrm_rescaled = []
for j in range(len(label_ts_nrm)):
    label_ts_nrm_rescaled += [rescale(label_ts_nrm[j], epochs_nrm.times,
                                      baseline=(None, -0.5), mode="zscore")]

label_ts_hyp_rescaled = []
for j in range(len(label_ts_hyp)):
    label_ts_hyp_rescaled += [rescale(label_ts_hyp[j], epochs_hyp.times,
                                      baseline=(None, -0.5), mode="zscore")]

from_time = np.abs(stcs_nrm[0].times + 0).argmin()
to_time = np.abs(stcs_nrm[0].times - 0.2).argmin()

label_ts_nrm_rescaled_crop = []
for j in range(len(label_ts_nrm)):
    label_ts_nrm_rescaled_crop +=\
        [label_ts_nrm_rescaled[j][:, from_time:to_time]]

label_ts_hyp_rescaled_crop = []
for j in range(len(label_ts_hyp)):
    label_ts_hyp_rescaled_crop +=\
       [label_ts_hyp_rescaled[j][:, from_time:to_time]]

# np.save("labelTsHypToneMean-flipZscore_resample_crop.npy",
#         label_ts_hyp_rescaled_crop)
# np.save("labelTsNrmToneMean-flipZscore_resample_crop.npy",
#         label_ts_nrm_rescaled_crop)
<|endoftext|>


<commit_before>
'''
MAP Client Plugin
'''
__version__ = '0.1.0'
__author__ = 'Ju Zhang'

# import class that derives itself from the step mountpoint.
from mapclientplugins.fieldworkgait2392geomstep import step

<commit_msg>added location and name to __init__<commit_after>
'''
MAP Client Plugin
'''
__version__ = '0.1.0'
__author__ = 'Ju Zhang'
__stepname__ = 'Fieldwork Gait2392 Geometry'
__location__ = 'https://github.com/mapclient-plugins/fieldworkgait2392geomstep/commits/master.zip'

# import class that derives itself from the step mountpoint.
from mapclientplugins.fieldworkgait2392geomstep import step

<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import sys
import threading
import traceback

from dmoj import packet, graders
from dmoj.config import Problem, InvalidInitException, BatchedTestCase
from dmoj.error import CompileError
from dmoj.judgeenv import env, get_problem_roots, get_supported_problems, startup_warnings
from dmoj.result import Result
from dmoj.utils.ansi import ansi_style

if os.name == 'posix':
    try:
        import readline
        from dmoj.utils.nix.signal_debugger import setup_all_debuggers
        setup_all_debuggers()
    except ImportError:
        pass

try:
    from watchdog.observers import Observer
    from watchdog.events import FileSystemEventHandler
except ImportError:
    startup_warnings.append('watchdog module not found, install it to automatically update problems')
    Observer = None

    class FileSystemEventHandler(object):
        pass

class BatchBegin(object):
    pass

class BatchEnd(object):
    pass

class TerminateGrading(Exception):
    pass

class SendProblemsHandler(FileSystemEventHandler):
    def __init__(self, judge):
        self.judge = judge

    def on_any_event(self, event):
        self.judge.update_problems()

class Judge(object):
    def __init__(self):
        global startup_warnings
        self.current_submission = None
        self.current_grader = None
        self.current_submission_thread = None
        self._terminate_grading = False
        if Observer is not None:
            handler = SendProblemsHandler(self)
            self._monitor = monitor = Observer()
            for dir in get_problem_roots():
                monitor.schedule(handler, dir, recursive=True)
            try:
                monitor.start()
            except OSError:
                startup_warnings.append('failed to start filesystem monitor')
                self._monitor = None
        else:
            self._monitor = None

    def _stop_monitor(self):
        if self._monitor is not None:
            self._monitor.stop()
            self._monitor.join(1)

    def update_problems(self):
        """
        Pushes current problem set to server.
        """
        self.packet_manager.supported_problems_packet(get_supported_problems())

    def begin_grading(self, id, problem_id, language, source_code, time_limit, memory_limit, short_circuit,
                      blocking=False):
        try:
            self.current_submission_thread.join()
        except AttributeError:
            pass
        self.current_submission = id
        self.current_submission_thread = threading.Thread(target=self._begin_grading,
                                                          args=(problem_id, language, source_code,
                                                                time_limit, memory_limit, short_circuit))
        self.current_submission_thread.daemon = True
        self.current_submission_thread.start()
        if blocking:
            self.current_submission_thread.join()

    def terminate_grading(self):
        """
        Forcefully terminates the current submission. Not necessarily safe.
        """
        if self.current_submission_thread:
            self._terminate_grading = True
            if self.current_grader:
                self.current_grader.terminate()
            self.current_submission_thread.join()
            self.current_submission_thread = None

    def _begin_grading(self, problem_id, language, source, time_limit, memory_limit, short_circuit):
        submission_id = self.current_submission
        print ansi_style('Start grading #ansi[%s](yellow)/#ansi[%s](green|bold) in %s...'
                         % (problem_id, submission_id, language))

        try:
            problem = Problem(problem_id, time_limit, memory_limit)
        except InvalidInitException:
            return self.internal_error()

        if 'grader' in problem.config:
            grader_class = graders.InteractiveGrader
        elif 'signature_grader' in problem.config:
            grader_class = graders.SignatureGrader
        elif 'custom_judge' in problem.config:
            grader_class = graders.CustomGrader
        else:
            grader_class = graders.StandardGrader

        if isinstance(source, unicode):
            source = source.encode('utf-8')

        try:
            grader = grader_class(self, problem, language, source)
        except CompileError as ce:
            print ansi_style('#ansi[Failed compiling submission!](red|bold)')
            print ce.message,  # don't print extra newline
            grader = None
        except:  # if custom grader failed to initialize, report it to the site
            return self.internal_error()

        binary = grader.binary if grader else None

        # the compiler may have failed, or an error could have happened while initializing a custom judge
        # either way, we can't continue
        if binary:
            self.packet_manager.begin_grading_packet()

            batch_counter = 1
            in_batch = False
            try:
                for case_number, result in enumerate(self.grade_cases(grader, problem.cases,
                                                                      short_circuit=short_circuit)):
                    if isinstance(result, BatchBegin):
                        self.packet_manager.batch_begin_packet()
                        print ansi_style("#ansi[Batch #%d](yellow|bold)" % batch_counter)
                        in_batch = True
                    elif isinstance(result, BatchEnd):
                        self.packet_manager.batch_end_packet()
                        batch_counter += 1
                        in_batch = False
                    else:
                        codes = result.readable_codes()

                        # here be cancer
                        is_sc = (result.result_flag & Result.SC)
                        colored_codes = map(lambda x: '#ansi[%s](%s|bold)' % ('--' if x == 'SC' else x,
                                                                              Result.COLORS_BYID[x]), codes)
                        colored_aux_codes = '{%s}' % ', '.join(colored_codes[1:]) if len(codes) > 1 else ''
                        colored_feedback = '(#ansi[%s](|underline)) ' % result.feedback if result.feedback else ''
                        case_info = '[%.3fs | %dkb] %s%s' % (result.execution_time, result.max_memory,
                                                             colored_feedback,
                                                             colored_aux_codes) if not is_sc else ''
                        case_padding = '  ' * in_batch
                        print ansi_style('%sTest case %2d %-3s %s' % (case_padding, case_number + 1,
                                                                      colored_codes[0], case_info))

                        # cases are indexed at 1
                        self.packet_manager.test_case_status_packet(
                            case_number + 1, result.points, result.case.points, result.result_flag,
                            result.execution_time,
                            result.max_memory,
                            result.proc_output[:result.case.output_prefix_length].decode('utf-8', 'replace'),
                            result.feedback)
            except TerminateGrading:
                self.packet_manager.submission_terminated_packet()
                print ansi_style('#ansi[Forcefully terminating grading. Temporary files may not be deleted.](red|bold)')
                pass
            except:
                self.internal_error()
            else:
                self.packet_manager.grading_end_packet()

        print ansi_style('Done grading #ansi[%s](yellow)/#ansi[%s](green|bold).' % (problem_id, submission_id))
        print
        self._terminate_grading = False
        self.current_submission_thread = None
        self.current_submission = None
        self.current_grader = None

    def grade_cases(self, grader, cases, short_circuit=False, is_short_circuiting=False):

        for case in cases:
            # Yield notifying objects for batch begin/end, and unwrap all cases inside the batches
            if isinstance(case, BatchedTestCase):
                yield BatchBegin()
                for _ in self.grade_cases(grader, case.batched_cases, short_circuit=True,
                                          is_short_circuiting=is_short_circuiting):
                    yield _
                yield BatchEnd()
                continue

            # Stop grading if we're short circuiting
            if is_short_circuiting:
                result = Result(case)
                result.result_flag = Result.SC
                yield result
                continue

            # Must check here because we might be interrupted mid-execution
            # If we don't bail out, we get an IR.
            # In Java's case, all the code after this will crash.
            if self._terminate_grading:
                raise TerminateGrading()

            result = grader.grade(case)

            # If the WA bit of result_flag is set and we are set to short-circuit (e.g., in a batch),
            # short circuit the rest of the cases.
            # Do the same if the case is a pretest (i.e. has 0 points)
            if (result.result_flag & Result.WA) > 0 and (short_circuit or case.points == 0):
                is_short_circuiting = True

            yield result

    def internal_error(self, exc=None):
        if exc:
            try:
                raise exc
            except:
                pass
        exc = sys.exc_info()

        traceback.print_exception(*exc)
        self.packet_manager.internal_error_packet(''.join(traceback.format_exception(*exc)))

    def listen(self):
        """
        Attempts to connect to the handler server specified in command line.
        """
        self.packet_manager.run()

    def __del__(self):
        del self.packet_manager

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, traceback):
        pass

    def murder(self):
        """
        End any submission currently executing, and exit the judge.
        """
        self.terminate_grading()
        self._stop_monitor()

class ClassicJudge(Judge):
    def __init__(self, host, port):
        self.packet_manager = packet.PacketManager(host, port, self, env['id'], env['key'])
        super(ClassicJudge, self).__init__()

def sanity_check():
    # Don't allow starting up without wbox/cptbox, saves cryptic errors later on
    if os.name == 'nt':
        try:
            import wbox
        except ImportError:
            print >> sys.stderr, "wbox must be compiled to grade!"
            return False

        # DMOJ needs to be run as admin on Windows
        import ctypes
        if ctypes.windll.shell32.IsUserAnAdmin() == 0:
            print >> sys.stderr, "can't start, the DMOJ judge must be ran as admin"
            return False
    else:
        try:
            import cptbox
        except ImportError:
            print >> sys.stderr, "cptbox must be compiled to grade!"
            return False

        # However running as root on Linux is a Bad Idea
        if os.getuid() == 0:
            startup_warnings.append('running the judge as root can be potentially unsafe, '
                                    'consider using an unprivileged user instead')

    # _checker implements standard checker functions in C
    # we fall back to a Python implementation if it's not compiled, but it's slower
    try:
        from checkers import _checker
    except ImportError:
        startup_warnings.append('native checker module not found, compile _checker for optimal performance')
    return True

def main():
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    if not sanity_check():
        return 1

    import logging
    from dmoj import judgeenv, executors

    judgeenv.load_env()

    # Emulate ANSI colors with colorama
    if os.name == 'nt' and not judgeenv.no_ansi_emu:
        try:
            from colorama import init
            init()
        except ImportError as ignored:
            pass

    executors.load_executors()

    print 'Running live judge...'

    logging.basicConfig(filename=judgeenv.log_file, level=logging.INFO,
                        format='%(levelname)s %(asctime)s %(module)s %(message)s')

    judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port)

    for warning in judgeenv.startup_warnings:
        print ansi_style('#ansi[Warning: %s](yellow)' % warning)
    del judgeenv.startup_warnings
    print

    with judge:
        try:
            judge.listen()
        except KeyboardInterrupt:
            pass
        except:
            traceback.print_exc()
        finally:
            judge.murder()

if __name__ == '__main__':
    main()
<commit_msg>Initialize debuggers; #173<commit_after>#!/usr/bin/python
import os
import sys
import threading
import traceback

from dmoj import packet, graders
from dmoj.config import Problem, InvalidInitException, BatchedTestCase
from dmoj.error import CompileError
from dmoj.judgeenv import env, get_problem_roots, get_supported_problems
from dmoj.result import Result
from dmoj.utils.ansi import ansi_style

if os.name == 'posix':
    try:
        import readline
        from dmoj.utils.debugger.nix import setup_all_debuggers
        setup_all_debuggers()
    except ImportError:
        pass
else:
    try:
        from dmoj.utils.debugger.win import setup_all_debuggers
        setup_all_debuggers()
    except ImportError:
        pass

try:
    from watchdog.observers import Observer
    from watchdog.events import FileSystemEventHandler
except ImportError:
    startup_warnings.append('watchdog module not found, install it to automatically update problems')
    Observer = None

    class FileSystemEventHandler(object):
        pass

class BatchBegin(object):
    pass

class BatchEnd(object):
    pass

class TerminateGrading(Exception):
    pass

class SendProblemsHandler(FileSystemEventHandler):
    def __init__(self, judge):
        self.judge = judge

    def on_any_event(self, event):
        self.judge.update_problems()

class Judge(object):
    def __init__(self):
        global startup_warnings
        self.current_submission = None
        self.current_grader = None
        self.current_submission_thread = None
        self._terminate_grading = False
        if Observer is not None:
            handler = SendProblemsHandler(self)
            self._monitor = monitor = Observer()
            for dir in get_problem_roots():
                monitor.schedule(handler, dir, recursive=True)
            try:
                monitor.start()
            except OSError:
                startup_warnings.append('failed to start filesystem monitor')
                self._monitor = None
        else:
            self._monitor = None

    def _stop_monitor(self):
        if self._monitor is not None:
            self._monitor.stop()
            self._monitor.join(1)

    def update_problems(self):
        """
        Pushes current problem set to server.
        """
        self.packet_manager.supported_problems_packet(get_supported_problems())

    def begin_grading(self, id, problem_id, language, source_code, time_limit, memory_limit, short_circuit,
                      blocking=False):
        try:
            self.current_submission_thread.join()
        except AttributeError:
            pass
        self.current_submission = id
        self.current_submission_thread = threading.Thread(target=self._begin_grading,
                                                          args=(problem_id, language, source_code,
                                                                time_limit, memory_limit, short_circuit))
        self.current_submission_thread.daemon = True
        self.current_submission_thread.start()
        if blocking:
            self.current_submission_thread.join()

    def terminate_grading(self):
        """
        Forcefully terminates the current submission. Not necessarily safe.
        """
        if self.current_submission_thread:
            self._terminate_grading = True
            if self.current_grader:
                self.current_grader.terminate()
            self.current_submission_thread.join()
            self.current_submission_thread = None

    def _begin_grading(self, problem_id, language, source, time_limit, memory_limit, short_circuit):
        submission_id = self.current_submission
        print ansi_style('Start grading #ansi[%s](yellow)/#ansi[%s](green|bold) in %s...'
                         % (problem_id, submission_id, language))

        try:
            problem = Problem(problem_id, time_limit, memory_limit)
        except InvalidInitException:
            return self.internal_error()

        if 'grader' in problem.config:
            grader_class = graders.InteractiveGrader
        elif 'signature_grader' in problem.config:
            grader_class = graders.SignatureGrader
        elif 'custom_judge' in problem.config:
            grader_class = graders.CustomGrader
        else:
            grader_class = graders.StandardGrader

        if isinstance(source, unicode):
            source = source.encode('utf-8')

        try:
            grader = grader_class(self, problem, language, source)
        except CompileError as ce:
            print ansi_style('#ansi[Failed compiling submission!](red|bold)')
            print ce.message,  # don't print extra newline
            grader = None
        except:  # if custom grader failed to initialize, report it to the site
            return self.internal_error()

        binary = grader.binary if grader else None

        # the compiler may have failed, or an error could have happened while initializing a custom judge
        # either way, we can't continue
        if binary:
            self.packet_manager.begin_grading_packet()

            batch_counter = 1
            in_batch = False
            try:
                for case_number, result in enumerate(self.grade_cases(grader, problem.cases,
                                                                      short_circuit=short_circuit)):
                    if isinstance(result, BatchBegin):
                        self.packet_manager.batch_begin_packet()
                        print ansi_style("#ansi[Batch #%d](yellow|bold)" % batch_counter)
                        in_batch = True
                    elif isinstance(result, BatchEnd):
                        self.packet_manager.batch_end_packet()
                        batch_counter += 1
                        in_batch = False
                    else:
                        codes = result.readable_codes()

                        # here be cancer
                        is_sc = (result.result_flag & Result.SC)
                        colored_codes = map(lambda x: '#ansi[%s](%s|bold)' % ('--' if x == 'SC' else x,
                                                                              Result.COLORS_BYID[x]), codes)
                        colored_aux_codes = '{%s}' % ', '.join(colored_codes[1:]) if len(codes) > 1 else ''
                        colored_feedback = '(#ansi[%s](|underline)) ' % result.feedback if result.feedback else ''
                        case_info = '[%.3fs | %dkb] %s%s' % (result.execution_time, result.max_memory,
                                                             colored_feedback,
                                                             colored_aux_codes) if not is_sc else ''
                        case_padding = '  ' * in_batch
                        print ansi_style('%sTest case %2d %-3s %s' % (case_padding, case_number + 1,
                                                                      colored_codes[0], case_info))

                        # cases are indexed at 1
                        self.packet_manager.test_case_status_packet(
                            case_number + 1, result.points, result.case.points, result.result_flag,
                            result.execution_time,
                            result.max_memory,
                            result.proc_output[:result.case.output_prefix_length].decode('utf-8', 'replace'),
                            result.feedback)
            except TerminateGrading:
                self.packet_manager.submission_terminated_packet()
                print ansi_style('#ansi[Forcefully terminating grading. Temporary files may not be deleted.](red|bold)')
                pass
            except:
                self.internal_error()
            else:
                self.packet_manager.grading_end_packet()

        print ansi_style('Done grading #ansi[%s](yellow)/#ansi[%s](green|bold).' % (problem_id, submission_id))
        print
        self._terminate_grading = False
        self.current_submission_thread = None
        self.current_submission = None
        self.current_grader = None

    def grade_cases(self, grader, cases, short_circuit=False, is_short_circuiting=False):

        for case in cases:
            # Yield notifying objects for batch begin/end, and unwrap all cases inside the batches
            if isinstance(case, BatchedTestCase):
                yield BatchBegin()
                for _ in self.grade_cases(grader, case.batched_cases, short_circuit=True,
                                          is_short_circuiting=is_short_circuiting):
                    yield _
                yield BatchEnd()
                continue

            # Stop grading if we're short circuiting
            if is_short_circuiting:
                result = Result(case)
                result.result_flag = Result.SC
                yield result
                continue

            # Must check here because we might be interrupted mid-execution
            # If we don't bail out, we get an IR.
            # In Java's case, all the code after this will crash.
            if self._terminate_grading:
                raise TerminateGrading()

            result = grader.grade(case)

            # If the WA bit of result_flag is set and we are set to short-circuit (e.g., in a batch),
            # short circuit the rest of the cases.
            # Do the same if the case is a pretest (i.e. has 0 points)
            if (result.result_flag & Result.WA) > 0 and (short_circuit or case.points == 0):
                is_short_circuiting = True

            yield result

    def internal_error(self, exc=None):
        if exc:
            try:
                raise exc
            except:
                pass
        exc = sys.exc_info()

        traceback.print_exception(*exc)
        self.packet_manager.internal_error_packet(''.join(traceback.format_exception(*exc)))

    def listen(self):
        """
        Attempts to connect to the handler server specified in command line.
        """
        self.packet_manager.run()

    def __del__(self):
        del self.packet_manager

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, traceback):
        pass

    def murder(self):
        """
        End any submission currently executing, and exit the judge.
        """
        self.terminate_grading()
        self._stop_monitor()

class ClassicJudge(Judge):
    def __init__(self, host, port):
        self.packet_manager = packet.PacketManager(host, port, self, env['id'], env['key'])
        super(ClassicJudge, self).__init__()

def sanity_check():
    # Don't allow starting up without wbox/cptbox, saves cryptic errors later on
    if os.name == 'nt':
        try:
            import wbox
        except ImportError:
            print >> sys.stderr, "wbox must be compiled to grade!"
            return False

        # DMOJ needs to be run as admin on Windows
        import ctypes
        if ctypes.windll.shell32.IsUserAnAdmin() == 0:
            print >> sys.stderr, "can't start, the DMOJ judge must be ran as admin"
            return False
    else:
        try:
            import cptbox
        except ImportError:
            print >> sys.stderr, "cptbox must be compiled to grade!"
            return False

        # However running as root on Linux is a Bad Idea
        if os.getuid() == 0:
            startup_warnings.append('running the judge as root can be potentially unsafe, '
                                    'consider using an unprivileged user instead')

    # _checker implements standard checker functions in C
    # we fall back to a Python implementation if it's not compiled, but it's slower
    try:
        from checkers import _checker
    except ImportError:
        startup_warnings.append('native checker module not found, compile _checker for optimal performance')
    return True

def main():
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    if not sanity_check():
        return 1

    import logging
    from dmoj import judgeenv, executors

    judgeenv.load_env()

    # Emulate ANSI colors with colorama
    if os.name == 'nt' and not judgeenv.no_ansi_emu:
        try:
            from colorama import init
            init()
        except ImportError as ignored:
            pass

    executors.load_executors()

    print 'Running live judge...'

    logging.basicConfig(filename=judgeenv.log_file, level=logging.INFO,
                        format='%(levelname)s %(asctime)s %(module)s %(message)s')

    judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port)

    for warning in judgeenv.startup_warnings:
        print ansi_style('#ansi[Warning: %s](yellow)' % warning)
    del judgeenv.startup_warnings
    print

    with judge:
        try:
            judge.listen()
        except KeyboardInterrupt:
            pass
        except:
            traceback.print_exc()
        finally:
            judge.murder()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django import forms
from django.core.urlresolvers import reverse, reverse_lazy
from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.template import Context
from website.utils.httpUtil import HttpRequestProcessor
from django.conf import settings as django_settings
from django.template.loader import get_template
from django.template import Context, RequestContext, Template
from django.conf import settings
from jinja2 import FileSystemLoader, Environment
import hashlib
from website.models import Question, QuestionCategory, Jurisdiction, GeographicArea
from website.models.reporting import where_clause_for_area, matches_for_area
from django.db import connection
from collections import OrderedDict
import json
import re
from django.db.models import Q
from django.db.models.sql import Query
from django.db import DEFAULT_DB_ALIAS
from localflavor.us.us_states import US_STATES
import urllib
from django.shortcuts import render_to_response
import autocomplete_light
autocomplete_light.autodiscover()
from website.utils import reporting

##############################################################################
#
# Display an individual report on a question_id
#
##############################################################################
def report_on(request, question_id=None, filter_id=None):
    # Check request for validity
    data = { 'current_nav': 'reporting',
             'reports': [] }
    reports = None
    if question_id:
      question_id = int(question_id)
      question = Question.objects.get(id=question_id)
      if not (question.id in reporting.reports_by_qid or question.display_template in reporting.reports_by_type):
          raise Http404
      data['question_id'] = question_id
      data['name'] = question.question
      data['description'] = question.description

    geo_filter = None
    if filter_id:
        data['geo_filter'] = GeographicArea.objects.get(pk=filter_id)
        geo_filter = data['geo_filter'].where()
    else:
        data['geo_filter'] = {}
        for key in ['states', 'jurisdictions']:
            p = request.GET.getlist(key)
            if p:
              data['geo_filter'][key] = p
        if data['geo_filter']:
            geo_filter = where_clause_for_area(**data['geo_filter'])
            data['geo_filter_matches'] = matches_for_area(**data['geo_filter']).count()

    if question_id:
        data['reports'] = [dict(r, **{'idx': idx})
                           for (idx, r) in enumerate(reporting.add_temporal_reports(reporting.run_reports(question,
                                                                                geo_filter=geo_filter)))]

    if 'HTTP_ACCEPT' in request.META and 'json' in request.META['HTTP_ACCEPT']: #hack
        return HttpResponse(json.dumps(data))
    data['reports_json'] = json.dumps(data['reports'])

    questions = Question.objects.filter(accepted='1').exclude(form_type="CF").order_by("category", "display_order")
    reports_index = []
    category_last_encountered = ''
    for question in questions:
        if question.category.name != category_last_encountered:
            category_last_encountered = question.category.name
            # the category level does not exist create it.
            reports_index.append({ "category": question.category.name.replace('_', ' ').title(),
                                   "reports_in_category": [] })
        # append this report's data to the list - with a link if it exists
        reports_index[-1]['reports_in_category'].append(question)
    data['reports_index'] = reports_index
    data['report_types'] = reporting.reports_by_type.keys()
    data['report_qids'] = reporting.reports_by_qid.keys()
    data['form'] = GeographicAreaForm()
    data['request'] = request
    return render_to_response('reporting/report_on.jinja', data)

class GeographicAreaForm(forms.ModelForm):
    filter_name = forms.CharField()
    states = forms.MultipleChoiceField(choices = US_STATES,
                                       required = False)
    jurisdictions = forms.ModelMultipleChoiceField(queryset = Jurisdiction.objects.all(),
                                                   widget = autocomplete_light.MultipleChoiceWidget('JurisdictionAutocomplete'),
                                                   required = False)
    def __init__(self, *args, **kwargs):
        instance = kwargs.get('instance', None)
        initial = kwargs.get('initial', {})
        if instance:
            initial['states'] = instance.states
            initial['jurisdictions'] = instance.jurisdictions
        kwargs['initial'] = initial
        super(GeographicAreaForm, self).__init__(*args, **kwargs)
    def save(self, commit=True):
        area = super(GeographicAreaForm, self).save(commit)
        if commit and 'jurisdictions' in self.cleaned_data:
            area.jurisdictions.add(*self.cleaned_data['jurisdictions'])
            area.save()
        return area
    class Meta:
        model = GeographicArea
        fields = ['filter_name', 'description', 'states', 'jurisdictions']

class GeographicAreaDetail(DetailView):
    queryset = GeographicArea.objects.all()
    template_name = 'geographic_area_detail.jinja'

class GeographicAreaList(ListView):
    queryset = GeographicArea.objects.all()
    template_name = 'geographic_area_list.jinja'

class GeographicAreaCreate(CreateView):
    model = GeographicArea
    template_name = 'geographic_area_form.jinja'
    form_class = GeographicAreaForm
    def get(self, request, *args, **kwargs):
        self.initial = {key: ",".join(request.GET.getlist(key, "")).split(",") for key in set(request.GET.keys()) & set(['states', 'jurisdictions'])}
        return super(GeographicAreaCreate, self).get(request, *args, **kwargs)
    def get_context_data(self, **kwargs):
        if 'question_id' in self.kwargs:
            kwargs['question_id'] = self.kwargs['question_id']
        return super(GeographicAreaCreate, self).get_context_data(**kwargs)
    def form_valid(self, form):
        response = super(GeographicAreaCreate, self).form_valid(form)
        if 'question_id' in self.kwargs:
            self.kwargs.update({'filter_id': form.instance.pk})
            return HttpResponseRedirect(reverse('report-with-filter',
                                                kwargs = self.kwargs))
        else:
            return response
    def form_invalid(self, form):
        if 'question_id' in self.kwargs:
            params = {}
            if 'states' in form.cleaned_data and form.cleaned_data['states']:
                params['states'] = ",".join(form.cleaned_data['states'])
            elif 'jurisdictions' in form.cleaned_data and form.cleaned_data['jurisdictions']:
                params['jurisdictions'] = ",".join([str(j.pk) for j in form.cleaned_data['jurisdictions']])
            return HttpResponseRedirect(reverse('report', kwargs=self.kwargs) + '?' + \
                                        urllib.urlencode(params))
        else:
            return super(GeographicAreaCreate, self).form_invalid(form)

class GeographicAreaUpdate(UpdateView):
    model = GeographicArea
    template_name = 'geographic_area_form.jinja'
    form_class = GeographicAreaForm

class GeographicAreaDelete(DeleteView):
    model = GeographicArea
    success_url = reverse_lazy('geoarea-list')
    template_name = 'geographic_area_confirm_delete.jinja'
<commit_msg>make the reporing view use a RequestContext (fixes #144)<commit_after># -*- coding: utf-8 -*-
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django import forms
from django.core.urlresolvers import reverse, reverse_lazy
from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.template import Context
from website.utils.httpUtil import HttpRequestProcessor
from django.conf import settings as django_settings
from django.template.loader import get_template
from django.template import Context, RequestContext, Template
from django.conf import settings
from jinja2 import FileSystemLoader, Environment
import hashlib
from website.models import Question, QuestionCategory, Jurisdiction, GeographicArea
from website.models.reporting import where_clause_for_area, matches_for_area
from django.db import connection
from collections import OrderedDict
import json
import re
from django.db.models import Q
from django.db.models.sql import Query
from django.db import DEFAULT_DB_ALIAS
from localflavor.us.us_states import US_STATES
import urllib
from django.shortcuts import render
import autocomplete_light
autocomplete_light.autodiscover()
from website.utils import reporting

##############################################################################
#
# Display an individual report on a question_id
#
##############################################################################
def report_on(request, question_id=None, filter_id=None):
    # Check request for validity
    data = { 'current_nav': 'reporting',
             'reports': [] }
    reports = None
    if question_id:
      question_id = int(question_id)
      question = Question.objects.get(id=question_id)
      if not (question.id in reporting.reports_by_qid or question.display_template in reporting.reports_by_type):
          raise Http404
      data['question_id'] = question_id
      data['name'] = question.question
      data['description'] = question.description

    geo_filter = None
    if filter_id:
        data['geo_filter'] = GeographicArea.objects.get(pk=filter_id)
        geo_filter = data['geo_filter'].where()
    else:
        data['geo_filter'] = {}
        for key in ['states', 'jurisdictions']:
            p = request.GET.getlist(key)
            if p:
              data['geo_filter'][key] = p
        if data['geo_filter']:
            geo_filter = where_clause_for_area(**data['geo_filter'])
            data['geo_filter_matches'] = matches_for_area(**data['geo_filter']).count()

    if question_id:
        data['reports'] = [dict(r, **{'idx': idx})
                           for (idx, r) in enumerate(reporting.add_temporal_reports(reporting.run_reports(question,
                                                                                geo_filter=geo_filter)))]

    if 'HTTP_ACCEPT' in request.META and 'json' in request.META['HTTP_ACCEPT']: #hack
        return HttpResponse(json.dumps(data))
    data['reports_json'] = json.dumps(data['reports'])

    questions = Question.objects.filter(accepted='1').exclude(form_type="CF").order_by("category", "display_order")
    reports_index = []
    category_last_encountered = ''
    for question in questions:
        if question.category.name != category_last_encountered:
            category_last_encountered = question.category.name
            # the category level does not exist create it.
            reports_index.append({ "category": question.category.name.replace('_', ' ').title(),
                                   "reports_in_category": [] })
        # append this report's data to the list - with a link if it exists
        reports_index[-1]['reports_in_category'].append(question)
    data['reports_index'] = reports_index
    data['report_types'] = reporting.reports_by_type.keys()
    data['report_qids'] = reporting.reports_by_qid.keys()
    data['form'] = GeographicAreaForm()
    data['request'] = request
    return render(request, 'reporting/report_on.jinja', data)

class GeographicAreaForm(forms.ModelForm):
    filter_name = forms.CharField()
    states = forms.MultipleChoiceField(choices = US_STATES,
                                       required = False)
    jurisdictions = forms.ModelMultipleChoiceField(queryset = Jurisdiction.objects.all(),
                                                   widget = autocomplete_light.MultipleChoiceWidget('JurisdictionAutocomplete'),
                                                   required = False)
    def __init__(self, *args, **kwargs):
        instance = kwargs.get('instance', None)
        initial = kwargs.get('initial', {})
        if instance:
            initial['states'] = instance.states
            initial['jurisdictions'] = instance.jurisdictions
        kwargs['initial'] = initial
        super(GeographicAreaForm, self).__init__(*args, **kwargs)
    def save(self, commit=True):
        area = super(GeographicAreaForm, self).save(commit)
        if commit and 'jurisdictions' in self.cleaned_data:
            area.jurisdictions.add(*self.cleaned_data['jurisdictions'])
            area.save()
        return area
    class Meta:
        model = GeographicArea
        fields = ['filter_name', 'description', 'states', 'jurisdictions']

class GeographicAreaDetail(DetailView):
    queryset = GeographicArea.objects.all()
    template_name = 'geographic_area_detail.jinja'

class GeographicAreaList(ListView):
    queryset = GeographicArea.objects.all()
    template_name = 'geographic_area_list.jinja'

class GeographicAreaCreate(CreateView):
    model = GeographicArea
    template_name = 'geographic_area_form.jinja'
    form_class = GeographicAreaForm
    def get(self, request, *args, **kwargs):
        self.initial = {key: ",".join(request.GET.getlist(key, "")).split(",") for key in set(request.GET.keys()) & set(['states', 'jurisdictions'])}
        return super(GeographicAreaCreate, self).get(request, *args, **kwargs)
    def get_context_data(self, **kwargs):
        if 'question_id' in self.kwargs:
            kwargs['question_id'] = self.kwargs['question_id']
        return super(GeographicAreaCreate, self).get_context_data(**kwargs)
    def form_valid(self, form):
        response = super(GeographicAreaCreate, self).form_valid(form)
        if 'question_id' in self.kwargs:
            self.kwargs.update({'filter_id': form.instance.pk})
            return HttpResponseRedirect(reverse('report-with-filter',
                                                kwargs = self.kwargs))
        else:
            return response
    def form_invalid(self, form):
        if 'question_id' in self.kwargs:
            params = {}
            if 'states' in form.cleaned_data and form.cleaned_data['states']:
                params['states'] = ",".join(form.cleaned_data['states'])
            elif 'jurisdictions' in form.cleaned_data and form.cleaned_data['jurisdictions']:
                params['jurisdictions'] = ",".join([str(j.pk) for j in form.cleaned_data['jurisdictions']])
            return HttpResponseRedirect(reverse('report', kwargs=self.kwargs) + '?' + \
                                        urllib.urlencode(params))
        else:
            return super(GeographicAreaCreate, self).form_invalid(form)

class GeographicAreaUpdate(UpdateView):
    model = GeographicArea
    template_name = 'geographic_area_form.jinja'
    form_class = GeographicAreaForm

class GeographicAreaDelete(DeleteView):
    model = GeographicArea
    success_url = reverse_lazy('geoarea-list')
    template_name = 'geographic_area_confirm_delete.jinja'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os

def amiss(logPrefix):
    global ignoreList
    crashIsKnown = False

    fn = logPrefix + "-crash"

    if os.path.exists(fn):
        currentFile = file(fn, "r")
        for line in currentFile:
            for ig in ignoreList:
                if line.find(ig) != -1:
                    print "Known crash: " + ig
                    crashIsKnown = True
        currentFile.close()
    else:
        print "detect_interesting_crashes didn't see a crash log!"

    return not crashIsKnown

def getIgnores():
    global ignoreList
    ignoreFile = open("known_crashes.txt", "r")
    for line in ignoreFile:
        line = line.strip()
        if ((len(line) > 0) and not line.startswith("#")):
            ignoreList.append(line)

def ignore(assertion):
    global ignoreList
    for ig in ignoreList:
        if assertion.find(ig) != -1:
            return True
    return False

ignoreList = []
getIgnores()

print "detect_interesting_crashes is ready (ignoring %d strings)" % (len(ignoreList))
<commit_msg>Make detect_interesting_crashes quiet on startup<commit_after>#!/usr/bin/env python

import os

def amiss(logPrefix):
    global ignoreList
    crashIsKnown = False

    fn = logPrefix + "-crash"

    if os.path.exists(fn):
        currentFile = file(fn, "r")
        for line in currentFile:
            for ig in ignoreList:
                if line.find(ig) != -1:
                    print "Known crash: " + ig
                    crashIsKnown = True
        currentFile.close()
    else:
        print "detect_interesting_crashes didn't see a crash log!"

    return not crashIsKnown

def getIgnores():
    global ignoreList
    ignoreFile = open("known_crashes.txt", "r")
    for line in ignoreFile:
        line = line.strip()
        if ((len(line) > 0) and not line.startswith("#")):
            ignoreList.append(line)

def ignore(assertion):
    global ignoreList
    for ig in ignoreList:
        if assertion.find(ig) != -1:
            return True
    return False

ignoreList = []
getIgnores()

#print "detect_interesting_crashes is ready (ignoring %d strings)" % (len(ignoreList))
<|endoftext|>