<commit_before>"""
F-O-A-A-S Skill.

"""
from __future__ import print_function

import json
import random
import urllib
import requests

# --------------- Helpers that build all of the responses ----------------------

def build_speechlet_response(title, output, reprompt_text, should_end_session):
    return {
        'outputSpeech': {
            'type': 'SSML',
            'ssml': wrap_into_ssml(output)
        },
        'card': {
            'type': 'Simple',
            'title': title,
            'content': output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'SSML',
                'ssml': wrap_into_ssml(reprompt_text)
            }
        },
        'shouldEndSession': should_end_session
    }

def build_response(session_attributes, speechlet_response):
    return {
        'version': '1.0',
        'sessionAttributes': session_attributes,
        'response': speechlet_response
    }

# --------------- Functions that control the skill's behavior ------------------

def get_welcome_response():
    """ If we wanted to initialize the session to have some attributes we could
    add those here
    """

    session_attributes = {}
    card_title = "Welcome"
    speech_output = "Welcome to the insult generator. " \
                    "Please tell me who to insult by saying: " \
                    "Insult Frank, or tell me an insult."
    # If the user either does not reply to the welcome message or says something
    # that is not understood, they will be prompted again with this text.
    reprompt_text = "Who shall I insult?"
    should_end_session = False
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_help_response():
    session_attributes = {}
    card_title = "Help"
    speech_output = "Tell me who to insult by saying: " \
                    "Alexa, ask the insult generator to insult Anthony."
    # If the user either does not reply to the welcome message or says something
    # that is not understood, they will be prompted again with this text.
    reprompt_text = "Who should I insult?"
    should_end_session = False
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_stop_response():
    session_attributes = {}
    card_title = "Bye"
    speech_output = "Ok."
    reprompt_text = ""
    should_end_session = True
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def handle_session_end_request():
    card_title = "Bye"
    speech_output = "Thank you for trying the insult generator. " \
                    "Have a nice day! "
    # Setting this to true ends the session and exits the skill.
    should_end_session = True
    return build_response({}, build_speechlet_response(
        card_title, speech_output, None, should_end_session))

# --------------- Code to communicate with foaas ------------------

def make_request(request):
  try:
    resp = requests.get(request, headers={ 'Accept': 'application/json' })
    if resp.status_code != 200:
      pass
    else:
      data = json.loads(resp.content)
      return data
  except:
    print("Request failed:", sys.exc_info()[0])
    return request
  return None

def get_random_endpoint_for_person(to_text, endpoints, from_text="Your F-O-A-A-S Skill"):
    return "http://foaas.com/{}/{}/{}".format(random.choice(endpoints), urllib.quote(to_text), urllib.quote(from_text))

def get_random_endpoint(endpoints, from_text="Your F-O-A-A-S Skill"):
    return "http://foaas.com/{}/{}".format(random.choice(endpoints), urllib.quote(from_text))

name_endpoints = list()
generic_endpoints = list()

def get_operations():
  global generic_endpoints, name_endpoints
  if len(name_endpoints)>0 and len(generic_endpoints)>0:
      return
  name_endpoints = list()
  generic_endpoints = list()
  try:
    data = make_request("http://foaas.com/operations/")
    for entry in data:
      #format has changed, now we have to look for the URL field and trim it.
      endpoint = entry['name']
      if 'url' in entry:
        s = entry['url']
        s = s[1:] #remove slash
        s = s[:s.find('/')] # remove everything after second slash
        endpoint = s
      if len(entry['fields']) == 1:
        generic_endpoints.append(endpoint)
      elif len(entry['fields']) == 2:
        name_endpoints.append(endpoint)
  except:
    print("Could not get operations:", sys.exc_info()[0])

def get_message(request, default_text):
  message = default_text
  try:
    message = make_request(request)['message']
  except:
    print("Get message failed", sys.exc_info()[0])
  return message

def get_insult(kw, default_text):
  global generic_endpoints, name_endpoints
  speech_output = default_text
  url = None
  retry = 2
  while retry > 0:
      retry = retry - 1
      get_operations()
      try:
        if kw:
          url = get_random_endpoint_for_person(kw, name_endpoints)
        else:
          url = get_random_endpoint(generic_endpoints)
        speech_output = get_message(url, speech_output)
        retry = 0
      except:
        speech_output = url

  return speech_output

# --------------- End of foaas code ------------------

def communicate_with_foaas(intent, session):
    card_title = "FOAAS Response"
    session_attributes = {}
    should_end_session = True
    speech_output = "I didn't understand a fucking word!"

    if intent['name'] == "AboutPerson":
        kw = intent['slots']['KeyWord']['value']
        speech_output = get_insult(kw, speech_output)
    else:
        speech_output = get_insult(None, speech_output)

    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, speech_output, should_end_session))

# --------------- Events ------------------

def on_session_started(session_started_request, session):
    """ Called when the session starts """

    print("on_session_started requestId=" + session_started_request['requestId']
          + ", sessionId=" + session['sessionId'])

def on_launch(launch_request, session):
    """ Called when the user launches the skill without specifying what they
    want
    """

    print("on_launch requestId=" + launch_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # Dispatch to your skill's launch
    return get_welcome_response()

def on_intent(intent_request, session):
    """ Called when the user specifies an intent for this skill """

    #print("on_intent requestId=" + intent_request['requestId'] +
    #      ", sessionId=" + session['sessionId'])

    intent = intent_request['intent']
    intent_name = intent_request['intent']['name']

    # Dispatch to your skill's intent handlers
    if intent_name == "AboutPerson":
        return communicate_with_foaas(intent, session)
    elif intent_name == "AMAZON.HelpIntent":
        return get_help_response()
    elif (intent_name == "AMAZON.StopIntent") or (intent_name == "AMAZON.CancelIntent"):
        return get_stop_response()
    else:
        raise ValueError("Invalid intent")

def wrap_into_ssml(speech_string):
    speech_string = speech_string.replace("fuck", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVk\">f**k</phoneme>")
    speech_string = speech_string.replace("Fuck", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVk\">F**k</phoneme>")
    speech_string = speech_string.replace("fucking", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVking\">f**king</phoneme>")
    speech_string = speech_string.replace("Fucking", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVking\">F**ing</phoneme>")
    speech_string = speech_string.replace("fucker", u"<phoneme alphabet=\"x-sampa\" ph=\"\'fVk3`r\\\">f**ker</phoneme>")
    speech_string = speech_string.replace("Fucker", u"<phoneme alphabet=\"x-sampa\" ph=\"\'fVk3`r\\\">F**ker</phoneme>")

    return "<speak>{}</speak>".format(speech_string)

def on_session_ended(session_ended_request, session):
    """ Called when the user ends the session.

    Is not called when the skill returns should_end_session=true
    """
    print("on_session_ended requestId=" + session_ended_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # add cleanup logic here

# --------------- Main handler ------------------

def lambda_handler(event, context):
    """ Route the incoming request based on type (LaunchRequest, IntentRequest,
    etc.) The JSON body of the request is provided in the event parameter.
    """
    print("event.session.application.applicationId=" +
          event['session']['application']['applicationId'])

    """
    Uncomment this if statement and populate with your skill's application ID to
    prevent someone else from configuring a skill that sends requests to this
    function.
    """
    # if (event['session']['application']['applicationId'] !=
    #         "amzn1.echo-sdk-ams.app.[unique-value-here]"):
    #     raise ValueError("Invalid Application ID")

    if event['session']['new']:
        on_session_started({'requestId': event['request']['requestId']},
                           event['session'])

    if event['request']['type'] == "LaunchRequest":
        return on_launch(event['request'], event['session'])
    elif event['request']['type'] == "IntentRequest":
        return on_intent(event['request'], event['session'])
    elif event['request']['type'] == "SessionEndedRequest":
        return on_session_ended(event['request'], event['session'])

<commit_msg>Return default text instead of URL on exception (URL leads to invalid response)<commit_after>"""
F-O-A-A-S Skill.

"""
from __future__ import print_function

import json
import random
import urllib
import requests

# --------------- Helpers that build all of the responses ----------------------

def build_speechlet_response(title, output, reprompt_text, should_end_session):
    return {
        'outputSpeech': {
            'type': 'SSML',
            'ssml': wrap_into_ssml(output)
        },
        'card': {
            'type': 'Simple',
            'title': title,
            'content': output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'SSML',
                'ssml': wrap_into_ssml(reprompt_text)
            }
        },
        'shouldEndSession': should_end_session
    }

def build_response(session_attributes, speechlet_response):
    return {
        'version': '1.0',
        'sessionAttributes': session_attributes,
        'response': speechlet_response
    }

# --------------- Functions that control the skill's behavior ------------------

def get_welcome_response():
    """ If we wanted to initialize the session to have some attributes we could
    add those here
    """

    session_attributes = {}
    card_title = "Welcome"
    speech_output = "Welcome to the insult generator. " \
                    "Please tell me who to insult by saying: " \
                    "Insult Frank, or tell me an insult."
    # If the user either does not reply to the welcome message or says something
    # that is not understood, they will be prompted again with this text.
    reprompt_text = "Who shall I insult?"
    should_end_session = False
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_help_response():
    session_attributes = {}
    card_title = "Help"
    speech_output = "Tell me who to insult by saying: " \
                    "Alexa, ask the insult generator to insult Anthony."
    # If the user either does not reply to the welcome message or says something
    # that is not understood, they will be prompted again with this text.
    reprompt_text = "Who should I insult?"
    should_end_session = False
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def get_stop_response():
    session_attributes = {}
    card_title = "Bye"
    speech_output = "Ok."
    reprompt_text = ""
    should_end_session = True
    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, reprompt_text, should_end_session))

def handle_session_end_request():
    card_title = "Bye"
    speech_output = "Thank you for trying the insult generator. " \
                    "Have a nice day! "
    # Setting this to true ends the session and exits the skill.
    should_end_session = True
    return build_response({}, build_speechlet_response(
        card_title, speech_output, None, should_end_session))

# --------------- Code to communicate with foaas ------------------

def make_request(request):
  try:
    resp = requests.get(request, headers={ 'Accept': 'application/json' })
    if resp.status_code != 200:
      pass
    else:
      data = json.loads(resp.content)
      return data
  except:
    print("Request failed:", sys.exc_info()[0])
    return request
  return None

def get_random_endpoint_for_person(to_text, endpoints, from_text="Your F-O-A-A-S Skill"):
    return "http://foaas.com/{}/{}/{}".format(random.choice(endpoints), urllib.quote(to_text), urllib.quote(from_text))

def get_random_endpoint(endpoints, from_text="Your F-O-A-A-S Skill"):
    return "http://foaas.com/{}/{}".format(random.choice(endpoints), urllib.quote(from_text))

name_endpoints = list()
generic_endpoints = list()

def get_operations():
  global generic_endpoints, name_endpoints
  if len(name_endpoints)>0 and len(generic_endpoints)>0:
      return
  name_endpoints = list()
  generic_endpoints = list()
  try:
    data = make_request("http://foaas.com/operations/")
    for entry in data:
      #format has changed, now we have to look for the URL field and trim it.
      endpoint = entry['name']
      if 'url' in entry:
        s = entry['url']
        s = s[1:] #remove slash
        s = s[:s.find('/')] # remove everything after second slash
        endpoint = s
      if len(entry['fields']) == 1:
        generic_endpoints.append(endpoint)
      elif len(entry['fields']) == 2:
        name_endpoints.append(endpoint)
  except:
    print("Could not get operations:", sys.exc_info()[0])

def get_message(request, default_text):
  message = default_text
  try:
    message = make_request(request)['message']
  except:
    print("Get message failed", sys.exc_info()[0])
  return message

def get_insult(kw, default_text):
  global generic_endpoints, name_endpoints
  speech_output = default_text
  url = None
  retry = 2
  while retry > 0:
      retry = retry - 1
      get_operations()
      try:
        if kw:
          url = get_random_endpoint_for_person(kw, name_endpoints)
        else:
          url = get_random_endpoint(generic_endpoints)
        speech_output = get_message(url, speech_output)
        retry = 0
      except:
	pass

  return speech_output

# --------------- End of foaas code ------------------

def communicate_with_foaas(intent, session):
    card_title = "FOAAS Response"
    session_attributes = {}
    should_end_session = True
    speech_output = "I didn't understand a fucking word!"

    if intent['name'] == "AboutPerson":
        kw = intent['slots']['KeyWord']['value']
        speech_output = get_insult(kw, speech_output)
    else:
        speech_output = get_insult(None, speech_output)

    return build_response(session_attributes, build_speechlet_response(
        card_title, speech_output, speech_output, should_end_session))

# --------------- Events ------------------

def on_session_started(session_started_request, session):
    """ Called when the session starts """

    print("on_session_started requestId=" + session_started_request['requestId']
          + ", sessionId=" + session['sessionId'])

def on_launch(launch_request, session):
    """ Called when the user launches the skill without specifying what they
    want
    """

    print("on_launch requestId=" + launch_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # Dispatch to your skill's launch
    return get_welcome_response()

def on_intent(intent_request, session):
    """ Called when the user specifies an intent for this skill """

    #print("on_intent requestId=" + intent_request['requestId'] +
    #      ", sessionId=" + session['sessionId'])

    intent = intent_request['intent']
    intent_name = intent_request['intent']['name']

    # Dispatch to your skill's intent handlers
    if intent_name == "AboutPerson":
        return communicate_with_foaas(intent, session)
    elif intent_name == "AMAZON.HelpIntent":
        return get_help_response()
    elif (intent_name == "AMAZON.StopIntent") or (intent_name == "AMAZON.CancelIntent"):
        return get_stop_response()
    else:
        raise ValueError("Invalid intent")

def wrap_into_ssml(speech_string):
    speech_string = speech_string.replace("fuck", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVk\">f**k</phoneme>")
    speech_string = speech_string.replace("Fuck", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVk\">F**k</phoneme>")
    speech_string = speech_string.replace("fucking", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVking\">f**king</phoneme>")
    speech_string = speech_string.replace("Fucking", "<phoneme alphabet=\"x-sampa\" ph=\"\'fVking\">F**ing</phoneme>")
    speech_string = speech_string.replace("fucker", u"<phoneme alphabet=\"x-sampa\" ph=\"\'fVk3`r\\\">f**ker</phoneme>")
    speech_string = speech_string.replace("Fucker", u"<phoneme alphabet=\"x-sampa\" ph=\"\'fVk3`r\\\">F**ker</phoneme>")

    return "<speak>{}</speak>".format(speech_string)

def on_session_ended(session_ended_request, session):
    """ Called when the user ends the session.

    Is not called when the skill returns should_end_session=true
    """
    print("on_session_ended requestId=" + session_ended_request['requestId'] +
          ", sessionId=" + session['sessionId'])
    # add cleanup logic here

# --------------- Main handler ------------------

def lambda_handler(event, context):
    """ Route the incoming request based on type (LaunchRequest, IntentRequest,
    etc.) The JSON body of the request is provided in the event parameter.
    """
    print("event.session.application.applicationId=" +
          event['session']['application']['applicationId'])

    """
    Uncomment this if statement and populate with your skill's application ID to
    prevent someone else from configuring a skill that sends requests to this
    function.
    """
    # if (event['session']['application']['applicationId'] !=
    #         "amzn1.echo-sdk-ams.app.[unique-value-here]"):
    #     raise ValueError("Invalid Application ID")

    if event['session']['new']:
        on_session_started({'requestId': event['request']['requestId']},
                           event['session'])

    if event['request']['type'] == "LaunchRequest":
        return on_launch(event['request'], event['session'])
    elif event['request']['type'] == "IntentRequest":
        return on_intent(event['request'], event['session'])
    elif event['request']['type'] == "SessionEndedRequest":
        return on_session_ended(event['request'], event['session'])

<|endoftext|>


<commit_before>

import sys
from time import time

class PID(object):

    def __init__(self):
        """initizes value for the PID"""
        self.kd = 0
        self.ki = 0
        self.kp = 1
        self.previous_error = 0
        self.integral_error = 0

    def set_k_values(self, kp, kd, ki):
        self.kp = kp
        self.ki = ki
        self.kd = kd

    def pid(self, target, process_var, timestep):
        current_error = (target + process_var)
        p_error = self.kp * current_error
        d_error = self.kd * (current_error - self.previous_error) \
            / timestep
        self.integral_error = (
            current_error + self.previous_error) / 2 \
            + self.integral_error
        i_error = self.ki * self.integral_error
        total_error = p_error + d_error + i_error
        self.previous_error = current_error
        return total_error
<commit_msg>Update to follower, reduce speed to motors.<commit_after>

import sys
from time import time

class PID(object):

    def __init__(self):
        """initizes value for the PID"""
        self.kd = 0
        self.ki = 0
        self.kp = 1
        self.previous_error = 0
        self.integral_error = 0

    def set_k_values(self, kp, kd, ki):
        self.kp = kp
        self.ki = ki
        self.kd = kd

    def pid(self, target, process_var, timestep):
        current_error = (target - process_var)
        p_error = self.kp * current_error
        d_error = self.kd * (current_error - self.previous_error) \
            / timestep
        self.integral_error = (
            current_error + self.previous_error) / 2 \
            + self.integral_error
        i_error = self.ki * self.integral_error
        total_error = p_error + d_error + i_error
        self.previous_error = current_error
        return total_error
<|endoftext|>


<commit_before>__author__ = 'sam'
import os
import numpy as np
import tables
import caleydo_server.range as ranges

import itertools

from caleydo_server.dataset_def import ADataSetEntry, ADataSetProvider

def assign_ids(ids, idtype):
  import caleydo_server.plugin

  manager = caleydo_server.plugin.lookup('idmanager')
  return np.array(manager(ids, idtype))

class HDFEntry(ADataSetEntry):
  def __init__(self, group, project):
    super(HDFEntry, self).__init__(group._v_title, project, group._v_attrs.type)
    self._group = group
    self._project = project
    self.path = self._group._v_pathname

class HDFMatrix(HDFEntry):
  def __init__(self, group, project):
    super(HDFMatrix, self).__init__(group, project)
    self._rowids = None
    self._colids = None
    self._range = None

  @property
  def rowtype(self):
    return self._group._v_attrs.rowtype

  @property
  def coltype(self):
    return self._group._v_attrs.coltype

  @property
  def value(self):
    return self._group._v_attrs.value

  @property
  def range(self):
    if 'range' in self._group._v_attrs:
      return self._group._v_attrs['range']
    if self._range is not None:
      return self._range
    print self.fqname
    d = self._group.data
    self._range = [np.nanmin(d), np.nanmax(d)]
    return self._range

  @property
  def shape(self):
    return self._group.data.shape

  def idtypes(self):
    return [self.rowtype, self.coltype]

  def to_description(self):
    r = super(HDFMatrix, self).to_description()
    r['rowtype'] = self.rowtype
    r['coltype'] = self.coltype
    r['value'] = v = dict(type=self.value)
    if self.value == 'real' or self.value == 'int':
      v['range'] = self.range
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      v['missing'] = self._group._v_attrs.missing
    elif self.value == 'categorical':
      cats = self._group._v_attrs.categories
      if 'names' in self._group._v_attrs:
        v['categories'] = [ dict(name=cat,label=name,color=col) for cat,name,col in itertools.izip(cats,self._group._v_attrs.names,self._group._v_attrs.colors) ]
      else:
        v['categories'] = cats

    if 'center' in self._group._v_attrs:
      v['center'] = self._group._v_attrs['center']
    r['size'] = self.shape
    return r

  def mask(self, arr):
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      missing = self._group._v_attrs.missing
      import numpy.ma as ma
      return ma.masked_equal(missing, arr)
    return arr

  def asnumpy(self, range=None):
    n = self._group.data
    if range is None:
      return self.mask(n)
    return self.mask(n[range[0].asslice(), range[1].asslice()])

  def rows(self, range=None):
    n = self._group.rows
    if range is None:
      return n
    return n[range.asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.rowtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def cols(self, range=None):
    n = self._group.cols
    if range is None:
      return n
    return n[range.asslice()]

  def colids(self, range=None):
    if self._colids is None:
      self._colids = assign_ids(self.cols(), self.coltype)
    n = self._colids
    if range is None:
      return n
    return n[range.asslice()]

  def filter(self, query):
    # perform the query on rows and cols and return a range with just the mathing one
    # np.argwhere
    np.arange(10)
    return ranges.all()

  def asjson(self, range=None):
    arr = self.asnumpy(range)
    rows = self.rows(None if range is None else range[0])
    cols = self.cols(None if range is None else range[1])
    rowids = self.rowids(None if range is None else range[1])
    colids = self.colids(None if range is None else range[1])

    r = dict(data=arr, rows=rows, cols=cols, rowIds=rowids, colids=colids)
    return r

class HDFVector(HDFEntry):
  def __init__(self, group, project):
    super(HDFVector, self).__init__(group, project)
    self._rowids = None
    self._range = None

  @property
  def idtype(self):
    return self._group._v_attrs.rowtype

  @property
  def value(self):
    return self._group._v_attrs.value

  @property
  def range(self):
    if 'range' in self._group._v_attrs:
      return self._group._v_attrs['range']
    if self._range is not None:
      return self._range
    d = self._group.data
    self._range = [np.nanmin(d), np.nanmax(d)]
    return self._range

  @property
  def shape(self):
    return len(self._group.data)

  def idtypes(self):
    return [self.idtype]

  def to_description(self):
    r = super(HDFVector, self).to_description()
    r['idtype'] = self.idtype
    r['value'] = dict(type=self.value,range=self.range)
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      r['value']['mssing'] = self._group._v_attrs.missing
    if 'center' in self._group._v_attrs:
      r['value']['center'] = self._group._v_attrs['center']
    r['size'] = [self.shape]
    return r

  def mask(self, arr):
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      missing = self._group._v_attrs.missing
      import numpy.ma as ma
      return ma.masked_equal(missing, arr)
    return arr

  def asnumpy(self, range=None):
    n = self._group.data
    if range is None:
      return self.mask(n)
    return self.mask(n[range[0].asslice()])

  def rows(self, range=None):
    n = self._group.rows
    if range is None:
      return n
    return n[range.asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.rowtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def asjson(self, range=None):
    arr = self.asnumpy(range)
    rows = self.rows(None if range is None else range[0])
    rowids = self.rowids(None if range is None else range[1])

    r = dict(data=arr, rows=rows, rowIds=rowids)
    return r

class HDFGroup(object):
  def __init__(self, name, range, color):
    self.name = name
    self.range = range
    self.color = color

  def __len__(self):
    return len(self.range)

  def dump(self):
    return dict(name=self.name, range=str(self.range), color=self.color)

def guess_color(name, i):
  name = name.lower()
  colors = dict(male='blue',female='red',deceased='#e41a1b',living='#377eb8')
  if name in colors:
    return colors[name]
  l = ['#8dd3c7', '#ffffb3', '#bebada', '#fb8072', '#80b1d3', '#fdb462', '#b3de69', '#fccde5', '#d9d9d9', '#bc80bd',
          '#ccebc5', '#ffed6f']
  return l[i%len(l)]

class HDFStratification(HDFEntry):
  def __init__(self, group, project):
    super(HDFStratification, self).__init__(group, project)
    self._rowids = None

  @property
  def idtype(self):
    return self._group._v_attrs.idtype

  def idtypes(self):
    return [self.idtype]

  def to_description(self):
    r = super(HDFStratification, self).to_description()
    r['idtype'] = self.idtype
    if 'origin' in self._group._v_attrs:
      r['origin'] = self._project + '/' + self._group._v_attrs.origin
    r['groups'] = [dict(name=gf._v_title, size=len(gf),color=gf._v_attrs['color'] if 'color' in gf._v_attrs else guess_color(gf._v_title, j))
                   for j,gf in enumerate(self._group._v_children.itervalues())]
    r['ngroups'] = len(r['groups'])
    r['size'] = [sum((g['size'] for g in r['groups']))]
    return r

  def _rows(self):
    return np.concatenate(self._group._v_children.values())

  def rows(self, range=None):
    n = self._rows()
    if range is None:
      return n
    return n[range[0].asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.idtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def groups(self):
    i = 0
    for j,g in enumerate(self._group._v_children.itervalues()):
      name = g._v_title
      color = g._v_attrs['color'] if 'color' in g._v_attrs else guess_color(name, j)
      l = len(g)
      yield HDFGroup(name, ranges.from_slice(i, i + l), color)
      i += l

  def __getitem__(self, item):
    group = getattr(self._group, item)
    return group

  def asjson(self, range = None):
    r = dict(rows=self.rows(range), rowIds=self.rowids(range), groups=[g.dump() for g in self.groups()])
    return r

class HDFColumn(object):
  def __init__(self, attrs, group):
    self._group = group
    self.key = attrs['key']
    self.name = attrs['name']
    self.type = attrs['type']
    self._converter = lambda x: x
    if self.type == 'categorical':
      self._converter = tables.misc.enum.Enum(attrs['categories'])
      self.categories = attrs['categories']
    elif self.type == 'int' or self.type == 'real':
      self.range = attrs['range'] if 'range' in attrs else self.compute_range()
      self.missing = attrs.get('missing',None)

  def compute_range(self):
    d = self._group.table.col(self.key)
    return [np.nanmin(d), np.nanmax(d)]

  def __call__(self, v):
    return self._converter(v)

  def mask(self, arr):
    if self.value == 'int' and self.missing is not None:
      import numpy.ma as ma
      return ma.masked_equal(self.missing, arr)
    return arr

  def asnumpy(self, range=None):
    n = self._group.table.col(self.key)
    if range is not None:
      n = n[range[0].asslice()]
    return self.mask(n)

  def dump(self):
    value = dict(type=self.type)
    if self.type == 'categorical':
      value['categories'] = self.categories
    if self.type == 'int' or self.type == 'real':
      value['range'] = self.range
    if self.type == 'int' and self.missing is not None:
      value['missing'] = self.missing
    return dict(name=self.name, value=value)

class HDFTable(HDFEntry):
  def __init__(self, group, project):
    super(HDFTable, self).__init__(group, project)

    self.columns = [HDFColumn(a, group) for a in group._v_attrs.columns]
    self._rowids = None

  @property
  def idtype(self):
    return self._group._v_attrs.rowtype

  def idtypes(self):
    return [self.idtype]

  def to_description(self):
    r = super(HDFTable, self).to_description()
    r['idtype'] = self.idtype
    r['columns'] = [d.dump() for d in self.columns]
    r['size'] = [len(self._group.table), len(self.columns)]
    return r

  def rows(self, range=None):
    n = self._group.rows
    if range is None:
      return n
    return n[range.asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.idtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def asnumpy(self, range=None):
    n = self._group.table
    if range is None:
      return n
    return n[range[0].asslice()]

  def filter(self, query):
    # perform the query on rows and cols and return a range with just the mathing one
    # np.argwhere
    np.arange(10)
    return ranges.all()

  def asjson(self, range=None):
    arr = self.asnumpy(range)
    rows = self.rows(None if range is None else range[0])
    rowids = self.rowids(None if range is None else range[0])

    dd = [[c(row[c.key]) for c in self.columns] for row in arr]
    r = dict(data=dd, rows=rows, rowIds = rowids)

    return r

class HDFProject(object):
  def __init__(self, filename, baseDir):
    self.filename = filename
    p = os.path.relpath(filename, baseDir)
    project,_ = os.path.splitext(p)
    project = project.replace('.','_')
    self._h = tables.open_file(filename, 'r')

    self.entries = []
    for group in self._h.walk_groups('/'):
      if 'type' not in group._v_attrs:
        continue
      t = group._v_attrs.type
      if t == 'matrix':
        self.entries.append(HDFMatrix(group, project))
      elif t == 'stratification':
        self.entries.append(HDFStratification(group, project))
      elif t == 'table':
        self.entries.append(HDFTable(group, project))
      elif t == 'vector':
        self.entries.append(HDFVector(group, project))

  def __iter__(self):
    return iter(self.entries)

  def __len__(self):
    return len(self.entries)

  def __getitem__(self, dataset_id):
    for f in self.entries:
      if f.id == dataset_id:
        return f
    return None

class HDFFilesProvider(ADataSetProvider):
  def __init__(self):
    import caleydo_server.config
    c = caleydo_server.config.view('caleydo_data_hdf')
    from caleydo_server.util import glob_recursivly
    baseDir = caleydo_server.config.get('dataDir','caleydo_server')
    self.files = [HDFProject(f, baseDir) for f in glob_recursivly(baseDir,c.glob)]

  def __len__(self):
    return sum((len(f) for f in self.files))

  def __iter__(self):
    return itertools.chain(*self.files)

  def __getitem__(self, dataset_id):
    for f in self.files:
      r = f[dataset_id]
      if r is not None:
        return r
    return None

if __name__ == '__main__':
  # app.debug1 = True

  c = HDFFilesProvider()

def create():
  return HDFFilesProvider()
<commit_msg>order stratification groups by name<commit_after>__author__ = 'sam'
import os
import numpy as np
import tables
import caleydo_server.range as ranges

import itertools

from caleydo_server.dataset_def import ADataSetEntry, ADataSetProvider

def assign_ids(ids, idtype):
  import caleydo_server.plugin

  manager = caleydo_server.plugin.lookup('idmanager')
  return np.array(manager(ids, idtype))

class HDFEntry(ADataSetEntry):
  def __init__(self, group, project):
    super(HDFEntry, self).__init__(group._v_title, project, group._v_attrs.type)
    self._group = group
    self._project = project
    self.path = self._group._v_pathname

class HDFMatrix(HDFEntry):
  def __init__(self, group, project):
    super(HDFMatrix, self).__init__(group, project)
    self._rowids = None
    self._colids = None
    self._range = None

  @property
  def rowtype(self):
    return self._group._v_attrs.rowtype

  @property
  def coltype(self):
    return self._group._v_attrs.coltype

  @property
  def value(self):
    return self._group._v_attrs.value

  @property
  def range(self):
    if 'range' in self._group._v_attrs:
      return self._group._v_attrs['range']
    if self._range is not None:
      return self._range
    print self.fqname
    d = self._group.data
    self._range = [np.nanmin(d), np.nanmax(d)]
    return self._range

  @property
  def shape(self):
    return self._group.data.shape

  def idtypes(self):
    return [self.rowtype, self.coltype]

  def to_description(self):
    r = super(HDFMatrix, self).to_description()
    r['rowtype'] = self.rowtype
    r['coltype'] = self.coltype
    r['value'] = v = dict(type=self.value)
    if self.value == 'real' or self.value == 'int':
      v['range'] = self.range
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      v['missing'] = self._group._v_attrs.missing
    elif self.value == 'categorical':
      cats = self._group._v_attrs.categories
      if 'names' in self._group._v_attrs:
        v['categories'] = [ dict(name=cat,label=name,color=col) for cat,name,col in itertools.izip(cats,self._group._v_attrs.names,self._group._v_attrs.colors) ]
      else:
        v['categories'] = cats

    if 'center' in self._group._v_attrs:
      v['center'] = self._group._v_attrs['center']
    r['size'] = self.shape
    return r

  def mask(self, arr):
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      missing = self._group._v_attrs.missing
      import numpy.ma as ma
      return ma.masked_equal(missing, arr)
    return arr

  def asnumpy(self, range=None):
    n = self._group.data
    if range is None:
      return self.mask(n)
    return self.mask(n[range[0].asslice(), range[1].asslice()])

  def rows(self, range=None):
    n = self._group.rows
    if range is None:
      return n
    return n[range.asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.rowtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def cols(self, range=None):
    n = self._group.cols
    if range is None:
      return n
    return n[range.asslice()]

  def colids(self, range=None):
    if self._colids is None:
      self._colids = assign_ids(self.cols(), self.coltype)
    n = self._colids
    if range is None:
      return n
    return n[range.asslice()]

  def filter(self, query):
    # perform the query on rows and cols and return a range with just the mathing one
    # np.argwhere
    np.arange(10)
    return ranges.all()

  def asjson(self, range=None):
    arr = self.asnumpy(range)
    rows = self.rows(None if range is None else range[0])
    cols = self.cols(None if range is None else range[1])
    rowids = self.rowids(None if range is None else range[1])
    colids = self.colids(None if range is None else range[1])

    r = dict(data=arr, rows=rows, cols=cols, rowIds=rowids, colids=colids)
    return r

class HDFVector(HDFEntry):
  def __init__(self, group, project):
    super(HDFVector, self).__init__(group, project)
    self._rowids = None
    self._range = None

  @property
  def idtype(self):
    return self._group._v_attrs.rowtype

  @property
  def value(self):
    return self._group._v_attrs.value

  @property
  def range(self):
    if 'range' in self._group._v_attrs:
      return self._group._v_attrs['range']
    if self._range is not None:
      return self._range
    d = self._group.data
    self._range = [np.nanmin(d), np.nanmax(d)]
    return self._range

  @property
  def shape(self):
    return len(self._group.data)

  def idtypes(self):
    return [self.idtype]

  def to_description(self):
    r = super(HDFVector, self).to_description()
    r['idtype'] = self.idtype
    r['value'] = dict(type=self.value,range=self.range)
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      r['value']['mssing'] = self._group._v_attrs.missing
    if 'center' in self._group._v_attrs:
      r['value']['center'] = self._group._v_attrs['center']
    r['size'] = [self.shape]
    return r

  def mask(self, arr):
    if self.value == 'int' and hasattr(self._group._v_attrs,'missing'):
      missing = self._group._v_attrs.missing
      import numpy.ma as ma
      return ma.masked_equal(missing, arr)
    return arr

  def asnumpy(self, range=None):
    n = self._group.data
    if range is None:
      return self.mask(n)
    return self.mask(n[range[0].asslice()])

  def rows(self, range=None):
    n = self._group.rows
    if range is None:
      return n
    return n[range.asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.rowtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def asjson(self, range=None):
    arr = self.asnumpy(range)
    rows = self.rows(None if range is None else range[0])
    rowids = self.rowids(None if range is None else range[1])

    r = dict(data=arr, rows=rows, rowIds=rowids)
    return r

class HDFGroup(object):
  def __init__(self, name, range, color):
    self.name = name
    self.range = range
    self.color = color

  def __len__(self):
    return len(self.range)

  def dump(self):
    return dict(name=self.name, range=str(self.range), color=self.color)

def guess_color(name, i):
  name = name.lower()
  colors = dict(male='blue',female='red',deceased='#e41a1b',living='#377eb8')
  if name in colors:
    return colors[name]
  l = ['#8dd3c7', '#ffffb3', '#bebada', '#fb8072', '#80b1d3', '#fdb462', '#b3de69', '#fccde5', '#d9d9d9', '#bc80bd',
          '#ccebc5', '#ffed6f']
  return l[i%len(l)]

class HDFStratification(HDFEntry):
  def __init__(self, group, project):
    super(HDFStratification, self).__init__(group, project)
    self._rowids = None

  @property
  def idtype(self):
    return self._group._v_attrs.idtype

  def idtypes(self):
    return [self.idtype]

  def to_description(self):
    r = super(HDFStratification, self).to_description()
    r['idtype'] = self.idtype
    if 'origin' in self._group._v_attrs:
      r['origin'] = self._project + '/' + self._group._v_attrs.origin
    r['groups'] = [dict(name=gf._v_title, size=len(gf),color=gf._v_attrs['color'] if 'color' in gf._v_attrs else guess_color(gf._v_title, j))
                   for j,gf in enumerate(self._sortedgroup())]

    r['ngroups'] = len(r['groups'])
    r['size'] = [sum((g['size'] for g in r['groups']))]
    return r

  def _sortedgroup(self):
    return sorted(self._group._v_children.itervalues(), key=lambda x: x._v_title)

  def _rows(self):
    return np.concatenate(self._sortedgroup())

  def rows(self, range=None):
    n = self._rows()
    if range is None:
      return n
    return n[range[0].asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.idtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def groups(self):
    i = 0
    for j,g in enumerate(self._sortedgroup()):
      name = g._v_title
      color = g._v_attrs['color'] if 'color' in g._v_attrs else guess_color(name, j)
      l = len(g)
      yield HDFGroup(name, ranges.from_slice(i, i + l), color)
      i += l

  def __getitem__(self, item):
    group = getattr(self._group, item)
    return group

  def asjson(self, range = None):
    r = dict(rows=self.rows(range), rowIds=self.rowids(range), groups=[g.dump() for g in self.groups()])
    return r

class HDFColumn(object):
  def __init__(self, attrs, group):
    self._group = group
    self.key = attrs['key']
    self.name = attrs['name']
    self.type = attrs['type']
    self._converter = lambda x: x
    if self.type == 'categorical':
      self._converter = tables.misc.enum.Enum(attrs['categories'])
      self.categories = attrs['categories']
    elif self.type == 'int' or self.type == 'real':
      self.range = attrs['range'] if 'range' in attrs else self.compute_range()
      self.missing = attrs.get('missing',None)

  def compute_range(self):
    d = self._group.table.col(self.key)
    return [np.nanmin(d), np.nanmax(d)]

  def __call__(self, v):
    return self._converter(v)

  def mask(self, arr):
    if self.value == 'int' and self.missing is not None:
      import numpy.ma as ma
      return ma.masked_equal(self.missing, arr)
    return arr

  def asnumpy(self, range=None):
    n = self._group.table.col(self.key)
    if range is not None:
      n = n[range[0].asslice()]
    return self.mask(n)

  def dump(self):
    value = dict(type=self.type)
    if self.type == 'categorical':
      value['categories'] = self.categories
    if self.type == 'int' or self.type == 'real':
      value['range'] = self.range
    if self.type == 'int' and self.missing is not None:
      value['missing'] = self.missing
    return dict(name=self.name, value=value)

class HDFTable(HDFEntry):
  def __init__(self, group, project):
    super(HDFTable, self).__init__(group, project)

    self.columns = [HDFColumn(a, group) for a in group._v_attrs.columns]
    self._rowids = None

  @property
  def idtype(self):
    return self._group._v_attrs.rowtype

  def idtypes(self):
    return [self.idtype]

  def to_description(self):
    r = super(HDFTable, self).to_description()
    r['idtype'] = self.idtype
    r['columns'] = [d.dump() for d in self.columns]
    r['size'] = [len(self._group.table), len(self.columns)]
    return r

  def rows(self, range=None):
    n = self._group.rows
    if range is None:
      return n
    return n[range.asslice()]

  def rowids(self, range=None):
    if self._rowids is None:
      self._rowids = assign_ids(self.rows(), self.idtype)
    n = self._rowids
    if range is None:
      return n
    return n[range.asslice()]

  def asnumpy(self, range=None):
    n = self._group.table
    if range is None:
      return n
    return n[range[0].asslice()]

  def filter(self, query):
    # perform the query on rows and cols and return a range with just the mathing one
    # np.argwhere
    np.arange(10)
    return ranges.all()

  def asjson(self, range=None):
    arr = self.asnumpy(range)
    rows = self.rows(None if range is None else range[0])
    rowids = self.rowids(None if range is None else range[0])

    dd = [[c(row[c.key]) for c in self.columns] for row in arr]
    r = dict(data=dd, rows=rows, rowIds = rowids)

    return r

class HDFProject(object):
  def __init__(self, filename, baseDir):
    self.filename = filename
    p = os.path.relpath(filename, baseDir)
    project,_ = os.path.splitext(p)
    project = project.replace('.','_')
    self._h = tables.open_file(filename, 'r')

    self.entries = []
    for group in self._h.walk_groups('/'):
      if 'type' not in group._v_attrs:
        continue
      t = group._v_attrs.type
      if t == 'matrix':
        self.entries.append(HDFMatrix(group, project))
      elif t == 'stratification':
        self.entries.append(HDFStratification(group, project))
      elif t == 'table':
        self.entries.append(HDFTable(group, project))
      elif t == 'vector':
        self.entries.append(HDFVector(group, project))

  def __iter__(self):
    return iter(self.entries)

  def __len__(self):
    return len(self.entries)

  def __getitem__(self, dataset_id):
    for f in self.entries:
      if f.id == dataset_id:
        return f
    return None

class HDFFilesProvider(ADataSetProvider):
  def __init__(self):
    import caleydo_server.config
    c = caleydo_server.config.view('caleydo_data_hdf')
    from caleydo_server.util import glob_recursivly
    baseDir = caleydo_server.config.get('dataDir','caleydo_server')
    self.files = [HDFProject(f, baseDir) for f in glob_recursivly(baseDir,c.glob)]

  def __len__(self):
    return sum((len(f) for f in self.files))

  def __iter__(self):
    return itertools.chain(*self.files)

  def __getitem__(self, dataset_id):
    for f in self.files:
      r = f[dataset_id]
      if r is not None:
        return r
    return None

if __name__ == '__main__':
  # app.debug1 = True

  c = HDFFilesProvider()

def create():
  return HDFFilesProvider()
<|endoftext|>


<commit_before>from django.conf.urls.defaults import patterns, url, include
from django.utils.translation import ugettext_lazy as _
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.conf import settings

from preferences import preferences

from foundry.models import Page
from foundry.forms import LoginForm, PasswordResetForm
from foundry.views import CategoryObjectDetailView, CategoryObjectListView

admin.autodiscover()

try:
    import object_tools
    object_tools.autodiscover()
except ImportError:
    pass

urlpatterns = patterns('',
    (r'^gallery/', include('gallery.urls')),
    (r'^googlesearch/', include('googlesearch.urls')),
    (r'^music/', include('music.urls')),
    (r'^jmbo/', include('jmbo.urls')),  # todo: paster makes this r'^$ which does not work
    (r'^chart/', include('chart.urls')),
    (r'^comments/', include('django.contrib.comments.urls')),
#    (r'^richcomments/', include('richcomments.urls')), # re-evaluate this product. It needs to degrade to non-ajax as well.
    (r'^likes/', include('likes.urls')),
    (r'^object-tools/', include(object_tools.tools.urls)),
    (r'^show/', include('show.urls')),
    (r'^event/', include('event.urls')),
    (r'^competition/', include('competition.urls')),
    (r'^ckeditor/', include('ckeditor.urls')),
    (r'^contact/', include('contact.urls')),
    (r'^post/', include('post.urls')),	# todo: add to paster
    (r'^poll/', include('poll.urls')),	# todo: add to paster
    (r'^simple-autocomplete/', include('simple_autocomplete.urls')),

    (r'^admin/', include(admin.site.urls)),

    url(
        r'^$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'base.html',
        },
        name='home'
    ),
    url(
        r'^logo/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/logo.html',
        },
        name='logo'
    ),
    url(
        r'^header/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/inclusion_tags/header.html',
        },
        name='header'
    ),
    url(
        r'^category/(?P<category_slug>[\w-]+)/$',
        CategoryObjectListView.as_view(),
        name='category_object_list'
    ),
    url(
        r'^category/(?P<category_slug>[\w-]+)/(?P<slug>[\w-]+)/$',
        CategoryObjectDetailView.as_view(),
        name='category_object_detail'
    ),

    # Join, login, password reset
    url(
        r'^join/$',
        'foundry.views.join',
        {},
        name='join',
    ),
    url(
        r'^join-finish/$',
        'foundry.views.join_finish',
        {},
        name='join-finish',
    ),
    url(
        r'^login/$',
        'django.contrib.auth.views.login',
        {'authentication_form':LoginForm},
        name='login',
    ),
    url(
        r'^logout/$',
        'django.contrib.auth.views.logout',
        {'next_page':'/'},
        name='logout',
    ),
    # Pre-empt password reset so we can use custom form
    (
        r'^auth/password_reset/$',
        'django.contrib.auth.views.password_reset',
        {
            'password_reset_form':PasswordResetForm,
        }
    ),
    (r'^auth/', include('django.contrib.auth.urls')),

    # Pages defined in preferences
    url(
        r'^about-us/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/static_page.html',
            'extra_context':{'content':lambda:preferences.GeneralPreferences.about_us, 'title':_("About us")}
        },
        name='about-us'
    ),
    url(
        r'^terms-and-conditions/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/static_page.html',
            'extra_context':{'content':lambda:preferences.GeneralPreferences.terms_and_conditions, 'title':_("Terms and conditions")}
        },
        name='terms-and-conditions'
    ),
    url(
        r'^privacy-policy/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/static_page.html',
            'extra_context':{'content':lambda:preferences.GeneralPreferences.privacy_policy, 'title':_("Privacy policy")}
        },
        name='privacy-policy'
    ),

    # Age gateway
    url(
        r'^age-gateway/$',
        'foundry.views.age_gateway',
        {},
        name='age-gateway',
    ),

    # Listing
    url(
        r'^listing/(?P<slug>[\w-]+)/$',
        'foundry.views.listing_detail',
        {},
        name='listing-detail'
    ),

    # Page detail
    url(
        r'^page/(?P<slug>[\w-]+)/$',
        'foundry.views.page_detail',
        {},
        name='page-detail'
    ),

    # Lorem ipsum
    url(
        r'^lorem-ipsum/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/lorem_ipsum.html',
        },
        name='lorem-ipsum'
    ),

    # Search
    url(
        r'^search/$',
        'foundry.views.search',
        {},
        name='search'
    ),

    # Comment reply form in case of no javascript
    url(
        r'^comment-reply-form/$',
        'foundry.views.comment_reply_form',
        {},
        name='comment-reply-form'
    ),

    # Chatroom detail
    url(
        r'^chatroom/(?P<slug>[\w-]+)/$',
        'foundry.views.chatroom_detail',
        {},
        name='chatroom-detail'
    ),

    # Create blogpost
    url(
        r'^create-blogpost/$',
        'foundry.views.create_blogpost',
        {},
        name='create-blogpost',
    ),

    # Blogpost list
    url(
        r'^blogposts/$',
        'foundry.views.blogpost_object_list',
        {'limit': 300},
        name='blogpost_object_list'
    ),

    # Blogpost detail
    url(
        r'^blogpost/(?P<slug>[\w-]+)/$',
        'foundry.views.blogpost_object_detail',
        {},
        name='blogpost_object_detail'
    ),

    # Member notifications
    url(
        r'^member-notifications/$',
        'foundry.views.member_notifications',
        {},
        name='member-notifications'
    ),

    # User detail page
    url(
        r'^users/(?P<username>[\w-]+)/$',
        'foundry.views.user_detail',
        {},
        name='user-detail'
    ),

    # Member detail page
    url(
        r'^members/(?P<username>[\w-]+)/$',
        'foundry.views.member_detail',
        {},
        name='member-detail'
    ),

    # Test views
    url(
        r'^test-plain-response/$',
        'foundry.views.test_plain_response',
        {},
        name='test-plain-response'
    ),
    url(
        r'^test-redirect/$',
        'foundry.views.test_redirect',
        {},
        name='test-redirect'
    ),
    url(
        r'^pages/$',
        'django.views.generic.list_detail.object_list',
        {'queryset':Page.permitted.all().order_by('title')},
        'page-list'
    ),

    # Admin
    url(
        r'^admin-row-create-ajax/$',
        'foundry.admin_views.row_create_ajax',
        {},
        name='admin-row-create-ajax',
    ),
    url(
        r'^admin-column-create-ajax/$',
        'foundry.admin_views.column_create_ajax',
        {},
        name='admin-column-create-ajax',
    ),
    url(
        r'^admin-tile-create-ajax/$',
        'foundry.admin_views.tile_create_ajax',
        {},
        name='admin-tile-create-ajax',
    ),
    url(
        r'^admin-row-edit-ajax/$',
        'foundry.admin_views.row_edit_ajax',
        {},
        name='admin-row-edit-ajax',
    ),
    url(
        r'^admin-column-edit-ajax/$',
        'foundry.admin_views.column_edit_ajax',
        {},
        name='admin-column-edit-ajax',
    ),
    url(
        r'^admin-tile-edit-ajax/$',
        'foundry.admin_views.tile_edit_ajax',
        {},
        name='admin-tile-edit-ajax',
    ),
    url(
        r'^admin-row-delete-ajax/$',
        'foundry.admin_views.row_delete_ajax',
        {},
        name='admin-row-delete-ajax',
    ),
    url(
        r'^admin-column-delete-ajax/$',
        'foundry.admin_views.column_delete_ajax',
        {},
        name='admin-column-delete-ajax',
    ),
    url(
        r'^admin-tile-delete-ajax/$',
        'foundry.admin_views.tile_delete_ajax',
        {},
        name='admin-tile-delete-ajax',
    ),
    url(
        r'^admin-persist-sort-ajax/$',
        'foundry.admin_views.persist_sort_ajax',
        {},
        name='admin-persist-sort-ajax',
    ),

)

urlpatterns += staticfiles_urlpatterns()

if settings.DEBUG:
    urlpatterns += patterns('',
    (r'^media/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
    )
<commit_msg>disable view modifier for gallery<commit_after>from django.conf.urls.defaults import patterns, url, include
from django.utils.translation import ugettext_lazy as _
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.conf import settings

from preferences import preferences

from foundry.models import Page
from foundry.forms import LoginForm, PasswordResetForm
from foundry.views import CategoryObjectDetailView, CategoryObjectListView

admin.autodiscover()

try:
    import object_tools
    object_tools.autodiscover()
except ImportError:
    pass

urlpatterns = patterns('',
    # Pre-empt url call since we want to disable view modifiers for gallery
    # until that product gets fixed.
    url(
        r'^gallery/(?P<slug>[\w-]+)/$',
        'gallery.views.object_detail',
        {'view_modifier': []},
        name='gallery_object_detail'
    ),

    (r'^gallery/', include('gallery.urls')),
    (r'^googlesearch/', include('googlesearch.urls')),
    (r'^music/', include('music.urls')),
    (r'^jmbo/', include('jmbo.urls')),  # todo: paster makes this r'^$ which does not work
    (r'^chart/', include('chart.urls')),
    (r'^comments/', include('django.contrib.comments.urls')),
#    (r'^richcomments/', include('richcomments.urls')), # re-evaluate this product. It needs to degrade to non-ajax as well.
    (r'^likes/', include('likes.urls')),
    (r'^object-tools/', include(object_tools.tools.urls)),
    (r'^show/', include('show.urls')),
    (r'^event/', include('event.urls')),
    (r'^competition/', include('competition.urls')),
    (r'^ckeditor/', include('ckeditor.urls')),
    (r'^contact/', include('contact.urls')),
    (r'^post/', include('post.urls')),	# todo: add to paster
    (r'^poll/', include('poll.urls')),	# todo: add to paster
    (r'^simple-autocomplete/', include('simple_autocomplete.urls')),

    (r'^admin/', include(admin.site.urls)),

    url(
        r'^$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'base.html',
        },
        name='home'
    ),
    url(
        r'^logo/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/logo.html',
        },
        name='logo'
    ),
    url(
        r'^header/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/inclusion_tags/header.html',
        },
        name='header'
    ),
    url(
        r'^category/(?P<category_slug>[\w-]+)/$',
        CategoryObjectListView.as_view(),
        name='category_object_list'
    ),
    url(
        r'^category/(?P<category_slug>[\w-]+)/(?P<slug>[\w-]+)/$',
        CategoryObjectDetailView.as_view(),
        name='category_object_detail'
    ),

    # Join, login, password reset
    url(
        r'^join/$',
        'foundry.views.join',
        {},
        name='join',
    ),
    url(
        r'^join-finish/$',
        'foundry.views.join_finish',
        {},
        name='join-finish',
    ),
    url(
        r'^login/$',
        'django.contrib.auth.views.login',
        {'authentication_form':LoginForm},
        name='login',
    ),
    url(
        r'^logout/$',
        'django.contrib.auth.views.logout',
        {'next_page':'/'},
        name='logout',
    ),
    # Pre-empt password reset so we can use custom form
    (
        r'^auth/password_reset/$',
        'django.contrib.auth.views.password_reset',
        {
            'password_reset_form':PasswordResetForm,
        }
    ),
    (r'^auth/', include('django.contrib.auth.urls')),

    # Pages defined in preferences
    url(
        r'^about-us/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/static_page.html',
            'extra_context':{'content':lambda:preferences.GeneralPreferences.about_us, 'title':_("About us")}
        },
        name='about-us'
    ),
    url(
        r'^terms-and-conditions/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/static_page.html',
            'extra_context':{'content':lambda:preferences.GeneralPreferences.terms_and_conditions, 'title':_("Terms and conditions")}
        },
        name='terms-and-conditions'
    ),
    url(
        r'^privacy-policy/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/static_page.html',
            'extra_context':{'content':lambda:preferences.GeneralPreferences.privacy_policy, 'title':_("Privacy policy")}
        },
        name='privacy-policy'
    ),

    # Age gateway
    url(
        r'^age-gateway/$',
        'foundry.views.age_gateway',
        {},
        name='age-gateway',
    ),

    # Listing
    url(
        r'^listing/(?P<slug>[\w-]+)/$',
        'foundry.views.listing_detail',
        {},
        name='listing-detail'
    ),

    # Page detail
    url(
        r'^page/(?P<slug>[\w-]+)/$',
        'foundry.views.page_detail',
        {},
        name='page-detail'
    ),

    # Lorem ipsum
    url(
        r'^lorem-ipsum/$',
        'django.views.generic.simple.direct_to_template',
        {
            'template':'foundry/lorem_ipsum.html',
        },
        name='lorem-ipsum'
    ),

    # Search
    url(
        r'^search/$',
        'foundry.views.search',
        {},
        name='search'
    ),

    # Comment reply form in case of no javascript
    url(
        r'^comment-reply-form/$',
        'foundry.views.comment_reply_form',
        {},
        name='comment-reply-form'
    ),

    # Chatroom detail
    url(
        r'^chatroom/(?P<slug>[\w-]+)/$',
        'foundry.views.chatroom_detail',
        {},
        name='chatroom-detail'
    ),

    # Create blogpost
    url(
        r'^create-blogpost/$',
        'foundry.views.create_blogpost',
        {},
        name='create-blogpost',
    ),

    # Blogpost list
    url(
        r'^blogposts/$',
        'foundry.views.blogpost_object_list',
        {'limit': 300},
        name='blogpost_object_list'
    ),

    # Blogpost detail
    url(
        r'^blogpost/(?P<slug>[\w-]+)/$',
        'foundry.views.blogpost_object_detail',
        {},
        name='blogpost_object_detail'
    ),

    # Member notifications
    url(
        r'^member-notifications/$',
        'foundry.views.member_notifications',
        {},
        name='member-notifications'
    ),

    # User detail page
    url(
        r'^users/(?P<username>[\w-]+)/$',
        'foundry.views.user_detail',
        {},
        name='user-detail'
    ),

    # Member detail page
    url(
        r'^members/(?P<username>[\w-]+)/$',
        'foundry.views.member_detail',
        {},
        name='member-detail'
    ),

    # Test views
    url(
        r'^test-plain-response/$',
        'foundry.views.test_plain_response',
        {},
        name='test-plain-response'
    ),
    url(
        r'^test-redirect/$',
        'foundry.views.test_redirect',
        {},
        name='test-redirect'
    ),
    url(
        r'^pages/$',
        'django.views.generic.list_detail.object_list',
        {'queryset':Page.permitted.all().order_by('title')},
        'page-list'
    ),

    # Admin
    url(
        r'^admin-row-create-ajax/$',
        'foundry.admin_views.row_create_ajax',
        {},
        name='admin-row-create-ajax',
    ),
    url(
        r'^admin-column-create-ajax/$',
        'foundry.admin_views.column_create_ajax',
        {},
        name='admin-column-create-ajax',
    ),
    url(
        r'^admin-tile-create-ajax/$',
        'foundry.admin_views.tile_create_ajax',
        {},
        name='admin-tile-create-ajax',
    ),
    url(
        r'^admin-row-edit-ajax/$',
        'foundry.admin_views.row_edit_ajax',
        {},
        name='admin-row-edit-ajax',
    ),
    url(
        r'^admin-column-edit-ajax/$',
        'foundry.admin_views.column_edit_ajax',
        {},
        name='admin-column-edit-ajax',
    ),
    url(
        r'^admin-tile-edit-ajax/$',
        'foundry.admin_views.tile_edit_ajax',
        {},
        name='admin-tile-edit-ajax',
    ),
    url(
        r'^admin-row-delete-ajax/$',
        'foundry.admin_views.row_delete_ajax',
        {},
        name='admin-row-delete-ajax',
    ),
    url(
        r'^admin-column-delete-ajax/$',
        'foundry.admin_views.column_delete_ajax',
        {},
        name='admin-column-delete-ajax',
    ),
    url(
        r'^admin-tile-delete-ajax/$',
        'foundry.admin_views.tile_delete_ajax',
        {},
        name='admin-tile-delete-ajax',
    ),
    url(
        r'^admin-persist-sort-ajax/$',
        'foundry.admin_views.persist_sort_ajax',
        {},
        name='admin-persist-sort-ajax',
    ),

)

urlpatterns += staticfiles_urlpatterns()

if settings.DEBUG:
    urlpatterns += patterns('',
    (r'^media/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
    )
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
usage:
    fq profile [options] <fq>...
    fq fetch <fq>...
    fq fastqc-dump <fastqc-group> [<fq>...]
    fq dump
    fq set <project> <kind>

options:
  -h --help                   Show this screen.
  --version                   Show version.
  --verbose                   Speak up
  --fastqc                    Gather fastqc statistics as well
  --kv=<k:v>                  Additional key-value pairs to add

"""

from docopt import docopt
from gcloud import datastore
import hashlib
import io
import os
from clint.textui import colored, puts_err, progress, indent
import fqprofile
from fq import autoconvert, json_serial, parse_fastqc
from fq import check_program_exists, fastqc_groups, fastqc_headers
from fq.fq_util import fastq_reader
import json
import os.path
import sys
from datetime import datetime
from math import log
import re
import time
import glob
from subprocess import Popen, PIPE
from tempfile import gettempdir
import socket

def fastqc(filename):
    t_dir = gettempdir()
    fastqc_results = filename.replace(".fq", "_fastqc") \
                             .replace(".fastq", "_fastqc") \
                             .replace("_fastqc.gz", "")

    comm = ["fastqc", "--out", t_dir, "--extract", filename]
    out, err = Popen(comm, stdout=PIPE, stderr=PIPE).communicate()
    fqc_file = t_dir + "/" + fastqc_results + "_fastqc/fastqc_data.txt"
    return parse_fastqc(fqc_file)

def test_fastqc(filename):
    return filename.endswith(".fq.gz") or filename.endswith(".fastq.gz")

def get_item(kind, name):
    return ds.get(ds.key(kind, name))

exclude_indices = ['most_abundant_sequence',
                   'fastqc_per_base_sequence_quality_data',
                   'fastqc_per_tile_sequence_quality_data',
                   'fastqc_per_sequence_quality_scores_data',
                   'fastqc_per_base_sequence_content_data',
                   'fastqc_per_sequence_gc_content_data',
                   'fastqc_per_base_n_content_data',
                   'fastqc_sequence_length_distribution_data',
                   'fastqc_sequence_duplication_levels_data',
                   'fastqc_overrepresented_sequences_data',
                   'fastqc_adapter_content_data',
                   'fastqc_kmer_content_data']

def update_item(kind, name, **kwargs):
    item = get_item(kind, name)
    if item is None:
        m = datastore.Entity(key=ds.key(kind, name),
                             exclude_from_indexes=exclude_indices)
    else:
        m = datastore.Entity(key=ds.key(kind, name),
                             exclude_from_indexes=exclude_indices)
        m.update(dict(item))
    for key, value in kwargs.items():
        if type(value) == str:
            m[key] = unicode(value)
        elif type(value) == list:
            if key in m:
                m[key] += value
            else:
                m[key] = value
            m[key] = list(set(m[key]))
        # If date created of file is earlier
        elif key == 'date_created' and item:
            vtimestamp = time.mktime(value.timetuple())
            dstimestamp = time.mktime(m['date_created'].timetuple())
            if vtimestamp < dstimestamp:
                m[key] = value

        else:
            m[key] = value
    if 'fq_profile_count' in m:
        m['fq_profile_count'] += 1
    else:
        m['fq_profile_count'] = 1
    ds.put(m)

def store_item(kind, name, **kwargs):
    m = datastore.Entity(key=ds.key(kind, name))
    for key, value in kwargs.items():
        if type(value) == str:
            m[key] = unicode(value)
        else:
            m[key] = value
    ds.put(m)

def query_item(kind, filters=None):
    # filters:
    # [("var_name", "=", 1)]
    query = ds.query(kind=kind)
    if filters:
        for var, op, val in filters:
            query.add_filter(var, op, val)
    return query.fetch()

def md5sum(src, length=io.DEFAULT_BUFFER_SIZE):
    calculated = 0
    md5 = hashlib.md5()
    with io.open(src, mode="rb") as fd:
        filesize = os.stat(src).st_size
        expected_size = (filesize / io.DEFAULT_BUFFER_SIZE) + 1
        for chunk in progress.bar(iter(lambda: fd.read(length), b''),
                                  expected_size=expected_size):
            md5.update(chunk)
            calculated += len(chunk)
    return md5

class checksums:
    def __init__(self):
        self.hashes = {}

    def get_or_update_checksum(self, filename):
        filename = os.path.abspath(filename)
        basename = os.path.basename(filename)
        base_dir = os.path.dirname(filename)
        checksum_file = base_dir + "/.checksum"
        if os.path.exists(checksum_file):
            hash_set = open(checksum_file, 'r').readlines()
            hash_set = [x.strip().split("\t") for x in hash_set]
            hash_set = {v: k for k, v in hash_set}
            self.hashes.update(hash_set)

        if filename in self.hashes.keys():
            if verbose:
                puts_err(colored.blue("\n" + basename + "\t[x] Using cached hash"))
        else:
            if verbose:
                puts_err(colored.blue("\n" + basename + "\t[ ] Generating hash"))

        if filename not in self.hashes.keys():
            hash = md5sum(filename).hexdigest()
            self.hashes.update({filename: hash})
            # Generate hash if it does not exist
            self.hashes.update({filename: hash})
            out = hash + "\t" + filename + "\n"
            open(checksum_file, 'a').write(out)
        return self.hashes[filename]

# Stack Overflow: 1094841
_suffixes = ['bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']

def file_size(size):
    order = int(log(float(size), 2) / 10) if size else 0
    return '{:.4g} {}'.format(size / (1 << (order * 10)), _suffixes[order])

def main():
    args = docopt(__doc__,
                  options_first=False)
    ck = checksums()
    settings_file = os.path.dirname(fqprofile.__file__) + "/.config"
    # Save settings
    if args["set"]:
        settings = {"project": args["<project>"], "kind": args["<kind>"]}
        open(settings_file, 'w').write(json.dumps(settings, indent=4))
        puts_err("Saved Settings")
        exit()
    else:
        if not os.path.exists(settings_file):
            with indent(4):
                exit(puts_err(colored.red("\nPlease set project and kind using 'fq set'\n")))
        # Load settings
        settings = json.loads(open(settings_file, 'r').read())
        project = settings['project']
        kind = settings['kind']
        output_str = "Project: {project} - Kind: {kind}".format(**locals())
        output_under = len(output_str) * "="
        puts_err(colored.blue("\n{output_str}\n{output_under}".format(**locals())))

    global ds
    global verbose
    verbose = args["--verbose"]
    ds = datastore.Client(project=project)

    if args["dump"]:
        fastq_dumped = query_item(kind)
        for i in fastq_dumped:
            print(json.dumps(i, default=json_serial, indent=4, sort_keys=True))

    if "*" in args["<fq>"] and len(args) == 1:
        fq_set = glob.glob(args["<fq>"])
    elif "-" in args["<fq>"]:
        fq_set = [x.strip() for x in sys.stdin.readlines()]
    else:
        fq_set = args["<fq>"]
    fq_set_exists = map(os.path.isfile, fq_set)
    fastqc_complient = map(test_fastqc, fq_set)

    if args["fastqc-dump"]:
        fastqc_group = "fastqc_" + args["<fastqc-group>"] + "_data"
        if args["<fastqc-group>"] not in fastqc_groups:
            with indent(4):
                puts_err(colored.blue("\nFastQC group not found. Available FastQC groups:\n\n" +
                                      '\n'.join(fastqc_groups) + "\n"))
                exit()

        if fq_set:
            # Output header
            print('\t'.join(['filename'] + fastqc_headers[args["<fastqc-group>"]]))
            for i in fq_set:
                hash = ck.get_or_update_checksum(i)
                fastqc_data = get_item(kind, hash)[fastqc_group].splitlines()
                out = '\n'.join([i + "\t" + x for x in fastqc_data])
                print(out)
        else:
            # Output header
            print('\t'.join(['hash', 'filename'] + fastqc_headers[args["<fastqc-group>"]]))
            # If no fq specified, dump everything:
            fastq_dumped = query_item(kind)
            for i in fastq_dumped:
                fastqc_data = i[fastqc_group].splitlines()
                fnames = i.key.name + "\t" + i['filename'][0]
                out = '\n'.join([fnames + "\t" + x for x in fastqc_data])
                print(out)
        exit()

    if args["--fastqc"]:
        check_program_exists("fastqc")
        if not all(fastqc_complient):
            err_msg = "\n--fastqc only works with fq.gz, fastq.gz," + \
                       " fq, and fastq extensions.\n"
            with indent(4):
                puts_err(colored.red(err_msg))
                exit()
    if not all(fq_set_exists):
        missing_files = [f for f, exists in zip(fq_set, fq_set_exists)
                         if exists is False]
        with indent(4):
            puts_err(colored.red("\nFile not found:\n\n" +
                             "\n".join(missing_files) + "\n"))
            exit()

    error_fqs = []
    for fastq in fq_set:
        basename = os.path.basename(fastq)
        hash = ck.get_or_update_checksum(fastq)
        fq = fastq_reader(fastq)
        if fq.error is True:
            error_fqs.append(fastq)
            with indent(4):
                puts_err(colored.red("\nDoes not appear to be a Fastq: " +
                     fastq + "\n"))
            continue

        if args["fetch"]:
            d = get_item(kind, hash)
            if d:
                print(json.dumps(d,
                                 default=json_serial,
                                 indent=4,
                                 sort_keys=True))
            else:
                puts_err(colored.red("{basename} has not been profiled. Profile with 'fq profile'".format(basename=basename)))
            continue

        nfq = get_item(kind, hash)
        kwdata = {}
        # Test if fq stats generated.
        if nfq is None or u"total_reads" not in nfq.keys():
            if verbose:
                puts_err(colored.blue(basename + "\t[ ] Profiling"))
            kwdata.update(fq.header)
            kwdata.update(fq.calculate_fastq_stats())
        else:
            if verbose:
                puts_err(colored.blue(basename + "\t[x] Already profiled"))

        # Test if fastq filename matches illumina conventions
        illumina_keys = None
        i1 = re.match(r"([^_]+)_([ATGC]+)_([L0-9]{4})_([R12]{2})_([0-9]{3}).(fastq|fq).gz", basename)
        i2 = re.match(r"([^_]+)_(S[0-9]+)_([L0-9]{4})_([R12]{2})_([0-9]{3}).(fastq|fq).gz", basename)
        if i1:
            r = i1
            illumina_keys = ["illumina_filename_sample",
                             "illumina_filename_barcode_sequence",
                             "illumina_filename_lane",
                             "illumina_filename_read",
                             "illumina_filename_set_number"]
        elif i2:
            r = i2
            illumina_keys = ["illumina_filename_sample",
                             "illumina_filename_sample_number",
                             "illumina_filename_lane",
                             "illumina_filename_read",
                             "illumina_filename_set_number"]
        if illumina_keys:
            puts_err(colored.blue(basename + "\tIllumina Filename"))
            illumina_values = map(autoconvert, r.groups())
            illumina_data = dict(zip(illumina_keys, illumina_values))
            kwdata.update(illumina_data)

        # File statistics
        file_stat = os.stat(fastq)
        kwdata['filesize'] = file_stat.st_size
        kwdata['hfilesize'] = file_size(file_stat.st_size)
        date_created = file_stat.st_ctime
        kwdata['date_created'] = datetime.fromtimestamp(date_created)

        # Add custom data
        if args['--kv']:
            puts_err(colored.blue(basename + "\tStoring Custom data"))
            kv = [x.split(":") for x in args['--kv'].split(",")]
            kv = {k: autoconvert(v) for k, v in kv}
            kwdata.update(kv)

        # FASTQC
        if args["--fastqc"]:
            if  nfq is None or u"fastqc_version" not in nfq.keys():
                if verbose:
                    puts_err(colored.blue(basename + "\t[ ] Running Fastqc"))
                fqc_data = fastqc(fastq)
                kwdata.update(fqc_data)
            else:
                if verbose:
                    puts_err(colored.blue(basename + "\t[x] FastQC already run"))

        filename = os.path.abspath(fastq)
        kwdata['hostname'] = [unicode(os.getlogin())]
        kwdata['basename'] = [unicode(basename)]
        kwdata['filename'] = [unicode(filename)]
        update_item(kind,
                    hash,
                    **kwdata)
        puts_err(colored.blue(basename + "\tComplete"))

    if error_fqs and len(fq_set) > 1:
        with indent(4):
            puts_err(colored.red("\nFastqs that errored:\n\n" +
                     '\n'.join(error_fqs) + "\n"))

if __name__ == '__main__':
    main()
<commit_msg>Fix storing custom data<commit_after>#! /usr/bin/env python
"""
usage:
    fq profile [options] <fq>...
    fq fetch <fq>...
    fq fastqc-dump <fastqc-group> [<fq>...]
    fq dump
    fq set <project> <kind>

options:
  -h --help                   Show this screen.
  --version                   Show version.
  --verbose                   Speak up
  --fastqc                    Gather fastqc statistics as well
  --kv=<k:v>                  Additional key-value pairs to add

"""

from docopt import docopt
from gcloud import datastore
import hashlib
import io
import os
from clint.textui import colored, puts_err, progress, indent
import fqprofile
from fq import autoconvert, json_serial, parse_fastqc
from fq import check_program_exists, fastqc_groups, fastqc_headers
from fq.fq_util import fastq_reader
import json
import os.path
import sys
from datetime import datetime
from math import log
import re
import time
import glob
from subprocess import Popen, PIPE
from tempfile import gettempdir
import socket

def fastqc(filename):
    t_dir = gettempdir()
    fastqc_results = filename.replace(".fq", "_fastqc") \
                             .replace(".fastq", "_fastqc") \
                             .replace("_fastqc.gz", "")

    comm = ["fastqc", "--out", t_dir, "--extract", filename]
    out, err = Popen(comm, stdout=PIPE, stderr=PIPE).communicate()
    fqc_file = t_dir + "/" + fastqc_results + "_fastqc/fastqc_data.txt"
    return parse_fastqc(fqc_file)

def test_fastqc(filename):
    return filename.endswith(".fq.gz") or filename.endswith(".fastq.gz")

def get_item(kind, name):
    return ds.get(ds.key(kind, name))

exclude_indices = ['most_abundant_sequence',
                   'fastqc_per_base_sequence_quality_data',
                   'fastqc_per_tile_sequence_quality_data',
                   'fastqc_per_sequence_quality_scores_data',
                   'fastqc_per_base_sequence_content_data',
                   'fastqc_per_sequence_gc_content_data',
                   'fastqc_per_base_n_content_data',
                   'fastqc_sequence_length_distribution_data',
                   'fastqc_sequence_duplication_levels_data',
                   'fastqc_overrepresented_sequences_data',
                   'fastqc_adapter_content_data',
                   'fastqc_kmer_content_data']

def update_item(kind, name, **kwargs):
    item = get_item(kind, name)
    if item is None:
        m = datastore.Entity(key=ds.key(kind, name),
                             exclude_from_indexes=exclude_indices)
    else:
        m = datastore.Entity(key=ds.key(kind, name),
                             exclude_from_indexes=exclude_indices)
        m.update(dict(item))
    for key, value in kwargs.items():
        if type(value) == str:
            m[key] = unicode(value)
        elif type(value) == list:
            if key in m:
                m[key] += value
            else:
                m[key] = value
            m[key] = list(set(m[key]))
        # If date created of file is earlier
        elif key == 'date_created' and item:
            vtimestamp = time.mktime(value.timetuple())
            dstimestamp = time.mktime(m['date_created'].timetuple())
            if vtimestamp < dstimestamp:
                m[key] = value

        else:
            m[key] = value
    if 'fq_profile_count' in m:
        m['fq_profile_count'] += 1
    else:
        m['fq_profile_count'] = 1
    ds.put(m)

def store_item(kind, name, **kwargs):
    m = datastore.Entity(key=ds.key(kind, name))
    for key, value in kwargs.items():
        if type(value) == str:
            m[key] = unicode(value)
        else:
            m[key] = value
    ds.put(m)

def query_item(kind, filters=None):
    # filters:
    # [("var_name", "=", 1)]
    query = ds.query(kind=kind)
    if filters:
        for var, op, val in filters:
            query.add_filter(var, op, val)
    return query.fetch()

def md5sum(src, length=io.DEFAULT_BUFFER_SIZE):
    calculated = 0
    md5 = hashlib.md5()
    with io.open(src, mode="rb") as fd:
        filesize = os.stat(src).st_size
        expected_size = (filesize / io.DEFAULT_BUFFER_SIZE) + 1
        for chunk in progress.bar(iter(lambda: fd.read(length), b''),
                                  expected_size=expected_size):
            md5.update(chunk)
            calculated += len(chunk)
    return md5

class checksums:
    def __init__(self):
        self.hashes = {}

    def get_or_update_checksum(self, filename):
        filename = os.path.abspath(filename)
        basename = os.path.basename(filename)
        base_dir = os.path.dirname(filename)
        checksum_file = base_dir + "/.checksum"
        if os.path.exists(checksum_file):
            hash_set = open(checksum_file, 'r').readlines()
            hash_set = [x.strip().split("\t") for x in hash_set]
            hash_set = {v: k for k, v in hash_set}
            self.hashes.update(hash_set)

        if filename in self.hashes.keys():
            if verbose:
                puts_err(colored.blue("\n" + basename + "\t[x] Using cached hash"))
        else:
            if verbose:
                puts_err(colored.blue("\n" + basename + "\t[ ] Generating hash"))

        if filename not in self.hashes.keys():
            hash = md5sum(filename).hexdigest()
            self.hashes.update({filename: hash})
            # Generate hash if it does not exist
            self.hashes.update({filename: hash})
            out = hash + "\t" + filename + "\n"
            open(checksum_file, 'a').write(out)
        return self.hashes[filename]

# Stack Overflow: 1094841
_suffixes = ['bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']

def file_size(size):
    order = int(log(float(size), 2) / 10) if size else 0
    return '{:.4g} {}'.format(size / (1 << (order * 10)), _suffixes[order])

def main():
    args = docopt(__doc__,
                  options_first=False)
    ck = checksums()
    settings_file = os.path.dirname(fqprofile.__file__) + "/.config"
    # Save settings
    if args["set"]:
        settings = {"project": args["<project>"], "kind": args["<kind>"]}
        open(settings_file, 'w').write(json.dumps(settings, indent=4))
        puts_err("Saved Settings")
        exit()
    else:
        if not os.path.exists(settings_file):
            with indent(4):
                exit(puts_err(colored.red("\nPlease set project and kind using 'fq set'\n")))
        # Load settings
        settings = json.loads(open(settings_file, 'r').read())
        project = settings['project']
        kind = settings['kind']
        output_str = "Project: {project} - Kind: {kind}".format(**locals())
        output_under = len(output_str) * "="
        puts_err(colored.blue("\n{output_str}\n{output_under}".format(**locals())))

    global ds
    global verbose
    verbose = args["--verbose"]
    ds = datastore.Client(project=project)

    if args["dump"]:
        fastq_dumped = query_item(kind)
        for i in fastq_dumped:
            print(json.dumps(i, default=json_serial, indent=4, sort_keys=True))

    if "*" in args["<fq>"] and len(args) == 1:
        fq_set = glob.glob(args["<fq>"])
    elif "-" in args["<fq>"]:
        fq_set = [x.strip() for x in sys.stdin.readlines()]
    else:
        fq_set = args["<fq>"]
    fq_set_exists = map(os.path.isfile, fq_set)
    fastqc_complient = map(test_fastqc, fq_set)

    if args["fastqc-dump"]:
        fastqc_group = "fastqc_" + args["<fastqc-group>"] + "_data"
        if args["<fastqc-group>"] not in fastqc_groups:
            with indent(4):
                puts_err(colored.blue("\nFastQC group not found. Available FastQC groups:\n\n" +
                                      '\n'.join(fastqc_groups) + "\n"))
                exit()

        if fq_set:
            # Output header
            print('\t'.join(['filename'] + fastqc_headers[args["<fastqc-group>"]]))
            for i in fq_set:
                hash = ck.get_or_update_checksum(i)
                fastqc_data = get_item(kind, hash)[fastqc_group].splitlines()
                out = '\n'.join([i + "\t" + x for x in fastqc_data])
                print(out)
        else:
            # Output header
            print('\t'.join(['hash', 'filename'] + fastqc_headers[args["<fastqc-group>"]]))
            # If no fq specified, dump everything:
            fastq_dumped = query_item(kind)
            for i in fastq_dumped:
                fastqc_data = i[fastqc_group].splitlines()
                fnames = i.key.name + "\t" + i['filename'][0]
                out = '\n'.join([fnames + "\t" + x for x in fastqc_data])
                print(out)
        exit()

    if args["--fastqc"]:
        check_program_exists("fastqc")
        if not all(fastqc_complient):
            err_msg = "\n--fastqc only works with fq.gz, fastq.gz," + \
                       " fq, and fastq extensions.\n"
            with indent(4):
                puts_err(colored.red(err_msg))
                exit()
    if not all(fq_set_exists):
        missing_files = [f for f, exists in zip(fq_set, fq_set_exists)
                         if exists is False]
        with indent(4):
            puts_err(colored.red("\nFile not found:\n\n" +
                             "\n".join(missing_files) + "\n"))
            exit()

    error_fqs = []
    for fastq in fq_set:
        basename = os.path.basename(fastq)
        hash = ck.get_or_update_checksum(fastq)
        fq = fastq_reader(fastq)
        if fq.error is True:
            error_fqs.append(fastq)
            with indent(4):
                puts_err(colored.red("\nDoes not appear to be a Fastq: " +
                     fastq + "\n"))
            continue

        if args["fetch"]:
            d = get_item(kind, hash)
            if d:
                print(json.dumps(d,
                                 default=json_serial,
                                 indent=4,
                                 sort_keys=True))
            else:
                puts_err(colored.red("{basename} has not been profiled. Profile with 'fq profile'".format(basename=basename)))
            continue

        nfq = get_item(kind, hash)
        kwdata = {}
        # Test if fq stats generated.
        if nfq is None or u"total_reads" not in nfq.keys():
            if verbose:
                puts_err(colored.blue(basename + "\t[ ] Profiling"))
            kwdata.update(fq.header)
            kwdata.update(fq.calculate_fastq_stats())
        else:
            if verbose:
                puts_err(colored.blue(basename + "\t[x] Already profiled"))

        # Test if fastq filename matches illumina conventions
        illumina_keys = None
        i1 = re.match(r"([^_]+)_([ATGC]+)_([L0-9]{4})_([R12]{2})_([0-9]{3}).(fastq|fq).gz", basename)
        i2 = re.match(r"([^_]+)_(S[0-9]+)_([L0-9]{4})_([R12]{2})_([0-9]{3}).(fastq|fq).gz", basename)
        if i1:
            r = i1
            illumina_keys = ["illumina_filename_sample",
                             "illumina_filename_barcode_sequence",
                             "illumina_filename_lane",
                             "illumina_filename_read",
                             "illumina_filename_set_number"]
        elif i2:
            r = i2
            illumina_keys = ["illumina_filename_sample",
                             "illumina_filename_sample_number",
                             "illumina_filename_lane",
                             "illumina_filename_read",
                             "illumina_filename_set_number"]
        if illumina_keys:
            puts_err(colored.blue(basename + "\tIllumina Filename"))
            illumina_values = map(autoconvert, r.groups())
            illumina_data = dict(zip(illumina_keys, illumina_values))
            kwdata.update(illumina_data)

        # File statistics
        file_stat = os.stat(fastq)
        kwdata['filesize'] = file_stat.st_size
        kwdata['hfilesize'] = file_size(file_stat.st_size)
        date_created = file_stat.st_ctime
        kwdata['date_created'] = datetime.fromtimestamp(date_created)

        # Add custom data
        if args['--kv']:
            if verbose:
                puts_err(colored.blue(basename + "\tStoring Custom data"))
            kv = [x.split(":") for x in args['--kv'].split(",")]
            kv = {k: autoconvert(v) for k, v in kv}
            kwdata.update(kv)

        # FASTQC
        if args["--fastqc"]:
            if  nfq is None or u"fastqc_version" not in nfq.keys():
                if verbose:
                    puts_err(colored.blue(basename + "\t[ ] Running Fastqc"))
                fqc_data = fastqc(fastq)
                kwdata.update(fqc_data)
            else:
                if verbose:
                    puts_err(colored.blue(basename + "\t[x] FastQC already run"))

        filename = os.path.abspath(fastq)
        kwdata['hostname'] = [unicode(os.getlogin())]
        kwdata['basename'] = [unicode(basename)]
        kwdata['filename'] = [unicode(filename)]
        update_item(kind,
                    hash,
                    **kwdata)
        puts_err(colored.blue(basename + "\tComplete"))

    if error_fqs and len(fq_set) > 1:
        with indent(4):
            puts_err(colored.red("\nFastqs that errored:\n\n" +
                     '\n'.join(error_fqs) + "\n"))

if __name__ == '__main__':
    main()
<|endoftext|>