<commit_before>from nn.contiguousLayer import ContiguousLayer
import numpy as np
from theano import config, shared, dot, function
import theano.tensor as t
from encoder import AutoEncoder

class ContractiveAutoEncoder(ContiguousLayer, AutoEncoder) :
    '''This class describes a Contractive AutoEncoder (CAE). This is a
       unsupervised learning technique which encodes an input (feedforward),
       and then decodes the output vector by sending backwards through the
       layer. This attempts to reconstruct the original input.

       If the decoded message matches the original input, the encoders is
       considered lossless. Otherwise the loss is calculated and the encoder
       is updated, so it can better encode the input when encountered again.

       CAEs can be used to initialize a Neural Network in a greedy layerwise
       fashion. This allows networks to encode regular patterns using
       non-labeled datasets.

       layerID          : unique name identifier for this layer
       input            : the input buffer for this layer
       inputSize        : number of elements in input buffer
       numNeurons       : number of neurons in this layer
       learningRate     : learning rate for all neurons
       contractionRate  : variance (dimensionality) reduction rate
       initialWeights   : weights to initialize the network
                          None generates random weights for the layer
       initialHidThresh : thresholds to initialize the forward network
                          None generates random thresholds for the layer
       initialVisThresh : thresholds to initialize the backward network
                          None generates random thresholds for the layer
       activation       : the sigmoid function to use for activation
                          this must be a function with a derivative form
       randomNumGen     : generator for the initial weight values
    '''
    def __init__ (self, layerID, input, inputSize, numNeurons,
                  learningRate=0.001, contractionRate=0.01,
                  initialWeights=None, initialHidThresh=None,
                  initialVisThresh=None, activation=t.nnet.sigmoid,
                  randomNumGen=None) :
        ContiguousLayer.__init__(self, layerID=layerID,
                                 input=input,
                                 inputSize=inputSize,
                                 numNeurons=numNeurons,
                                 learningRate=learningRate,
                                 initialWeights=initialHidThresh,
                                 activation=activation,
                                 randomNumGen=randomNumGen)
        AutoEncoder.__init__(contractionRate)
        self._weightsBack = self._weights.T

        # setup initial values for the hidden thresholds
        if initialVisThresh is None :
            initialVisThresh = np.zeros((self.inputSize[1],),
                                        dtype=config.floatX)
        self._thresholdsBack = shared(value=initialVisThresh, borrow=True)

        # setup the decoder --
        # this take the output of the feedforward process as input and
        # and runs the output back through the network in reverse. The net
        # effect is to reconstruct the input, and ultimately to see how well
        # the network is at encoding the message.
        out = dot(self.output, self._weightsBack) + self._thresholdsBack
        self._decodedInput = out if activation is None else activation(out)
        self._reconstruction = function([self.input], self._decodedInput)

        # compute the jacobian cost of the reconstructed input
        jacobianMat = t.reshape(self.output * (1 - self.output),
                                (self.inputSize[0], 1, self.numNeurons)) * \
                      t.reshape(self._weights,
                                (1, self.numNeurons, self.inputSize[1]))
        self._jacobianCost = (t.mean(t.sum(jacobianMat ** 2) //
                             self.inputSize[0])) * self._contractionFactor

        # create the negative log likelihood function --
        # this is our cost function with respect to the original input
        self._nll = \
            t.mean(-t.sum(self.input * t.log(self._decodedInput) +
                          (1 - self.input) * t.log(1 - self._decodedInput),
                          axis=1))

        gradients = t.grad(self._nll + self._jacobianCost, self.getWeights())
        self._updates = [(weights, weights - learningRate * gradient)
                         for weights, gradient in zip(self.getWeights(),
                                                      gradients)]

        # TODO: this needs to be stackable and take the input to the first
        #       layer, not just the input of this layer. This will ensure
        #       the other layers are activated to get the input to this layer
        self._trainLayer = t.function([self.input],
                                      [self._jacobianCost, self._nll],
                                      updates=self._updates)

    def getWeights(self) :
        '''Update to account for the decode thresholds.'''
        return [self._weights, self._thresholds, self._thresholdsBack]
    def getUpdates(self) :
        '''This allows the Stacker to build the layerwise training.'''
        return ([self._jacobianCost, self._nll], self._updates)
    def train(self, image) :
        self._trainLayer(image)

<commit_msg>update comments<commit_after>from nn.contiguousLayer import ContiguousLayer
import numpy as np
from theano import config, shared, dot, function
import theano.tensor as t
from encoder import AutoEncoder

class ContractiveAutoEncoder(ContiguousLayer, AutoEncoder) :
    '''This class describes a Contractive AutoEncoder (CAE). This is a
       unsupervised learning technique which encodes an input (feedforward),
       and then decodes the output vector by sending backwards through the
       layer. This attempts to reconstruct the original input.

       If the decoded message matches the original input, the encoders is
       considered lossless. Otherwise the loss is calculated and the encoder
       is updated, so it can better encode the input when encountered again.
       Over time the object will extract regular patterns in the data which
       are frequently encountered.

       CAEs can be used to initialize a Neural Network in a greedy layerwise
       fashion. This should be used to better regularize the weight
       initialization, and can be used when unlabeled data far outweighs
       the amount of labeled.

       layerID          : unique name identifier for this layer
       input            : the input buffer for this layer
       inputSize        : number of elements in input buffer
       numNeurons       : number of neurons in this layer
       learningRate     : learning rate for all neurons
       contractionRate  : variance (dimensionality) reduction rate
       initialWeights   : weights to initialize the network
                          None generates random weights for the layer
       initialHidThresh : thresholds to initialize the forward network
                          None generates random thresholds for the layer
       initialVisThresh : thresholds to initialize the backward network
                          None generates random thresholds for the layer
       activation       : the sigmoid function to use for activation
                          this must be a function with a derivative form
       randomNumGen     : generator for the initial weight values
    '''
    def __init__ (self, layerID, input, inputSize, numNeurons,
                  learningRate=0.001, contractionRate=0.01,
                  initialWeights=None, initialHidThresh=None,
                  initialVisThresh=None, activation=t.nnet.sigmoid,
                  randomNumGen=None) :
        ContiguousLayer.__init__(self, layerID=layerID,
                                 input=input,
                                 inputSize=inputSize,
                                 numNeurons=numNeurons,
                                 learningRate=learningRate,
                                 initialWeights=initialHidThresh,
                                 activation=activation,
                                 randomNumGen=randomNumGen)
        AutoEncoder.__init__(contractionRate)
        self._weightsBack = self._weights.T

        # setup initial values for the hidden thresholds
        if initialVisThresh is None :
            initialVisThresh = np.zeros((self.inputSize[1],),
                                        dtype=config.floatX)
        self._thresholdsBack = shared(value=initialVisThresh, borrow=True)

        # setup the decoder --
        # this take the output of the feedforward process as input and
        # and runs the output back through the network in reverse. The net
        # effect is to reconstruct the input, and ultimately to see how well
        # the network is at encoding the message.
        out = dot(self.output, self._weightsBack) + self._thresholdsBack
        self._decodedInput = out if activation is None else activation(out)
        self._reconstruction = function([self.input], self._decodedInput)

        # compute the jacobian cost of the reconstructed input
        jacobianMat = t.reshape(self.output * (1 - self.output),
                                (self.inputSize[0], 1, self.numNeurons)) * \
                      t.reshape(self._weights,
                                (1, self.numNeurons, self.inputSize[1]))
        self._jacobianCost = (t.mean(t.sum(jacobianMat ** 2) //
                             self.inputSize[0])) * self._contractionFactor

        # create the negative log likelihood function --
        # this is our cost function with respect to the original input
        self._nll = \
            t.mean(-t.sum(self.input * t.log(self._decodedInput) +
                          (1 - self.input) * t.log(1 - self._decodedInput),
                          axis=1))

        gradients = t.grad(self._nll + self._jacobianCost, self.getWeights())
        self._updates = [(weights, weights - learningRate * gradient)
                         for weights, gradient in zip(self.getWeights(),
                                                      gradients)]

        # TODO: this needs to be stackable and take the input to the first
        #       layer, not just the input of this layer. This will ensure
        #       the other layers are activated to get the input to this layer
        self._trainLayer = t.function([self.input],
                                      [self._jacobianCost, self._nll],
                                      updates=self._updates)

    def getWeights(self) :
        '''Update to account for the decode thresholds.'''
        return [self._weights, self._thresholds, self._thresholdsBack]
    def getUpdates(self) :
        '''This allows the Stacker to build the layerwise training.'''
        return ([self._jacobianCost, self._nll], self._updates)
    def train(self, image) :
        self._trainLayer(image)

<|endoftext|>


<commit_before>from lib.config import Config
from slackclient import SlackClient
from flask import abort
from youtube import Youtube
from random import randint
from datetime import datetime
from lib.mysql import MySQL
from json import loads
from ast import literal_eval

class Tubey():

    buttons = [{
                    "name": "send",
                    "text": "Send",
                    "type": "button",
                    "style": "primary",
                    "value": "Send that pineapple"
                },
                {
                    "name": "shuffle",
                    "text": "Shuffle",
                    "type": "button",
                    "value": "Shuffle it up!"
                },
                {
                    "name": "next",
                    "text": "Next",
                    "type": "button",
                    "value": "NEXT!"
                },
                {
                    "name": "cancel",
                    "text": "Cancel",
                    "type": "button",
                    "style": "danger",
                    "value": "Cancel dis plz"
                 }]

    _youtube = Youtube()

    def __init__(self):
        # Cache the client in memory
        self._client = None
        self._mysql = MySQL(Config.get_variable('mysql_db', 'host'),
                             Config.get_variable('mysql_db', 'user'),
                             Config.get_variable('mysql_db', 'password'),
                             int(Config.get_variable('mysql_db', 'port')))

    def send_message(self, params, type="Message"):
        # Sends message to the user/channel
        # Type should either be Message or Ephemeral

        client = self.__get_client__()
        return client.api_call("chat.post" + type, **params)

    def __get_client__(self):
        # Fetch a cached slack client or create one and return it
        # type is either 'bot' or 'user'
        if self._client is not None:
            return self._client

        token = Config.get_variable('tubey', 'oauth_token')
        sc = SlackClient(token)
        self._client = sc
        return sc

    def __insert_search__(self, user_info, team_name, videos, query):
        mysql = self._mysql
        mysql.execute("USE tubey;")
        video_ids = str([x['id']['videoId'] for x in videos]).replace("'", "\\'")

        mysql.execute("INSERT INTO video_suggestions (user_name, user_id, search_query, videos, team_name)"
                      " VALUES ('{}', '{}', '{}', '{}', '{}')".format(user_info['username'], user_info['user_id'],
                                                                query, video_ids, team_name))
        mysql.commit()
        mysql.execute("SELECT * FROM video_suggestions WHERE search_id= LAST_INSERT_ID()")
        row_inserted = mysql.fetchone()
        return row_inserted[0]

    def __build_message__(self, suggested_video, channel, username, search_id, index=0):

        published_date = datetime.strptime(suggested_video['snippet']['publishedAt'][0:10], "%Y-%m-%d").date().strftime(
            '%B %d, %Y')
        channel_name = suggested_video['snippet']['channelTitle']
        video_title = suggested_video['snippet']['title']
        description = suggested_video['snippet']['description']
        thumbnail = suggested_video['snippet']['thumbnails']['high']['url']
        id = suggested_video['id']['videoId']

        self.buttons[0]['value'] = id
        self.buttons[1]['value'] = '{{"index": {}, "search_id": {}}}'.format(index, search_id)
        self.buttons[2]['value'] = '{{"index": {}, "search_id": {}}}'.format(index, search_id)

        params = {
            'unfurl_links': False,
            'channel': channel,
            'user': username,
            "attachments": [{
                "title": "Video title: {}".format(video_title),
                "text": "Channel name: {}\nPublished date: {}\nVideo description: {}".format(channel_name, published_date, description),
                "title_link": "https://www.youtube.com/watch?v=".format(id),
                "image_url": thumbnail,
                "fallback": "You need to upgrade your Slack client to use this command!",
                "color": "#CD201F",
                "attachment_type": "default",
                "actions": self.buttons,
                "callback_id": "primary_menu"
            }]
        }

        return params

    def suggest_video(self, user_info, team_info, channel_info, query="", action_info={}):
        # Sends a video suggestion in an ephemeral message
        if 'name' in action_info.keys() and action_info['name'] == 'shuffle':
            button_value = loads(action_info['value'])
            search_id = button_value['search_id']
            channel = channel_info['id']
            username = user_info['name']
            self._mysql.execute("USE tubey;")
            self._mysql.execute("select videos from video_suggestions where search_id = {}".format(search_id))
            videos = literal_eval(self._mysql.fetchone()[0])
            num_vids = len(videos)
            index = randint(0, num_vids) % num_vids
            suggested_video = self._youtube.get_video_metadata(videos[index])
            suggested_video['id'] = { "videoId": suggested_video['id'] }
            message_to_send = self.__build_message__(suggested_video, channel=channel,
                                                       username=username, search_id=search_id, index=index)
            message_to_send['replace_original'] = True
            return message_to_send
        elif 'name' in action_info.keys() and action_info['name'] == 'next':
            button_value = loads(action_info['value'])
            index = button_value['index']
            search_id = button_value['search_id']
            channel = channel_info['id']
            username = user_info['name']
            self._mysql.execute("USE tubey;")
            self._mysql.execute("select videos from video_suggestions where search_id = {}".format(search_id))
            videos = literal_eval(self._mysql.fetchone()[0])
            index += 1
            suggested_video = self._youtube.get_video_metadata(videos[index])
            suggested_video['id'] = {"videoId": suggested_video['id']}
            message_to_send = self.__build_message__(suggested_video, channel=channel,
                                                     username=username, search_id=search_id, index=index)
            message_to_send['replace_original'] = True
            return message_to_send

        # This is the first call
        else:
            videos = self.search(query)
            suggested_video = videos[0]
            search_id = self.__insert_search__(videos=videos, query=query, user_info=user_info, team_name=team_info)
            message_to_send = self.__build_message__(suggested_video, channel=channel_info,
                                                       username=user_info['user_id'], search_id=search_id)
            response = self.send_message(message_to_send, type="Ephemeral")
            print(response)

    def send_video(self, user, video_id, channel):
        # Sends a chosen video to the channel
        params =  {
            "channel": channel,
            "text": "@{} shared a video!\nhttps://www.youtube.com/watch?v={}".format(user, video_id),
            "link_names": True
        }

        response = self.send_message(params)

        return {"delete_original": True}

    def send_channel(self, channel_id, channel_name, thumbnail_url):
        # Sends the youtube channel to the active user/slack channel
        params = {'channel': 'tubeydev',
                  'text': "Here's the channel:",
                  'attachments': [
                      {'fallback': channel_name + ' thumbnail',
                       'title': channel_name,
                       "title_link": "https://www.youtube.com/channel/{}".format(channel_id),
                       "image_url": thumbnail_url
                       }]
                  }
        self.send_message(params)

    def search(self, search_query):
        results = self._youtube.query(search_query)

        return results['videos']

    def verify_token(self, token):
        verif_token = Config.get_variable('tubey', 'verif_token')

        # Validate the request parameters
        if token != verif_token:
            abort(401)  # Unauthorized request. If you're not Slack, go away

if __name__ == "__main__":
    # Should probably put a legitimate sample run here...I'll start it off
    pass
<commit_msg>Change button order<commit_after>from lib.config import Config
from slackclient import SlackClient
from flask import abort
from youtube import Youtube
from random import randint
from datetime import datetime
from lib.mysql import MySQL
from json import loads
from ast import literal_eval

class Tubey():

    buttons = [{
                    "name": "send",
                    "text": "Send",
                    "type": "button",
                    "style": "primary",
                    "value": "Send that pineapple"
                },
                {
                    "name": "next",
                    "text": "Next",
                    "type": "button",
                    "value": "NEXT!"
                },
                {
                    "name": "shuffle",
                    "text": "Shuffle",
                    "type": "button",
                    "value": "Shuffle it up!"
                },
                {
                    "name": "cancel",
                    "text": "Cancel",
                    "type": "button",
                    "style": "danger",
                    "value": "Cancel dis plz"
                 }]

    _youtube = Youtube()

    def __init__(self):
        # Cache the client in memory
        self._client = None
        self._mysql = MySQL(Config.get_variable('mysql_db', 'host'),
                             Config.get_variable('mysql_db', 'user'),
                             Config.get_variable('mysql_db', 'password'),
                             int(Config.get_variable('mysql_db', 'port')))

    def send_message(self, params, type="Message"):
        # Sends message to the user/channel
        # Type should either be Message or Ephemeral

        client = self.__get_client__()
        return client.api_call("chat.post" + type, **params)

    def __get_client__(self):
        # Fetch a cached slack client or create one and return it
        # type is either 'bot' or 'user'
        if self._client is not None:
            return self._client

        token = Config.get_variable('tubey', 'oauth_token')
        sc = SlackClient(token)
        self._client = sc
        return sc

    def __insert_search__(self, user_info, team_name, videos, query):
        mysql = self._mysql
        mysql.execute("USE tubey;")
        video_ids = str([x['id']['videoId'] for x in videos]).replace("'", "\\'")

        mysql.execute("INSERT INTO video_suggestions (user_name, user_id, search_query, videos, team_name)"
                      " VALUES ('{}', '{}', '{}', '{}', '{}')".format(user_info['username'], user_info['user_id'],
                                                                query, video_ids, team_name))
        mysql.commit()
        mysql.execute("SELECT * FROM video_suggestions WHERE search_id= LAST_INSERT_ID()")
        row_inserted = mysql.fetchone()
        return row_inserted[0]

    def __build_message__(self, suggested_video, channel, username, search_id, index=0):

        published_date = datetime.strptime(suggested_video['snippet']['publishedAt'][0:10], "%Y-%m-%d").date().strftime(
            '%B %d, %Y')
        channel_name = suggested_video['snippet']['channelTitle']
        video_title = suggested_video['snippet']['title']
        description = suggested_video['snippet']['description']
        thumbnail = suggested_video['snippet']['thumbnails']['high']['url']
        id = suggested_video['id']['videoId']

        self.buttons[0]['value'] = id
        self.buttons[1]['value'] = '{{"index": {}, "search_id": {}}}'.format(index, search_id)
        self.buttons[2]['value'] = '{{"index": {}, "search_id": {}}}'.format(index, search_id)

        params = {
            'unfurl_links': False,
            'channel': channel,
            'user': username,
            "attachments": [{
                "title": "Video title: {}".format(video_title),
                "text": "Channel name: {}\nPublished date: {}\nVideo description: {}".format(channel_name, published_date, description),
                "title_link": "https://www.youtube.com/watch?v=".format(id),
                "image_url": thumbnail,
                "fallback": "You need to upgrade your Slack client to use this command!",
                "color": "#CD201F",
                "attachment_type": "default",
                "actions": self.buttons,
                "callback_id": "primary_menu"
            }]
        }

        return params

    def suggest_video(self, user_info, team_info, channel_info, query="", action_info={}):
        # Sends a video suggestion in an ephemeral message
        if 'name' in action_info.keys() and action_info['name'] == 'shuffle':
            button_value = loads(action_info['value'])
            search_id = button_value['search_id']
            channel = channel_info['id']
            username = user_info['name']
            self._mysql.execute("USE tubey;")
            self._mysql.execute("select videos from video_suggestions where search_id = {}".format(search_id))
            videos = literal_eval(self._mysql.fetchone()[0])
            num_vids = len(videos)
            index = randint(0, num_vids) % num_vids
            suggested_video = self._youtube.get_video_metadata(videos[index])
            suggested_video['id'] = { "videoId": suggested_video['id'] }
            message_to_send = self.__build_message__(suggested_video, channel=channel,
                                                       username=username, search_id=search_id, index=index)
            message_to_send['replace_original'] = True
            return message_to_send
        elif 'name' in action_info.keys() and action_info['name'] == 'next':
            button_value = loads(action_info['value'])
            index = button_value['index']
            search_id = button_value['search_id']
            channel = channel_info['id']
            username = user_info['name']
            self._mysql.execute("USE tubey;")
            self._mysql.execute("select videos from video_suggestions where search_id = {}".format(search_id))
            videos = literal_eval(self._mysql.fetchone()[0])
            index += 1
            suggested_video = self._youtube.get_video_metadata(videos[index])
            suggested_video['id'] = {"videoId": suggested_video['id']}
            message_to_send = self.__build_message__(suggested_video, channel=channel,
                                                     username=username, search_id=search_id, index=index)
            message_to_send['replace_original'] = True
            return message_to_send

        # This is the first call
        else:
            videos = self.search(query)
            suggested_video = videos[0]
            search_id = self.__insert_search__(videos=videos, query=query, user_info=user_info, team_name=team_info)
            message_to_send = self.__build_message__(suggested_video, channel=channel_info,
                                                       username=user_info['user_id'], search_id=search_id)
            response = self.send_message(message_to_send, type="Ephemeral")
            print(response)

    def send_video(self, user, video_id, channel):
        # Sends a chosen video to the channel
        params =  {
            "channel": channel,
            "text": "@{} shared a video!\nhttps://www.youtube.com/watch?v={}".format(user, video_id),
            "link_names": True
        }

        response = self.send_message(params)

        return {"delete_original": True}

    def send_channel(self, channel_id, channel_name, thumbnail_url):
        # Sends the youtube channel to the active user/slack channel
        params = {'channel': 'tubeydev',
                  'text': "Here's the channel:",
                  'attachments': [
                      {'fallback': channel_name + ' thumbnail',
                       'title': channel_name,
                       "title_link": "https://www.youtube.com/channel/{}".format(channel_id),
                       "image_url": thumbnail_url
                       }]
                  }
        self.send_message(params)

    def search(self, search_query):
        results = self._youtube.query(search_query)

        return results['videos']

    def verify_token(self, token):
        verif_token = Config.get_variable('tubey', 'verif_token')

        # Validate the request parameters
        if token != verif_token:
            abort(401)  # Unauthorized request. If you're not Slack, go away

if __name__ == "__main__":
    # Should probably put a legitimate sample run here...I'll start it off
    pass
<|endoftext|>


<commit_before># coding=utf-8
"""
© 2013 LinkedIn Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import numpy
import os
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.ticker as ticker
from mpl_toolkits.axes_grid1 import host_subplot
import mpl_toolkits.axisartist as AA
import logging
import naarad.naarad_constants as CONSTANTS

logger = logging.getLogger('naarad.graphing.matplotlib')

def convert_to_mdate(date_str):
  try:
    mdate = mdates.strpdate2num('%Y-%m-%d %H:%M:%S.%f')(date_str)
  except:
    mdate = mdates.strpdate2num('%Y-%m-%d %H:%M:%S')(date_str)
  return mdate

# MPL-WA-07
# matplotlib does not rotate colors correctly when using multiple y axes. This method fills in that gap.
def get_current_color(index):
  return CONSTANTS.COLOR_PALETTE[index % len(CONSTANTS.COLOR_PALETTE)]

def get_graph_metadata(plots):
  height = 0
  width = 0
  title = ''
  for plot in plots:
    if plot.graph_height > height:
      height = plot.graph_height
    if plot.graph_width > width:
      width = plot.graph_width
    if title == '':
      title = plot.graph_title
    elif title != plot.graph_title:
      title = title + ',' + plot.graph_title
  return height/80, width/80, title

def curate_plot_list(plots):
  delete_nodes = []
  for plot in plots:
    if os.path.exists(plot.input_csv):
      if not os.path.getsize(plot.input_csv):
        logger.warning("%s file is empty. No plot corresponding to this file will be generated", plot.input_csv)
        delete_nodes.append(plot)
    else:
      logger.warning("%s file does not exist. No plot corresponding to this file will be generated", plot.input_csv)
      delete_nodes.append(plot)
  for node in delete_nodes:
    plots.remove(node)
  return plots

#  plot_type: currently supports 'time' for time-series data and 'cdf' for CDF data
def graph_data(list_of_plots, output_directory, resource_path, output_filename, plot_type = 'time'):
  plots = curate_plot_list(list_of_plots)
  plot_count = len(plots)
  if plot_count == 0:
    return False, None
  graph_height, graph_width, graph_title = get_graph_metadata(list_of_plots)
  current_plot_count = 0

  fig, axis = plt.subplots()
  fig.set_size_inches(graph_width, graph_height)
  if plot_count < 2:
    fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET, right=CONSTANTS.SUBPLOT_RIGHT_OFFSET)
  else:
    fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET, right=CONSTANTS.SUBPLOT_RIGHT_OFFSET - CONSTANTS.Y_AXIS_OFFSET * (plot_count - 2))
  current_axis = axis
  for plot in plots:
    current_plot_count += 1
    logger.info('Processing: ' + plot.input_csv + ' [ ' + output_filename + ' ]')
    if plot_type == 'time':
      xval, yval = numpy.loadtxt(plot.input_csv, unpack=True, delimiter=',', converters={0: convert_to_mdate})  #time -- extracts an array of timstamps and an array of yval
    elif plot_type == 'cdf':
      xval, yval = numpy.loadtxt(plot.input_csv, unpack=True, delimiter=',')  #cdf -- extracts an array of percentiles and an array of yval
    maximum_yvalue = numpy.amax(yval) * (1.0 + CONSTANTS.ZOOM_FACTOR * current_plot_count)
    minimum_yvalue = numpy.amin(yval) * (1.0 - CONSTANTS.ZOOM_FACTOR * current_plot_count)

    if current_plot_count == 0:
      current_axis.yaxis.set_ticks_position('left')
    if current_plot_count > 1:
      current_axis = axis.twinx()
      current_axis.yaxis.grid(False)
      #Set right y-axis for additional plots
      current_axis.yaxis.set_ticks_position('right')
      #Offset the right y axis to avoid overlap
      current_axis.spines['right'].set_position(('axes', 1 + CONSTANTS.Y_AXIS_OFFSET * (current_plot_count-2)))
      current_axis.spines['right'].set_smart_bounds(False)
      current_axis.spines['right'].set_color(get_current_color(current_plot_count))
      current_axis.set_frame_on(True)
      current_axis.patch.set_visible(False)
    current_axis.set_ylabel(plot.y_label, color=get_current_color(current_plot_count), fontsize=CONSTANTS.Y_LABEL_FONTSIZE)
    current_axis.set_ylim([minimum_yvalue, maximum_yvalue])
    if plot_type == 'time':  #time -- extracts an array of timstamps and an array of yval
      if plot.graph_type == 'line':
        current_axis.plot_date(x=xval, y=yval, linestyle='-', marker=None, color=get_current_color(current_plot_count))
      else:
        current_axis.plot_date(x=xval, y=yval, marker='.', color=get_current_color(current_plot_count))
    elif plot_type == 'cdf': #cdf -- extracts an array of percentiles and an array of yval
      if plot.graph_type == 'line':
        current_axis.plot(xval, yval, '-')
      else:
        current_axis.plot(xval, yval, '.')
    y_ticks = current_axis.get_yticklabels()
    for y_tick in y_ticks:
      y_tick.set_color(get_current_color(current_plot_count))
      y_tick.set_fontsize(CONSTANTS.Y_TICKS_FONTSIZE)
    for x_tick in current_axis.get_xticklabels():
      x_tick.set_fontsize(CONSTANTS.X_TICKS_FONTSIZE)
  axis.yaxis.grid(True)
  axis.xaxis.grid(True)
  axis.set_title(graph_title)
  if plot_type == 'time':
    axis.set_xlabel('Time')  #time-- sets the label of the x-axis as 'time'
    x_date_format = mdates.DateFormatter(CONSTANTS.X_TICKS_DATEFORMAT)
    axis.xaxis.set_major_formatter(x_date_format)
  elif plot_type == 'cdf':
    axis.set_xlabel('Percentile')
    #x_formatter = ticker.ScalarFormatter(useOffset=False)
    #axis.xaxis.set_major_formatter(x_formatter)
  plot_file_name = os.path.join(output_directory, output_filename + ".png")
  fig.savefig(plot_file_name)
  plt.close()
  #Create html fragment to be used for creation of the report
  with open(os.path.join(output_directory, output_filename + '.div'), 'w') as div_file: #cdf--
    div_file.write('') #cdf--
  return True, os.path.join(output_directory, output_filename + '.div') #cdf--
<commit_msg>add CDF support to Naarad: generate CDF graphs for important metrics in the summary page<commit_after># coding=utf-8
"""
© 2013 LinkedIn Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import numpy
import os
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from mpl_toolkits.axes_grid1 import host_subplot
import mpl_toolkits.axisartist as AA
import logging
import naarad.naarad_constants as CONSTANTS

logger = logging.getLogger('naarad.graphing.matplotlib')

def convert_to_mdate(date_str):
  try:
    mdate = mdates.strpdate2num('%Y-%m-%d %H:%M:%S.%f')(date_str)
  except:
    mdate = mdates.strpdate2num('%Y-%m-%d %H:%M:%S')(date_str)
  return mdate

# MPL-WA-07
# matplotlib does not rotate colors correctly when using multiple y axes. This method fills in that gap.
def get_current_color(index):
  return CONSTANTS.COLOR_PALETTE[index % len(CONSTANTS.COLOR_PALETTE)]

def get_graph_metadata(plots):
  height = 0
  width = 0
  title = ''
  for plot in plots:
    if plot.graph_height > height:
      height = plot.graph_height
    if plot.graph_width > width:
      width = plot.graph_width
    if title == '':
      title = plot.graph_title
    elif title != plot.graph_title:
      title = title + ',' + plot.graph_title
  return height/80, width/80, title

def curate_plot_list(plots):
  delete_nodes = []
  for plot in plots:
    if os.path.exists(plot.input_csv):
      if not os.path.getsize(plot.input_csv):
        logger.warning("%s file is empty. No plot corresponding to this file will be generated", plot.input_csv)
        delete_nodes.append(plot)
    else:
      logger.warning("%s file does not exist. No plot corresponding to this file will be generated", plot.input_csv)
      delete_nodes.append(plot)
  for node in delete_nodes:
    plots.remove(node)
  return plots

def graph_data(list_of_plots, output_directory, resource_path, output_filename, plot_type = 'time'):
  """
  graph_data current supports two plot types
  'time': generate a time-series plot (the x-axis is a time series)
  'cdf': generate a CDF plot (the x-axis shows percentiles)
  """
  plots = curate_plot_list(list_of_plots)
  plot_count = len(plots)
  if plot_count == 0:
    return False, None
  graph_height, graph_width, graph_title = get_graph_metadata(list_of_plots)
  current_plot_count = 0
  fig, axis = plt.subplots()
  fig.set_size_inches(graph_width, graph_height)
  if plot_count < 2:
    fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET, right=CONSTANTS.SUBPLOT_RIGHT_OFFSET)
  else:
    fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET, right=CONSTANTS.SUBPLOT_RIGHT_OFFSET - CONSTANTS.Y_AXIS_OFFSET * (plot_count - 2))
  current_axis = axis
  for plot in plots:
    current_plot_count += 1
    logger.info('Processing: ' + plot.input_csv + ' [ ' + output_filename + ' ]')
    if plot_type == 'time':
      xval, yval = numpy.loadtxt(plot.input_csv, unpack=True, delimiter=',', converters={0: convert_to_mdate})
    elif plot_type == 'cdf':
      xval, yval = numpy.loadtxt(plot.input_csv, unpack=True, delimiter=',')
    maximum_yvalue = numpy.amax(yval) * (1.0 + CONSTANTS.ZOOM_FACTOR * current_plot_count)
    minimum_yvalue = numpy.amin(yval) * (1.0 - CONSTANTS.ZOOM_FACTOR * current_plot_count)

    if current_plot_count == 0:
      current_axis.yaxis.set_ticks_position('left')
    if current_plot_count > 1:
      current_axis = axis.twinx()
      current_axis.yaxis.grid(False)
      #Set right y-axis for additional plots
      current_axis.yaxis.set_ticks_position('right')
      #Offset the right y axis to avoid overlap
      current_axis.spines['right'].set_position(('axes', 1 + CONSTANTS.Y_AXIS_OFFSET * (current_plot_count-2)))
      current_axis.spines['right'].set_smart_bounds(False)
      current_axis.spines['right'].set_color(get_current_color(current_plot_count))
      current_axis.set_frame_on(True)
      current_axis.patch.set_visible(False)
    current_axis.set_ylabel(plot.y_label, color=get_current_color(current_plot_count), fontsize=CONSTANTS.Y_LABEL_FONTSIZE)
    current_axis.set_ylim([minimum_yvalue, maximum_yvalue])
    if plot_type == 'time':
      if plot.graph_type == 'line':
        current_axis.plot_date(x=xval, y=yval, linestyle='-', marker=None, color=get_current_color(current_plot_count))
      else:
        current_axis.plot_date(x=xval, y=yval, marker='.', color=get_current_color(current_plot_count))
    elif plot_type == 'cdf':
      if plot.graph_type == 'line':
        current_axis.plot(xval, yval, '-')
      else:
        current_axis.plot(xval, yval, '.')
    y_ticks = current_axis.get_yticklabels()
    for y_tick in y_ticks:
      y_tick.set_color(get_current_color(current_plot_count))
      y_tick.set_fontsize(CONSTANTS.Y_TICKS_FONTSIZE)
    for x_tick in current_axis.get_xticklabels():
      x_tick.set_fontsize(CONSTANTS.X_TICKS_FONTSIZE)
  axis.yaxis.grid(True)
  axis.xaxis.grid(True)
  axis.set_title(graph_title)
  if plot_type == 'time':
    axis.set_xlabel('Time')
    x_date_format = mdates.DateFormatter(CONSTANTS.X_TICKS_DATEFORMAT)
    axis.xaxis.set_major_formatter(x_date_format)
  elif plot_type == 'cdf':
    axis.set_xlabel('Percentile')
  plot_file_name = os.path.join(output_directory, output_filename + ".png")
  fig.savefig(plot_file_name)
  plt.close()
  #Create html fragment to be used for creation of the report
  with open(os.path.join(output_directory, output_filename + '.div'), 'w') as div_file:
    div_file.write('')
  return True, os.path.join(output_directory, output_filename + '.div')
<|endoftext|>


<commit_before>import json
import re
import datetime

from django import http
from django.shortcuts import render
from django.contrib.auth.models import Permission
from django.contrib.sites.models import RequestSite
from django.core.urlresolvers import reverse
from django.conf import settings
from django import forms
# explicit import because django.forms has an __all__
from django.forms.forms import DeclarativeFieldsMetaclass

from ratelimit.decorators import ratelimit
from waffle.decorators import waffle_switch

import crashstats.supersearch.models
from crashstats.crashstats import models
from crashstats.crashstats import utils
from crashstats.tokens.models import Token
from .cleaner import Cleaner

# List of all modules that contain models we want to expose.
MODELS_MODULES = (
    models,
    crashstats.supersearch.models,
)

# See http://www.iana.org/assignments/http-status-codes
REASON_PHRASES = {
    100: 'CONTINUE',
    101: 'SWITCHING PROTOCOLS',
    102: 'PROCESSING',
    200: 'OK',
    201: 'CREATED',
    202: 'ACCEPTED',
    203: 'NON-AUTHORITATIVE INFORMATION',
    204: 'NO CONTENT',
    205: 'RESET CONTENT',
    206: 'PARTIAL CONTENT',
    207: 'MULTI-STATUS',
    208: 'ALREADY REPORTED',
    226: 'IM USED',
    300: 'MULTIPLE CHOICES',
    301: 'MOVED PERMANENTLY',
    302: 'FOUND',
    303: 'SEE OTHER',
    304: 'NOT MODIFIED',
    305: 'USE PROXY',
    306: 'RESERVED',
    307: 'TEMPORARY REDIRECT',
    400: 'BAD REQUEST',
    401: 'UNAUTHORIZED',
    402: 'PAYMENT REQUIRED',
    403: 'FORBIDDEN',
    404: 'NOT FOUND',
    405: 'METHOD NOT ALLOWED',
    406: 'NOT ACCEPTABLE',
    407: 'PROXY AUTHENTICATION REQUIRED',
    408: 'REQUEST TIMEOUT',
    409: 'CONFLICT',
    410: 'GONE',
    411: 'LENGTH REQUIRED',
    412: 'PRECONDITION FAILED',
    413: 'REQUEST ENTITY TOO LARGE',
    414: 'REQUEST-URI TOO LONG',
    415: 'UNSUPPORTED MEDIA TYPE',
    416: 'REQUESTED RANGE NOT SATISFIABLE',
    417: 'EXPECTATION FAILED',
    418: "I'M A TEAPOT",
    422: 'UNPROCESSABLE ENTITY',
    423: 'LOCKED',
    424: 'FAILED DEPENDENCY',
    426: 'UPGRADE REQUIRED',
    428: 'PRECONDITION REQUIRED',
    429: 'TOO MANY REQUESTS',
    431: 'REQUEST HEADER FIELDS TOO LARGE',
    500: 'INTERNAL SERVER ERROR',
    501: 'NOT IMPLEMENTED',
    502: 'BAD GATEWAY',
    503: 'SERVICE UNAVAILABLE',
    504: 'GATEWAY TIMEOUT',
    505: 'HTTP VERSION NOT SUPPORTED',
    506: 'VARIANT ALSO NEGOTIATES',
    507: 'INSUFFICIENT STORAGE',
    508: 'LOOP DETECTED',
    510: 'NOT EXTENDED',
    511: 'NETWORK AUTHENTICATION REQUIRED',
}

class APIWhitelistError(Exception):
    pass

class MultipleStringField(forms.TypedMultipleChoiceField):
    """Field that do not validate if the field values are in self.choices"""

    def to_python(self, value):
        """Override checking method"""
        return map(self.coerce, value)

    def validate(self, value):
        """Nothing to do here"""
        if self.required and not value:
            raise forms.ValidationError(self.error_messages['required'])

TYPE_MAP = {
    basestring: forms.CharField,
    list: MultipleStringField,
    datetime.date: forms.DateField,
    datetime.datetime: forms.DateTimeField,
    int: forms.IntegerField,
}

def fancy_init(self, model, *args, **kwargs):
    self.model = model
    self.__old_init__(*args, **kwargs)
    for parameter in model.get_annotated_params():
        required = parameter['required']
        name = parameter['name']

        if parameter['type'] not in TYPE_MAP:
            raise NotImplementedError(parameter['type'])
        field_class = TYPE_MAP[parameter['type']]
        self.fields[name] = field_class(required=required)

class FormWrapperMeta(DeclarativeFieldsMetaclass):
    def __new__(cls, name, bases, attrs):
        attrs['__old_init__'] = bases[0].__init__
        attrs['__init__'] = fancy_init
        return super(FormWrapperMeta, cls).__new__(cls, name, bases, attrs)

class FormWrapper(forms.Form):
    __metaclass__ = FormWrapperMeta

# Names of models we don't want to serve at all
BLACKLIST = (
    # not because it's sensitive but because it's only used for writes
    'ReleasesFeatured',
    # because it's only used for the admin
    'Field',
)

def _skip_ratelimit(request):
    return request.user.is_authenticated()

def has_permissions(user, permissions):
    for permission in permissions:
        if not user.has_perm(permission):
            return False
    return True

@waffle_switch('app_api_all')
@ratelimit(
    method=['GET', 'POST', 'PUT'],
    rate='10/m',
    skip_if=_skip_ratelimit
)
@utils.add_CORS_header  # must be before `utils.json_view`
@utils.json_view
def model_wrapper(request, model_name):
    if model_name in BLACKLIST:
        raise http.Http404("Don't know what you're talking about!")

    for source in MODELS_MODULES:
        try:
            model = getattr(source, model_name)
            break
        except AttributeError:
            pass
    else:
        raise http.Http404('no model called `%s`' % model_name)

    required_permissions = getattr(model, 'API_REQUIRED_PERMISSIONS', None)
    if isinstance(required_permissions, basestring):
        required_permissions = [required_permissions]
    if (
        required_permissions and
        not has_permissions(request.user, required_permissions)
    ):
        permission_names = []
        for permission in required_permissions:
            codename = permission.split('.', 1)[1]
            try:
                permission_names.append(
                    Permission.objects.get(
                        codename=codename
                    ).name
                )
            except Permission.DoesNotExist:
                permission_names.append(codename)
        # you're not allowed to use this model
        return http.HttpResponseForbidden(
            "Use of this endpoint requires the '%s' permission\n" %
            (', '.join(permission_names))
        )

    # XXX use RatelimitMiddleware instead of this in case
    # we ratelimit multiple views
    if getattr(request, 'limited', False):
        # http://tools.ietf.org/html/rfc6585#page-3
        return http.HttpResponse('Too Many Requests', status=429)

    # it being set to None means it's been deliberately disabled
    if getattr(model, 'API_WHITELIST', False) is False:
        raise APIWhitelistError('No API_WHITELIST defined for %r' % model)

    instance = model()
    if request.method == 'POST':
        function = instance.post
    else:
        function = instance.get

    form = FormWrapper(model, request.REQUEST)
    if form.is_valid():
        try:
            result = function(**form.cleaned_data)
        except models.BadStatusCodeError as e:
            error_code = e.status
            message = e.message
            if error_code >= 400 and error_code < 500:
                # if the error message looks like JSON,
                # carry that forward in the response
                try:
                    json.loads(message)
                    return http.HttpResponse(
                        message,
                        status=error_code,
                        mimetype='application/json; charset=UTF-8'
                    )
                except ValueError:
                    # The error from the middleware was not a JSON error.
                    # Not much more we can do.
                    reason = REASON_PHRASES.get(
                        error_code,
                        'UNKNOWN STATUS CODE'
                    )
                    return http.HttpResponse(reason, status=error_code)
            if error_code >= 500:
                # special case
                reason = REASON_PHRASES[424]
                return http.HttpResponse(
                    reason,
                    status=424,
                    mimetype='text/plain'
                )
            raise
        except ValueError as e:
            if 'No JSON object could be decoded' in e:
                return http.HttpResponse(
                    'Not a valid JSON response',
                    status=400
                )
            raise

        if not request.user.has_perm('crashstats.view_pii'):
            clean_scrub = getattr(model, 'API_CLEAN_SCRUB', None)
            if result and model.API_WHITELIST:
                cleaner = Cleaner(
                    model.API_WHITELIST,
                    clean_scrub=clean_scrub,
                    # if True, uses warnings.warn() to show fields
                    # not whitelisted
                    debug=settings.DEBUG,
                )
                cleaner.start(result)

    else:
        # custom override of the status code
        return {'errors': dict(form.errors)}, 400

    return result

@waffle_switch('app_api_all')
def documentation(request):
    endpoints = [
    ]

    all_models = []
    for source in MODELS_MODULES:
        all_models += [getattr(source, x) for x in dir(source)]

    for model in all_models:
        try:
            if not issubclass(model, models.SocorroMiddleware):
                continue
            if model is models.SocorroMiddleware:
                continue
            if model.__name__ in BLACKLIST:
                continue
        except TypeError:
            # most likely a builtin class or something
            continue
        if (
            model.API_REQUIRED_PERMISSIONS and
            not has_permissions(request.user, model.API_REQUIRED_PERMISSIONS)
        ):
            continue
        endpoints.append(_describe_model(model))

    base_url = (
        '%s://%s' % (request.is_secure() and 'https' or 'http',
                     RequestSite(request).domain)
    )
    if request.user.is_authenticated():
        your_tokens = Token.objects.active().filter(user=request.user)
    else:
        your_tokens = Token.objects.none()
    context = {
        'endpoints': endpoints,
        'base_url': base_url,
        'count_tokens': your_tokens.count()
    }
    return render(request, 'api/documentation.html', context)

def _describe_model(model):
    params = list(model.get_annotated_params())
    params.sort(key=lambda x: (not x['required'], x['name']))
    methods = []
    if model.get:
        methods.append('GET')
    elif model.post:
        methods.append('POST')

    docstring = model.__doc__
    if docstring:
        docstring = dedent_left(docstring.rstrip(), 4)

    required_permissions = []
    if model.API_REQUIRED_PERMISSIONS:
        permissions = model.API_REQUIRED_PERMISSIONS
        if isinstance(permissions, basestring):
            permissions = [permissions]
        for permission in permissions:
            codename = permission.split('.', 1)[1]
            required_permissions.append(
                Permission.objects.get(codename=codename).name
            )

    data = {
        'name': model.__name__,
        'url': reverse('api:model_wrapper', args=(model.__name__,)),
        'parameters': params,
        'defaults': getattr(model, 'defaults', {}),
        'methods': methods,
        'docstring': docstring,
        'required_permissions': required_permissions,
    }
    return data

def dedent_left(text, spaces):
    """
    If the string is:
        '   One\n'
        '     Two\n'
        'Three\n'

    And you set @spaces=2
    Then return this:
        ' One\n'
        '   Two\n'
        'Three\n'
    """
    lines = []
    regex = re.compile('^\s{%s}' % spaces)
    for line in text.splitlines():
        line = regex.sub('', line)
        lines.append(line)
    return '\n'.join(lines)
<commit_msg>Fixes bug 1013494 - Remove Query model from the public API.<commit_after>import json
import re
import datetime

from django import http
from django.shortcuts import render
from django.contrib.auth.models import Permission
from django.contrib.sites.models import RequestSite
from django.core.urlresolvers import reverse
from django.conf import settings
from django import forms
# explicit import because django.forms has an __all__
from django.forms.forms import DeclarativeFieldsMetaclass

from ratelimit.decorators import ratelimit
from waffle.decorators import waffle_switch

import crashstats.supersearch.models
from crashstats.crashstats import models
from crashstats.crashstats import utils
from crashstats.tokens.models import Token
from .cleaner import Cleaner

# List of all modules that contain models we want to expose.
MODELS_MODULES = (
    models,
    crashstats.supersearch.models,
)

# See http://www.iana.org/assignments/http-status-codes
REASON_PHRASES = {
    100: 'CONTINUE',
    101: 'SWITCHING PROTOCOLS',
    102: 'PROCESSING',
    200: 'OK',
    201: 'CREATED',
    202: 'ACCEPTED',
    203: 'NON-AUTHORITATIVE INFORMATION',
    204: 'NO CONTENT',
    205: 'RESET CONTENT',
    206: 'PARTIAL CONTENT',
    207: 'MULTI-STATUS',
    208: 'ALREADY REPORTED',
    226: 'IM USED',
    300: 'MULTIPLE CHOICES',
    301: 'MOVED PERMANENTLY',
    302: 'FOUND',
    303: 'SEE OTHER',
    304: 'NOT MODIFIED',
    305: 'USE PROXY',
    306: 'RESERVED',
    307: 'TEMPORARY REDIRECT',
    400: 'BAD REQUEST',
    401: 'UNAUTHORIZED',
    402: 'PAYMENT REQUIRED',
    403: 'FORBIDDEN',
    404: 'NOT FOUND',
    405: 'METHOD NOT ALLOWED',
    406: 'NOT ACCEPTABLE',
    407: 'PROXY AUTHENTICATION REQUIRED',
    408: 'REQUEST TIMEOUT',
    409: 'CONFLICT',
    410: 'GONE',
    411: 'LENGTH REQUIRED',
    412: 'PRECONDITION FAILED',
    413: 'REQUEST ENTITY TOO LARGE',
    414: 'REQUEST-URI TOO LONG',
    415: 'UNSUPPORTED MEDIA TYPE',
    416: 'REQUESTED RANGE NOT SATISFIABLE',
    417: 'EXPECTATION FAILED',
    418: "I'M A TEAPOT",
    422: 'UNPROCESSABLE ENTITY',
    423: 'LOCKED',
    424: 'FAILED DEPENDENCY',
    426: 'UPGRADE REQUIRED',
    428: 'PRECONDITION REQUIRED',
    429: 'TOO MANY REQUESTS',
    431: 'REQUEST HEADER FIELDS TOO LARGE',
    500: 'INTERNAL SERVER ERROR',
    501: 'NOT IMPLEMENTED',
    502: 'BAD GATEWAY',
    503: 'SERVICE UNAVAILABLE',
    504: 'GATEWAY TIMEOUT',
    505: 'HTTP VERSION NOT SUPPORTED',
    506: 'VARIANT ALSO NEGOTIATES',
    507: 'INSUFFICIENT STORAGE',
    508: 'LOOP DETECTED',
    510: 'NOT EXTENDED',
    511: 'NETWORK AUTHENTICATION REQUIRED',
}

class APIWhitelistError(Exception):
    pass

class MultipleStringField(forms.TypedMultipleChoiceField):
    """Field that do not validate if the field values are in self.choices"""

    def to_python(self, value):
        """Override checking method"""
        return map(self.coerce, value)

    def validate(self, value):
        """Nothing to do here"""
        if self.required and not value:
            raise forms.ValidationError(self.error_messages['required'])

TYPE_MAP = {
    basestring: forms.CharField,
    list: MultipleStringField,
    datetime.date: forms.DateField,
    datetime.datetime: forms.DateTimeField,
    int: forms.IntegerField,
}

def fancy_init(self, model, *args, **kwargs):
    self.model = model
    self.__old_init__(*args, **kwargs)
    for parameter in model.get_annotated_params():
        required = parameter['required']
        name = parameter['name']

        if parameter['type'] not in TYPE_MAP:
            raise NotImplementedError(parameter['type'])
        field_class = TYPE_MAP[parameter['type']]
        self.fields[name] = field_class(required=required)

class FormWrapperMeta(DeclarativeFieldsMetaclass):
    def __new__(cls, name, bases, attrs):
        attrs['__old_init__'] = bases[0].__init__
        attrs['__init__'] = fancy_init
        return super(FormWrapperMeta, cls).__new__(cls, name, bases, attrs)

class FormWrapper(forms.Form):
    __metaclass__ = FormWrapperMeta

# Names of models we don't want to serve at all
BLACKLIST = (
    # not because it's sensitive but because it's only used for writes
    'ReleasesFeatured',
    # because it's only used for the admin
    'Field',
    # because it's very sensitive and we don't want to expose it
    'Query',
)

def _skip_ratelimit(request):
    return request.user.is_authenticated()

def has_permissions(user, permissions):
    for permission in permissions:
        if not user.has_perm(permission):
            return False
    return True

@waffle_switch('app_api_all')
@ratelimit(
    method=['GET', 'POST', 'PUT'],
    rate='10/m',
    skip_if=_skip_ratelimit
)
@utils.add_CORS_header  # must be before `utils.json_view`
@utils.json_view
def model_wrapper(request, model_name):
    if model_name in BLACKLIST:
        raise http.Http404("Don't know what you're talking about!")

    for source in MODELS_MODULES:
        try:
            model = getattr(source, model_name)
            break
        except AttributeError:
            pass
    else:
        raise http.Http404('no model called `%s`' % model_name)

    required_permissions = getattr(model, 'API_REQUIRED_PERMISSIONS', None)
    if isinstance(required_permissions, basestring):
        required_permissions = [required_permissions]
    if (
        required_permissions and
        not has_permissions(request.user, required_permissions)
    ):
        permission_names = []
        for permission in required_permissions:
            codename = permission.split('.', 1)[1]
            try:
                permission_names.append(
                    Permission.objects.get(
                        codename=codename
                    ).name
                )
            except Permission.DoesNotExist:
                permission_names.append(codename)
        # you're not allowed to use this model
        return http.HttpResponseForbidden(
            "Use of this endpoint requires the '%s' permission\n" %
            (', '.join(permission_names))
        )

    # XXX use RatelimitMiddleware instead of this in case
    # we ratelimit multiple views
    if getattr(request, 'limited', False):
        # http://tools.ietf.org/html/rfc6585#page-3
        return http.HttpResponse('Too Many Requests', status=429)

    # it being set to None means it's been deliberately disabled
    if getattr(model, 'API_WHITELIST', False) is False:
        raise APIWhitelistError('No API_WHITELIST defined for %r' % model)

    instance = model()
    if request.method == 'POST':
        function = instance.post
    else:
        function = instance.get

    form = FormWrapper(model, request.REQUEST)
    if form.is_valid():
        try:
            result = function(**form.cleaned_data)
        except models.BadStatusCodeError as e:
            error_code = e.status
            message = e.message
            if error_code >= 400 and error_code < 500:
                # if the error message looks like JSON,
                # carry that forward in the response
                try:
                    json.loads(message)
                    return http.HttpResponse(
                        message,
                        status=error_code,
                        mimetype='application/json; charset=UTF-8'
                    )
                except ValueError:
                    # The error from the middleware was not a JSON error.
                    # Not much more we can do.
                    reason = REASON_PHRASES.get(
                        error_code,
                        'UNKNOWN STATUS CODE'
                    )
                    return http.HttpResponse(reason, status=error_code)
            if error_code >= 500:
                # special case
                reason = REASON_PHRASES[424]
                return http.HttpResponse(
                    reason,
                    status=424,
                    mimetype='text/plain'
                )
            raise
        except ValueError as e:
            if 'No JSON object could be decoded' in e:
                return http.HttpResponse(
                    'Not a valid JSON response',
                    status=400
                )
            raise

        if not request.user.has_perm('crashstats.view_pii'):
            clean_scrub = getattr(model, 'API_CLEAN_SCRUB', None)
            if result and model.API_WHITELIST:
                cleaner = Cleaner(
                    model.API_WHITELIST,
                    clean_scrub=clean_scrub,
                    # if True, uses warnings.warn() to show fields
                    # not whitelisted
                    debug=settings.DEBUG,
                )
                cleaner.start(result)

    else:
        # custom override of the status code
        return {'errors': dict(form.errors)}, 400

    return result

@waffle_switch('app_api_all')
def documentation(request):
    endpoints = [
    ]

    all_models = []
    for source in MODELS_MODULES:
        all_models += [getattr(source, x) for x in dir(source)]

    for model in all_models:
        try:
            if not issubclass(model, models.SocorroMiddleware):
                continue
            if model is models.SocorroMiddleware:
                continue
            if model.__name__ in BLACKLIST:
                continue
        except TypeError:
            # most likely a builtin class or something
            continue
        if (
            model.API_REQUIRED_PERMISSIONS and
            not has_permissions(request.user, model.API_REQUIRED_PERMISSIONS)
        ):
            continue
        endpoints.append(_describe_model(model))

    base_url = (
        '%s://%s' % (request.is_secure() and 'https' or 'http',
                     RequestSite(request).domain)
    )
    if request.user.is_authenticated():
        your_tokens = Token.objects.active().filter(user=request.user)
    else:
        your_tokens = Token.objects.none()
    context = {
        'endpoints': endpoints,
        'base_url': base_url,
        'count_tokens': your_tokens.count()
    }
    return render(request, 'api/documentation.html', context)

def _describe_model(model):
    params = list(model.get_annotated_params())
    params.sort(key=lambda x: (not x['required'], x['name']))
    methods = []
    if model.get:
        methods.append('GET')
    elif model.post:
        methods.append('POST')

    docstring = model.__doc__
    if docstring:
        docstring = dedent_left(docstring.rstrip(), 4)

    required_permissions = []
    if model.API_REQUIRED_PERMISSIONS:
        permissions = model.API_REQUIRED_PERMISSIONS
        if isinstance(permissions, basestring):
            permissions = [permissions]
        for permission in permissions:
            codename = permission.split('.', 1)[1]
            required_permissions.append(
                Permission.objects.get(codename=codename).name
            )

    data = {
        'name': model.__name__,
        'url': reverse('api:model_wrapper', args=(model.__name__,)),
        'parameters': params,
        'defaults': getattr(model, 'defaults', {}),
        'methods': methods,
        'docstring': docstring,
        'required_permissions': required_permissions,
    }
    return data

def dedent_left(text, spaces):
    """
    If the string is:
        '   One\n'
        '     Two\n'
        'Three\n'

    And you set @spaces=2
    Then return this:
        ' One\n'
        '   Two\n'
        'Three\n'
    """
    lines = []
    regex = re.compile('^\s{%s}' % spaces)
    for line in text.splitlines():
        line = regex.sub('', line)
        lines.append(line)
    return '\n'.join(lines)
<|endoftext|>


<commit_before>
import collections

__all__ = [
    "MergeConflict",
    "ValueNotKnownError",
    "ValueAmbiguousError",
    "root",
]

class State(object):

    def __init__(self, root, parent=None, owner=None):
        self.root = root
        self.parent = parent
        self.slot_values = {}
        self.slot_positions = collections.defaultdict(lambda: set())
        self.owner = owner

    def merge_children(self, children, or_none=False):
        pass

    def _create_child(self, owner=None):
        return State(self.root, self, owner)

    def _child_context(self, owner, auto_merge):
        previous = self.root.current_state
        new = self._create_child(owner)
        class Context(object):
            def __enter__(context):
                self.root.current_state = new
                return new
            def __exit__(context, exc_type, exc_value, traceback):
                if auto_merge and exc_type is None:
                    self.merge_children([new])
                self.root.current_state = previous
        return Context()

    def fork(self, owner=None):
        return self._child_context(owner, auto_merge=False)

    def transaction(self, owner=None):
        return self._child_context(owner, auto_merge=True)

    def set_slot(self, slot, value, position=None):
        self.slot_values[slot] = value
        self.slot_positions[slot] = set(
            [position] if position is not None else []
        )

    def get_slot_value(self, slot):
        current = self
        while current is not None:
            try:
                return current.slot_values[slot]
            except KeyError:
                current = self.parent
            return Slot.NOT_KNOWN

class Slot(object):
    # we will compare by reference to this thing to detect the "don't know"
    # case.
    NOT_KNOWN = type("not_known", (object,), {
        "__repr__": lambda self: "datafork.Slot.NOT_KNOWN"
    })()

    def __init__(
        self,
        root,
        owner=None,
        initial_value=NOT_KNOWN,
    ):
        self.owner = owner
        self.root = root
        self.set_value(
            initial_value,
        )

    @property
    def value(self):
        try:
            return Slot.prepare_return_value(
                self,
                self.final_value,
            )
        except AttributeError:
            return Slot.prepare_return_value(
                self,
                self.root.current_state.get_slot_value(self),
            )

    @property
    def positions(self):
        try:
            return self.final_positions
        except AttributeError:
            return self.root.current_state.get_slot_positions(self)

    @value.setter
    def value(self, value):
        self.set_value(value)

    def set_value(self, value, position=None):
        if hasattr(self, "final_value"):
            # should never happen
            raise Exception(
                "Can't set value on slot %r: it has been finalized" % self,
            )
        else:
            self.root.current_state.set_slot(
                self,
                value,
                position=position,
            )

    def set_value_not_known(self, position=None):
        self.set_value(Slot.NOT_KNOWN, position=position)

    @property
    def value_is_known(self):
        try:
            value = self.value
        except ValueNotKnownError:
            return False
        else:
            return True

    @classmethod
    def prepare_return_value(cls, slot, value):
        if type(value) is MergeConflict:
            raise ValueAmbiguousError(slot, value)
        elif value is cls.NOT_KNOWN:
            raise ValueNotKnownError(slot)
        else:
            return value

class Root(State):
    def __init__(self, root_owner=None, slot_type=Slot):
        State.__init__(self, self, None, root_owner)
        self.current_state = self
        self.slot_type = slot_type
        self.slots = set()

    def slot(
        self,
        owner=None,
        initial_value=Slot.NOT_KNOWN,
        initial_position=None,
    ):
        """
        Creates a new slot in this root.
        """
        slot = self.slot_type(self, owner, initial_value, initial_position)
        self.slots.add(slot)
        return slot

    def slotted_object(self):
        return SlottedObject(self)

    def slotted_mapping(self):
        return SlottedMapping(self)

    def slotted_sequence(self):
        return SlottedSequence(self)

    def slotted_set(self):
        return SlottedSet(self)

    def finalize_data(self):
        for slot in self.slots:
            slot.final_value = slot.value
            slot.final_position = slot.position
            # sever the connection from the slot to the root so that
            # the root can be garbage collected after the with block exits.
            # The slot doesn't need the root anymore.
            del slot.root

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.finalize_data()

def root(owner=None):
    new = Root(root_owner=owner)
    class Context(object):
        def __enter__(self):
            return new
        def __exit__(self, exc_type, exc_value, traceback):
            new.finalize_data()
    return Context()

class MergeConflict(object):

    def __init__(self, possibilities):
        self.possibilities = possibilities

    def __repr__(self):
        return "<MergeConflict %r>" % self.possibilities

class ValueNotKnownError(Exception):
    def __init__(self, slot):
        Exception.__init__(self, 'Slot %r value not known' % slot)
        self.slot = slot

class ValueAmbiguousError(ValueNotKnownError):
    def __init__(self, slot, conflict):
        Exception.__init__(self, 'Slot %r value is ambiguous' % slot)
        self.slot = sloft
        self.conflict = conflict

class NoActiveRoot(Exception):
    pass
<commit_msg>Correct typo of "slot" as "sloft".<commit_after>
import collections

__all__ = [
    "MergeConflict",
    "ValueNotKnownError",
    "ValueAmbiguousError",
    "root",
]

class State(object):

    def __init__(self, root, parent=None, owner=None):
        self.root = root
        self.parent = parent
        self.slot_values = {}
        self.slot_positions = collections.defaultdict(lambda: set())
        self.owner = owner

    def merge_children(self, children, or_none=False):
        pass

    def _create_child(self, owner=None):
        return State(self.root, self, owner)

    def _child_context(self, owner, auto_merge):
        previous = self.root.current_state
        new = self._create_child(owner)
        class Context(object):
            def __enter__(context):
                self.root.current_state = new
                return new
            def __exit__(context, exc_type, exc_value, traceback):
                if auto_merge and exc_type is None:
                    self.merge_children([new])
                self.root.current_state = previous
        return Context()

    def fork(self, owner=None):
        return self._child_context(owner, auto_merge=False)

    def transaction(self, owner=None):
        return self._child_context(owner, auto_merge=True)

    def set_slot(self, slot, value, position=None):
        self.slot_values[slot] = value
        self.slot_positions[slot] = set(
            [position] if position is not None else []
        )

    def get_slot_value(self, slot):
        current = self
        while current is not None:
            try:
                return current.slot_values[slot]
            except KeyError:
                current = self.parent
            return Slot.NOT_KNOWN

class Slot(object):
    # we will compare by reference to this thing to detect the "don't know"
    # case.
    NOT_KNOWN = type("not_known", (object,), {
        "__repr__": lambda self: "datafork.Slot.NOT_KNOWN"
    })()

    def __init__(
        self,
        root,
        owner=None,
        initial_value=NOT_KNOWN,
    ):
        self.owner = owner
        self.root = root
        self.set_value(
            initial_value,
        )

    @property
    def value(self):
        try:
            return Slot.prepare_return_value(
                self,
                self.final_value,
            )
        except AttributeError:
            return Slot.prepare_return_value(
                self,
                self.root.current_state.get_slot_value(self),
            )

    @property
    def positions(self):
        try:
            return self.final_positions
        except AttributeError:
            return self.root.current_state.get_slot_positions(self)

    @value.setter
    def value(self, value):
        self.set_value(value)

    def set_value(self, value, position=None):
        if hasattr(self, "final_value"):
            # should never happen
            raise Exception(
                "Can't set value on slot %r: it has been finalized" % self,
            )
        else:
            self.root.current_state.set_slot(
                self,
                value,
                position=position,
            )

    def set_value_not_known(self, position=None):
        self.set_value(Slot.NOT_KNOWN, position=position)

    @property
    def value_is_known(self):
        try:
            value = self.value
        except ValueNotKnownError:
            return False
        else:
            return True

    @classmethod
    def prepare_return_value(cls, slot, value):
        if type(value) is MergeConflict:
            raise ValueAmbiguousError(slot, value)
        elif value is cls.NOT_KNOWN:
            raise ValueNotKnownError(slot)
        else:
            return value

class Root(State):
    def __init__(self, root_owner=None, slot_type=Slot):
        State.__init__(self, self, None, root_owner)
        self.current_state = self
        self.slot_type = slot_type
        self.slots = set()

    def slot(
        self,
        owner=None,
        initial_value=Slot.NOT_KNOWN,
        initial_position=None,
    ):
        """
        Creates a new slot in this root.
        """
        slot = self.slot_type(self, owner, initial_value, initial_position)
        self.slots.add(slot)
        return slot

    def slotted_object(self):
        return SlottedObject(self)

    def slotted_mapping(self):
        return SlottedMapping(self)

    def slotted_sequence(self):
        return SlottedSequence(self)

    def slotted_set(self):
        return SlottedSet(self)

    def finalize_data(self):
        for slot in self.slots:
            slot.final_value = slot.value
            slot.final_position = slot.position
            # sever the connection from the slot to the root so that
            # the root can be garbage collected after the with block exits.
            # The slot doesn't need the root anymore.
            del slot.root

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.finalize_data()

def root(owner=None):
    new = Root(root_owner=owner)
    class Context(object):
        def __enter__(self):
            return new
        def __exit__(self, exc_type, exc_value, traceback):
            new.finalize_data()
    return Context()

class MergeConflict(object):

    def __init__(self, possibilities):
        self.possibilities = possibilities

    def __repr__(self):
        return "<MergeConflict %r>" % self.possibilities

class ValueNotKnownError(Exception):
    def __init__(self, slot):
        Exception.__init__(self, 'Slot %r value not known' % slot)
        self.slot = slot

class ValueAmbiguousError(ValueNotKnownError):
    def __init__(self, slot, conflict):
        Exception.__init__(self, 'Slot %r value is ambiguous' % slot)
        self.slot = slot
        self.conflict = conflict

class NoActiveRoot(Exception):
    pass
<|endoftext|>


<commit_before>from celery.utils.log import get_task_logger

from admin_recommendation.celery import app
from crawler.models import App, UserApps, SimilarApp, User

logger = get_task_logger('celery.tasks')

@app.task(name='recommend_to_user')
def recommend_to_user(user_id):
    logger.info("Finding user: {}".format(user_id))
    user = User.objects.get(id=user_id)
    if not user:
        logger.info("No user found")
        return

    logger.info("User found: {}".format(user))
    logger.info("Finding user apps")
    user_apps = UserApps.objects.filter(user=user).all()
    apps = set()
    if not user_apps:
        logger.info("No user apps found")
        return

    for user_app in user_apps:
        package_name = user_app.package_name
        logger.info("Finding similar to {}".format(package_name))
        similar_apps = SimilarApp.objects.filter(source_package=package_name).all()
        for similar_app in similar_apps:
            similar_app_package = similar_app.similar_package
            if not UserApps.objects.filter(package_name=similar_app_package).exists():
                app_similar = App.objects.filter(package_name=similar_app_package).first()
                if app_similar:
                    logger.info("Similar to {} found: {}".format(package_name, app_similar.package_name))
                    apps.add(app_similar)

        other_similar_apps = SimilarApp.objects.filter(similar_package=package_name).all()
        for similar_app in other_similar_apps:
            similar_app_package = similar_app.similar_package
            if not UserApps.objects.filter(package_name=similar_app_package).exists():
                app_similar = App.objects.filter(package_name=similar_app_package).first()
                if app_similar:
                    logger.info("Similar to {} found: {}".format(package_name, app_similar.package_name))
                    apps.add(app_similar)

    if len(apps) > 0:
        user.recommended_apps = list(apps)
        user.save()
<commit_msg>When recommending not removing apps that user already has installed<commit_after>from celery.utils.log import get_task_logger

from admin_recommendation.celery import app
from crawler.models import App, UserApps, SimilarApp, User

logger = get_task_logger('celery.tasks')

@app.task(name='recommend_to_user')
def recommend_to_user(user_id):
    logger.info("Finding user: {}".format(user_id))
    user = User.objects.get(id=user_id)
    if not user:
        logger.info("No user found")
        return

    logger.info("User found: {}".format(user))
    logger.info("Finding user apps")
    user_apps = UserApps.objects.filter(user=user).all()
    apps = set()
    if not user_apps:
        logger.info("No user apps found")
        return

    for user_app in user_apps:
        package_name = user_app.package_name
        logger.info("Finding similar to {}".format(package_name))
        similar_apps = SimilarApp.objects.filter(source_package=package_name).all()
        for similar_app in similar_apps:
            similar_app_package = similar_app.similar_package
            app_similar = App.objects.filter(package_name=similar_app_package).first()
            if app_similar:
                logger.info("Similar to {} found: {}".format(package_name, app_similar.package_name))
                apps.add(app_similar)

        other_similar_apps = SimilarApp.objects.filter(similar_package=package_name).all()
        for similar_app in other_similar_apps:
            similar_app_package = similar_app.similar_package
            app_similar = App.objects.filter(package_name=similar_app_package).first()
            if app_similar:
                logger.info("Similar to {} found: {}".format(package_name, app_similar.package_name))
                apps.add(app_similar)

    if len(apps) > 0:
        user.recommended_apps = list(apps)
        user.save()
<|endoftext|>


<commit_before>import logging
import os

__author__ = 'Jonas'

class BuildDocument:
    def __init__(self, name, subdir, idx, bib):
        self.subdir = subdir
        self.idx = idx + '.idx'
        self.bib = bib
        self.tex = name + '.tex'
        self.pdf = name + '.pdf'
        self.log = name + '.log'

class DocumentParser:
    def __init__(self, task):
        self._task = task

    def documents(self):
        """Return all builds from the .intexration file."""
        path = os.path.join(self._task.build_dir, '.intexration')
        documents = []
        if os.path.exists(path):
            parser = configparser.ConfigParser()
            parser.read(path)
            for build_name in parser.sections():
                if parser.has_option(build_name, 'dir'):
                    subdir = parser[build_name]['dir']
                else:
                    subdir = ''
                if parser.has_option(build_name, 'idx'):
                    idx = parser[build_name]['idx']
                else:
                    idx = build_name
                if parser.has_option(build_name, 'bib'):
                    bib = parser[build_name]['bib']
                else:
                    bib = build_name
                documents.append(BuildDocument(build_name, subdir, idx, bib))
        else:
            logging.error("No .intexration file found for %s", self._task.title())
        return documents

class Document:
    def __init__(self, name, root):
        self.name = name
        self.root = root
        self._lines = self._read_log()

    def log_file(self):
        return os.path.join(self.root, self.name + '.log')

    def pdf_file(self):
        return os.path.join(self.root, self.name + '.pdf')

    def _read_log(self):
        """Read all lines form log file"""
        path = self.log_file()
        if not os.path.exists(path):
            raise RuntimeError("The logfile does not exist")
        log_file = open(path, "r", encoding='latin-1')
        return log_file.readlines()

    def get_warnings(self):
        """Parse warnings from log file."""
        warnings = []
        multi_line_error = False
        for line in self._lines:
            if multi_line_error and line == config.LOG_NEW_LINE_CHAR:
                multi_line_error = False
            if config.LOG_WARNING_STRING in line or multi_line_error:
                warnings.append(line)
                multi_line_error = True
        return warnings

    def get_errors(self):
        """Parse errors from logfile."""
        errors = []
        multi_line_error = False
        for line in self._lines:
            if multi_line_error and line == config.LOG_NEW_LINE_CHAR:
                multi_line_error = False
            if line.startswith(config.LOG_ERROR_STRING) or multi_line_error:
                errors.append(line.replace(config.LOG_ERROR_STRING, ""))
                multi_line_error = True
        return errors

    def get_log(self):
        return self._lines

class DocumentExplorer:
    def __init__(self, root):
        self.root = root

    def all_owners(self):
        return os.listdir(self.root)

    def all_repos(self, owner):
        return os.listdir(os.path.join(self.root, owner))

    def all_names(self, owner, repo):
        names = []
        for file in os.listdir(os.path.join(self.root, owner, repo)):
            if file.endswith('.log'):
                names.append(file)
        return names

    def all_documents(self):
        documents = []
        for owner in self.all_owners():
            for repo in self.all_repos(owner):
                for name in self.all_names(owner, repo):
                    path = os.path.join(self.root, owner, repo)
                    documents.append(Document(name, path))
        return documents

<commit_msg>Debug statement added.<commit_after>import logging
import os

__author__ = 'Jonas'

class BuildDocument:
    def __init__(self, name, subdir, idx, bib):
        self.subdir = subdir
        self.idx = idx + '.idx'
        self.bib = bib
        self.tex = name + '.tex'
        self.pdf = name + '.pdf'
        self.log = name + '.log'

class DocumentParser:
    def __init__(self, task):
        self._task = task

    def documents(self):
        """Return all builds from the .intexration file."""
        path = os.path.join(self._task.build_dir, '.intexration')
        documents = []
        if os.path.exists(path):
            parser = configparser.ConfigParser()
            parser.read(path)
            for build_name in parser.sections():
                if parser.has_option(build_name, 'dir'):
                    subdir = parser[build_name]['dir']
                else:
                    subdir = ''
                if parser.has_option(build_name, 'idx'):
                    idx = parser[build_name]['idx']
                else:
                    idx = build_name
                if parser.has_option(build_name, 'bib'):
                    bib = parser[build_name]['bib']
                else:
                    bib = build_name
                documents.append(BuildDocument(build_name, subdir, idx, bib))
        else:
            logging.error("No .intexration file found for %s", self._task.title())
        return documents

class Document:
    def __init__(self, name, root):
        self.name = name
        self.root = root
        self._lines = self._read_log()

    def log_file(self):
        return os.path.join(self.root, self.name + '.log')

    def pdf_file(self):
        return os.path.join(self.root, self.name + '.pdf')

    def _read_log(self):
        """Read all lines form log file"""
        path = self.log_file()
        if not os.path.exists(path):
            raise RuntimeError("The logfile does not exist")
        log_file = open(path, "r", encoding='latin-1')
        return log_file.readlines()

    def get_warnings(self):
        """Parse warnings from log file."""
        warnings = []
        multi_line_error = False
        for line in self._lines:
            if multi_line_error and line == config.LOG_NEW_LINE_CHAR:
                multi_line_error = False
            if config.LOG_WARNING_STRING in line or multi_line_error:
                warnings.append(line)
                multi_line_error = True
        return warnings

    def get_errors(self):
        """Parse errors from logfile."""
        errors = []
        multi_line_error = False
        for line in self._lines:
            if multi_line_error and line == config.LOG_NEW_LINE_CHAR:
                multi_line_error = False
            if line.startswith(config.LOG_ERROR_STRING) or multi_line_error:
                errors.append(line.replace(config.LOG_ERROR_STRING, ""))
                multi_line_error = True
        return errors

    def get_log(self):
        return self._lines

class DocumentExplorer:
    def __init__(self, root):
        self.root = root

    def all_owners(self):
        return os.listdir(self.root)

    def all_repos(self, owner):
        return os.listdir(os.path.join(self.root, owner))

    def all_names(self, owner, repo):
        names = []
        for file in os.listdir(os.path.join(self.root, owner, repo)):
            if file.endswith('.log'):
                names.append(file)
        return names

    def all_documents(self):
        documents = []
        for owner in self.all_owners():
            for repo in self.all_repos(owner):
                for name in self.all_names(owner, repo):
                    path = os.path.join(self.root, owner, repo)
                    logging.debug(path)
                    documents.append(Document(name, path))
        return documents

<|endoftext|>


<commit_before>"""
========================================
Regression on continuous data (rER[P/F])
========================================

This demonstrates how rERPs/regressing the continuous data is a
generalisation of traditional averaging. If all preprocessing steps
are the same and if no overlap between epochs exists and if all
predictors are binary, regression is virtually identical to traditional
averaging.
If overlap exists and/or predictors are continuous, traditional averaging
is inapplicable, but regression can estimate, including those of
continuous predictors.

References
----------
See Smith, N. J., & Kutas, M. (2015). Regression-based estimation of ERP
waveforms: II. Non-linear effects, overlap correction, and practical
considerations. Psychophysiology, 52(2), 169-189.

Authors: Jona Sassenhagen <jona.sassenhagen@gmail.de>
License: BSD (3-clause)
"""

import numpy as np
import matplotlib.pyplot as plt

import mne
from mne.datasets import spm_face
from mne.stats.regression import linear_regression_raw

mne.set_log_level(False)

# Preprocess data
data_path = spm_face.data_path()
# Load and filter data, set up epochs
raw_fname = data_path + '/MEG/spm/SPM_CTF_MEG_example_faces1_3D_raw.fif'

raw = mne.io.Raw(raw_fname % 1, preload=True)  # Take first run

picks = mne.pick_types(raw.info, meg=True, exclude='bads')
raw.filter(1, 45, method='iir')

events = mne.find_events(raw, stim_channel='UPPT001')
event_id = dict(faces=1, scrambled=2)
tmin, tmax = -.1, .5

raw.pick_types(meg=True)

# regular epoching
epochs = mne.Epochs(raw, events, event_id, tmin, tmax, reject=None,
                    baseline=None, preload=True, verbose=False)

# rERF
evoked_dict = linear_regression_raw(raw, events=events, event_id=event_id,
                                    reject=False, tmin=tmin, tmax=tmax)

# plot both results
cond = "faces"
plt.title("Traditional ERF")
epochs[cond].average().plot()
plt.title("rERF")
evoked_dict[cond].plot()  # linear_regression_raw returns a dict of evokeds
<commit_msg>Update plot_regress_continuous.py<commit_after>"""
========================================
Regression on continuous data (rER[P/F])
========================================

This demonstrates how rERPs/regressing the continuous data is a
generalisation of traditional averaging. If all preprocessing steps
are the same and if no overlap between epochs exists and if all
predictors are binary, regression is virtually identical to traditional
averaging.
If overlap exists and/or predictors are continuous, traditional averaging
is inapplicable, but regression can estimate, including those of
continuous predictors.

References
----------
See Smith, N. J., & Kutas, M. (2015). Regression-based estimation of ERP
waveforms: II. Non-linear effects, overlap correction, and practical
considerations. Psychophysiology, 52(2), 169-189.

Authors: Jona Sassenhagen <jona.sassenhagen@gmail.de>
License: BSD (3-clause)
"""

import matplotlib.pyplot as plt

import mne
from mne.datasets import spm_face
from mne.stats.regression import linear_regression_raw

mne.set_log_level(False)

# Preprocess data
data_path = spm_face.data_path()
# Load and filter data, set up epochs
raw_fname = data_path + '/MEG/spm/SPM_CTF_MEG_example_faces1_3D_raw.fif'

raw = mne.io.Raw(raw_fname % 1, preload=True)  # Take first run

picks = mne.pick_types(raw.info, meg=True, exclude='bads')
raw.filter(1, 45, method='iir')

events = mne.find_events(raw, stim_channel='UPPT001')
event_id = dict(faces=1, scrambled=2)
tmin, tmax = -.1, .5

raw.pick_types(meg=True)

# regular epoching
epochs = mne.Epochs(raw, events, event_id, tmin, tmax, reject=None,
                    baseline=None, preload=True, verbose=False)

# rERF
evoked_dict = linear_regression_raw(raw, events=events, event_id=event_id,
                                    reject=False, tmin=tmin, tmax=tmax)

# plot both results
cond = "faces"
plt.title("Traditional ERF")
epochs[cond].average().plot()
plt.title("rERF")
evoked_dict[cond].plot()  # linear_regression_raw returns a dict of evokeds
<|endoftext|>


<commit_before>"""
-------------------------------------------------------------------------------
Student Name: Hannah Richstein
Professor Name: Dr. Frinchaboy
Class: PHYS 50733
HW 1, Problem 2
Last edited: 29 Jan 2017

Overview:
---------
This program asks the user for an orbital period in minutes and calculates the
altitude a satellite must have in meters to remain in orbit around the Earth.

Input:
------
Period, in minutes.

Output:
-------
Altitude, in meters. Scientific notation with three significant figures.

Program Limitations:
--------------------
N/A

Significant Program Variables:
------------------------------
T: float-like
    The period of the orbiting satellite in minutes

h: float-like
    The required altitude in meters of a satellite orbiting Earth with period T

"""

from __future__ import division, absolute_import
from numpy import pi

#Setting constants
G = 6.67e-11     #m**3 kg**-1 s**-2, Newton's gravitational constant
M = 5.97e24      #kg, Mass of the Earth
R = 6371 * 10**3 #m, Radius of the Earth

#Asking for user input
T = input("Please enter the desired period, T, in minutes: ")
T *= 60

#Calculating the desired value
h = ( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) - R

#Printing the value to the screen
print("The required altitude h, in meters is {0:1.2e}".format(h))

#END PROGRAM

"""
2a)

For a planet or satellite orbiting a central body of mass M,

(T / (2*pi) )**2 = a**3 / (G * M)

where T is period in seconds, a is the radius of a circular orbit, and G is
Newton's gravitational constant.

(From this equation, we see Kepler's third law of planetary motion, that the
square of the orbital period is proportional to the cube of the semi-major
axis.)

We split a, the full radius, to be h + R, where R is the radius of the Earth
and h is the altitude the satellite must be above the Earth's surface.

(T / (2*pi) )**2 = (R + h)**3 / (G*M)

We then solve for h.

T**2 / (4*(pi**2)) * (G*M) = (R + h)**3

G*M*(T**2) / (4*(pi**2)) = (R + h)**3

( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) = R + h

( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) - R = h

h = ( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) - R

which matches equation (1) on the assignment sheet.

2c)

T = 24*60 min (1 day),  h =  3.59e+07
T = 90 min,             h =  2.79e+05
T = 45 min,             h = -2.18+06

From the last of these calculations, we can conclude that there cannot be a
satellite which orbits the Earth every 45 minutes.

2d)

The sidereal day is how long it takes for the Earth to make a full rotation on
its axis relative to very distant, or "fixed," stars.  It takes into account
Earth's rotation as well as its progress in its orbit around the Sun.

T = 23.93 * 60 min (1 sidereal day), h = 3.58e+07

This made a difference in required altitude of about 0.01e+07, or 1.00e+05 m.
A geosynchronous satellite requires an altitude of about 100,000 meters less
than one which has a period of 24 hours, a solar day.
"""<commit_msg>The linter doesn't appreciate the fact that I'm trying to keep my parentheses under control.<commit_after>"""
-------------------------------------------------------------------------------
Student Name: Hannah Richstein
Professor Name: Dr. Frinchaboy
Class: PHYS 50733
HW 1, Problem 2
Last edited: 29 Jan 2017

Overview:
---------
This program asks the user for an orbital period in minutes and calculates the
altitude a satellite must have in meters to remain in orbit around the Earth.

Input:
------
Period, in minutes.

Output:
-------
Altitude, in meters. Scientific notation with three significant figures.

Program Limitations:
--------------------
N/A

Significant Program Variables:
------------------------------
T: float-like
    The period of the orbiting satellite in minutes

h: float-like
    The required altitude in meters of a satellite orbiting Earth with period T

"""

from __future__ import division, absolute_import
from numpy import pi

# Setting constants
G = 6.67e-11      # m**3 kg**-1 s**-2, Newton's gravitational constant
M = 5.97e24       # kg, Mass of the Earth
R = 6371 * 10**3  # m, Radius of the Earth

# Asking for user input
T  = input("Please enter the desired period, T, in minutes: ")
T *= 60

# Calculating the desired value
h  = ( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) - R

# Printing the value to the screen
print("The required altitude h, in meters is {0:1.2e}".format(h))

# END PROGRAM

"""
2a)

For a planet or satellite orbiting a central body of mass M,

(T / (2*pi) )**2 = a**3 / (G * M)

where T is period in seconds, a is the radius of a circular orbit, and G is
Newton's gravitational constant.

(From this equation, we see Kepler's third law of planetary motion, that the
square of the orbital period is proportional to the cube of the semi-major
axis.)

We split a, the full radius, to be h + R, where R is the radius of the Earth
and h is the altitude the satellite must be above the Earth's surface.

(T / (2*pi) )**2 = (R + h)**3 / (G*M)

We then solve for h.

T**2 / (4*(pi**2)) * (G*M) = (R + h)**3

G*M*(T**2) / (4*(pi**2)) = (R + h)**3

( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) = R + h

( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) - R = h

h = ( (G*M*(T**2)) / (4*(pi**2)) )**(1/3) - R

which matches equation (1) on the assignment sheet.

2c)

T = 24*60 min (1 day),  h =  3.59e+07
T = 90 min,             h =  2.79e+05
T = 45 min,             h = -2.18+06

From the last of these calculations, we can conclude that there cannot be a
satellite which orbits the Earth every 45 minutes.

2d)

The sidereal day is how long it takes for the Earth to make a full rotation on
its axis relative to very distant, or "fixed," stars.  It takes into account
Earth's rotation as well as its progress in its orbit around the Sun.

T = 23.93 * 60 min (1 sidereal day), h = 3.58e+07

This made a difference in required altitude of about 0.01e+07, or 1.00e+05 m.
A geosynchronous satellite requires an altitude of about 100,000 meters less
than one which has a period of 24 hours, a solar day.
"""
<|endoftext|>