<commit_before>import numpy as np

""" Diagnostics.py

A collection of utility functions for diagnostics on FMRI data

See test_* functions in this directory for nose tests
"""

def vol_std(data):
    """ Return standard deviation across voxels for 4D array `data`

    Parameters
    ----------
    data : 4D array
        4D array from FMRI run with last axis indexing volumes.  Call the shape
        of this array (M, N, P, T) where T is the number of volumes.

    Returns
    -------
    std_values : array shape (T,)
        One dimensonal array where ``std_values[i]`` gives the standard
        deviation of all voxels contained in ``data[..., i]``.
    """
    vol_1d = [np.std(np.ravel(data[..., i])) for i in range(data.shape[-1])]
    return(vol_1d)

def iqr_outliers(arr_1d, iqr_scale=1.5):
    """ Return indices of outliers identified by interquartile range

    Parameters
    ----------
    arr_1d : 1D array
        One-dimensional numpy array, from which we will identify outlier
        values.
    iqr_scale : float, optional
        Scaling for IQR to set low and high thresholds.  Low threshold is given
        by 25th centile value minus ``iqr_scale * IQR``, and high threshold id
        given by 75 centile value plus ``iqr_scale * IQR``.

    Returns
    -------
    outlier_indices : array
        Array containing indices in `arr_1d` that contain outlier values.
    lo_hi_thresh : tuple
        Tuple containing 2 values (low threshold, high thresold) as described
        above.
    """
    IQR = np.percentile(arr_1d, [75,25])[0] - np.percentile(arr_1d, [75,25])[1]
    max = np.percentile(arr_1d, [75,25])[0]+(IQR*iqr_scale)
    min = np.percentile(arr_1d, [75,25])[1]-(IQR*iqr_scale)
    bin = [0 if arr_1d[i] <= max and arr_1d[i] >= min else 1 for i in range(len(arr_1d))]
    return np.nonzero(bin)[0], (min,max)

def vol_rms_diff(arr_4d):
    """ Return root mean square of differences between sequential volumes

    Parameters
    ----------
    data : 4D array
        4D array from FMRI run with last axis indexing volumes.  Call the shape
        of this array (M, N, P, T) where T is the number of volumes.

    Returns
    -------
    rms_values : array shape (T-1,)
        One dimensonal array where ``rms_values[i]`` gives the square root of
        the mean (across voxels) of the squared difference between volume i and
        volume i + 1.
    """
    rms_values = np.array([np.sqrt(np.mean((arr_4d[..., i + 1] - arr_4d[..., i]) ** 2)) for i in range(arr_4d.shape[-1]-1) ])
    return rms_values

def extend_diff_outliers(diff_indices):
    """ Extend difference-based outlier indices `diff_indices` by pairing

    Parameters
    ----------
    diff_indices : array
        Array of indices of differences that have been detected as outliers.  A
        difference index of ``i`` refers to the difference between volume ``i``
        and volume ``i + 1``.

    Returns
    -------
    extended_indices : array
        Array where each index ``j`` in `diff_indices has been replaced by two
        indices, ``j`` and ``j+1``, unless ``j+1`` is present in
        ``diff_indices``.  For example, if the input was ``[3, 7, 8, 12, 20]``,
        ``[3, 4, 7, 8, 9, 12, 13, 20, 21]``.
    """
    noel = zip(diff_indices, diff_indices + 1)
    return np.unique(np.array(noel).ravel())
<commit_msg>Fixing the failing build<commit_after>""" Diagnostics.py

A collection of utility functions for diagnostics on FMRI data

See test_* functions in this directory for nose tests
"""

import numpy as np

def vol_std(data):
    """ Return standard deviation across voxels for 4D array `data`

    Parameters
    ----------
    data : 4D array
        4D array from FMRI run with last axis indexing volumes.  Call the shape
        of this array (M, N, P, T) where T is the number of volumes.

    Returns
    -------
    std_values : array shape (T,)
        One dimensonal array where ``std_values[i]`` gives the standard
        deviation of all voxels contained in ``data[..., i]``.
    """
    T = data.shape[-1]
    data_2d = np.reshape(data, (-1, T))
    return np.std(data_2d, axis=0)

def iqr_outliers(arr_1d, iqr_scale=1.5):
    """ Return indices of outliers identified by interquartile range

    Parameters
    ----------
    arr_1d : 1D array
        One-dimensional numpy array, from which we will identify outlier
        values.
    iqr_scale : float, optional
        Scaling for IQR to set low and high thresholds.  Low threshold is given
        by 25th centile value minus ``iqr_scale * IQR``, and high threshold id
        given by 75 centile value plus ``iqr_scale * IQR``.

    Returns
    -------
    outlier_indices : array
        Array containing indices in `arr_1d` that contain outlier values.
    lo_hi_thresh : tuple
        Tuple containing 2 values (low threshold, high thresold) as described
        above.
    """
    # Hint : np.lookfor('centile')
    # Hint : np.lookfor('nonzero')
    pct_25, pct_75 = np.percentile(arr_1d, [25, 75])
    iqr = pct_75 - pct_25
    lo_thresh = pct_25 - iqr * iqr_scale
    hi_thresh = pct_75 + iqr * iqr_scale
    is_outlier = (arr_1d < lo_thresh) | (arr_1d > hi_thresh)
    return np.nonzero(is_outlier)[0], (lo_thresh, hi_thresh)

def vol_rms_diff(arr_4d):
    """ Return root mean square of differences between sequential volumes

    Parameters
    ----------
    data : 4D array
        4D array from FMRI run with last axis indexing volumes.  Call the shape
        of this array (M, N, P, T) where T is the number of volumes.

    Returns
    -------
    rms_values : array shape (T-1,)
        One dimensonal array where ``rms_values[i]`` gives the square root of
        the mean (across voxels) of the squared difference between volume i and
        volume i + 1.
    """
    T = arr_4d.shape[-1]
    diff_data = np.diff(arr_4d, axis=-1)
    diff_data_2d = np.reshape(diff_data, (-1, T-1))
    return np.sqrt(np.mean(diff_data_2d ** 2, axis=0))

def extend_diff_outliers(diff_indices):
    """ Extend difference-based outlier indices `diff_indices` by pairing

    Parameters
    ----------
    diff_indices : array
        Array of indices of differences that have been detected as outliers.  A
        difference index of ``i`` refers to the difference between volume ``i``
        and volume ``i + 1``.

    Returns
    -------
    extended_indices : array
        Array where each index ``j`` in `diff_indices has been replaced by two
        indices, ``j`` and ``j+1``, unless ``j+1`` is present in
        ``diff_indices``.  For example, if the input was ``[3, 7, 8, 12, 20]``,
        ``[3, 4, 7, 8, 9, 12, 13, 20, 21]``.
    """
    # Make two columns with the second being the first plus 1
    ext_outliers = np.column_stack((diff_indices, diff_indices + 1))
    # Use numpy reshape ordering to get the values from the first row, then the
    # second row etc...
    return np.unique(ext_outliers.ravel())
<|endoftext|>


<commit_before>from cssdbpy import Connection
from SSDB import SSDB
from time import time

def cssdbpy_test():
	conn = Connection('192.168.0.103', 8888)
	start_time = time()
	map(lambda x: conn.execute('hset','cssdbpy',str(i), 1), [i for i in xrange(10000,90000)])
	print 'hset', time()-start_time
	start_time = time()
	conn.execute('hscan','cssdbpy','','', -1)
	print 'hscan', time()-start_time
	for action in ['hget', 'hdel']:
		start_time = time()
		map(lambda x: conn.execute(action,'cssdbpy',str(i)), [i for i in xrange(10000,90000)])
		print action, time()-start_time

def SSDB_test():
	conn = SSDB('192.168.0.103', 8888)
	start_time = time()
	map(lambda x: conn.request('hset',['cssdbpy',str(i), 1]), [i for i in xrange(10000,90000)])
	print 'hset', time()-start_time
	start_time = time()
	conn.request('hscan',['cssdbpy','','', -1])
	print 'hscan', time()-start_time
	for action in ['hget', 'hdel']:
		start_time = time()
		map(lambda x: conn.request(action,['cssdbpy',str(i)]), [i for i in xrange(10000,90000)])
		print action, time()-start_time

if __name__ == '__main__':
	SSDB_test()
	cssdbpy_test()<commit_msg>#160906 IP SSDB on localhost<commit_after>from cssdbpy import Connection
from SSDB import SSDB
from time import time

def cssdbpy_test():
	conn = Connection('127.0.0.1', 8888)
	start_time = time()
	map(lambda x: conn.execute('hset','cssdbpy',str(i), 1), [i for i in xrange(10000,90000)])
	print 'hset', time()-start_time
	start_time = time()
	conn.execute('hscan','cssdbpy','','', -1)
	print 'hscan', time()-start_time
	for action in ['hget', 'hdel']:
		start_time = time()
		map(lambda x: conn.execute(action,'cssdbpy',str(i)), [i for i in xrange(10000,90000)])
		print action, time()-start_time

def SSDB_test():
	conn = SSDB('127.0.0.1', 8888)
	start_time = time()
	map(lambda x: conn.request('hset',['cssdbpy',str(i), 1]), [i for i in xrange(10000,90000)])
	print 'hset', time()-start_time
	start_time = time()
	conn.request('hscan',['cssdbpy','','', -1])
	print 'hscan', time()-start_time
	for action in ['hget', 'hdel']:
		start_time = time()
		map(lambda x: conn.request(action,['cssdbpy',str(i)]), [i for i in xrange(10000,90000)])
		print action, time()-start_time

if __name__ == '__main__':
	SSDB_test()
	cssdbpy_test()<|endoftext|>


<commit_before># coding: utf-8

import json
import requests

class VkAudio(object):

	def __init__(self, token):
		self.session = requests.session()
		self.session.params = {'access_token': token, 'v': '5.52',}

	def _get(self, url, *args, **kwargs):
		return self.session.get(url, *args, **kwargs)

	def get_by_url(self, url):
		name = url.split("/")[-1]
		user_res = self._get("https://api.vk.com/method/users.get",
		                     params={"user_ids": name, "fields": "id",}).json()
		print(user_res)
		if 'error' not in user_res:
			return self.get_user_audio(user_res['response'][0]['id'])

		group_res = self._get("https://api.vk.com/method/groups.getById",
		                     params={"group_ids": name, "fields": "id",}).json()
		if 'error' not in group_res:
			return self.get_group_audio(group_res['response'][0]['id'])
		return []

	def get_user_audio(self, user_id):
		response = self._get("https://api.vk.com/method/audio.get",
		                     params={"owner_id": user_id,
		                             "need_user": 0,})
		result = []
		try:
			data = response.json()['response']['items']
		except:
			return result
		for song in data:
			result.append({'artist': song['artist'],
			               'title': song['title'],
			               'url': song['url'],})
		return result

	def get_group_audio(self, user_id):
		response = self._get("https://api.vk.com/method/audio.get",
		                     params={"owner_id": "-" + str(user_id),
		                             "need_user": 0,})
		result = []
		try:
			data = response.json()['response']['items']
		except:
			return result
		for song in data:
			result.append({'artist': song['artist'],
			               'title': song['title'],
			               'url': song['url'],})
		return result

<commit_msg>Update vkaudio.py<commit_after># coding: utf-8

import json
import requests

class VkAudio(object):

	def __init__(self, token):
		self.session = requests.session()
		self.session.params = {'access_token': token, 'v': '5.52',}

	def _get(self, url, *args, **kwargs):
		return self.session.get(url, *args, **kwargs)

	def get_by_url(self, url):
		name = url.split("/")[-1]
		user_res = self._get("https://api.vk.com/method/users.get",
		                     params={"user_ids": name, "fields": "id",}).json()
		if 'error' not in user_res:
			return self.get_user_audio(user_res['response'][0]['id'])

		group_res = self._get("https://api.vk.com/method/groups.getById",
		                     params={"group_ids": name, "fields": "id",}).json()
		if 'error' not in group_res:
			return self.get_group_audio(group_res['response'][0]['id'])
		return []

	def get_user_audio(self, user_id):
		response = self._get("https://api.vk.com/method/audio.get",
		                     params={"owner_id": user_id,
		                             "need_user": 0,})
		result = []
		try:
			data = response.json()['response']['items']
		except:
			return result
		for song in data:
			result.append({'artist': song['artist'],
			               'title': song['title'],
			               'url': song['url'],})
		return result

	def get_group_audio(self, user_id):
		response = self._get("https://api.vk.com/method/audio.get",
		                     params={"owner_id": "-" + str(user_id),
		                             "need_user": 0,})
		result = []
		try:
			data = response.json()['response']['items']
		except:
			return result
		for song in data:
			result.append({'artist': song['artist'],
			               'title': song['title'],
			               'url': song['url'],})
		return result

<|endoftext|>


<commit_before>
<commit_msg>Force right matplotlib backend<commit_after>import matplotlib
matplotlib.use('Agg')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import sys
import numpy
from optparse import OptionParser

global output_file_contents
global num_cores

def rowSearch(heading, key):
   global output_file_contents
   global num_cores

   key += "(.*)"
   heading_found = False

   for line in output_file_contents:
      if heading_found:
         match_key = re.search(key, line)
         if match_key:
            counts = line.split('|')
            # print len(counts)
            event_counts = counts[1:num_cores+1]
            for i in range(0, num_cores):
               if (len(event_counts[i].split()) == 0):
                  event_counts[i] = "0.0"
               #elif (event_counts[i].split()[0] == "nan"):
               #   event_counts[i] = "0.0"
            # print event_counts
            # print "%d" % len(event_counts)
            return map(lambda x: float(x), event_counts)
      else:
         if (re.search(heading, line)):
            heading_found = True

   print "ERROR: Could not find key [%s,%s]" % (heading, key)
   sys.exit(1)

def getTime(key):
   global output_file_contents

   key += "\s+([0-9]+)\s*"
   for line in output_file_contents:
      match_key = re.search(key, line)
      if match_key:
         return float(match_key.group(1))
   print "ERROR: Could not find key [%s]" % (key)
   sys.exit(2)

parser = OptionParser()
parser.add_option("--input-file", dest="input_file", help="Graphite Input File")
parser.add_option("--stats-file", dest="stats_file", help="Statistics File")
parser.add_option("--num-cores", dest="num_cores", type="int", help="Number of Cores")
(options,args) = parser.parse_args()

# Read Results Files
try:
   output_file_contents = open(options.input_file, 'r').readlines()
except IOError:
   print "ERROR: Could not open file (%s)" % (options.input_file)
   sys.exit(3)

print "Parsing simulation output file: %s" % (options.input_file)
num_cores = options.num_cores

# Total Instructions
target_instructions = sum(rowSearch("Core Summary", "Total Instructions"))

# Completion Time - In nanoseconds
target_time = rowSearch("Core Summary", "Completion Time \(in nanoseconds\)")[0]

# Host Time
host_time = getTime("Shutdown Time \(in microseconds\)")
host_initialization_time = getTime("Start Time \(in microseconds\)")
host_working_time = getTime("Stop Time \(in microseconds\)") - getTime("Start Time \(in microseconds\)")
host_shutdown_time = getTime("Shutdown Time \(in microseconds\)") - getTime("Stop Time \(in microseconds\)")

# Write event counters to a file
stats_file = open(options.stats_file, 'w')
stats_file.write("Target-Instructions = %f\n" % (target_instructions))
stats_file.write("Target-Time = %f\n" % (target_time))
stats_file.write("Host-Time = %f\n" % (host_time))
stats_file.write("Host-Initialization-Time = %f\n" % (host_initialization_time))
stats_file.write("Host-Working-Time = %f\n" % (host_working_time))
stats_file.write("Host-Shutdown-Time = %f\n" % (host_shutdown_time))
stats_file.close()

print "Written stats file: %s" % (options.stats_file)
<commit_msg>[tools] Added more capability to tools/parse_output.py to get energy<commit_after>#!/usr/bin/env python

import re
import sys
import numpy
from optparse import OptionParser

global output_file_contents
global num_cores

def searchKey(key, line):
   global num_cores
   key += "(.*)"
   match_key = re.search(key, line)
   if match_key:
      counts = line.split('|')
      event_counts = counts[1:num_cores+1]
      for i in range(0, num_cores):
         if (len(event_counts[i].split()) == 0):
            event_counts[i] = "0.0"
      return map(lambda x: float(x), event_counts)
   return None

def rowSearch1(heading, key):
   global output_file_contents
   heading_found = False
   for line in output_file_contents:
      if heading_found:
         value = searchKey(key, line)
         if value:
            return value
      else:
         heading_found = (re.search(heading, line) != None)

   print "ERROR: Could not find key [%s,%s]" % (heading, key)
   sys.exit(1)

def rowSearch2(heading, sub_heading, key):
   global output_file_contents
   heading_found = False
   sub_heading_found = False
   for line in output_file_contents:
      if heading_found:
         if sub_heading_found:
            value = searchKey(key, line)
            if value:
               return value
         else:
            sub_heading_found = (re.search(sub_heading, line) != None)
      else:
         heading_found = (re.search(heading, line) != None)

   print "ERROR: Could not find key [%s,%s,%s]" % (heading, sub_heading, key)
   sys.exit(1)

def getTime(key):
   global output_file_contents

   key += "\s+([0-9]+)\s*"
   for line in output_file_contents:
      match_key = re.search(key, line)
      if match_key:
         return float(match_key.group(1))
   print "ERROR: Could not find key [%s]" % (key)
   sys.exit(2)

parser = OptionParser()
parser.add_option("--results-dir", dest="results_dir", help="Graphite Results Directory")
parser.add_option("--num-cores", dest="num_cores", type="int", help="Number of Cores")
(options,args) = parser.parse_args()

# Read Results Files
try:
   output_file_contents = open("%s/sim.out" % (options.results_dir), 'r').readlines()
except IOError:
   print "ERROR: Could not open file (%s/sim.out)" % (options.results_dir)
   sys.exit(3)

print "Parsing simulation output file: %s/sim.out" % (options.results_dir)
num_cores = options.num_cores

# Total Instructions
target_instructions = sum(rowSearch1("Core Summary", "Total Instructions"))

# Completion Time - In nanoseconds
target_time = max(rowSearch1("Core Summary", "Completion Time \(in nanoseconds\)"))
# Energy - In joules
core_energy = sum(rowSearch2("Tile Energy Monitor Summary", "Core", "Total Energy \(in J\)"))
cache_hierarchy_energy = sum(rowSearch2("Tile Energy Monitor Summary", "Cache Hierarchy \(L1-I, L1-D, L2\)", "Total Energy \(in J\)"))
networks_energy = sum(rowSearch2("Tile Energy Monitor Summary", "Networks \(User, Memory\)", "Total Energy \(in J\)"))
target_energy = core_energy + cache_hierarchy_energy + networks_energy

# Host Time
host_time = getTime("Shutdown Time \(in microseconds\)")
host_initialization_time = getTime("Start Time \(in microseconds\)")
host_working_time = getTime("Stop Time \(in microseconds\)") - getTime("Start Time \(in microseconds\)")
host_shutdown_time = getTime("Shutdown Time \(in microseconds\)") - getTime("Stop Time \(in microseconds\)")

# Write event counters to a file
stats_file = open("%s/stats.out" % (options.results_dir), 'w')

stats_file.write("Target-Instructions = %f\n" % (target_instructions))
stats_file.write("Target-Time = %f\n" % (target_time))
stats_file.write("Target-Energy = %f\n" % (target_energy))
stats_file.write("Target-Core-Energy = %f\n" % (core_energy))
stats_file.write("Target-Cache-Hierarchy-Energy = %f\n" % (cache_hierarchy_energy))
stats_file.write("Target-Networks-Energy = %f\n" % (networks_energy))

stats_file.write("Host-Time = %f\n" % (host_time))
stats_file.write("Host-Initialization-Time = %f\n" % (host_initialization_time))
stats_file.write("Host-Working-Time = %f\n" % (host_working_time))
stats_file.write("Host-Shutdown-Time = %f\n" % (host_shutdown_time))
stats_file.close()

print "Written stats file: %s/stats.out" % (options.results_dir)
<|endoftext|>


<commit_before>
"""
Set up the plot figures, axes, and items to be done for each frame.

This module is imported by the plotting routines and then the
function setplot is called to set the plot parameters.

"""

import os

import numpy as np
import matplotlib.pyplot as plt
import datetime

from clawpack.geoclaw.util import fetch_noaa_tide_data
from clawpack.visclaw import colormaps, gaugetools
import clawpack.clawutil.data
import clawpack.amrclaw.data
import clawpack.geoclaw.data

import clawpack.geoclaw.surge.plot as surge

try:
    from setplotfg import setplotfg
except:
    setplotfg = None

def setplot(plotdata):
    r"""Setplot function for surge plotting"""

    plotdata.clearfigures()  # clear any old figures,axes,items data
    plotdata.format = 'binary'

    # Load data from output
    clawdata = clawpack.clawutil.data.ClawInputData(2)
    clawdata.read(os.path.join(plotdata.outdir,'claw.data'))
    physics = clawpack.geoclaw.data.GeoClawData()
    physics.read(os.path.join(plotdata.outdir,'geoclaw.data'))
    surge_data = clawpack.geoclaw.data.SurgeData()
    surge_data.read(os.path.join(plotdata.outdir,'surge.data'))
    friction_data = clawpack.geoclaw.data.FrictionData()
    friction_data.read(os.path.join(plotdata.outdir,'friction.data'))

    # Load storm track32
    track = surge.track_data(os.path.join(plotdata.outdir,'fort.track'))

    # Calculate landfall time
    landfall_dt = datetime.datetime(2005, 8, 29, 11, 10) \
                    - datetime.datetime(2005, 1, 1, 0, 0)
    landfall = landfall_dt.days * 24.0 * 60**2 + landfall_dt.seconds

    # Set afteraxes function
    surge_afteraxes = lambda cd: surge.surge_afteraxes(cd,
                                        track, landfall, plot_direction=False)

    # Limits for plots
    regions = [{"name": "Full Domain",
                "limits": [[clawdata.lower[0], clawdata.upper[0]],
                           [clawdata.lower[1], clawdata.upper[1]]]},
               {"name": "New Orleans - Zoom",
                "limits": [[-91.0, -88.5], [28.5, 30.5]]},
               {"name": "New Orleans - Region",
                "limits": [[-91, -87], [28.5, 32.0]]}]

    full_xlimits = regions[0]['limits'][0]
    full_ylimits = regions[0]['limits'][1]

    # Color limits
    surface_limits = [physics.sea_level - 5.0, physics.sea_level + 5.0]
    surface_ticks = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]
    surface_labels = [str(value) for value in surface_ticks]
    speed_limits = [0.0, 3.0]
    speed_ticks = [0, 1, 2, 3]
    speed_labels = [str(value) for value in speed_ticks]
    wind_limits = [0,40]
    pressure_limits = [966,1013]
    friction_bounds = [0.01,0.04]

    # ==========================================================================
    # ==========================================================================
    #   Plot specifications
    # ==========================================================================
    # ==========================================================================

    # Loop over region specifications ploting both surface and speeds
    for region in regions:
        name = region['name']
        xlimits = region['limits'][0]
        ylimits = region['limits'][1]
        # ======================================================================
        #  Surface Elevations
        # ======================================================================
        plotfigure = plotdata.new_plotfigure(name='Surface - %s' % name)
        plotfigure.show = True

        # Set up for axes in this figure:
        plotaxes = plotfigure.new_plotaxes()
        plotaxes.title = 'Surface'
        plotaxes.scaled = True
        plotaxes.xlimits = xlimits
        plotaxes.ylimits = ylimits
        plotaxes.afteraxes = surge_afteraxes

        surge.add_surface_elevation(plotaxes, bounds=surface_limits)
        surge.add_land(plotaxes)

        # ======================================================================
        #  Water Speed
        # ======================================================================
        plotfigure = plotdata.new_plotfigure(name='Currents - %s'% name)
        plotfigure.show = True

        # Set up for axes in this figure:
        plotaxes = plotfigure.new_plotaxes()
        plotaxes.title = 'Currents'
        plotaxes.scaled = True
        plotaxes.xlimits = xlimits
        plotaxes.ylimits = ylimits
        plotaxes.afteraxes = surge_afteraxes

        surge.add_speed(plotaxes, bounds=speed_limits)
        surge.add_land(plotaxes)

    # ========================================================================
    # Hurricane forcing - Entire gulf
    # ========================================================================
    # Friction field
    plotfigure = plotdata.new_plotfigure(name='Friction')
    plotfigure.show = friction_data.variable_friction and False

    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = full_xlimits
    plotaxes.ylimits = full_ylimits
    plotaxes.title = "Manning's N Coefficients"
    plotaxes.afteraxes = surge_afteraxes
    plotaxes.scaled = True

    surge.add_friction(plotaxes,bounds=friction_bounds)

    # Pressure field
    plotfigure = plotdata.new_plotfigure(name='Pressure')
    plotfigure.show = surge_data.pressure_forcing and True

    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = full_xlimits
    plotaxes.ylimits = full_ylimits
    plotaxes.title = "Pressure Field"
    plotaxes.afteraxes = surge_afteraxes
    plotaxes.scaled = True

    surge.add_pressure(plotaxes,bounds=pressure_limits)
    # add_pressure(plotaxes)
    surge.add_land(plotaxes)

    # Wind field
    plotfigure = plotdata.new_plotfigure(name='Wind Speed')
    plotfigure.show = surge_data.wind_forcing and True

    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = full_xlimits
    plotaxes.ylimits = full_ylimits
    plotaxes.title = "Wind Field"
    plotaxes.afteraxes = surge_afteraxes
    plotaxes.scaled = True

    surge.add_wind(plotaxes,bounds=wind_limits,plot_type='imshow')
    surge.add_land(plotaxes)

    # ========================================================================
    #  Figures for gauges
    # ========================================================================
    plotfigure = plotdata.new_plotfigure(name='Surface & topo', figno=300, \
                    type='each_gauge')
    plotfigure.show = True
    plotfigure.clf_each_gauge = True

    stations = [('8761724', 'Grand Isle, LA'),
                ('8760922', 'Pilots Station East, SW Pass, LA'),
                ('8735180', 'Dauphin Island, AL')]

    landfall_time = np.datetime64('2005-08-29T11:10')
    begin_date = datetime.datetime(2005, 8, 26)
    end_date = datetime.datetime(2005, 8, 31)

    def get_actual_water_levels(station_id):
        # Fetch water levels and tide predictions for given station
        date_time, water_level, tide = fetch_noaa_tide_data(station_id,
                begin_date, end_date)

        # Calculate times relative to landfall
        days_rel_landfall = (date_time - landfall_time) / np.timedelta64(1, 'D')

        # Subtract tide predictions from measured water levels
        water_level -= tide

        return days_rel_landfall, water_level

    def gauge_afteraxes(cd):
        station_id, station_name = stations[cd.gaugeno - 1]
        days_rel_landfall, actual_level = get_actual_water_levels(station_id)

        axes = plt.gca()
        surge.plot_landfall_gauge(cd.gaugesoln, axes, landfall=landfall)
        axes.plot(days_rel_landfall, actual_level, 'g')

        # Fix up plot - in particular fix time labels
        axes.set_title(station_name)
        axes.set_xlabel('Days relative to landfall')
        axes.set_ylabel('Surface (m)')
        axes.set_xlim([-3, 1])
        axes.set_ylim([-0.5, 2.5])
        axes.set_xticks([-3, -2, -1, 0, 1])
        axes.set_xticklabels([r"$-3$", r"$-2$", r"$-1$", r"$0$", r"$1$"])
        axes.grid(True)

    # Set up for axes in this figure:
    plotaxes = plotfigure.new_plotaxes()
    plotaxes.afteraxes = gauge_afteraxes

    # Plot surface as blue curve:
    plotitem = plotaxes.new_plotitem(plot_type='1d_plot')
    plotitem.plot_var = 3
    plotitem.plotstyle = 'b-'

    #
    #  Gauge Location Plot
    #
    def gauge_location_afteraxes(cd):
        plt.subplots_adjust(left=0.12, bottom=0.06, right=0.97, top=0.97)
        surge_afteraxes(cd)
        gaugetools.plot_gauge_locations(cd.plotdata, gaugenos='all',
                                        format_string='ko', add_labels=True)

    plotfigure = plotdata.new_plotfigure(name="Gauge Locations")
    plotfigure.show = True

    # Set up for axes in this figure:
    plotaxes = plotfigure.new_plotaxes()
    plotaxes.title = 'Gauge Locations'
    plotaxes.scaled = True
    plotaxes.xlimits = [-90.5, -87.5]
    plotaxes.ylimits = [28.0, 31.0]
    plotaxes.afteraxes = gauge_location_afteraxes

    surge.add_surface_elevation(plotaxes, bounds=surface_limits)
    surge.add_land(plotaxes)

    #-----------------------------------------

    # Parameters used only when creating html and/or latex hardcopy
    # e.g., via pyclaw.plotters.frametools.printframes:

    plotdata.printfigs = True                # print figures
    plotdata.print_format = 'png'            # file format
    plotdata.print_framenos = 'all'          # list of frames to print
    plotdata.print_gaugenos = 'all'          # list of gauges to print
    plotdata.print_fignos = 'all'            # list of figures to print
    plotdata.html = True                     # create html files of plots?
    plotdata.html_homelink = '../README.html'   # pointer for top of index
    plotdata.latex = True                    # create latex file of plots?
    plotdata.latex_figsperline = 2           # layout of plots
    plotdata.latex_framesperline = 1         # layout of plots
    plotdata.latex_makepdf = False           # also run pdflatex?

    return plotdata

<commit_msg>Fix colorbar issue<commit_after>
"""
Set up the plot figures, axes, and items to be done for each frame.

This module is imported by the plotting routines and then the
function setplot is called to set the plot parameters.

"""

import os

import numpy as np
import matplotlib.pyplot as plt
import datetime

from clawpack.geoclaw.util import fetch_noaa_tide_data
from clawpack.visclaw import colormaps, gaugetools
import clawpack.clawutil.data
import clawpack.amrclaw.data
import clawpack.geoclaw.data

import clawpack.geoclaw.surge.plot as surge

try:
    from setplotfg import setplotfg
except:
    setplotfg = None

def setplot(plotdata):
    r"""Setplot function for surge plotting"""

    plotdata.clearfigures()  # clear any old figures,axes,items data
    plotdata.format = 'binary'

    # Load data from output
    clawdata = clawpack.clawutil.data.ClawInputData(2)
    clawdata.read(os.path.join(plotdata.outdir,'claw.data'))
    physics = clawpack.geoclaw.data.GeoClawData()
    physics.read(os.path.join(plotdata.outdir,'geoclaw.data'))
    surge_data = clawpack.geoclaw.data.SurgeData()
    surge_data.read(os.path.join(plotdata.outdir,'surge.data'))
    friction_data = clawpack.geoclaw.data.FrictionData()
    friction_data.read(os.path.join(plotdata.outdir,'friction.data'))

    # Load storm track32
    track = surge.track_data(os.path.join(plotdata.outdir,'fort.track'))

    # Calculate landfall time
    landfall_dt = datetime.datetime(2005, 8, 29, 11, 10) \
                    - datetime.datetime(2005, 1, 1, 0, 0)
    landfall = landfall_dt.days * 24.0 * 60**2 + landfall_dt.seconds

    # Set afteraxes function
    surge_afteraxes = lambda cd: surge.surge_afteraxes(cd,
                                        track, landfall, plot_direction=False)

    # Limits for plots
    regions = [{"name": "Full Domain",
                "limits": [[clawdata.lower[0], clawdata.upper[0]],
                           [clawdata.lower[1], clawdata.upper[1]]]},
               {"name": "New Orleans",
                "limits": [[-92.5, -87.5], [27.5, 31.5]]}]

    full_xlimits = regions[0]['limits'][0]
    full_ylimits = regions[0]['limits'][1]

    # Color limits
    surface_limits = [physics.sea_level - 5.0, physics.sea_level + 5.0]
    surface_ticks = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]
    surface_labels = [str(value) for value in surface_ticks]
    speed_limits = [0.0, 3.0]
    speed_ticks = [0, 1, 2, 3]
    speed_labels = [str(value) for value in speed_ticks]
    wind_limits = [0,40]
    pressure_limits = [966,1013]
    friction_bounds = [0.01,0.04]

    # ==========================================================================
    # ==========================================================================
    #   Plot specifications
    # ==========================================================================
    # ==========================================================================

    # Loop over region specifications ploting both surface and speeds
    for region in regions:
        name = region['name']
        xlimits = region['limits'][0]
        ylimits = region['limits'][1]
        # ======================================================================
        #  Surface Elevations
        # ======================================================================
        plotfigure = plotdata.new_plotfigure(name='Surface - %s' % name)
        plotfigure.show = True

        # Set up for axes in this figure:
        plotaxes = plotfigure.new_plotaxes()
        plotaxes.title = 'Surface'
        plotaxes.scaled = True
        plotaxes.xlimits = xlimits
        plotaxes.ylimits = ylimits
        plotaxes.afteraxes = surge_afteraxes

        surge.add_bathy_contours(plotaxes)
        surge.add_surface_elevation(plotaxes, bounds=surface_limits)
        plotaxes.plotitem_dict['surface'].amr_patchedges_show = [0] * 10

        # ======================================================================
        #  Water Speed
        # ======================================================================
        plotfigure = plotdata.new_plotfigure(name='Currents - %s'% name)
        plotfigure.show = True

        # Set up for axes in this figure:
        plotaxes = plotfigure.new_plotaxes()
        plotaxes.title = 'Currents'
        plotaxes.scaled = True
        plotaxes.xlimits = xlimits
        plotaxes.ylimits = ylimits
        plotaxes.afteraxes = surge_afteraxes

        surge.add_bathy_contours(plotaxes)
        surge.add_speed(plotaxes, bounds=speed_limits)
        plotaxes.plotitem_dict['speed'].amr_patchedges_show = [0] * 10

    # ========================================================================
    # Hurricane forcing - Entire gulf
    # ========================================================================
    # Friction field
    plotfigure = plotdata.new_plotfigure(name='Friction')
    plotfigure.show = friction_data.variable_friction and False

    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = full_xlimits
    plotaxes.ylimits = full_ylimits
    plotaxes.title = "Manning's N Coefficients"
    plotaxes.afteraxes = surge_afteraxes
    plotaxes.scaled = True

    surge.add_friction(plotaxes,bounds=friction_bounds)

    # Pressure field
    plotfigure = plotdata.new_plotfigure(name='Pressure')
    plotfigure.show = surge_data.pressure_forcing and True

    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = full_xlimits
    plotaxes.ylimits = full_ylimits
    plotaxes.title = "Pressure Field"
    plotaxes.afteraxes = surge_afteraxes
    plotaxes.scaled = True

    surge.add_bathy_contours(plotaxes)
    surge.add_pressure(plotaxes, bounds=pressure_limits)
    plotaxes.plotitem_dict['pressure'].amr_patchedges_show = [0] * 10

    # Wind field
    plotfigure = plotdata.new_plotfigure(name='Wind Speed')
    plotfigure.show = surge_data.wind_forcing and True

    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = full_xlimits
    plotaxes.ylimits = full_ylimits
    plotaxes.title = "Wind Field"
    plotaxes.afteraxes = surge_afteraxes
    plotaxes.scaled = True

    surge.add_bathy_contours(plotaxes)
    surge.add_wind(plotaxes, bounds=wind_limits, plot_type='imshow')
    plotaxes.plotitem_dict['wind'].amr_patchedges_show = [0] * 10

    # ========================================================================
    #  Figures for gauges
    # ========================================================================
    plotfigure = plotdata.new_plotfigure(name='Surface & topo', figno=300, \
                    type='each_gauge')
    plotfigure.show = True
    plotfigure.clf_each_gauge = True

    stations = [('8761724', 'Grand Isle, LA'),
                ('8760922', 'Pilots Station East, SW Pass, LA'),
                ('8735180', 'Dauphin Island, AL')]

    landfall_time = np.datetime64('2005-08-29T11:10')
    begin_date = datetime.datetime(2005, 8, 26)
    end_date = datetime.datetime(2005, 8, 31)

    def get_actual_water_levels(station_id):
        # Fetch water levels and tide predictions for given station
        date_time, water_level, tide = fetch_noaa_tide_data(station_id,
                begin_date, end_date)

        # Calculate times relative to landfall
        days_rel_landfall = (date_time - landfall_time) / np.timedelta64(1, 'D')

        # Subtract tide predictions from measured water levels
        water_level -= tide

        return days_rel_landfall, water_level

    def gauge_afteraxes(cd):
        station_id, station_name = stations[cd.gaugeno - 1]
        days_rel_landfall, actual_level = get_actual_water_levels(station_id)

        axes = plt.gca()
        surge.plot_landfall_gauge(cd.gaugesoln, axes, landfall=landfall)
        axes.plot(days_rel_landfall, actual_level, 'g')

        # Fix up plot - in particular fix time labels
        axes.set_title(station_name)
        axes.set_xlabel('Days relative to landfall')
        axes.set_ylabel('Surface (m)')
        axes.set_xlim([-3, 1])
        axes.set_ylim([-0.5, 2.5])
        axes.set_xticks([-3, -2, -1, 0, 1])
        axes.set_xticklabels([r"$-3$", r"$-2$", r"$-1$", r"$0$", r"$1$"])
        axes.grid(True)

    # Set up for axes in this figure:
    plotaxes = plotfigure.new_plotaxes()
    plotaxes.afteraxes = gauge_afteraxes

    # Plot surface as blue curve:
    plotitem = plotaxes.new_plotitem(plot_type='1d_plot')
    plotitem.plot_var = 3
    plotitem.plotstyle = 'b-'

    #
    #  Gauge Location Plot
    #
    def gauge_location_afteraxes(cd):
        plt.subplots_adjust(left=0.12, bottom=0.06, right=0.97, top=0.97)
        surge_afteraxes(cd)
        gaugetools.plot_gauge_locations(cd.plotdata, gaugenos='all',
                                        format_string='ko', add_labels=True)

    plotfigure = plotdata.new_plotfigure(name="Gauge Locations")
    plotfigure.show = True

    # Set up for axes in this figure:
    plotaxes = plotfigure.new_plotaxes()
    plotaxes.title = 'Gauge Locations'
    plotaxes.scaled = True
    plotaxes.xlimits = [-91.5, -87.5]
    plotaxes.ylimits = [28.0, 31.0]
    plotaxes.afteraxes = gauge_location_afteraxes

    surge.add_bathy_contours(plotaxes)
    surge.add_surface_elevation(plotaxes, bounds=surface_limits)
    plotaxes.plotitem_dict['surface'].amr_patchedges_show = [0] * 10

    #-----------------------------------------

    # Parameters used only when creating html and/or latex hardcopy
    # e.g., via pyclaw.plotters.frametools.printframes:

    plotdata.printfigs = True                # print figures
    plotdata.print_format = 'png'            # file format
    plotdata.print_framenos = 'all'          # list of frames to print
    plotdata.print_gaugenos = 'all'          # list of gauges to print
    plotdata.print_fignos = 'all'            # list of figures to print
    plotdata.html = True                     # create html files of plots?
    plotdata.html_homelink = '../README.html'   # pointer for top of index
    plotdata.latex = True                    # create latex file of plots?
    plotdata.latex_figsperline = 2           # layout of plots
    plotdata.latex_framesperline = 1         # layout of plots
    plotdata.latex_makepdf = False           # also run pdflatex?

    return plotdata

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from datetime import datetime
# import json

import re
import requests
import requests_cache
from requests import ConnectionError

class MovieNotFound(Exception):
    pass

class Movie(object):
    __api_url = 'http://www.omdbapi.com'
    __headers = {'user-agent': 'cinema/0.0.8'}

    def __init__(self, name, exact_match=False, year=0):
        assert type(name) is str
        assert type(year) is int

        name = name.strip()

        self._exact_match = exact_match
        self._q_name = name
        self._q_year = year
        self._title = None
        self._year = None
        self._released = None
        self._runtime = None
        self._genre = ()
        self._imdb_rating = None
        self._meta_score = None
        self._tomato_meter = None
        self._rated = None
        self._directors = ()
        self._writers = ()
        self._language = ()
        self._country = ()
        self._poster = None
        self._awards = None
        self._cast = ()
        self._plot = None
        self._imdb_id = None
        self._action()

    @property
    def title(self):
        """
        Title
        :type: str
        """
        return self._title

    @property
    def year(self):
        """
        Year
        :type: int
        """
        return self._year

    @property
    def released(self):
        """
        Year
        :type: date
        """
        return self._released

    @property
    def runtime(self):
        """
        Runtime
        :type: int
        """
        return self._runtime

    @property
    def genre(self):
        """
        Genre
        :type: tuple
        """
        return self._genre

    @property
    def imdb_rating(self):
        """
        IMDB Rating
        :type: float
        """
        return self._imdb_rating

    @property
    def metascore(self):
        """
        Metascore
        :type: int
        """
        return self._meta_score

    @property
    def tomatometer(self):
        """
        TOMATOMETER
        :type: int
        """
        return self._tomato_meter

    @property
    def rated(self):
        """
        Rated
        :type: str
        """
        return self._rated

    @property
    def directors(self):
        """
        Directors
        :type: tuple
        """
        return self._directors

    @property
    def writers(self):
        """
        Writers
        :type: tuple
        """
        return self._writers

    @property
    def language(self):
        """
        Language
        :type: tuple
        """
        return self._language

    @property
    def country(self):
        """
        Country
        :type: tuple
        """
        return self._country

    @property
    def poster(self):
        """
        Poster URL
        :type: str
        """
        return self._poster

    @property
    def awards(self):
        """
        Awards
        :type: str
        """
        return self._awards

    @property
    def cast(self):
        """
        Cast
        :type: tuple
        """
        return self._cast

    @property
    def plot(self):
        """
        Plot
        :type: str
        """
        return self._plot

    @property
    def imdb_id(self):
        """
        IMDB ID
        :type: str
        """
        return self._imdb_id

    def _search_for_imdb_id(self, movie_name):
        """
        Searches for the movie name in google and tries to find its imdb id
        :param movie_name: The name of the movie to search for
        :return: IMDB unique identifier
        """
        imdb_id = ""
        payload = {'q': movie_name + ' imdb'}
        response = requests.get('https://www.google.com/search', params=payload)
        if response.status_code == requests.codes.ok:
            html = response.text
            regex = re.compile("www.imdb.com/title/.+?/")
            search_result = regex.search(html)
            if search_result is not None:
                imdb_id = (search_result.group(0).split('/'))[2]
        return imdb_id

    def _action(self):
        requests_cache.install_cache('omdb_cache', expire_after=300, backend='memory')

        payload = {'plot': 'full',
                   'r': 'json',
                   'tomatoes': 'true',
                   'type': 'movie',
                   'v': '1'}
        if self._exact_match:
            payload['t'] = self._q_name
        else:
            payload['s'] = self._q_name
        if self._q_year != 0:
            payload['y'] = self._q_year

        result = requests.get(self.__api_url, headers=self.__headers, params=payload)

        if result.status_code != requests.codes.ok:
            raise ConnectionError

        data = result.json()
        imdb_id = ''
        if data['Response'] == 'False':
            # Trying searching for the movie's name in google
            imdb_id = self._search_for_imdb_id(self._q_name)
            if imdb_id == '':
                raise MovieNotFound

        # print result.request.url
        # print json.dumps(data, indent=4, sort_keys=True)
        payload.pop('s', None)
        if imdb_id != '':
            payload['i'] = imdb_id
        else:
            data = data["Search"][0]
            payload['t'] = data['Title']
        result = requests.get(self.__api_url, headers=self.__headers, params=payload)
        if result.status_code != requests.codes.ok:
            raise ConnectionError

        data = result.json()
        if data['Response'] == 'False':
            raise MovieNotFound

        # print result.request.url
        # print json.dumps(data, indent=4, sort_keys=True)

        self._title = data['Title']
        self._poster = data['Poster']
        self._year = int(data['Year'])
        self._imdb_id = data['imdbID']
        self._imdb_rating = float(data['imdbRating'])
        try:
            self._meta_score = int(data['Metascore'])
        except ValueError:
            self._meta_score = 0
        try:
            self._tomato_meter = int(data['tomatoMeter'])
        except ValueError:
            self._tomato_meter = 0
        self._released = datetime.strptime(data['Released'], '%d %b %Y').date()
        try:
            self._runtime = int(data['Runtime'].split()[0])
        except ValueError:
            self._runtime = 0
        self._rated = data['Rated']
        self._awards = data['Awards']
        self._plot = data['Plot']
        self._genre = tuple(data['Genre'].split(', '))
        self._directors = tuple(data['Director'].split(', '))
        self._writers = tuple(data['Writer'].split(', '))
        self._language = tuple(data['Language'].split(', '))
        self._country = tuple(data['Country'].split(', '))
        self._cast = tuple(data['Actors'].encode('utf-8').decode('utf-8').split(', '))

    def __repr__(self):
        return 'Movie(%s - %d)' % (self.title, self.year)

    def __str__(self):
        return '%s (%d)' % (self.title, self.year)
<commit_msg>Remove redundant encode and decode utf-8<commit_after># -*- coding: utf-8 -*-
from datetime import datetime
# import json

import re
import requests
import requests_cache
from requests import ConnectionError

class MovieNotFound(Exception):
    pass

class Movie(object):
    __api_url = 'http://www.omdbapi.com'
    __headers = {'user-agent': 'cinema/0.0.8'}

    def __init__(self, name, exact_match=False, year=0):
        assert type(name) is str
        assert type(year) is int

        name = name.strip()

        self._exact_match = exact_match
        self._q_name = name
        self._q_year = year
        self._title = None
        self._year = None
        self._released = None
        self._runtime = None
        self._genre = ()
        self._imdb_rating = None
        self._meta_score = None
        self._tomato_meter = None
        self._rated = None
        self._directors = ()
        self._writers = ()
        self._language = ()
        self._country = ()
        self._poster = None
        self._awards = None
        self._cast = ()
        self._plot = None
        self._imdb_id = None
        self._action()

    @property
    def title(self):
        """
        Title
        :type: str
        """
        return self._title

    @property
    def year(self):
        """
        Year
        :type: int
        """
        return self._year

    @property
    def released(self):
        """
        Year
        :type: date
        """
        return self._released

    @property
    def runtime(self):
        """
        Runtime
        :type: int
        """
        return self._runtime

    @property
    def genre(self):
        """
        Genre
        :type: tuple
        """
        return self._genre

    @property
    def imdb_rating(self):
        """
        IMDB Rating
        :type: float
        """
        return self._imdb_rating

    @property
    def metascore(self):
        """
        Metascore
        :type: int
        """
        return self._meta_score

    @property
    def tomatometer(self):
        """
        TOMATOMETER
        :type: int
        """
        return self._tomato_meter

    @property
    def rated(self):
        """
        Rated
        :type: str
        """
        return self._rated

    @property
    def directors(self):
        """
        Directors
        :type: tuple
        """
        return self._directors

    @property
    def writers(self):
        """
        Writers
        :type: tuple
        """
        return self._writers

    @property
    def language(self):
        """
        Language
        :type: tuple
        """
        return self._language

    @property
    def country(self):
        """
        Country
        :type: tuple
        """
        return self._country

    @property
    def poster(self):
        """
        Poster URL
        :type: str
        """
        return self._poster

    @property
    def awards(self):
        """
        Awards
        :type: str
        """
        return self._awards

    @property
    def cast(self):
        """
        Cast
        :type: tuple
        """
        return self._cast

    @property
    def plot(self):
        """
        Plot
        :type: str
        """
        return self._plot

    @property
    def imdb_id(self):
        """
        IMDB ID
        :type: str
        """
        return self._imdb_id

    def _search_for_imdb_id(self, movie_name):
        """
        Searches for the movie name in google and tries to find its imdb id
        :param movie_name: The name of the movie to search for
        :return: IMDB unique identifier
        """
        imdb_id = ""
        payload = {'q': movie_name + ' imdb'}
        response = requests.get('https://www.google.com/search', params=payload)
        if response.status_code == requests.codes.ok:
            html = response.text
            regex = re.compile("www.imdb.com/title/.+?/")
            search_result = regex.search(html)
            if search_result is not None:
                imdb_id = (search_result.group(0).split('/'))[2]
        return imdb_id

    def _action(self):
        requests_cache.install_cache('omdb_cache', expire_after=300, backend='memory')

        payload = {'plot': 'full',
                   'r': 'json',
                   'tomatoes': 'true',
                   'type': 'movie',
                   'v': '1'}
        if self._exact_match:
            payload['t'] = self._q_name
        else:
            payload['s'] = self._q_name
        if self._q_year != 0:
            payload['y'] = self._q_year

        result = requests.get(self.__api_url, headers=self.__headers, params=payload)

        if result.status_code != requests.codes.ok:
            raise ConnectionError

        data = result.json()
        imdb_id = ''
        if data['Response'] == 'False':
            # Trying searching for the movie's name in google
            imdb_id = self._search_for_imdb_id(self._q_name)
            if imdb_id == '':
                raise MovieNotFound

        # print result.request.url
        # print json.dumps(data, indent=4, sort_keys=True)
        payload.pop('s', None)
        if imdb_id != '':
            payload['i'] = imdb_id
        else:
            data = data["Search"][0]
            payload['t'] = data['Title']
        result = requests.get(self.__api_url, headers=self.__headers, params=payload)
        if result.status_code != requests.codes.ok:
            raise ConnectionError

        data = result.json()
        if data['Response'] == 'False':
            raise MovieNotFound

        # print result.request.url
        # print json.dumps(data, indent=4, sort_keys=True)

        self._title = data['Title']
        self._poster = data['Poster']
        self._year = int(data['Year'])
        self._imdb_id = data['imdbID']
        self._imdb_rating = float(data['imdbRating'])
        try:
            self._meta_score = int(data['Metascore'])
        except ValueError:
            self._meta_score = 0
        try:
            self._tomato_meter = int(data['tomatoMeter'])
        except ValueError:
            self._tomato_meter = 0
        self._released = datetime.strptime(data['Released'], '%d %b %Y').date()
        try:
            self._runtime = int(data['Runtime'].split()[0])
        except ValueError:
            self._runtime = 0
        self._rated = data['Rated']
        self._awards = data['Awards']
        self._plot = data['Plot']
        self._genre = tuple(data['Genre'].split(', '))
        self._directors = tuple(data['Director'].split(', '))
        self._writers = tuple(data['Writer'].split(', '))
        self._language = tuple(data['Language'].split(', '))
        self._country = tuple(data['Country'].split(', '))
        self._cast = tuple(data['Actors'].split(', '))

    def __repr__(self):
        return 'Movie(%s - %d)' % (self.title, self.year)

    def __str__(self):
        return '%s (%d)' % (self.title, self.year)
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import argparse
import collections
import unidecode
import shutil

import compmusic.file
import compmusic.musicbrainz
import musicbrainzngs as mb
mb.set_useragent("Dunya", "0.1")
mb.set_rate_limit(False)
mb.set_hostname("sitar.s.upf.edu:8090")

class MakamScore(object):
    mapping = collections.defaultdict(list)
    targetdir = "results"

    def __init__(self, scores, pdfs, midis, audio):
        self.scores = scores
        self.pdfs = pdfs
        self.midis = midis
        self.audio = audio
        if self.audio:
            self.map_files_to_recording(self.audio)

    def map_files_to_recording(self, source_dir):
        for root, dirs, files in os.walk(source_dir):
            if len(files):
                for f in files:
                    if compmusic.file.is_mp3_file(f):
                        fname = os.path.join(root, f)
                        metadata = compmusic.file.file_metadata(fname)
                        recordingid = metadata["meta"]["recordingid"]
                        if recordingid:
                            self.mapping[recordingid].append(fname)

    def save_scores(self, fname, workid):
        # make a dir based on fname
        target = os.path.join(self.targetdir, fname)
        try:
            os.makedirs(target)
        except:
            pass
        try:
            shutil.copy(os.path.join(self.scores, fname+".txt"), target)
        except IOError:
            pass
        try:
            shutil.copy(os.path.join(self.pdfs, fname+".pdf"), target)
        except IOError:
            pass
        try:
            shutil.copy(os.path.join(self.midis, fname+".mid"), target)
        except IOError:
            pass

        work = mb.get_work_by_id(workid, includes=["recording-rels"])
        work = work["work"]
        for rel in work.get("recording-relation-list", []):
            recid = rel["target"]
            for fname in self.mapping.get(recid, []):
                if os.path.exists(os.path.join(target, os.path.basename(fname))):
                    name = os.path.basename(fname)
                    sourcedir = os.path.dirname(fname)
                    n, ext = os.path.splitext(name)
                    newname = n + "_1" + ext
                    newdest = os.path.join(target, newname)
                    print name, "exists, copying", target, "to", newname, "instead"
                    shutil.copy(fname, newdest)
                else:
                    shutil.copy(fname, target)

    def match(self, a, b):
        if isinstance(a, unicode):
            a = unidecode.unidecode(a)
        if isinstance(b, unicode):
            b = unidecode.unidecode(b)
        import re
        bey = re.compile(r'\bbey\b')
        efendi = re.compile(r'\befendi\b')
        a = a.lower()
        b = b.lower()

        # Remove bey and efendi honorifics
        a = bey.sub("", a)
        b = bey.sub("", b)
        a = efendi.sub("", a)
        b = efendi.sub("", b)

        # lowercase and remove all non-letters (spaces, ', -, etc)
        a = re.sub(r"[^a-z]", "", a)
        b = re.sub(r"[^a-z]", "", b)

        # if a match, return
        if a == b:
            return True

        # otherwise, do edit distance.
        # an edit of <= 3 at the end of a string counts as a match
        if len(a) > len(b):
            a, b = b, a
        # now, a is the shorter one. If a starts in b at position 0
        #  then we have the overlap at the end
        if b.startswith(a):
            if len(b) - len(a) <= 3:
                return True

        # otherwise, lev dand listance of <= 3
        import Levenshtein
        return Levenshtein.distance(a, b) <= 3

    def test_file(self, fname):
        fname = os.path.basename(fname)
        fname = os.path.splitext(fname)[0]
        parts = fname.split("--")
        if len(parts) != 5:
            print "wrong number of parts", parts
            return
        makam, form, usul, name, composer = parts
        composer = composer.replace("_", " ")

        if not name:
            name = makam + ' ' + form

        name = name.replace("_", " ")
        works = mb.search_works(name)["work-list"]
        for w in works:
            for a in w.get("artist-relation-list", []):
                a = a["artist"]
                aname = a["name"]
                wname = w["title"]
                if isinstance(aname, unicode):
                    aname = unidecode.unidecode(aname)
                if isinstance(wname, unicode):
                    wname = unidecode.unidecode(wname)
                if self.match(composer, aname) and self.match(name, wname):
                    print "  match: %s by %s - http://musicbrainz.org/work/%s" % (w["title"], aname, w["id"])
                    self.save_scores(fname, w["id"])

    def test_dir(self, dirname):
        for i, f in enumerate(os.listdir(dirname)):
            print i, f
            fname = os.path.join(dirname, f)
            self.test_file(fname)

def main(args):
    m = MakamScore(args.s, args.p, args.m, args.a)
    if args.f:
        m.test_file(args.f)
    else:
        m.test_dir(args.s)

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("-s", help="symbtr txt")
    p.add_argument("-p", help="symbtr pdf")
    p.add_argument("-m", help="symbtr mid")
    p.add_argument("-a", help="audio files")
    p.add_argument("-f", help="a single file to test")
    args = p.parse_args()
    main(args)
<commit_msg>add some memoisation, do better person matching<commit_after>#!/usr/bin/python

import sys
import os
import argparse
import collections
import unidecode
import shutil
import random
import functools32
import json
import string

import compmusic.file
import compmusic.musicbrainz
import musicbrainzngs as mb
mb.set_useragent("Dunya", "0.1")
mb.set_rate_limit(False)
mb.set_hostname("sitar.s.upf.edu:8090")

class MakamScore(object):
    mapping = collections.defaultdict(list)
    targetdir = "results"

    def __init__(self, scores, pdfs, midis, audio):
        self.scores = scores
        self.pdfs = pdfs
        self.midis = midis
        self.audio = audio
        if self.audio:
            self.map_files_to_recording(self.audio)

    def map_files_to_recording(self, source_dir):
        if os.path.exists("recording_to_file.json"):
            self.mapping = json.load(open("recording_to_file.json"))
            return
        for root, dirs, files in os.walk(source_dir):
            if len(files):
                for f in files:
                    fname = os.path.join(root, f)
                    if compmusic.file.is_mp3_file(fname):
                        metadata = compmusic.file.file_metadata(fname)
                        recordingid = metadata["meta"]["recordingid"]
                        if recordingid:
                            self.mapping[recordingid].append(fname)
        json.dump(self.mapping, open("recording_to_file.json", "w"))

    def save_scores(self, fname, recordingids):
        # make a dir based on fname
        target = os.path.join(self.targetdir, fname)
        try:
            os.makedirs(target)
        except:
            pass
        try:
            shutil.copy(os.path.join(self.scores, fname+".txt"), target)
        except IOError:
            pass
        try:
            shutil.copy(os.path.join(self.pdfs, fname+".pdf"), target)
        except IOError:
            pass
        try:
            shutil.copy(os.path.join(self.midis, fname+".mid"), target)
        except IOError:
            pass

        for recid in recordingids:
            for fname in self.mapping.get(recid, []):
                name = os.path.basename(fname)
                n, ext = os.path.splitext(name)
                if os.path.exists(os.path.join(target, n)):
                    sourcedir = os.path.dirname(fname)
                    rand = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(4))
                    ndir = "%s_%s" % (n, rand)
                    newname = "%s%s" % (ndir, ext)
                    newdest = os.path.join(target, ndir, newname)
                    try:
                        os.makedirs(os.path.dirname(newdest))
                    except:
                        pass
                    print name, "exists, copying", target, "to", newname, "instead"
                    shutil.copy(fname, newdest)
                else:
                    innertarget = os.path.join(target, n)
                    try:
                        os.makedirs(innertarget)
                    except:
                        pass
                    shutil.copy(fname, innertarget)

    @functools32.lru_cache(2000)
    def get_performance_credit_for_recording(self, recordingid):
        recording = mb.get_recording_by_id(recordingid, includes=["releases"])
        recording = recording["recording"]
        ret = []
        for release in recording.get("release-list", []):
            relid = release["id"]
            mbrelease = mb.get_release_by_id(relid, includes=["artist-credits", "recordings"])
            mbrelease = mbrelease["release"]
            for medium in mbrelease.get("medium-list", []):
                for track in medium.get("track-list", []):
                    if track["recording"]["id"] == recordingid:
                        ret.append(track["recording"]["artist-credit-phrase"])
        return list(set(ret))

    @functools32.lru_cache(2000)
    def aliases_for_artist(self, artistid):
        a = mb.get_artist_by_id(artistid, includes=["aliases"])
        a = a["artist"]
        ret = []
        for alias in a.get("alias-list", []):
            ret.append(alias["alias"])
        return list(set(ret))

    def match(self, a, b):
        if isinstance(a, unicode):
            a = unidecode.unidecode(a)
        if isinstance(b, unicode):
            b = unidecode.unidecode(b)
        import re
        bey = re.compile(r'\bbey\b')
        efendi = re.compile(r'\befendi\b')
        a = a.lower()
        b = b.lower()

        # Remove bey and efendi honorifics
        a = bey.sub("", a)
        b = bey.sub("", b)
        a = efendi.sub("", a)
        b = efendi.sub("", b)

        # lowercase and remove all non-letters (spaces, ', -, etc)
        a = re.sub(r"[^a-z]", "", a)
        b = re.sub(r"[^a-z]", "", b)

        # if a match, return
        if a == b:
            return True

        # otherwise, do edit distance.
        # an edit of <= 3 at the end of a string counts as a match
        if len(a) > len(b):
            a, b = b, a
        # now, a is the shorter one. If a starts in b at position 0
        #  then we have the overlap at the end
        if b.startswith(a):
            if len(b) - len(a) <= 3:
                return True

        # otherwise, lev dand listance of <= 3
        import Levenshtein
        return Levenshtein.distance(a, b) <= 3

    def test_file(self, fname):
        fname = os.path.basename(fname)
        fname = os.path.splitext(fname)[0]
        parts = fname.split("--")
        if len(parts) != 5:
            print "wrong number of parts", parts
            return
        makam, form, usul, name, composer = parts
        composer = composer.replace("_", " ")

        if not name:
            name = makam + ' ' + form

        name = name.replace("_", " ")
        works = mb.search_works(name)["work-list"]
        for w in works:
            for a in w.get("artist-relation-list", []):
                a = a["artist"]
                aname = a["name"]
                wname = w["title"]
                if isinstance(aname, unicode):
                    aname = unidecode.unidecode(aname)
                if isinstance(wname, unicode):
                    wname = unidecode.unidecode(wname)
                anywork = self.match(name, wname)
                anyname = self.match(composer, aname)
                reclist = self.recordingids_for_work(w["id"])
                for n in self.aliases_for_artist(a["id"]):
                    anyname = anyname or self.match(composer, n)
                for r in reclist:
                    for n in self.get_performance_credit_for_recording(r):
                        anyname = anyname or self.match(composer, n)
                if anywork and anyname:
                    print "  match: %s by %s - http://musicbrainz.org/work/%s" % (w["title"], aname, w["id"])
                    self.save_scores(fname, reclist)

    @functools32.lru_cache(2000)
    def recordingids_for_work(self, workid):
        work = mb.get_work_by_id(workid, includes=["recording-rels"])
        work = work["work"]
        ret = []
        for rel in work.get("recording-relation-list", []):
            recid = rel["target"]
            ret.append(recid)
        return ret

    def test_dir(self, dirname):
        for i, f in enumerate(os.listdir(dirname)):
            print i, f
            fname = os.path.join(dirname, f)
            self.test_file(fname)

def main(args):
    m = MakamScore(args.s, args.p, args.m, args.a)
    if args.f:
        m.test_file(args.f)
    else:
        m.test_dir(args.s)

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("-s", help="symbtr txt")
    p.add_argument("-p", help="symbtr pdf")
    p.add_argument("-m", help="symbtr mid")
    p.add_argument("-a", help="audio files")
    p.add_argument("-f", help="a single file to test")
    args = p.parse_args()
    main(args)
<|endoftext|>


<commit_before>from __future__ import with_statement

import os, stat, logging, site
from os.path import join, exists, abspath, realpath, splitext, split
from fs import listdir, filters
import hashlib, json, datetime, time
import mimetypes

from browsers import *

class FileExpander(object):
	"""
	Used to expand files in the site into the Redis cache.
	"""
	def __init__(self,base,relpath,prefix=None,config=None):
		self.config = config
		self.prefix = prefix
		self.base = base
		self.path = relpath
		self.name, self.ext = splitext(split(relpath)[1])

		path = abspath(join(base,relpath))
		stat_result = os.stat(path)
		self.modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME])

		content = None
		with open(path, "rb") as f:
			content = f.read()

		self.expandScss = False
		self.expandDocument = False

		self.domain = self.config["domain"]

		# Load file into *header* and *content*
		if self.ext in self.FILE_EXTENSIONS:
			self.FILE_EXTENSIONS[self.ext](self,self.path)
		else:
			self._default_file(self.path)

	def __repr__(self):
		return "%s %s %s %s" % (self.urlpath,self.name,self.ext,self.header)

	def _get_published(self):
		if "published" in self.header:
			return self.header["published"]
		return True

	published = property(_get_published)

	def _get_matter_and_content(self,relpath,defaultheader):
		content = None
		with open(abspath(join(self.base,relpath)), "rb") as f:
			content = f.read()
		return self.config.split_matter_and_utf8_content(content,defaultheader)

	def _scss_file(self,relpath):
		header = {
			"Content-Type": "text/css",
		}
		self.header,self.content = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]
		else:
			self.urlpath = "/%s.css" % splitext(self.path)[0]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = self.urlpath
		self.expandScss = True

	def _xml_file(self,relpath):
		header = {
			"Content-Type": "text/xml",
		}
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
		else:
			self.urlpath = "/" + self.path

		self.header,self.content = self._get_matter_and_content(relpath,header)
		self.content = self.content.lstrip()

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = self.urlpath
		if "document" in self.header and self.header["document"]:
			self.expandDocument = True

	def _text_file(self,relpath):
		header = {
			"Content-Type": "text/plain",
		}
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
			if self.urlpath[-1] != "/": self.urlpath += "/"
		else:
			self.urlpath = "/" + self.path

		self.header,self.content = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = self.urlpath
		if "document" in self.header:
			self.expandDocument = True

	def _html_file(self,relpath):
		header = {
			"Content-Type": "text/html",
		}
		outpath = None
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
			if self.urlpath[-1] != "/": self.urlpath += "/"
			outpath = self.urlpath + "index.html"
		else:
			self.urlpath = "/" + splitext(self.path)[0] + "/"

		self.header,self.content = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = outpath or self.urlpath
		if "document" in self.header:
			self.expandDocument = True

	def _markdown_file(self,relpath):
		header = {
			"Content-Type": "text/html",
		}
		outpath = None
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
			if self.urlpath[-1] != "/": self.urlpath += "/"
			outpath = self.urlpath + "index.html"
		else:
			self.urlpath = "/" + splitext(self.path)[0] + "/"

		self.header,rest = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = outpath or self.urlpath

		if "document" in self.header:
			self.expandDocument = True

		import markdown2
		extras = ["code-friendly","wiki-tables","cuddled-lists"]
		if "markdown-extras" in self.header:
			extras = self.header["markdown-extras"]
		self.content = markdown2.markdown(rest,extras)

	def _default_file(self,relpath):
		header = {
			"Content-Type": "text/plain",
		}
		mime_type,encoding = mimetypes.guess_type(self.path)
		if mime_type:
			header["Content-Type"] = mime_type
		content = None
		with open(abspath(join(self.base,relpath)), "rb") as f:
			content = f.read()
		self.urlpath = "/" + self.path
		self.outpath = self.urlpath
		self.header = header
		self.content = content

	def _appcache_file(self,relpath):
		header = {
			"Content-Type": "text/cache-manifest",
		}
		mime_type,encoding = mimetypes.guess_type(self.path)
		if mime_type:
			header["Content-Type"] = mime_type

		self.header,rest = self._get_matter_and_content(relpath,header)
		self.content = rest.lstrip()

		self.urlpath = "/" + self.path
		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

	FILE_EXTENSIONS = {
		".scss" : _scss_file,
		".txt" : _text_file,
		".text" : _text_file,
		".html" : _html_file,
		".xml" : _xml_file,
		".md" : _markdown_file,
		".mdown" : _markdown_file,
		".markdown" : _markdown_file,
		".appcache" : _appcache_file,
	}

def save_expander(expander,browser,config):
	base_path = config["output"]
	file_path = join(base_path,browser.browser_type,expander.outpath[1:])

	if not expander.published and exists(file_path):
		os.remove(file_path)
		return
	if not exists(join(base_path,browser.browser_type)):
		os.makedirs(join(base_path,browser.browser_type))

	if expander.expandScss:
		relpath = join(browser.browser_type,expander.path)
		if exists(join(expander.base,relpath)):
			header,content = expander._get_matter_and_content(relpath,expander.header)
		else:
			header,content = expander.header,expander.content
		header = browser.expandHeader(header,config=expander.config)
		content = browser.expandScss(header,content,config=expander.config)
		#expander.update_lists(header,{ "offline": [expander.urlpath] })
	elif expander.expandDocument:
		header = browser.expandHeader(expander.header,config=expander.config)
		content, lists = browser.expandDocument(header,expander.content,config=expander.config)
		#expander.update_lists(header,lists)
	else:
		header = expander.header
		content = expander.content
		#expander.update_lists(header,{ "offline": [expander.urlpath] })

	with open(file_path,"wb") as f:
		# print "writing", file_path
		f.write(content)

# populate with files that have an extension and do not start with _
def populate(expander_writer,config):

	if site.SCSS_DIR:
		try:
			import scss
			setattr(scss,"LOAD_PATHS",site.SCSS_DIR)
		except ImportError:
			pass
	""" TODO:
LOAD_PATHS = os.path.join(PROJECT_ROOT, 'sass/frameworks/')
# Assets path, where new sprite files are created:
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static/')
# Assets path, where new sprite files are created:
ASSETS_ROOT = os.path.join(PROJECT_ROOT, 'static/assets/')
# Urls for the static and assets:
STATIC_URL = '/static/'
ASSETS_URL = '/static/assets/'
"""

	base_filters = [config.exclude_filter(),filters.no_directories,filters.no_hidden,filters.no_system]

	if site.SCSS_DIR:
		for relpath in listdir(site.SCSS_DIR,filters=base_filters+[filters.fnmatch("*.scss"),]):
			expander = FileExpander(site.SCSS_DIR,relpath,config=config,prefix="css")
			#TODO ensure that _x.scss is not published
			#setattr(scss,"LOAD_PATHS",site.SCSS_DIR)
			for browser in browsers:
				expander_writer(expander,browser,config)
			logging.info("Cached %s for %s as %s" % (relpath,expander.domain,repr(expander)))

	for relpath in listdir(site.SITE_DIR,recursed=True,filters=base_filters):
		expander = FileExpander(site.SITE_DIR,relpath,config=config)
		if relpath[0] != "_":
			for browser in browsers:
				expander_writer(expander,browser,config)
			logging.info("Cached %s for %s as %s" % (relpath,expander.domain,repr(expander)))
		#TODO generate descr entries for urllists
	# TODO track deleted files removing them from cache

<commit_msg>makedirs css destination<commit_after>from __future__ import with_statement

import os, stat, logging, site
from os.path import join, exists, abspath, realpath, splitext, split, dirname
from fs import listdir, filters
import hashlib, json, datetime, time
import mimetypes

from browsers import *

class FileExpander(object):
	"""
	Used to expand files in the site into the Redis cache.
	"""
	def __init__(self,base,relpath,prefix=None,config=None):
		self.config = config
		self.prefix = prefix
		self.base = base
		self.path = relpath
		self.name, self.ext = splitext(split(relpath)[1])

		path = abspath(join(base,relpath))
		stat_result = os.stat(path)
		self.modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME])

		content = None
		with open(path, "rb") as f:
			content = f.read()

		self.expandScss = False
		self.expandDocument = False

		self.domain = self.config["domain"]

		# Load file into *header* and *content*
		if self.ext in self.FILE_EXTENSIONS:
			self.FILE_EXTENSIONS[self.ext](self,self.path)
		else:
			self._default_file(self.path)

	def __repr__(self):
		return "%s %s %s %s" % (self.urlpath,self.name,self.ext,self.header)

	def _get_published(self):
		if "published" in self.header:
			return self.header["published"]
		return True

	published = property(_get_published)

	def _get_matter_and_content(self,relpath,defaultheader):
		content = None
		with open(abspath(join(self.base,relpath)), "rb") as f:
			content = f.read()
		return self.config.split_matter_and_utf8_content(content,defaultheader)

	def _scss_file(self,relpath):
		header = {
			"Content-Type": "text/css",
		}
		self.header,self.content = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]
		else:
			self.urlpath = "/%s.css" % splitext(self.path)[0]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = self.urlpath
		self.expandScss = True

	def _xml_file(self,relpath):
		header = {
			"Content-Type": "text/xml",
		}
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
		else:
			self.urlpath = "/" + self.path

		self.header,self.content = self._get_matter_and_content(relpath,header)
		self.content = self.content.lstrip()

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = self.urlpath
		if "document" in self.header and self.header["document"]:
			self.expandDocument = True

	def _text_file(self,relpath):
		header = {
			"Content-Type": "text/plain",
		}
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
			if self.urlpath[-1] != "/": self.urlpath += "/"
		else:
			self.urlpath = "/" + self.path

		self.header,self.content = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = self.urlpath
		if "document" in self.header:
			self.expandDocument = True

	def _html_file(self,relpath):
		header = {
			"Content-Type": "text/html",
		}
		outpath = None
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
			if self.urlpath[-1] != "/": self.urlpath += "/"
			outpath = self.urlpath + "index.html"
		else:
			self.urlpath = "/" + splitext(self.path)[0] + "/"

		self.header,self.content = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = outpath or self.urlpath
		if "document" in self.header:
			self.expandDocument = True

	def _markdown_file(self,relpath):
		header = {
			"Content-Type": "text/html",
		}
		outpath = None
		if self.name == "index":
			self.urlpath = "/" + split(self.path)[0]
			if self.urlpath[-1] != "/": self.urlpath += "/"
			outpath = self.urlpath + "index.html"
		else:
			self.urlpath = "/" + splitext(self.path)[0] + "/"

		self.header,rest = self._get_matter_and_content(relpath,header)

		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

		self.outpath = outpath or self.urlpath

		if "document" in self.header:
			self.expandDocument = True

		import markdown2
		extras = ["code-friendly","wiki-tables","cuddled-lists"]
		if "markdown-extras" in self.header:
			extras = self.header["markdown-extras"]
		self.content = markdown2.markdown(rest,extras)

	def _default_file(self,relpath):
		header = {
			"Content-Type": "text/plain",
		}
		mime_type,encoding = mimetypes.guess_type(self.path)
		if mime_type:
			header["Content-Type"] = mime_type
		content = None
		with open(abspath(join(self.base,relpath)), "rb") as f:
			content = f.read()
		self.urlpath = "/" + self.path
		self.outpath = self.urlpath
		self.header = header
		self.content = content

	def _appcache_file(self,relpath):
		header = {
			"Content-Type": "text/cache-manifest",
		}
		mime_type,encoding = mimetypes.guess_type(self.path)
		if mime_type:
			header["Content-Type"] = mime_type

		self.header,rest = self._get_matter_and_content(relpath,header)
		self.content = rest.lstrip()

		self.urlpath = "/" + self.path
		if "permalink" in self.header:
			#TODO if it doesn't start with / look up special meaning keyword
			#TODO replace :xxx with value of xxx (year,month,day,title,i_day,i_month,categories,tags)
			self.urlpath = self.header["permalink"]
		elif "url" in self.header:
			self.urlpath = "/" + self.header["url"]

		if self.prefix:
			self.urlpath = "/%s%s" % (self.prefix,self.urlpath)

	FILE_EXTENSIONS = {
		".scss" : _scss_file,
		".txt" : _text_file,
		".text" : _text_file,
		".html" : _html_file,
		".xml" : _xml_file,
		".md" : _markdown_file,
		".mdown" : _markdown_file,
		".markdown" : _markdown_file,
		".appcache" : _appcache_file,
	}

def save_expander(expander,browser,config):
	base_path = config["output"]
	file_path = join(base_path,browser.browser_type,expander.outpath[1:])
	dir_path = dirname(file_path)

	if not expander.published and exists(file_path):
		os.remove(file_path)
		return
	if not exists(dir_path):
		os.makedirs(dir_path)

	if expander.expandScss:
		relpath = join(browser.browser_type,expander.path)
		if exists(join(expander.base,relpath)):
			header,content = expander._get_matter_and_content(relpath,expander.header)
		else:
			header,content = expander.header,expander.content
		header = browser.expandHeader(header,config=expander.config)
		content = browser.expandScss(header,content,config=expander.config)
		#expander.update_lists(header,{ "offline": [expander.urlpath] })
	elif expander.expandDocument:
		header = browser.expandHeader(expander.header,config=expander.config)
		content, lists = browser.expandDocument(header,expander.content,config=expander.config)
		#expander.update_lists(header,lists)
	else:
		header = expander.header
		content = expander.content
		#expander.update_lists(header,{ "offline": [expander.urlpath] })

	with open(file_path,"wb") as f:
		# print "writing", file_path
		f.write(content)

# populate with files that have an extension and do not start with _
def populate(expander_writer,config):

	if site.SCSS_DIR:
		try:
			import scss
			setattr(scss,"LOAD_PATHS",site.SCSS_DIR)
		except ImportError:
			pass
	""" TODO:
LOAD_PATHS = os.path.join(PROJECT_ROOT, 'sass/frameworks/')
# Assets path, where new sprite files are created:
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static/')
# Assets path, where new sprite files are created:
ASSETS_ROOT = os.path.join(PROJECT_ROOT, 'static/assets/')
# Urls for the static and assets:
STATIC_URL = '/static/'
ASSETS_URL = '/static/assets/'
"""

	base_filters = [config.exclude_filter(),filters.no_directories,filters.no_hidden,filters.no_system]

	if site.SCSS_DIR:
		for relpath in listdir(site.SCSS_DIR,filters=base_filters+[filters.fnmatch("*.scss"),]):
			expander = FileExpander(site.SCSS_DIR,relpath,config=config,prefix="css")
			#TODO ensure that _x.scss is not published
			#setattr(scss,"LOAD_PATHS",site.SCSS_DIR)
			for browser in browsers:
				expander_writer(expander,browser,config)
			logging.info("Cached %s for %s as %s" % (relpath,expander.domain,repr(expander)))

	for relpath in listdir(site.SITE_DIR,recursed=True,filters=base_filters):
		expander = FileExpander(site.SITE_DIR,relpath,config=config)
		if relpath[0] != "_":
			for browser in browsers:
				expander_writer(expander,browser,config)
			logging.info("Cached %s for %s as %s" % (relpath,expander.domain,repr(expander)))
		#TODO generate descr entries for urllists
	# TODO track deleted files removing them from cache

<|endoftext|>


<commit_before>import json
import os

from cl import settings
from cl.lib import magic
from cl.lib import search_utils
from cl.lib.sunburnt import sunburnt
from cl.search.models import Court
from cl.stats import tally_stat

from django.http import Http404, HttpResponse, JsonResponse
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext

def annotate_courts_with_counts(courts, court_count_tuples):
    """Solr gives us a response like:

        court_count_tuples = [
            ('ca2', 200),
            ('ca1', 42),
            ...
        ]

    Here we add an attribute to our court objects so they have these values.
    """
    # Convert the tuple to a dict
    court_count_dict = {}
    for court_str, count in court_count_tuples:
        court_count_dict[court_str] = count

    for court in courts:
        court.count = court_count_dict.get(court.pk, 0)

    return courts

def make_court_variable():
    courts = Court.objects.exclude(jurisdiction='T')  # Non-testing courts
    conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL, mode='r')
    response = conn.raw_query(
        **search_utils.build_court_count_query()).execute()
    court_count_tuples = response.facet_counts.facet_fields['court_exact']
    courts = annotate_courts_with_counts(courts, court_count_tuples)
    return courts

def court_index(request):
    """Shows the information we have available for the courts."""
    courts = make_court_variable()
    return render_to_response(
        'jurisdictions.html',
        {'courts': courts,
         'private': False},
        RequestContext(request)
    )

def rest_index(request):
    courts = make_court_variable()
    court_count = len(courts)
    return render_to_response(
        'rest-docs-latest.html',
        {'court_count': court_count,
         'courts': courts,
         'private': False},
        RequestContext(request)
    )

def rest_index_v1(request):
    courts = make_court_variable()
    court_count = len(courts)
    return render_to_response(
        'rest-docs-v1.html',
        {'court_count': court_count,
         'courts': courts,
         'private': False},
        RequestContext(request)
    )

def api_index(request):
    court_count = Court.objects.exclude(
        jurisdiction='T'
    ).count()  # Non-testing courts
    return render_to_response(
        'docs.html',
        {'court_count': court_count,
         'private': False},
        RequestContext(request)
    )

def bulk_data_index(request):
    """Shows an index page for the dumps."""
    courts = make_court_variable()
    court_count = len(courts)
    return render_to_response(
        'bulk-data.html',
        {'court_count': court_count,
         'courts': courts,
         'private': False},
        RequestContext(request)
    )

def serve_pagerank_file(request):
    """Serves the bulk pagerank file from the bulk data directory."""
    file_loc = settings.BULK_DATA_DIR + "external_pagerank"
    file_name = file_loc.split('/')[-1]
    try:
        mimetype = magic.from_file(file_loc, mime=True)
    except IOError:
        raise Http404('Unable to locate external_pagerank file in %s' % settings.BULK_DATA_DIR)
    response = HttpResponse()
    response['X-Sendfile'] = os.path.join(file_loc)
    response['Content-Disposition'] = 'attachment; filename="%s"' % file_name.encode('utf-8')
    response['Content-Type'] = mimetype
    tally_stat('bulk_data.pagerank.served')
    return response

def strip_trailing_zeroes(data):
    """Removes zeroes from the end of the court data

    Some courts only have values through to a certain date, but we don't
    check for that in our queries. Instead, we truncate any zero-values that
    occur at the end of their stats.
    """
    i = len(data) - 1
    while i > 0:
        if data[i][1] == 0:
            i -= 1
        else:
            break

    return data[:i + 1]

def coverage_data(request, version, court):
    """Provides coverage data for a court.

    Responds to either AJAX or regular requests.
    """

    if court != 'all':
        court_str = get_object_or_404(Court, pk=court).pk
    else:
        court_str = 'all'
    q = request.GET.get('q')
    conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL, mode='r')
    start_year = search_utils.get_court_start_year(conn, court_str)
    response = conn.raw_query(
        **search_utils.build_coverage_query(court_str, start_year, q)
    ).execute()
    counts = response.facet_counts.facet_ranges[0][1][0][1]
    counts = strip_trailing_zeroes(counts)

    # Calculate the totals
    annual_counts = {}
    total_docs = 0
    for date_string, count in counts:
        annual_counts[date_string[:4]] = count
        total_docs += count
    response = {
        'annual_counts': annual_counts,
        'total': total_docs,
    }

    return JsonResponse(json.dumps(response), safe=False)

def deprecated_api(request, v):
    return JsonResponse(
        {
            "meta": {
                "status": "This endpoint is deprecated. Please upgrade to the "
                          "newest version of the API.",
            },
            "objects": []
        },
        safe=False,
    )
<commit_msg>Adds a 410 status code to the deprecated API warning page.<commit_after>import json
import os

from cl import settings
from cl.lib import magic
from cl.lib import search_utils
from cl.lib.sunburnt import sunburnt
from cl.search.models import Court
from cl.stats import tally_stat

from django.http import Http404, HttpResponse, JsonResponse
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext

def annotate_courts_with_counts(courts, court_count_tuples):
    """Solr gives us a response like:

        court_count_tuples = [
            ('ca2', 200),
            ('ca1', 42),
            ...
        ]

    Here we add an attribute to our court objects so they have these values.
    """
    # Convert the tuple to a dict
    court_count_dict = {}
    for court_str, count in court_count_tuples:
        court_count_dict[court_str] = count

    for court in courts:
        court.count = court_count_dict.get(court.pk, 0)

    return courts

def make_court_variable():
    courts = Court.objects.exclude(jurisdiction='T')  # Non-testing courts
    conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL, mode='r')
    response = conn.raw_query(
        **search_utils.build_court_count_query()).execute()
    court_count_tuples = response.facet_counts.facet_fields['court_exact']
    courts = annotate_courts_with_counts(courts, court_count_tuples)
    return courts

def court_index(request):
    """Shows the information we have available for the courts."""
    courts = make_court_variable()
    return render_to_response(
        'jurisdictions.html',
        {'courts': courts,
         'private': False},
        RequestContext(request)
    )

def rest_index(request):
    courts = make_court_variable()
    court_count = len(courts)
    return render_to_response(
        'rest-docs-latest.html',
        {'court_count': court_count,
         'courts': courts,
         'private': False},
        RequestContext(request)
    )

def rest_index_v1(request):
    courts = make_court_variable()
    court_count = len(courts)
    return render_to_response(
        'rest-docs-v1.html',
        {'court_count': court_count,
         'courts': courts,
         'private': False},
        RequestContext(request)
    )

def api_index(request):
    court_count = Court.objects.exclude(
        jurisdiction='T'
    ).count()  # Non-testing courts
    return render_to_response(
        'docs.html',
        {'court_count': court_count,
         'private': False},
        RequestContext(request)
    )

def bulk_data_index(request):
    """Shows an index page for the dumps."""
    courts = make_court_variable()
    court_count = len(courts)
    return render_to_response(
        'bulk-data.html',
        {'court_count': court_count,
         'courts': courts,
         'private': False},
        RequestContext(request)
    )

def serve_pagerank_file(request):
    """Serves the bulk pagerank file from the bulk data directory."""
    file_loc = settings.BULK_DATA_DIR + "external_pagerank"
    file_name = file_loc.split('/')[-1]
    try:
        mimetype = magic.from_file(file_loc, mime=True)
    except IOError:
        raise Http404('Unable to locate external_pagerank file in %s' % settings.BULK_DATA_DIR)
    response = HttpResponse()
    response['X-Sendfile'] = os.path.join(file_loc)
    response['Content-Disposition'] = 'attachment; filename="%s"' % file_name.encode('utf-8')
    response['Content-Type'] = mimetype
    tally_stat('bulk_data.pagerank.served')
    return response

def strip_trailing_zeroes(data):
    """Removes zeroes from the end of the court data

    Some courts only have values through to a certain date, but we don't
    check for that in our queries. Instead, we truncate any zero-values that
    occur at the end of their stats.
    """
    i = len(data) - 1
    while i > 0:
        if data[i][1] == 0:
            i -= 1
        else:
            break

    return data[:i + 1]

def coverage_data(request, version, court):
    """Provides coverage data for a court.

    Responds to either AJAX or regular requests.
    """

    if court != 'all':
        court_str = get_object_or_404(Court, pk=court).pk
    else:
        court_str = 'all'
    q = request.GET.get('q')
    conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL, mode='r')
    start_year = search_utils.get_court_start_year(conn, court_str)
    response = conn.raw_query(
        **search_utils.build_coverage_query(court_str, start_year, q)
    ).execute()
    counts = response.facet_counts.facet_ranges[0][1][0][1]
    counts = strip_trailing_zeroes(counts)

    # Calculate the totals
    annual_counts = {}
    total_docs = 0
    for date_string, count in counts:
        annual_counts[date_string[:4]] = count
        total_docs += count
    response = {
        'annual_counts': annual_counts,
        'total': total_docs,
    }

    return JsonResponse(json.dumps(response), safe=False)

def deprecated_api(request, v):
    return JsonResponse(
        {
            "meta": {
                "status": "This endpoint is deprecated. Please upgrade to the "
                          "newest version of the API.",
            },
            "objects": []
        },
        safe=False,
        status=410,  # Gone
    )
<|endoftext|>