<commit_before>#! /usr/bin/env python
from __future__ import with_statement

# sfi -- slice-based facility interface

import sys
import os, os.path
from optparse import OptionParser
from util.cert import Keypair, Certificate
from util.credential import Credential
from util.geniclient import GeniClient

sfi_dir = os.path.expanduser("~/.sfi/")
slicemgr = None
registry = None
user = None
authority = None
verbose = False

#
# Establish Connection to SliceMgr and Registry Servers
#
def set_servers(options):
   global slicemgr
   global registry
   global user
   global authority

   # Set SliceMgr URL
   if (options.sm is not None):
      sm_url = options.sm
   elif ("SFI_SM" in os.environ):
      sm_url = os.environ["SFI_SM"]
   else:
      print "No Known Slice Manager"
      sys.exit(1)

   # Set Registry URL
   if (options.registry is not None):
      reg_url = options.registry
   elif ("SFI_REGISTRY" in os.environ):
      reg_url = os.environ["SFI_REGISTRY"]
   else:
      print "No Known Registry Server"
      sys.exit(1)

   if options.verbose :
      print "Contacting Slice Manager at:", sm_url
      print "Contacting Registry at:", registry_url

   # Set user HRN
   if (options.user is not None):
      user = options.user
   elif ("SFI_USER" in os.environ):
      user = os.environ["SFI_USER"]
   else:
      print "No Known User Name"
      sys.exit(1)

   # Set authority HRN
   if (options.auth is not None):
      authority = options.auth
   elif ("SFI_AUTH" in os.environ):
      authority = os.environ["SFI_AUTH"]
   else:
      authority = None

   # Get key and certificate
   key_file = get_key_file()
   cert_file = get_cert_file(key_file)

   # Establish connection to server(s)
   slicemgr = GeniClient(sm_url, key_file, cert_file)
   registry = GeniClient(registry_url, key_file, cert_file)
   return

#
# Get various credential and spec files
#
# Establishes limiting conventions
#   - conflates MAs and SAs
#   - assumes last token in slice name is unique
#
# Bootstraps credentials
#   - bootstrap user credential from self-signed certificate
#   - bootstrap authority credential from user credential
#   - bootstrap slice credential from user credential
#

def get_leaf(name):
   parts = name.split(".")
   return parts[-1]

def get_key_file():
   file = os.path.join(sfi_dir, get_leaf(user) + ".pkey")
   if (os.path.isfile(file)):
      return file
   else:
      print "Key file", file, "does not exist"
      sys.exit(-1)
   return

def get_cert_file(key_file):
   global verbose

   file = os.path.join(sfi_dir, get_leaf(user) + ".cert")
   if (os.path.isfile(file)):
      return file
   else:
      k = Keypair(filename = key_file)
      cert = Certificate(subject=user)
      cert.set_pubkey(k)
      cert.set_issuer(k, user)
      cert.sign()
      if verbose :
         print "Writing self-signed certificate to", file
      cert.save_to_file(file)
      return file

def get_user_cred():
   global user

   file = os.path.join(sfi_dir, get_leaf(user) + ".cred")
   if (os.path.isfile(file)):
      user_cred = Credential(filename=file)
      return user_cred
   else:
      # bootstrap user credential
      user_cred = get_credential(None, "user", user)
      if user_cred:
         user_cred.save_to_file(file, save_parents=True)
         if verbose:
            print "Writing user credential to", file
         return user_cred
      else:
         print "Failed to get user credential"
         sys.exit(-1)

def get_auth_cred():
   global authority

   file = os.path.join(sfi_dir, "authority.cred")
   if (os.path.isfile(file)):
      auth_cred = Credential(filename=file)
      return auth_cred
   else:
      # bootstrap authority credential from user credential
      user_cred = get_user_cred()
      auth_cred = get_credential(user_cred, "sa", authority)
      if auth_cred:
         auth_cred.save_to_file(file, save_parents=True)
         if verbose:
            print "Writing authority credential to", file
         return auth_cred
      else:
         print "Failed to get authority credential"
         sys.exit(-1)

def get_slice_cred(name):
   file = os.path.join(sfi_dir, "slice_" + get_leaf(name) + ".cred")
   if (os.path.isfile(file)):
      slice_cred = Credential(filename=file)
      return slice_cred
   else:
      # bootstrap slice credential from user credential
      user_cred = get_user_cred()
      slice_cred = get_credential(user_cred, "slice", name)
      if slice_cred:
         slice_cred.save_to_file(file, save_parents=True)
         if verbose:
            print "Writing slice credential to", file
         return slice_cred
      else:
         print "Failed to get slice credential"
         sys.exit(-1)

def get_rspec_file(rspec):
   if (os.path.isabs(rspec)):
      file = rspec
   else:
      file = os.path.join(sfi_dir, rspec)
   if (os.path.isfile(file)):
      return file
   else:
      print "No such rspec file"
      sys.exit(1)

def get_record_file(record):
   if (os.path.isabs(record)):
      file = record
   else:
      file = os.path.join(sfi_dir, record)
   if (os.path.isfile(file)):
      return file
   else:
      print "No such registry record file"
      sys.exit(1)

#
# Generate sub-command parser
#
def create_cmd_parser(command):
   cmdargs = {"list": "name",
              "show": "name",
              "remove": "name",
              "add": "name record",
              "update": "name record",
              "nodes": "[name]",
              "slices": "",
              "resources": "name",
              "create": "name rspec",
              "delete": "name",
              "reset": "name",
              "start": "name",
              "stop": "name"
             }
   if command not in cmdargs:
      print "Invalid command\n"
      print "Commands:list,show,remove,add,update,nodes,slices,resources,create,delete,start,stop,reset"
      sys.exit(2)

   parser = OptionParser(usage="sfi [sfi_options] %s [options] %s" \
      % (command, cmdargs[command]))
   if command in ("nodes", "resources"):
      parser.add_option("-f", "--format", dest="format",type="choice",
           help="output format (dns|ip|hrn|rspec)",default="rspec",
           choices=("dns","ip","hrn","rspec"))
   elif command in ("list", "show", "remove"):
      parser.add_option("-t", "--type", dest="type",type="choice",
           help="type filter (user|slice|sa|ma|node|aggregate)",
           choices=("user","slice","sa","ma","node","aggregate", "all"),
           default="all")
   return parser

#
# Main: parse arguments and dispatch to command
#
def main():
   global verbose

   # Generate command line parser
   parser = OptionParser(usage="sfi [options] command [command_options] [command_args]",
        description="Commands: list,show,remove,add,update,nodes,slices,resources,create,delete,start,stop,reset")
   parser.add_option("-r", "--registry", dest="registry",
        help="root registry", metavar="URL", default=None)
   parser.add_option("-s", "--slicemgr", dest="sm",
        help="slice manager", metavar="URL", default=None)
   parser.add_option("-d", "--dir", dest="dir",
        help="working directory", metavar="PATH", default = sfi_dir)
   parser.add_option("-u", "--user", dest="user",
        help="user name", metavar="HRN", default=None)
   parser.add_option("-a", "--auth", dest="auth",
        help="authority name", metavar="HRN", default=None)
   parser.add_option("-v", "--verbose",
        action="store_true", dest="verbose", default=False,
        help="verbose mode")
   parser.disable_interspersed_args()
   (options, args) = parser.parse_args()
   command = args[0]
   (cmd_opts, cmd_args) = create_cmd_parser(command).parse_args(args[1:])
   verbose = options.verbose
   if verbose :
      print options.registry, options.sm, options.dir, options.verbose,\
         options.user, options.auth
      print command
      if command in ("nodes", "resources"):
         print cmd_opts.format
      elif command in ("list","show","remove"):
         print cmd_opts.type
      print cmd_args

   set_servers(options)

   # Dispatch to selected command
   try:
      globals()[command](cmd_opts, cmd_args)
   except KeyError:
      print "Command not found:", command
      sys.exit(1)
   return

#
# Following functions implement the commands
#
# Registry-related commands
#

# list entires in named authority registry
def list(opts, args):
   global registry
   user_cred = get_user_cred()
   result = registry.list(user_cred, args[0])
   display_record(opts.type, results)
   return

# show named registry record
def show(opts, args):
   global registry
   user_cred = get_user_cred()
   result = reg_chan.resolve(user_cred, args[0])
   display_record(opts.type, results)
   return

# removed named registry record
#   - have to first retrieve the record to be removed
def remove(opts, args):
   global registry
   auth_cred = get_auth_cred()
   results = registry.resolve(auth_cred, args[0])
   record = filter_record(opts.type, results)
   return registry.remove(auth_cred, record)

# add named registry record
def add(opts, args):
   global registry
   auth_cred = get_auth_cred()
   rec_file = get_record_file(args[1])
   with open(rec_file) as f:
      record = f.read()
   return registry.register(auth_cred, record)

# update named registry entry
def update(opts, args):
   global registry
   user_cred = get_user_cred()
   rec_file = get_record_file(args[1])
   with open(rec_file) as f:
      record = f.read()
   return registry.update(user_cred, record)

#
# Slice-related commands
#

# list available nodes
def nodes(opts, args):
   global slicemgr
   user_cred = get_user_cred()
   if (len(args) == 0):
      context = None
   else:
      context = args[0]
   result = slicemgr.list_nodes(user_cred, context)
   display_rspec(opts.format, result)
   return

# list instantiated slices
def slices(opts, args):
   global slicemgr
   user_cred = get_user_cred()
   result = slicemgr.list_slices(user_cred)
   display_rspec(opts.format, results)
   return

# show rspec for named slice
def resources(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   result = slicemgr.get_slice_resources(slice_cred, args[0])
   display_rspec(opts.format, result)
   return

# created named slice with given rspec
def create(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   rspec_file = get_rspec_file(args[1])
   with open(rspec_file) as f:
      rspec = f.read()
   return slicemgr.create_slice(slice_cred, rspec)

# delete named slice
def delete(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.delete_slice(slice_cred)

# start named slice
def start(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.start_slice(slice_cred)

# stop named slice
def stop(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.stop_slice(slice_cred)

# reset named slice
def reset(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.reset_slice(slice_cred)

#
#
# Display and Filter RSpecs and Records
#   - to be replace by EMF-generated routines
#
#

def display_rspec(format, rspec):
   print "display rspec"
   return

def display_record(type, record):
   rec = filter_record(type, record)
   print "display record"
   return

def filter_record(type, record):
   print "filter record"
   return

if __name__=="__main__":
   main()
<commit_msg>added 'save XML' option<commit_after>#! /usr/bin/env python
from __future__ import with_statement

# sfi -- slice-based facility interface

import sys
import os, os.path
from optparse import OptionParser
from util.cert import Keypair, Certificate
from util.credential import Credential
from util.geniclient import GeniClient

sfi_dir = os.path.expanduser("~/.sfi/")
slicemgr = None
registry = None
user = None
authority = None
verbose = False

#
# Establish Connection to SliceMgr and Registry Servers
#
def set_servers(options):
   global slicemgr
   global registry
   global user
   global authority

   # Set SliceMgr URL
   if (options.sm is not None):
      sm_url = options.sm
   elif ("SFI_SM" in os.environ):
      sm_url = os.environ["SFI_SM"]
   else:
      print "No Known Slice Manager"
      sys.exit(1)

   # Set Registry URL
   if (options.registry is not None):
      reg_url = options.registry
   elif ("SFI_REGISTRY" in os.environ):
      reg_url = os.environ["SFI_REGISTRY"]
   else:
      print "No Known Registry Server"
      sys.exit(1)

   if options.verbose :
      print "Contacting Slice Manager at:", sm_url
      print "Contacting Registry at:", registry_url

   # Set user HRN
   if (options.user is not None):
      user = options.user
   elif ("SFI_USER" in os.environ):
      user = os.environ["SFI_USER"]
   else:
      print "No Known User Name"
      sys.exit(1)

   # Set authority HRN
   if (options.auth is not None):
      authority = options.auth
   elif ("SFI_AUTH" in os.environ):
      authority = os.environ["SFI_AUTH"]
   else:
      authority = None

   # Get key and certificate
   key_file = get_key_file()
   cert_file = get_cert_file(key_file)

   # Establish connection to server(s)
   slicemgr = GeniClient(sm_url, key_file, cert_file)
   registry = GeniClient(registry_url, key_file, cert_file)
   return

#
# Get various credential and spec files
#
# Establishes limiting conventions
#   - conflates MAs and SAs
#   - assumes last token in slice name is unique
#
# Bootstraps credentials
#   - bootstrap user credential from self-signed certificate
#   - bootstrap authority credential from user credential
#   - bootstrap slice credential from user credential
#

def get_leaf(name):
   parts = name.split(".")
   return parts[-1]

def get_key_file():
   file = os.path.join(sfi_dir, get_leaf(user) + ".pkey")
   if (os.path.isfile(file)):
      return file
   else:
      print "Key file", file, "does not exist"
      sys.exit(-1)
   return

def get_cert_file(key_file):
   global verbose

   file = os.path.join(sfi_dir, get_leaf(user) + ".cert")
   if (os.path.isfile(file)):
      return file
   else:
      k = Keypair(filename = key_file)
      cert = Certificate(subject=user)
      cert.set_pubkey(k)
      cert.set_issuer(k, user)
      cert.sign()
      if verbose :
         print "Writing self-signed certificate to", file
      cert.save_to_file(file)
      return file

def get_user_cred():
   global user

   file = os.path.join(sfi_dir, get_leaf(user) + ".cred")
   if (os.path.isfile(file)):
      user_cred = Credential(filename=file)
      return user_cred
   else:
      # bootstrap user credential
      user_cred = get_credential(None, "user", user)
      if user_cred:
         user_cred.save_to_file(file, save_parents=True)
         if verbose:
            print "Writing user credential to", file
         return user_cred
      else:
         print "Failed to get user credential"
         sys.exit(-1)

def get_auth_cred():
   global authority

   file = os.path.join(sfi_dir, "authority.cred")
   if (os.path.isfile(file)):
      auth_cred = Credential(filename=file)
      return auth_cred
   else:
      # bootstrap authority credential from user credential
      user_cred = get_user_cred()
      auth_cred = get_credential(user_cred, "sa", authority)
      if auth_cred:
         auth_cred.save_to_file(file, save_parents=True)
         if verbose:
            print "Writing authority credential to", file
         return auth_cred
      else:
         print "Failed to get authority credential"
         sys.exit(-1)

def get_slice_cred(name):
   file = os.path.join(sfi_dir, "slice_" + get_leaf(name) + ".cred")
   if (os.path.isfile(file)):
      slice_cred = Credential(filename=file)
      return slice_cred
   else:
      # bootstrap slice credential from user credential
      user_cred = get_user_cred()
      slice_cred = get_credential(user_cred, "slice", name)
      if slice_cred:
         slice_cred.save_to_file(file, save_parents=True)
         if verbose:
            print "Writing slice credential to", file
         return slice_cred
      else:
         print "Failed to get slice credential"
         sys.exit(-1)

def get_rspec_file(rspec):
   if (os.path.isabs(rspec)):
      file = rspec
   else:
      file = os.path.join(sfi_dir, rspec)
   if (os.path.isfile(file)):
      return file
   else:
      print "No such rspec file"
      sys.exit(1)

def get_record_file(record):
   if (os.path.isabs(record)):
      file = record
   else:
      file = os.path.join(sfi_dir, record)
   if (os.path.isfile(file)):
      return file
   else:
      print "No such registry record file"
      sys.exit(1)

#
# Generate sub-command parser
#
def create_cmd_parser(command):
   cmdargs = {"list": "name",
              "show": "name",
              "remove": "name",
              "add": "name record",
              "update": "name record",
              "nodes": "[name]",
              "slices": "",
              "resources": "name",
              "create": "name rspec",
              "delete": "name",
              "reset": "name",
              "start": "name",
              "stop": "name"
             }
   if command not in cmdargs:
      print "Invalid command\n"
      print "Commands:list,show,remove,add,update,nodes,slices,resources,create,delete,start,stop,reset"
      sys.exit(2)

   parser = OptionParser(usage="sfi [sfi_options] %s [options] %s" \
      % (command, cmdargs[command]))
   if command in ("nodes", "resources"):
      parser.add_option("-f", "--format", dest="format",type="choice",
           help="display format (dns|ip|hrn|rspec)",default="rspec",
           choices=("dns","ip","hrn","rspec"))
   if command in ("list", "show", "remove"):
      parser.add_option("-t", "--type", dest="type",type="choice",
           help="type filter (user|slice|sa|ma|node|aggregate)",
           choices=("user","slice","sa","ma","node","aggregate", "all"),
           default="all")
   if command in ("show", "nodes", "resources"):
      parser.add_option("-o", "--output", dest="file",
           help="output XML to file", metavar="FILE", default=None)
   return parser

#
# Main: parse arguments and dispatch to command
#
def main():
   global verbose

   # Generate command line parser
   parser = OptionParser(usage="sfi [options] command [command_options] [command_args]",
        description="Commands: list,show,remove,add,update,nodes,slices,resources,create,delete,start,stop,reset")
   parser.add_option("-r", "--registry", dest="registry",
        help="root registry", metavar="URL", default=None)
   parser.add_option("-s", "--slicemgr", dest="sm",
        help="slice manager", metavar="URL", default=None)
   parser.add_option("-d", "--dir", dest="dir",
        help="working directory", metavar="PATH", default = sfi_dir)
   parser.add_option("-u", "--user", dest="user",
        help="user name", metavar="HRN", default=None)
   parser.add_option("-a", "--auth", dest="auth",
        help="authority name", metavar="HRN", default=None)
   parser.add_option("-v", "--verbose",
        action="store_true", dest="verbose", default=False,
        help="verbose mode")
   parser.disable_interspersed_args()
   (options, args) = parser.parse_args()
   command = args[0]
   (cmd_opts, cmd_args) = create_cmd_parser(command).parse_args(args[1:])
   verbose = options.verbose
   if verbose :
      print options.registry, options.sm, options.dir, options.verbose,\
         options.user, options.auth
      print command
      if command in ("nodes", "resources"):
         print cmd_opts.format
      elif command in ("list","show","remove"):
         print cmd_opts.type
      print cmd_args

   set_servers(options)

   # Dispatch to selected command
   try:
      globals()[command](cmd_opts, cmd_args)
   except KeyError:
      print "Command not found:", command
      sys.exit(1)
   return

#
# Following functions implement the commands
#
# Registry-related commands
#

# list entires in named authority registry
def list(opts, args):
   global registry
   user_cred = get_user_cred()
   result = registry.list(user_cred, args[0])
   display_record(opts.type, results)
   return

# show named registry record
def show(opts, args):
   global registry
   user_cred = get_user_cred()
   result = reg_chan.resolve(user_cred, args[0])
   display_record(opts.type, results)
   if (opts.file is not None):
      save_record_to_file(opts.file, result)
   return

# removed named registry record
#   - have to first retrieve the record to be removed
def remove(opts, args):
   global registry
   auth_cred = get_auth_cred()
   results = registry.resolve(auth_cred, args[0])
   record = filter_record(opts.type, results)
   return registry.remove(auth_cred, record)

# add named registry record
def add(opts, args):
   global registry
   auth_cred = get_auth_cred()
   rec_file = get_record_file(args[1])
   with open(rec_file) as f:
      record = f.read()
   return registry.register(auth_cred, record)

# update named registry entry
def update(opts, args):
   global registry
   user_cred = get_user_cred()
   rec_file = get_record_file(args[1])
   with open(rec_file) as f:
      record = f.read()
   return registry.update(user_cred, record)

#
# Slice-related commands
#

# list available nodes
def nodes(opts, args):
   global slicemgr
   user_cred = get_user_cred()
   if (len(args) == 0):
      context = None
   else:
      context = args[0]
   result = slicemgr.list_nodes(user_cred, context)
   display_rspec(opts.format, result)
   if (opts.file is not None):
      save_rspec_to_file(opts.file, result)
   return

# list instantiated slices
def slices(opts, args):
   global slicemgr
   user_cred = get_user_cred()
   result = slicemgr.list_slices(user_cred)
   display_rspec(opts.format, results)
   return

# show rspec for named slice
def resources(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   result = slicemgr.get_slice_resources(slice_cred, args[0])
   display_rspec(opts.format, result)
   if (opts.file is not None):
      save_rspec_to_file(opts.file, result)
   return

# created named slice with given rspec
def create(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   rspec_file = get_rspec_file(args[1])
   with open(rspec_file) as f:
      rspec = f.read()
   return slicemgr.create_slice(slice_cred, rspec)

# delete named slice
def delete(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.delete_slice(slice_cred)

# start named slice
def start(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.start_slice(slice_cred)

# stop named slice
def stop(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.stop_slice(slice_cred)

# reset named slice
def reset(opts, args):
   global slicemgr
   slice_cred = get_slice_cred(args[0])
   return slicemgr.reset_slice(slice_cred)

#
#
# Display, Save, and Filter RSpecs and Records
#   - to be replace by EMF-generated routines
#
#

def display_rspec(format, rspec):
   print "display rspec"
   return

def save_rspec_to_file(file, rspec):
   print "save rspec"
   return

def display_record(type, record):
   rec = filter_record(type, record)
   print "display record"
   return

def filter_record(type, record):
   print "filter record"
   return

def save_record_to_file(file, record):
   print "save record"
   return

if __name__=="__main__":
   main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from cStringIO import StringIO
from os.path import getsize
from struct import pack, unpack, error as StructError
from .log import log
from .structures import fields

class DBHeader(object):
	"""
	Base DBFile header class
	"""
	def __repr__(self):
		return "%s(%s)" % (self.__class__.__name__, ", ".join("%s=%r" % (k, self.__dict__[k]) for k in self.__dict__))

class DBFile(object):
	"""
	Base class for WDB and DBC files
	"""

	def __init__(self, file, build, structure, environment):
		if isinstance(file, basestring):
			file = open(file, "r")
		self.file = file
		self.build = build
		self._addresses = {}
		self._values = {}
		self.environment = environment

		self._rowDynamicFields = 0 # Dynamic fields index, used when parsing a row

	def __repr__(self):
		return "%s(file=%r, build=%r)" % (self.__class__.__name__, self.file, self.build)

	def __contains__(self, id):
		return id in self._addresses

	def __getitem__(self, item):
		if isinstance(item, slice):
			keys = sorted(self._addresses.keys())[item]
			return [self[k] for k in keys]

		if item not in self._values:
			self._parse_row(item)

		return self._values[item]

	def __setitem__(self, item, value):
		if not isinstance(item, int):
			raise TypeError("DBFile indices must be integers, not %s" % (type(item)))

		if isinstance(value, DBRow):
			self._values[item] = value
			self._addresses[item] = -1
		else:
			# FIXME technically we should allow DBRow, but this is untested and will need resetting parent
			raise TypeError("Unsupported type for DBFile.__setitem__: %s" % (type(value)))

	def __delitem__(self, item):
		if item in self._values:
			del self._values[item]
		del self._addresses[item]

	def __iter__(self):
		return self._addresses.__iter__()

	def __len__(self):
		return len(self._addresses)

	def _add_row(self, id, address, reclen):
		if id in self._addresses: # Something's wrong here
			log.warning("Multiple instances of row %r found" % (id))
		self._addresses[id] = (address, reclen)

	def _parse_field(self, data, field, row=None):
		"""
		Parse a single field in stream.
		"""
		if field.dyn > self._rowDynamicFields:
			return None # The column doesn't exist in this row, we set it to None

		ret = None
		try:
			if isinstance(field, fields.StringField):
				ret = self._parse_string(data)

			elif isinstance(field, fields.DataField): # wowcache.wdb
				length = getattr(row, field.master)
				ret = data.read(length)

			elif isinstance(field, fields.DynamicMaster):
				ret, = unpack("<I", data.read(4))
				self._rowDynamicFields = ret

			else:
				ret, = unpack("<%s" % (field.char), data.read(field.size))
		except StructError:
			log.warning("Field %s could not be parsed properly" % (field))
			ret = None

		return ret

	def supportsSeeking(self):
		return hasattr(self.file, "seek")

	def append(self, row):
		"""
		Append a row at the end of the file.
		If the row does not have an id, one is automatically assigned.
		"""
		i = len(self) + 1 # FIXME this wont work properly in incomplete files
		if "_id" not in row:
			row["_id"] = i
		self[i] = row

	def clear(self):
		"""
		Delete every row in the file
		"""
		for k in self.keys(): # Use key, otherwise we get RuntimeError: dictionary changed size during iteration
			del self[k]

	def keys(self):
		return self._addresses.keys()

	def items(self):
		return [(k, self[k]) for k in self]

	def parse_row(self, data, reclen=0):
		"""
		Assign data to a DBRow instance
		"""
		return DBRow(self, data=data, reclen=reclen)

	def values(self):
		"""
		Return a list of the file's values
		"""
		return [self[id] for id in self]

	def setRow(self, key, **values):
		self.__setitem__(key, DBRow(self, columns=values))

	def update(self, other):
		"""
		Update file from iterable other
		"""
		for k in other:
			self[k] = other[k]

	def write(self, filename=""):
		"""
		Write the file data on disk. If filename is not given, use currently opened file.
		"""
		_filename = filename or self.file.name

		data = self.header.data() + self.data() + self.eof()

		f = open(_filename, "wb") # Don't open before calling data() as uncached rows would be empty
		f.write(data)
		f.close()
		log.info("Written %i bytes at %s" % (getsize(f.name), f.name))

		if not filename: # Reopen self.file, we modified it
			# XXX do we need to wipe self._values here?
			self.file.close()
			self.file = open(f.name, "rb")

class DBRow(list):
	"""
	A database row.
	Names of the variables of that class should not be used in field names of structures
	"""
	initialized = False

	def __init__(self, parent, data=None, columns=None, reclen=0):
		self._parent = parent
		self._values = {} # Columns values storage
		self.structure = parent.structure

		self.initialized = True # needed for __setattr__

		if columns:
			if type(columns) == list:
				self.extend(columns)

			elif type(columns) == dict:
				self._default()
				_cols = [k.name for k in self.structure]
				for k in columns:
					try:
						self[_cols.index(k)] = columns[k]
					except ValueError:
						log.warning("Column %r not found" % (k))

		elif data:
			dynfields = 0
			data = StringIO(data)
			for field in self.structure:
				_data = parent._parse_field(data, field, self)
				self.append(_data)

			if reclen:
				real_reclen = reclen + self._parent.row_header_size
				if data.tell() != real_reclen:
					log.warning("Reclen not respected for row %r. Expected %i, read %i. (%+i)" % (self.id, real_reclen, data.tell(), real_reclen-data.tell()))

	def __dir__(self):
		result = self.__dict__.keys()
		result.extend(self.structure.column_names)
		return result

	def __getattr__(self, attr):
		if attr in self.structure:
			return self._get_value(attr)

		if attr in self.structure._abstractions: # Union abstractions etc
			field, func = self.structure._abstractions[attr]
			return func(field, self)

		if "__" in attr:
			return self.__get_deep_relation(attr)

		return super(DBRow, self).__getattribute__(attr)

	def __int__(self):
		return self.id

	def __setattr__(self, attr, value):
		# Do not preserve the value in DBRow! Use the save method to save.
		if self.initialized and attr in self.structure:
			self._set_value(attr, value)
		return super(DBRow, self).__setattr__(attr, value)

	def __setitem__(self, index, value):
		if not isinstance(index, int):
			raise TypeError("Expected int instance, got %s instead (%r)" % (type(index), index))
		list.__setitem__(self, index, value)
		col = self.structure[index]
		try:
			self._values[col.name] = col.to_python(value, row=self)
		except fields.UnresolvedRelation:
			self._values[col.name] = value

	def __get_reverse_relation(self, table, field):
		"""
		Return a list of rows matching the reverse relation
		"""
		if not hasattr(self._parent, "_reverse_relation_cache"):
			self._parent._reverse_relation_cache = {}
		cache = self._parent._reverse_relation_cache

		tfield = table + "__" + field
		if tfield not in cache:
			cache[tfield] = {}
			# First time lookup, let's build the cache
			table = self._parent.environment[table]
			for row in table:
				row = table[row]
				id = row._raw(field)
				if id not in cache[tfield]:
					cache[tfield][id] = []
				cache[tfield][id].append(row)

		return cache[tfield].get(self.id, None)

	def __get_deep_relation(self, rel):
		"""
		Parse a django-like multilevel relationship
		"""
		rels = rel.split("__")
		if "" in rels: # empty string
			raise ValueError("Invalid relation string")

		first = rels[0]
		if not hasattr(self, first):
			if first in self._parent.environment:
				remainder = rel[len(first + "__"):]
				return self.__get_reverse_relation(first, remainder)
			raise ValueError("Invalid relation string")

		ret = self
		rels = rels[::-1]
		while rels:
			ret = getattr(ret, rels.pop())

		return ret

	def _set_value(self, name, value):
		index = self.structure.index(name)
		col = self.structure[index]
		try:
			self._values[name] = col.to_python(value, self)
		except fields.UnresolvedRelation:
			self._values[name] = value
		self[index] = value

	def _get_value(self, name):
		if name not in self._values:
			raw_value = self[self.structure.index(name)]

			try:
				self._set_value(name, raw_value)
			except fields.UnresolvedRelation as e:
				return fields.UnresolvedObjectRef(e.reference)
			except fields.RelationError:
				return None # Key doesn't exist, or equals 0

		return self._values[name]

	def _raw(self, name):
		"""
		Returns the raw value from field 'name'
		"""
		index = self.structure.index(name)
		return self[index]

	def _save(self):
		for name in self._values:
			index = self.structure.index(name)
			col = self.structure[index]
			self[index] = col.from_python(self._values[name])

	def _field(self, name):
		"""
		Returns the field 'name'
		"""
		index = self.structure.index(name)
		return self.structure[index]

	def _default(self):
		"""
		Change all fields to their default values
		"""
		del self[:]
		self._values = {}
		for col in self.structure:
			char = col.char
			if col.dyn:
				self.append(None)
			elif char == "s":
				self.append("")
			elif char == "f":
				self.append(0.0)
			else:
				self.append(0)

	def dict(self):
		"""
		Return a dict of the row as colname: value
		"""
		return dict(zip(self.structure.column_names, self))

	def update(self, other):
		for k in other:
			self[k] = other[k]

	@property
	def id(self):
		"Temporary hack to transition between _id and id"
		return self._id
<commit_msg>Rename some __methods<commit_after># -*- coding: utf-8 -*-

from cStringIO import StringIO
from os.path import getsize
from struct import pack, unpack, error as StructError
from .log import log
from .structures import fields

class DBHeader(object):
	"""
	Base DBFile header class
	"""
	def __repr__(self):
		return "%s(%s)" % (self.__class__.__name__, ", ".join("%s=%r" % (k, self.__dict__[k]) for k in self.__dict__))

class DBFile(object):
	"""
	Base class for WDB and DBC files
	"""

	def __init__(self, file, build, structure, environment):
		if isinstance(file, basestring):
			file = open(file, "r")
		self.file = file
		self.build = build
		self._addresses = {}
		self._values = {}
		self.environment = environment

		self._rowDynamicFields = 0 # Dynamic fields index, used when parsing a row

	def __repr__(self):
		return "%s(file=%r, build=%r)" % (self.__class__.__name__, self.file, self.build)

	def __contains__(self, id):
		return id in self._addresses

	def __getitem__(self, item):
		if isinstance(item, slice):
			keys = sorted(self._addresses.keys())[item]
			return [self[k] for k in keys]

		if item not in self._values:
			self._parse_row(item)

		return self._values[item]

	def __setitem__(self, item, value):
		if not isinstance(item, int):
			raise TypeError("DBFile indices must be integers, not %s" % (type(item)))

		if isinstance(value, DBRow):
			self._values[item] = value
			self._addresses[item] = -1
		else:
			# FIXME technically we should allow DBRow, but this is untested and will need resetting parent
			raise TypeError("Unsupported type for DBFile.__setitem__: %s" % (type(value)))

	def __delitem__(self, item):
		if item in self._values:
			del self._values[item]
		del self._addresses[item]

	def __iter__(self):
		return self._addresses.__iter__()

	def __len__(self):
		return len(self._addresses)

	def _add_row(self, id, address, reclen):
		if id in self._addresses: # Something's wrong here
			log.warning("Multiple instances of row %r found" % (id))
		self._addresses[id] = (address, reclen)

	def _parse_field(self, data, field, row=None):
		"""
		Parse a single field in stream.
		"""
		if field.dyn > self._rowDynamicFields:
			return None # The column doesn't exist in this row, we set it to None

		ret = None
		try:
			if isinstance(field, fields.StringField):
				ret = self._parse_string(data)

			elif isinstance(field, fields.DataField): # wowcache.wdb
				length = getattr(row, field.master)
				ret = data.read(length)

			elif isinstance(field, fields.DynamicMaster):
				ret, = unpack("<I", data.read(4))
				self._rowDynamicFields = ret

			else:
				ret, = unpack("<%s" % (field.char), data.read(field.size))
		except StructError:
			log.warning("Field %s could not be parsed properly" % (field))
			ret = None

		return ret

	def supportsSeeking(self):
		return hasattr(self.file, "seek")

	def append(self, row):
		"""
		Append a row at the end of the file.
		If the row does not have an id, one is automatically assigned.
		"""
		i = len(self) + 1 # FIXME this wont work properly in incomplete files
		if "_id" not in row:
			row["_id"] = i
		self[i] = row

	def clear(self):
		"""
		Delete every row in the file
		"""
		for k in self.keys(): # Use key, otherwise we get RuntimeError: dictionary changed size during iteration
			del self[k]

	def keys(self):
		return self._addresses.keys()

	def items(self):
		return [(k, self[k]) for k in self]

	def parse_row(self, data, reclen=0):
		"""
		Assign data to a DBRow instance
		"""
		return DBRow(self, data=data, reclen=reclen)

	def values(self):
		"""
		Return a list of the file's values
		"""
		return [self[id] for id in self]

	def setRow(self, key, **values):
		self.__setitem__(key, DBRow(self, columns=values))

	def update(self, other):
		"""
		Update file from iterable other
		"""
		for k in other:
			self[k] = other[k]

	def write(self, filename=""):
		"""
		Write the file data on disk. If filename is not given, use currently opened file.
		"""
		_filename = filename or self.file.name

		data = self.header.data() + self.data() + self.eof()

		f = open(_filename, "wb") # Don't open before calling data() as uncached rows would be empty
		f.write(data)
		f.close()
		log.info("Written %i bytes at %s" % (getsize(f.name), f.name))

		if not filename: # Reopen self.file, we modified it
			# XXX do we need to wipe self._values here?
			self.file.close()
			self.file = open(f.name, "rb")

class DBRow(list):
	"""
	A database row.
	Names of the variables of that class should not be used in field names of structures
	"""
	initialized = False

	def __init__(self, parent, data=None, columns=None, reclen=0):
		self._parent = parent
		self._values = {} # Columns values storage
		self.structure = parent.structure

		self.initialized = True # needed for __setattr__

		if columns:
			if type(columns) == list:
				self.extend(columns)

			elif type(columns) == dict:
				self._default()
				_cols = [k.name for k in self.structure]
				for k in columns:
					try:
						self[_cols.index(k)] = columns[k]
					except ValueError:
						log.warning("Column %r not found" % (k))

		elif data:
			dynfields = 0
			data = StringIO(data)
			for field in self.structure:
				_data = parent._parse_field(data, field, self)
				self.append(_data)

			if reclen:
				real_reclen = reclen + self._parent.row_header_size
				if data.tell() != real_reclen:
					log.warning("Reclen not respected for row %r. Expected %i, read %i. (%+i)" % (self.id, real_reclen, data.tell(), real_reclen-data.tell()))

	def __dir__(self):
		result = self.__dict__.keys()
		result.extend(self.structure.column_names)
		return result

	def __getattr__(self, attr):
		if attr in self.structure:
			return self._get_value(attr)

		if attr in self.structure._abstractions: # Union abstractions etc
			field, func = self.structure._abstractions[attr]
			return func(field, self)

		if "__" in attr:
			return self._get_deep_relation(attr)

		return super(DBRow, self).__getattribute__(attr)

	def __int__(self):
		return self.id

	def __setattr__(self, attr, value):
		# Do not preserve the value in DBRow! Use the save method to save.
		if self.initialized and attr in self.structure:
			self._set_value(attr, value)
		return super(DBRow, self).__setattr__(attr, value)

	def __setitem__(self, index, value):
		if not isinstance(index, int):
			raise TypeError("Expected int instance, got %s instead (%r)" % (type(index), index))
		list.__setitem__(self, index, value)
		col = self.structure[index]
		try:
			self._values[col.name] = col.to_python(value, row=self)
		except fields.UnresolvedRelation:
			self._values[col.name] = value

	def _get_reverse_relation(self, table, field):
		"""
		Return a list of rows matching the reverse relation
		"""
		if not hasattr(self._parent, "_reverse_relation_cache"):
			self._parent._reverse_relation_cache = {}
		cache = self._parent._reverse_relation_cache

		tfield = table + "__" + field
		if tfield not in cache:
			cache[tfield] = {}
			# First time lookup, let's build the cache
			table = self._parent.environment[table]
			for row in table:
				row = table[row]
				id = row._raw(field)
				if id not in cache[tfield]:
					cache[tfield][id] = []
				cache[tfield][id].append(row)

		return cache[tfield].get(self.id, None)

	def _get_deep_relation(self, rel):
		"""
		Parse a django-like multilevel relationship
		"""
		rels = rel.split("__")
		if "" in rels: # empty string
			raise ValueError("Invalid relation string")

		first = rels[0]
		if not hasattr(self, first):
			if first in self._parent.environment:
				remainder = rel[len(first + "__"):]
				return self._get_reverse_relation(first, remainder)
			raise ValueError("Invalid relation string")

		ret = self
		rels = rels[::-1]
		while rels:
			ret = getattr(ret, rels.pop())

		return ret

	def _set_value(self, name, value):
		index = self.structure.index(name)
		col = self.structure[index]
		try:
			self._values[name] = col.to_python(value, self)
		except fields.UnresolvedRelation:
			self._values[name] = value
		self[index] = value

	def _get_value(self, name):
		if name not in self._values:
			raw_value = self[self.structure.index(name)]

			try:
				self._set_value(name, raw_value)
			except fields.UnresolvedRelation as e:
				return fields.UnresolvedObjectRef(e.reference)
			except fields.RelationError:
				return None # Key doesn't exist, or equals 0

		return self._values[name]

	def _raw(self, name):
		"""
		Returns the raw value from field 'name'
		"""
		index = self.structure.index(name)
		return self[index]

	def _save(self):
		for name in self._values:
			index = self.structure.index(name)
			col = self.structure[index]
			self[index] = col.from_python(self._values[name])

	def _field(self, name):
		"""
		Returns the field 'name'
		"""
		index = self.structure.index(name)
		return self.structure[index]

	def _default(self):
		"""
		Change all fields to their default values
		"""
		del self[:]
		self._values = {}
		for col in self.structure:
			char = col.char
			if col.dyn:
				self.append(None)
			elif char == "s":
				self.append("")
			elif char == "f":
				self.append(0.0)
			else:
				self.append(0)

	def dict(self):
		"""
		Return a dict of the row as colname: value
		"""
		return dict(zip(self.structure.column_names, self))

	def update(self, other):
		for k in other:
			self[k] = other[k]

	@property
	def id(self):
		"Temporary hack to transition between _id and id"
		return self._id
<|endoftext|>


<commit_before>import aiohttp

# Reddit command
async def ex(c, pch, dch, m, a):
    a = a.split(' ')
    if len(a) == 0:
        async with aiohttp.ClientSession() as s:
            async with s.get('https://reddit.com/.json') as r:
                d = await r.json()
                r = '''Hot 10 Submissions in Reddit's Front Page:
'''
                cnt = 1
                for s in d['data']['children']:
                    r += '''{}) {}
    {}
'''.format(cnt, s['data']['secure_media']['oembed']['title'])
                    cnt += 1
                await c.send_message(pch, r)
                await c.send_message(dch, '{}, I sent the list in a private message.'.format(m))
    else:
        a = a[0]
        async with aiohttp.ClientSession() as s:
            async with s.get('https://reddit.com/r/{}/.json'.format(a)) as r:
                d = await r.json()
                r = '''10 Hot Submissions in /r/{}:
'''.format(a)
                cnt = 1
                for sr in d['data']['children']:
                    if sr['data']['domain'] != 'self.movies':
                        r += '''{}) {}
    {}
'''.format(cnt, sr['data']['secure_media']['oembed']['title'])
                    cnt += 1
                await c.send_message(pch, r)
                await c.send_message(dch, '{}, I sent the list in a private message.'.format(m))
<commit_msg>Crushing bug...<commit_after>import aiohttp

# Reddit command
async def ex(c, pch, dch, m, a):
    a = a.split(' ')
    if len(a) == 1:
        async with aiohttp.ClientSession() as s:
            async with s.get('https://reddit.com/.json') as r:
                d = await r.json()
                r = '''Hot 10 Submissions in Reddit's Front Page:
'''
                cnt = 1
                for sr in d['data']['children']:
                    title = sr['data']['title']
                    url = sr['data']['permalink']
                    r += '''{}) {}
    <https://reddit.com{}>
'''.format(cnt, title, url)
                    cnt += 1
                await c.send_message(pch, r)
                await c.send_message(dch, '{}, I sent the list in a private message.'.format(m))
    else:
        a = a[1]
        async with aiohttp.ClientSession() as s:
            async with s.get('https://reddit.com/r/{}/.json'.format(a)) as r:
                d = await r.json()
                r = '''10 Hot Submissions in /r/{}:
'''.format(a)
                cnt = 1
                for sr in d['data']['children']:
                    if not sr['data']['author'] != 'AutoModerator':
                        title = sr['data']['title']
                        url = sr['data']['permalink']
                        r += '''{}) {}
    <https://reddit.com{}>
'''.format(cnt, title, url)
                    cnt += 1
                await c.send_message(pch, r)
                await c.send_message(dch, '{}, I sent the list in a private message.'.format(m))
<|endoftext|>


<commit_before>
import os
import shutil

from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_json

from yunohost.tools import Migration
from yunohost.app import (_initialize_apps_catalog_system,
                          _update_apps_catalog,
                          APPS_CATALOG_CACHE,
                          APPS_CATALOG_CONF)

logger = getActionLogger('yunohost.migration')

LEGACY_APPS_CATALOG_CONF = '/etc/yunohost/appslists.json'
LEGACY_APPS_CATALOG_CONF_BACKUP = LEGACY_APPS_CATALOG_CONF + ".old"

class MyMigration(Migration):

    "Migrate to the new future-proof apps catalog system"

    def run(self):

        if not os.path.exists(LEGACY_APPS_CATALOG_CONF):
            logger.info("No need to do anything")

        # Destroy old lecacy cache
        if os.path.exists(APPS_CATALOG_CACHE):
            shutil.rmtree(APPS_CATALOG_CACHE)

        # and legacy cron
        if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
            os.remove("/etc/cron.daily/yunohost-fetch-appslists")

        # Backup the legacy file
        try:
            legacy_catalogs = read_json(LEGACY_APPS_CATALOG_CONF)
            # If there's only one catalog, we assume it's just the old official catalog
            # Otherwise, warn the (power-?)users that they should migrate their old catalogs manually
            if len(legacy_catalogs) > 1:
                logger.warning("It looks like you had additional apps_catalog in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom apps_catalog. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPS_CATALOG_CONF, APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP))
        except Exception as e:
            logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPS_CATALOG_CONF, str(e)))

        os.rename(LEGACY_APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP)

        _initialize_apps_catalog_system()
        _update_apps_catalog()
<commit_msg>Improve robustness / idempotency(?) of new app catalog migration<commit_after>
import os
import shutil

from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_json

from yunohost.tools import Migration
from yunohost.app import (_initialize_apps_catalog_system,
                          _update_apps_catalog,
                          APPS_CATALOG_CACHE,
                          APPS_CATALOG_CONF)

logger = getActionLogger('yunohost.migration')

LEGACY_APPS_CATALOG_CONF = '/etc/yunohost/appslists.json'
LEGACY_APPS_CATALOG_CONF_BACKUP = LEGACY_APPS_CATALOG_CONF + ".old"

class MyMigration(Migration):

    "Migrate to the new future-proof apps catalog system"

    def run(self):

        if not os.path.exists(LEGACY_APPS_CATALOG_CONF):
            logger.info("No need to do anything")

        # Destroy old lecacy cache
        if os.path.exists(APPS_CATALOG_CACHE):
            shutil.rmtree(APPS_CATALOG_CACHE)

        # and legacy cron
        if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
            os.remove("/etc/cron.daily/yunohost-fetch-appslists")

        # Backup the legacy file
        try:
            legacy_catalogs = read_json(LEGACY_APPS_CATALOG_CONF)
            # If there's only one catalog, we assume it's just the old official catalog
            # Otherwise, warn the (power-?)users that they should migrate their old catalogs manually
            if len(legacy_catalogs) > 1:
                logger.warning("It looks like you had additional apps_catalog in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom apps_catalog. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPS_CATALOG_CONF, APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP))
        except Exception as e:
            logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPS_CATALOG_CONF, str(e)))

        if os.path.exists(LEGACY_APPS_CATALOG_CONF):
            os.rename(LEGACY_APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP)

        _initialize_apps_catalog_system()
        _update_apps_catalog()
<|endoftext|>


<commit_before>import random
from django.conf import settings
from django.core.urlresolvers import reverse

def inject_settings(request):
    """Use this function to inject specific settings into every page."""
    return {
        'DEBUG': settings.DEBUG,
        'MIN_DONATION': settings.MIN_DONATION
    }

info_tips = (
    # Other Projects
    'RECAP is our browser extension that saves you money whenever you use PACER.',
    'Using the RECAP project means never paying for the same PACER document twice.',
    'CourtListener is powered by more than 200 screen scrapers.',
    'We are collecting all of the court seals in the U.S. You can help contribute seals.',
    'We have a collection of more than 200 judicial portraits and we need volunteers to help collect them all.',

    # History
    'CourtListener was started in 2009 to create alerts for the Federal Appeals Courts. It has since grown into the user-supported non-profit Free Law Project.',

    # Non-profit and fundraising
    'Free Law Project is a 501(c)(3) non-profit that relies on your support to operate. Please donate to support this site.',
    'CourtListener gets more than two million visits per year, but has a lean staff of only a few developers. Please donate to support this site.',
    'CourtListener is supported by user donations and small grants. More donations result in less time spent seeking grants and more time adding features.',
    'Free Law Project is a member of the Free Access to Law Movement and relies heavily on your donations.',
    'Using smile.amazon.com, you can donate 0.5% of every purchase you make on Amazon to Free Law Project, the non-profit that sponsors CourtListener.',

    # Recognition
    'Free Law Project\'s founders were selected as FastCase 50 winners in 2014.',
    'Oral Arguments were selected as a Top Ten Legal Hack of 2014.',

    # Open source
    'All of code powering CourtListener is open source and can be copied, shared, and contributed to.',
    'We need volunteers to help us with coding, design and legal research. Contact us for more info or check out our help wanted board to get started.',
    'The current design of CourtListener was created by a volunteer.',

    # Neutral Citations
    'WestLaw currently has a monopoly on citations. This hinders legal innovation but few courts have adopted neutral citations.',

    # Features
    'Create alerts for any query to receive an email if the query has new results.',
    'There is an RSS feed for every query so you can easily stay up to date.',
    'A podcast is created for every oral argument query that you make.',
    'CourtListener has an API so anybody can easily use our data.' % reverse("api_index"),
    'Oral Arguments are available in variety of apps like Stitcher Radio and Google Music.' % reverse('podcasts'),
    'Search Relevancy on CourtListener is highly tuned and is powered by the citation network between cases.',
    'You can make sophisticated queries using a number of advanced search features.' % reverse('advanced_search'),
    'You can get an alert whenever an opinion is cited by using a Citation Search.',
    'We partnered with University of Baltimore to make a system of visualizing Supreme Court Cases.',
    'Information from the Supreme Court Database is available for nearly every SCOTUS case, making it easy to get in-depth analysis.',

    # RECAP
    'PACER was created by Congress and makes around $130,000,000 selling public domain legal documents.',
    'With an estimated 1 billion documents, PACER is likely the largest paywall in the world.',
    'The average PACER document is about 9.1 pages long. The longest we\'ve seen is over 4,000 pages.',
    'We extract text from every PACER document so that it is searchable. This takes months!',
)
def inject_random_tip(request):
    return {'TIP': random.choice(info_tips)}
<commit_msg>Cleanup and additions to the random info tips.<commit_after>import random

from django.conf import settings
from django.core.urlresolvers import reverse

def inject_settings(request):
    """Use this function to inject specific settings into every page."""
    return {
        'DEBUG': settings.DEBUG,
        'MIN_DONATION': settings.MIN_DONATION
    }

info_tips = (
    # Other Projects
    'RECAP is our browser extension that saves you money whenever you use PACER.',
    'Using the RECAP project means never paying for the same PACER document twice.',
    'CourtListener is powered by more than 200 screen scrapers.',
    'We are collecting all of the court seals in the U.S. You can help contribute seals.',
    'We have a collection of more than 200 judicial portraits and we need volunteers to help collect them all.',

    # History
    'CourtListener was started in 2009 to create alerts for the Federal Appeals Courts. It has since grown into the user-supported non-profit Free Law Project.',

    # Non-profit and fundraising
    'Free Law Project is a 501(c)(3) non-profit that relies on your support to operate. Please donate to support this site.',
    'CourtListener gets more than two million visits per year, but has a lean staff of only a few developers. Please donate to support this site.',
    'CourtListener is supported by user donations and small grants. More donations result in less time spent seeking grants and more time adding features.',
    'Free Law Project is a member of the Free Access to Law Movement and relies heavily on your donations.',
    'Using smile.amazon.com, you can donate 0.5% of every purchase you make on Amazon to Free Law Project, the non-profit that sponsors CourtListener.',

    # Recognition
    'Free Law Project\'s founders were selected as FastCase 50 winners in 2014.',
    'Oral Arguments were selected as a Top Ten Legal Hack of 2014.',

    # Open source
    'All of code powering CourtListener is open source and can be copied, shared, and contributed to.',
    'We need volunteers to help us with coding, design and legal research. Contact us for more info or check out our help wanted board to get started.',
    'The current design of CourtListener was created by a volunteer.',

    # Neutral Citations
    'WestLaw currently has a monopoly on citations. This hinders legal innovation but few courts have adopted neutral citations.',

    # Features
    'Create alerts for any query to receive an email if the query has new results.',
    'There is an RSS feed for every query so you can easily stay up to date.',
    'A podcast is created for every oral argument query that you make.',
    'CourtListener has an API so anybody can easily use our data.' % reverse("api_index"),
    'Oral Arguments are available in variety of apps like Stitcher Radio and Google Music.' % reverse('podcasts'),
    'Search Relevancy on CourtListener is highly tuned and is powered by the citation network between cases.',
    'You can make sophisticated queries using a number of advanced search features.' % reverse('advanced_search'),
    'You can get an alert whenever an opinion is cited by using a Citation Search.',
    'We partnered with University of Baltimore to make a system of visualizing Supreme Court Cases.',
    'Information from the Supreme Court Database is available for nearly every SCOTUS case, making it easy to get in-depth analysis.',

    # RECAP & PACER
    'RECAP was originally created at The Center for Information Technology Policy at Princeton University.',
    'PACER was created by Congress and makes around $140,000,000/year selling public domain legal documents.',
    'With an estimated 1 billion documents, PACER is the largest paywall in the world.',
    'The average PACER document is 9.1 pages long. The longest we\'ve seen so far is over 4,000 pages.',
    'Downloading the everything in PACER would cost around $1,000,000,000.',
    'We extract text from every PACER document so that it is searchable. This takes months!',
)
def inject_random_tip(request):
    return {'TIP': random.choice(info_tips)}
<|endoftext|>


<commit_before>import subprocess

from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger

from yunohost.tools import Migration
from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory

logger = getActionLogger('yunohost.migration')

class MyMigration(Migration):
    "Migrate DBs from Postgresql 9.4 to 9.6 after migrating to Stretch"

    def migrate(self):

        if not self.package_is_installed("postgresql-9.4"):
            logger.warning(m18n.n("migration_0005_postgresql_94_not_installed"))
            return

        if not self.package_is_installed("postgresql-9.6"):
            raise MoulinetteError(m18n.n("migration_0005_postgresql_96_not_installed"))

        if not space_used_by_directory("/var/lib/postgresql/9.4") < free_space_in_directory("/var/lib/postgresql"):
            raise MoulinetteError(m18n.n("migration_0005_not_enough_space", path="/var/lib/postgresql/"))

        subprocess.check_call("service postgresql stop", shell=True)
        subprocess.check_call("pg_dropcluster --stop 9.6 main", shell=True)
        subprocess.check_call("pg_upgradecluster -m upgrade 9.4 main", shell=True)
        subprocess.check_call("pg_dropcluster --stop 9.4 main", shell=True)
        subprocess.check_call("service postgresql start", shell=True)

    def backward(self):

        pass

    def package_is_installed(self, package_name):

        p = subprocess.Popen("dpkg --list | grep -q -w {}".format(package_name), shell=True)
        p.communicate()
        return p.returncode == 0
<commit_msg>Condition typo :<<commit_after>import subprocess

from moulinette import m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger

from yunohost.tools import Migration
from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory

logger = getActionLogger('yunohost.migration')

class MyMigration(Migration):
    "Migrate DBs from Postgresql 9.4 to 9.6 after migrating to Stretch"

    def migrate(self):

        if not self.package_is_installed("postgresql-9.4"):
            logger.warning(m18n.n("migration_0005_postgresql_94_not_installed"))
            return

        if not self.package_is_installed("postgresql-9.6"):
            raise MoulinetteError(m18n.n("migration_0005_postgresql_96_not_installed"))

        if not space_used_by_directory("/var/lib/postgresql/9.4") > free_space_in_directory("/var/lib/postgresql"):
            raise MoulinetteError(m18n.n("migration_0005_not_enough_space", path="/var/lib/postgresql/"))

        subprocess.check_call("service postgresql stop", shell=True)
        subprocess.check_call("pg_dropcluster --stop 9.6 main", shell=True)
        subprocess.check_call("pg_upgradecluster -m upgrade 9.4 main", shell=True)
        subprocess.check_call("pg_dropcluster --stop 9.4 main", shell=True)
        subprocess.check_call("service postgresql start", shell=True)

    def backward(self):

        pass

    def package_is_installed(self, package_name):

        p = subprocess.Popen("dpkg --list | grep -q -w {}".format(package_name), shell=True)
        p.communicate()
        return p.returncode == 0
<|endoftext|>


<commit_before>__author__ = 'stuart'

class JoinedObject(object):
    def __init__(self, left, right):
        self.left = left
        self.right = right

    def __getattr__(self, attr):
        if attr == 'left':
            return self.left
        elif attr == 'right':
            return self.right
        else:
            return self.get_from_sources(attr)

    def __repr__(self):
        return '<{} object at {}>'.format(
            self.left.__class__.__name__ + self.right.__class__.__name__,
            id(self))

    def get_from_sources(self, attr):
        if hasattr(self.left, attr):
            return getattr(self.left, attr)
        elif hasattr(self.right, attr):
            return getattr(self.right, attr)
        else:
            raise AttributeError(
                "Neither of joined object's parents ({}, {}), have attribute "
                "'{}'".format(self.left.__class__.__name__,
                              self.left.__class__.__name__, attr))
<commit_msg>Add proper `__dir__` reporting, fix bug in AttributeError<commit_after>__author__ = 'stuart'

class JoinedObject(object):
    def __init__(self, left, right):
        self.left = left
        self.right = right

    def __getattr__(self, attr):
        if attr == 'left':
            return self.left
        elif attr == 'right':
            return self.right
        else:
            return self.get_from_sources(attr)

    def __repr__(self):
        return '<{} object at {}>'.format(
            self.left.__class__.__name__ + self.right.__class__.__name__,
            id(self))

    def __dir__(self):
        attrs = list(set(dir(self.left) + dir(self.right) + ['left', 'right']))
        return sorted(attrs)

    def get_from_sources(self, attr):
        if hasattr(self.left, attr):
            return getattr(self.left, attr)
        elif hasattr(self.right, attr):
            return getattr(self.right, attr)
        else:
            raise AttributeError(
                "Neither of joined object's parents ({}, {}), have attribute "
                "'{}'".format(self.left.__class__.__name__,
                              self.right.__class__.__name__, attr))
<|endoftext|>


<commit_before># Copyright (c) 2008, Aldo Cortesi. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
    A command shell for Qtile.
"""
import readline, sys, pprint, re
import fcntl, termios, struct
import command

def terminalWidth():
    width = None
    try:
        cr = struct.unpack('hh', fcntl.ioctl(0, termios.TIOCGWINSZ, '1234'))
        width = int(cr[1])
    except (IOError, ImportError):
        pass
    return width or 80

class QSh:
    fd = sys.stdout
    def __init__(self, client):
        self.clientroot, self.current = client, client
        self.termwidth = terminalWidth()

    @property
    def prompt(self):
        s = self.current.selectors[:]
        if self.current.name:
            s += [(self.current.name, self.current.myselector)]
        return "%s> "%command.formatSelector(s)

    def printColumns(self, lst):
        lst = [str(i) for i in lst]
        mx = max([len(i) for i in lst])
        cols = self.termwidth / (mx+2)
        if not cols:
            cols = 1
        for i in range(len(lst)/cols):
            sl = lst[i*cols:(i+1)*cols]
            sl = [x + " "*(mx-len(x)) for x in sl]
            print >> self.fd, "  ".join(sl)
        if len(lst)%cols:
            sl = lst[-(len(lst)%cols):]
            sl = [x + " "*(mx-len(x)) for x in sl]
            print >> self.fd, "  ".join(sl)

    def smartLs(self, obj):
        """
            Returns an (attrs, keys) tuple.
        """
        if obj.parent and obj.myselector is None:
            t, itms = obj.parent.items(obj.name)
            attrs = obj._contains if t else None
            return attrs, itms
        else:
            return obj._contains, None
        sub = obj.parent

    def _cd(self, *arg):
        attrs, itms = self.smartLs(self.current)
        next = None
        if arg[0] == "..":
            next = self.current.parent or self.current
        else:
            for trans in [str, int]:
                try:
                    targ = trans(arg[0])
                except ValueError:
                    continue
                if attrs and targ in attrs:
                    next = getattr(self.current, targ)
                elif itms and targ in itms:
                    next = self.current[targ]
        if next:
            self.current = next
            if arg[1:]:
                return self._cd(*arg[1:])
            else:
                return
        else:
            return "No such item: %s"%arg

    def do_cd(self, arg):
        check = self.current
        v = self._cd(*[i for i in arg.split("/") if i])
        if v:
            self.current = check
            print >> self.fd, v

    def do_ls(self, arg):
        attrs, itms = self.smartLs(self.current)
        all = []
        if attrs:
            all.extend(attrs)
        if itms:
            all.extend(itms)
        all = ["%s/"%i for i in all]
        self.printColumns(all)

    def do_help(self, arg):
        self.printColumns(self.current.commands())

    def do_exit(self, args):
        sys.exit(0)
    do_quit = do_exit
    do_q = do_exit

    def _call(self, cmd, args):
        try:
            cmds = self.current.commands()
        except command.CommandError:
            cmds = []
        if cmd not in cmds:
            print >> self.fd, "No such command: %s"%cmd
            return None

        cmd = getattr(self.current, cmd)
        if args:
            args = "".join(args)
        else:
            args = "()"
        try:
            val = eval(
                    "cmd%s"%"".join(args),
                    {},
                    dict(cmd=cmd)
                )
            return val
        except SyntaxError, v:
            print >> self.fd, "Syntax error in expression: %s"%v.text
            return None
        except command.CommandException, val:
            print >> self.fd, "Command exception:\n", val

    def loop(self):
        while True:
            try:
                line = raw_input(self.prompt)
            except (EOFError, KeyboardInterrupt):
                return
            if not line:
                continue

            match = re.search(r"\W", line)
            if match:
                cmd, args = line[:match.start()].strip(), line[match.start():].strip()
            else:
                cmd, args = line, ""

            builtin = getattr(self, "do_"+cmd, None)
            if builtin:
                builtin(args)
            else:
                val = self._call(cmd, args)
                if val:
                    pprint.pprint(val)

<commit_msg>Make printColumns not crash on empty lists.<commit_after># Copyright (c) 2008, Aldo Cortesi. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
    A command shell for Qtile.
"""
import readline, sys, pprint, re
import fcntl, termios, struct
import command

def terminalWidth():
    width = None
    try:
        cr = struct.unpack('hh', fcntl.ioctl(0, termios.TIOCGWINSZ, '1234'))
        width = int(cr[1])
    except (IOError, ImportError):
        pass
    return width or 80

class QSh:
    fd = sys.stdout
    def __init__(self, client):
        self.clientroot, self.current = client, client
        self.termwidth = terminalWidth()

    @property
    def prompt(self):
        s = self.current.selectors[:]
        if self.current.name:
            s += [(self.current.name, self.current.myselector)]
        return "%s> "%command.formatSelector(s)

    def printColumns(self, lst):
        if lst:
            lst = [str(i) for i in lst]
            mx = max([len(i) for i in lst])
            cols = self.termwidth / (mx+2)
            if not cols:
                cols = 1
            for i in range(len(lst)/cols):
                sl = lst[i*cols:(i+1)*cols]
                sl = [x + " "*(mx-len(x)) for x in sl]
                print >> self.fd, "  ".join(sl)
            if len(lst)%cols:
                sl = lst[-(len(lst)%cols):]
                sl = [x + " "*(mx-len(x)) for x in sl]
                print >> self.fd, "  ".join(sl)

    def smartLs(self, obj):
        """
            Returns an (attrs, keys) tuple.
        """
        if obj.parent and obj.myselector is None:
            t, itms = obj.parent.items(obj.name)
            attrs = obj._contains if t else None
            return attrs, itms
        else:
            return obj._contains, None
        sub = obj.parent

    def _cd(self, *arg):
        attrs, itms = self.smartLs(self.current)
        next = None
        if arg[0] == "..":
            next = self.current.parent or self.current
        else:
            for trans in [str, int]:
                try:
                    targ = trans(arg[0])
                except ValueError:
                    continue
                if attrs and targ in attrs:
                    next = getattr(self.current, targ)
                elif itms and targ in itms:
                    next = self.current[targ]
        if next:
            self.current = next
            if arg[1:]:
                return self._cd(*arg[1:])
            else:
                return
        else:
            return "No such item: %s"%arg

    def do_cd(self, arg):
        check = self.current
        v = self._cd(*[i for i in arg.split("/") if i])
        if v:
            self.current = check
            print >> self.fd, v

    def do_ls(self, arg):
        attrs, itms = self.smartLs(self.current)
        all = []
        if attrs:
            all.extend(attrs)
        if itms:
            all.extend(itms)
        all = ["%s/"%i for i in all]
        self.printColumns(all)

    def do_help(self, arg):
        self.printColumns(self.current.commands())

    def do_exit(self, args):
        sys.exit(0)
    do_quit = do_exit
    do_q = do_exit

    def _call(self, cmd, args):
        try:
            cmds = self.current.commands()
        except command.CommandError:
            cmds = []
        if cmd not in cmds:
            print >> self.fd, "No such command: %s"%cmd
            return None

        cmd = getattr(self.current, cmd)
        if args:
            args = "".join(args)
        else:
            args = "()"
        try:
            val = eval(
                    "cmd%s"%"".join(args),
                    {},
                    dict(cmd=cmd)
                )
            return val
        except SyntaxError, v:
            print >> self.fd, "Syntax error in expression: %s"%v.text
            return None
        except command.CommandException, val:
            print >> self.fd, "Command exception:\n", val

    def loop(self):
        while True:
            try:
                line = raw_input(self.prompt)
            except (EOFError, KeyboardInterrupt):
                return
            if not line:
                continue

            match = re.search(r"\W", line)
            if match:
                cmd, args = line[:match.start()].strip(), line[match.start():].strip()
            else:
                cmd, args = line, ""

            builtin = getattr(self, "do_"+cmd, None)
            if builtin:
                builtin(args)
            else:
                val = self._call(cmd, args)
                if val:
                    pprint.pprint(val)

<|endoftext|>


<commit_before>#!/usr/bin/python
from bcc import BPF
import json
import sys
import time

if len(sys.argv) < 2:
    print("Usage: %s <duration-seconds>" % (sys.argv[0]))
    sys.exit(1)

prog="""
#include <linux/types.h>
#include <linux/sched.h>
#include <uapi/linux/ptrace.h>

struct scheduled_out_state_t {
    int running;
    int sleeping;
    int uninterruptible;
    int unknown;
};

BPF_TABLE("hash", pid_t, struct scheduled_out_state_t, scheduled_out_states, 1024);

int trace_finish_task_switch(struct pt_regs *ctx, struct task_struct *prev) {

    pid_t prev_pid = prev->pid;

    struct scheduled_out_state_t *states = scheduled_out_states.lookup(&prev_pid);
    if (states == 0) {
        struct scheduled_out_state_t new_state = {.running = 0, .sleeping = 0, .uninterruptible = 0, .unknown = 0};
        states = &new_state;
        scheduled_out_states.update(&prev_pid, states);
    }

    if (prev->state == 0) {
        states->running++;
    } else if (prev->state == 1) {
        states->sleeping++;
    } else if (prev->state == 2) {
        states->uninterruptible++;
    } else {
        states->unknown++;
    }

    return 0;
};
"""

b = BPF(text=prog)
b.attach_kprobe(event="finish_task_switch", fn_name="trace_finish_task_switch")

time.sleep(int(sys.argv[1]))
results = dict()
for k,v in b["scheduled_out_states"].iteritems():
    tid_stats = dict()
    tid_stats['R'] = v.running
    tid_stats['S'] = v.sleeping
    tid_stats['D'] = v.uninterruptible
    tid_stats['U'] = v.unknown
    total = v.running + v.sleeping + v.uninterruptible + v.unknown
    tid_stats['total'] = total
    if total != 0:
        results[int(k.value)] = tid_stats

    json.dump(results, open(sys.argv[2], 'w'))
<commit_msg>add usurpers by pid<commit_after>#!/usr/bin/python
from bcc import BPF
import json
import sys
import time

if len(sys.argv) < 2:
    print("Usage: %s <duration-seconds>" % (sys.argv[0]))
    sys.exit(1)

prog="""
#include <linux/types.h>
#include <linux/sched.h>
#include <uapi/linux/ptrace.h>

struct scheduled_out_state_t {
    int running;
    int sleeping;
    int uninterruptible;
    int unknown;
};

struct proc_name_t {
    char comm[TASK_COMM_LEN];
};

struct proc_counter_t {
    int count;
//    char *proc_name[TASK_COMM_LEN];
};

BPF_TABLE("hash", pid_t, struct proc_counter_t, usurpers, 1024);
BPF_TABLE("hash", pid_t, struct scheduled_out_state_t, scheduled_out_states, 1024);

int trace_finish_task_switch(struct pt_regs *ctx, struct task_struct *prev) {

    pid_t prev_pid = prev->pid;
    pid_t parent_pid = prev->parent->pid;
    pid_t incoming_pid = bpf_get_current_pid_tgid();

    struct proc_name_t pname = {};
    bpf_get_current_comm(&pname.comm, sizeof(pname.comm));

    struct proc_counter_t *counter = usurpers.lookup(&incoming_pid);
    if (counter == 0) {
        // TODO - strcopy (need to make child pid to proc name)?
        struct proc_counter_t new_counter = {/*.proc_name = &pname.comm, */.count = 0};
        counter = &new_counter;
        usurpers.update(&incoming_pid, counter);
    }
    counter->count++;

    struct scheduled_out_state_t *states = scheduled_out_states.lookup(&prev_pid);
    if (states == 0) {
        struct scheduled_out_state_t new_state = {.running = 0, .sleeping = 0, .uninterruptible = 0, .unknown = 0};
        states = &new_state;
        scheduled_out_states.update(&prev_pid, states);
    }

    if (prev->state == 0) {
        states->running++;
    } else if (prev->state == 1) {
        states->sleeping++;
    } else if (prev->state == 2) {
        states->uninterruptible++;
    } else {
        states->unknown++;
    }

    return 0;
};
"""

b = BPF(text=prog)
b.attach_kprobe(event="finish_task_switch", fn_name="trace_finish_task_switch")

time.sleep(int(sys.argv[1]))
results = dict()

for k, v in b["usurpers"].iteritems():
    print str(k) + " -> " + str(v.count)

for k,v in b["scheduled_out_states"].iteritems():
    tid_stats = dict()
    tid_stats['R'] = v.running
    tid_stats['S'] = v.sleeping
    tid_stats['D'] = v.uninterruptible
    tid_stats['U'] = v.unknown
    total = v.running + v.sleeping + v.uninterruptible + v.unknown
    tid_stats['total'] = total
    if total != 0:
        results[int(k.value)] = tid_stats

    json.dump(results, open(sys.argv[2], 'w'))
<|endoftext|>


<commit_before><commit_msg>Add cifar10.py<commit_after>import numpy as np
from scipy.misc import toimage
import matplotlib.pyplot as plt

from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import np_utils
from keras.utils.visualize_util import plot

def plot_cifar10(X, y):
    plt.figure()

    # 画像を描画
    nclasses = 10
    pos = 1
    for targetClass in range(nclasses):
        targetIdx = []
        # クラスclassIDの画像のインデックスリストを取得
        for i in range(len(y)):
            if y[i][0] == targetClass:
                targetIdx.append(i)

        # 各クラスからランダムに選んだ最初の10個の画像を描画
        np.random.shuffle(targetIdx)
        for idx in targetIdx[:10]:
            img = toimage(X[idx])
            plt.subplot(10, 10, pos)
            plt.imshow(img)
            plt.axis('off')
            pos += 1

    plt.savefig('result_cifar10/plot.png')

def plot_history(history):
    # 精度の履歴をプロット
    plt.figure()
    plt.plot(history.history['acc'])
    plt.plot(history.history['val_acc'])
    plt.title('model accuracy')
    plt.xlabel('epoch')
    plt.ylabel('accuracy')
    plt.legend(['train', 'test'], loc='upper left')
    plt.savefig('result_cifar10/acc.png')

    # 損失の履歴をプロット
    plt.figure()
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.legend(['train', 'test'], loc='upper left')
    plt.savefig('result_cifar10/loss.png')

if __name__ == '__main__':
    batch_size = 32
    nb_classes = 10
    nb_epoch = 25
    data_augmentation = False

    # 入力画像の次元
    img_rows, img_cols = 32, 32

    # チャネル数（RGBなので3）
    img_channels = 3

    # CIFAR-10データをロード
    # (nb_samples, nb_rows, nb_cols, nb_channel) = tf
    (X_train, y_train), (X_test, y_test) = cifar10.load_data()

    # ランダムに画像をプロット
    plot_cifar10(X_train, y_train)

    # 画素値を0-1に変換
    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')
    X_train /= 255.0
    X_test /= 255.0

    # クラスラベル（0-9）をone-hotエンコーディング形式に変換
    Y_train = np_utils.to_categorical(y_train, nb_classes)
    Y_test = np_utils.to_categorical(y_test, nb_classes)

    # CNNを構築
    model = Sequential()

    model.add(Convolution2D(32, 3, 3, border_mode='same', input_shape=X_train.shape[1:]))
    model.add(Activation('relu'))

    model.add(Convolution2D(32, 3, 3))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Convolution2D(64, 3, 3, border_mode='same'))
    model.add(Activation('relu'))

    model.add(Convolution2D(64, 3, 3))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))

    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))

    # 最適化はSGD+モーメンタム（オリジナルのやり方らしい）
    sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(loss='categorical_crossentropy',
                  optimizer=sgd,
                  metrics=['accuracy'])

    # モデルのサマリを表示
    model.summary()
    plot(model, show_shapes=True, to_file='result_cifar10/model.png')

    # TODO: validation_split=0.1にするとlossがまったく下がらない
    if not data_augmentation:
        print('Not using data augmentation')
        history = model.fit(X_train, Y_train,
                    batch_size=batch_size,
                    nb_epoch=nb_epoch,
                    verbose=1,
                    validation_data=[X_test, Y_test],
                    shuffle=True)

    # TODO: 学習したモデルの保存

    # 学習履歴をプロット
    plot_history(history)

    # モデルの評価
    loss, acc = model.evaluate(X_test, Y_test, verbose=0)
    print('Test loss:', loss)
    print('Test acc:', acc)
<|endoftext|>


<commit_before>import requests
import base64
import json
import time as time_module

TOKEN_URL = 'https://api.vasttrafik.se/token'
API_BASE_URL = 'https://api.vasttrafik.se/bin/rest.exe/v2'

def fetchToken():
	f = open('code/credentials.txt', 'r')
	CONSUMER_KEY = f.readline().split('\'')[1]
	CONSUMER_SECRET = f.readline().split('\'')[1]

	headers = {
		'Content-Type': 'application/x-www-form-urlencoded',
		'Authorization': 'Basic ' + base64.b64encode(CONSUMER_KEY + ':' + CONSUMER_SECRET)
	}
	data = {'grant_type': 'client_credentials'}

	response = requests.post(TOKEN_URL, data=data, headers=headers)
	obj = json.loads(response.content)
	return obj['access_token']

class Client:

	def __init__(self, format):
		self.token = fetchToken()
		if format == 'JSON':
			self.format = '&format=json'
		else:
			# default is XML
			self.format = ''

	## /journeyDetail endpoint
	def get_journey_detail(self, ref, query_params=None):
		return ''

	## /location endpoint
	# /location.allstops
	def get_all_stops(self, query_params=None):
		return ''

	# /location.nearbystops
	def get_nearby_stops(self, lat, long, query_params=None):
		return ''

	# /location.name
	def get_stops_by_name(self, query, query_params=None):
		data = self.get('/location.name?input=' + query)
		return data['LocationList']['StopLocation']

	# /location.nearbyaddress
	def get_nearby_address(self, lat, long, query_params=None):
		return ''

	## /arrivalBoard endpoint
	def get_arrivals(self, stopID, date=None, time=None, query_params=None):
		if date is not None and time is not None:
			data = self.get('/arrivalBoard?id=' + str(stopID) + '&date=' + date + '&time=' + time)
		else:
			data = self.get('/arrivalBoard?id=' + str(stopID) + '&date=' + time_module.strftime("%Y-%m-%d") +
			'&time=' + time_module.strftime("%H:%M"))
		return data['ArrivalBoard']['Arrival']

	## /departureBoard endpoint
	def get_departures(self, stopID, date=None, time=None, query_params=None):
		if date is not None and time is not None:
			data = self.get('/departureBoard?id=' + str(stopID) + '&date=' + date + '&time=' + time)
		else:
			data = self.get('/departureBoard?id=' + str(stopID) + '&date=' + time_module.strftime("%Y-%m-%d") +
			'&time=' + time_module.strftime("%H:%M"))
		return data['DepartureBoard']['Departure']

	## /trip endpoint
	def calculate_trip(self, query_params=None):
		return ''

	## request builder
	def get(self, endpoint, query_params=None):
		url = API_BASE_URL + endpoint + self.format

		if query_params is not None:
			for key in query_params:
				url += '&' + key + '=' + query_params[key]

		headers = {
			'Authorization': 'Bearer ' + self.token
		}

		res = requests.get(url, headers=headers)
		if res.status_code == 200:
			return json.loads(res.content, 'UTF-8')
		else:
			raise Exception('Error: ' + str(res.status_code) + res.read())
<commit_msg>Forgot to add client.py to last commit<commit_after>import requests
import base64
import json
import time as time_module

TOKEN_URL = 'https://api.vasttrafik.se/token'
API_BASE_URL = 'https://api.vasttrafik.se/bin/rest.exe/v2'
CONSUMER_KEY = '<insert your CONSUMER_KEY here>'
CONSUMER_SECRET = '<insert your CONSUMER_SECRET here>'

def fetchToken():
	headers = {
		'Content-Type': 'application/x-www-form-urlencoded',
		'Authorization': 'Basic ' + base64.b64encode(CONSUMER_KEY + ':' + CONSUMER_SECRET)
	}
	data = {'grant_type': 'client_credentials'}

	response = requests.post(TOKEN_URL, data=data, headers=headers)
	obj = json.loads(response.content)
	return obj['access_token']

class Client:

	def __init__(self, format):
		self.token = fetchToken()
		if format == 'JSON':
			self.format = '&format=json'
		else:
			# default is XML
			self.format = ''

	## /journeyDetail endpoint
	def get_journey_detail(self, ref, query_params=None):
		return ''

	## /location endpoint
	# /location.allstops
	def get_all_stops(self, query_params=None):
		return ''

	# /location.nearbystops
	def get_nearby_stops(self, lat, long, query_params=None):
		return ''

	# /location.name
	def get_stops_by_name(self, query, query_params=None):
		data = self.get('/location.name?input=' + query)
		return data['LocationList']['StopLocation']

	# /location.nearbyaddress
	def get_nearby_address(self, lat, long, query_params=None):
		return ''

	## /arrivalBoard endpoint
	def get_arrivals(self, stopID, date=None, time=None, query_params=None):
		if date is not None and time is not None:
			data = self.get('/arrivalBoard?id=' + str(stopID) + '&date=' + date + '&time=' + time)
		else:
			data = self.get('/arrivalBoard?id=' + str(stopID) + '&date=' + time_module.strftime("%Y-%m-%d") +
			'&time=' + time_module.strftime("%H:%M"))
		return data['ArrivalBoard']['Arrival']

	## /departureBoard endpoint
	def get_departures(self, stopID, date=None, time=None, query_params=None):
		if date is not None and time is not None:
			data = self.get('/departureBoard?id=' + str(stopID) + '&date=' + date + '&time=' + time)
		else:
			data = self.get('/departureBoard?id=' + str(stopID) + '&date=' + time_module.strftime("%Y-%m-%d") +
			'&time=' + time_module.strftime("%H:%M"))
		return data['DepartureBoard']['Departure']

	## /trip endpoint
	def calculate_trip(self, query_params=None):
		return ''

	## request builder
	def get(self, endpoint, query_params=None):
		url = API_BASE_URL + endpoint + self.format

		if query_params is not None:
			for key in query_params:
				url += '&' + key + '=' + query_params[key]

		headers = {
			'Authorization': 'Bearer ' + self.token
		}

		res = requests.get(url, headers=headers)
		if res.status_code == 200:
			return json.loads(res.content, 'UTF-8')
		else:
			raise Exception('Error: ' + str(res.status_code) + res.read())
<|endoftext|>