<commit_before>import sys
import re
import StringIO
import tempfile
import numpy as np
import zlib
import web
import h5py
import os
import cStringIO

import empaths
import restargs
import anncube
import anndb
import dbconfig
import annproj

#
#  annrest: RESTful interface to annotations
#

#
#  Build the returned braincube.  Called by all methods
#   that then refine the output.
#
def cutout ( imageargs, dbcfg, annoproj ):

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  args.setResolution ( annoproj.getResolution() )
  args.cutoutArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

  #Load the database
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  # Perform the cutout
  return annodb.cutout ( corner, dim, resolution )

#
#  Return a Numpy Pickle zipped
#
def numpyZip ( imageargs, dbcfg, annoproj ):
  """Return a web readable Numpy Pickle zipped"""

  try:
    cube = cutout ( imageargs, dbcfg, annoproj )
  except restargs.RESTRangeError:
    return web.notfound()
  except restargs.RESTBadArgsError:
    return web.badrequest()

  try:
    # Create the compressed cube
    fileobj = StringIO.StringIO ()
    np.save ( fileobj, cube.data )
    cdz = zlib.compress (fileobj.getvalue())
  except:
    return web.notfound()

  # Package the object as a Web readable file handle
  fileobj = StringIO.StringIO ( cdz )
  fileobj.seek(0)
  web.header('Content-type', 'application/zip')
  return fileobj.read()

#
#  Return a HDF5 file
#
def HDF5 ( imageargs, dbcfg ):
  """Return a web readable HDF5 file"""

  try:
    cube = cutout ( imageargs, dbcfg, annoproj )
  except restargs.RESTRangeError:
    return web.notfound()
  except restargs.RESTBadArgsError:
    return web.badrequest()

  # Create an in-memory HDF5 file
  tmpfile = tempfile.NamedTemporaryFile ()
  fh5out = h5py.File ( tmpfile.name )
  ds = fh5out.create_dataset ( "cube", tuple(cube.data.shape), np.uint8,\
                                 compression='gzip', data=cube.data )
  fh5out.close()
  tmpfile.seek(0)
  return tmpfile.read()

#
#  **Image return a readable png object
#    where ** is xy, xz, yz
#
def xyImage ( imageargs, dbcfg, annoproj ):
  """Return an xy plane fileobj.read()"""

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  # Don't love this, but must either parse or set resolution before xyArgs call
  args.setResolution ( annoproj.getResolution() )
  args.xyArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

#RBRM reinstate try/catch block
#  try:
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  cb = annodb.cutout ( corner, dim, resolution )
  fileobj = StringIO.StringIO ( )
  cb.xySlice ( fileobj )
#  except:
#    print "Exception"
#    return web.notfound()

  web.header('Content-type', 'image/png')
  fileobj.seek(0)
  return fileobj.read()

def xzImage ( imageargs, dbcfg, annoproj ):
  """Return an xz plane fileobj.read()"""

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  args.setResolution ( annoproj.getResolution() )
  args.xzArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

#RBRM reinstate try/catch block
#  try:
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  cb = annodb.cutout ( corner, dim, resolution )
  fileobj = StringIO.StringIO ( )
  cb.xzSlice ( fileobj )
#  except:
#    print "Exception"
#    return web.notfound()

  web.header('Content-type', 'image/png')
  fileobj.seek(0)
  return fileobj.read()

def yzImage ( imageargs, dbcfg, annoproj ):
  """Return an yz plane fileobj.read()"""

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  args.setResolution ( annoproj.getResolution() )
  args.yzArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

#RBRM reinstate try/catch block
#  try:
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  cb = annodb.cutout ( corner, dim, resolution )
  fileobj = StringIO.StringIO ( )
  cb.yzSlice ( fileobj )
#  except:
#    print "Exception"
#    return web.notfound()

  web.header('Content-type', 'image/png')
  fileobj.seek(0)
  return fileobj.read()

#
#  annId
#    return the annotation identifier of a pixel
#
def annId ( imageargs, dbcfg, annoproj ):
  """Return the annotation identifier of a voxel"""

  # Perform argument processing
  voxel = restargs.voxel ( imageargs, dbcfg, annoproj.getResolution() )

  # Get the identifier
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  return annodb.getVoxel ( voxel )

#
#  Select the service that you want.
#  Truncate this from the arguments and past
#  the rest of the RESTful arguments to the
#  appropriate function.  At this point, we have a
#  data set and a service.
#
def selectService ( webargs, dbcfg, annoproj ):
  """Parse the first arg and call service, HDF5, mpz, etc."""

  [ service, sym, rangeargs ] = webargs.partition ('/')

  if service == 'xy':
    return xyImage ( rangeargs, dbcfg, annoproj )

  elif service == 'xz':
    return xzImage ( rangeargs, dbcfg, annoproj)

  elif service == 'yz':
    return yzImage ( rangeargs, dbcfg, annoproj )

  elif service == 'hdf5':
    return HDF5 ( rangeargs, dbcfg, annoproj )

  elif service == 'npz':
    return  numpyZip ( rangeargs, dbcfg, annoproj )

  elif service == 'annid':
    return annId ( rangeargs, dbcfg, annoproj )

  else:
    return web.badrequest()

#
#  Select the service that you want.
#  Truncate this from the arguments and past
#  the rest of the RESTful arguments to the
#  appropriate function.  At this point, we have a
#  data set and a service.
#
def selectPost ( webargs, dbcfg, annoproj ):
  """Parse the first arg and call the right post service"""

  [ service, sym, postargs ] = webargs.partition ('/')

  # choose to overwrite (default), preserve, or make exception lists
  #  when voxels conflict
  # Perform argument processing

  if service == 'npnew':

    conflictopt = restargs.conflictOption ( postargs )

    try:
      # Grab the voxel list
      fileobj = cStringIO.StringIO ( web.data() )
      voxlist = np.load ( fileobj )

      # Make the annotation to the database
      annoDB = anndb.AnnotateDB ( dbcfg, annoproj )
      entityid = annoDB.newEntity ( voxlist, conflictopt )

    except:
      return web.BadRequest()

    return str(entityid)

  elif service == 'npextend':

    [ entity, sym, conflictargs ] = postargs.partition ('/')
    conflictopt = restargs.conflictOption ( conflictargs )

    try:
      # Grab the voxel list
      fileobj = cStringIO.StringIO ( web.data() )
      voxlist = np.load ( fileobj )

      # Make the annotation to the database
      annoDB = anndb.AnnotateDB ( dbcfg, annoproj )
      entityid = annoDB.extendEntity ( int(entity), voxlist, conflictopt )

    except:
      return web.BadRequest()

    return str(entityid)

  elif service == 'npadd':

    [ entity, sym, conflictargs ] = postargs.partition ('/')
    conflictopt = restargs.conflictOption ( conflictargs )

    try:
      # Grab the voxel list
      fileobj = cStringIO.StringIO ( web.data() )
      voxlist = np.load ( fileobj )

      # Make the annotation to the database
      annoDB = anndb.AnnotateDB ( dbcfg, annoproj )
      entityid = annoDB.addEntity ( int(entity), voxlist, conflictopt )

    except:
      return web.BadRequest()

    return str(entityid)

  #RBTODO HDF5 for matlab users?
  elif service == 'HDF5':
    return "Not yet"
    pass

  elif service == 'csv':
    return "No csv format specified yet"

  else:
    return web.notfound()

def switchDataset ( annoproj ):
  """Load the appropriate dbconfig project based on the dataset name"""

  print "dataset = ", annoproj.getDataset()
  # Switch on the dataset
  if annoproj.getDataset() == 'hayworth5nm':
    import dbconfighayworth5nm
    return dbconfighayworth5nm.dbConfigHayworth5nm()
  elif annoproj.getDataset() == 'bock11':
    import dbconfigbock11
    return dbconfigbock11.dbConfigBock11()
  elif annoproj.getDataset() == 'kasthuri11':
    import dbconfigkasthuri11
    return dbconfigkasthuri11.dbConfigKasthuri11()
  else:
    return web.badrequest()

#
#  Interface to annotation by project.
#   Lookup the project token in the database and figure out the
#   right database to load.
#
def annoget ( webargs ):
  """Interface to the cutout service for annotations.
      Load the annotation project and invoke the appropriate
      dataset."""

  [ token, sym, rangeargs ] = webargs.partition ('/')
  annprojdb = annproj.AnnotateProjectsDB()
  annoproj = annprojdb.getAnnoProj ( token )
  dbcfg = switchDataset ( annoproj )
  return selectService ( rangeargs, dbcfg, annoproj )

def annopost ( webargs ):
  """Interface to the annotation write service
      Load the annotation project and invoke the appropriate
      dataset."""

  [ token, sym, rangeargs ] = webargs.partition ('/')
  annprojdb = annproj.AnnotateProjectsDB()
  annoproj = annprojdb.getAnnoProj ( token )
  dbcfg = switchDataset ( annoproj )
  return selectPost ( rangeargs, dbcfg, annoproj )

<commit_msg>Removed print.<commit_after>import sys
import re
import StringIO
import tempfile
import numpy as np
import zlib
import web
import h5py
import os
import cStringIO

import empaths
import restargs
import anncube
import anndb
import dbconfig
import annproj

#
#  annrest: RESTful interface to annotations
#

#
#  Build the returned braincube.  Called by all methods
#   that then refine the output.
#
def cutout ( imageargs, dbcfg, annoproj ):

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  args.setResolution ( annoproj.getResolution() )
  args.cutoutArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

  #Load the database
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  # Perform the cutout
  return annodb.cutout ( corner, dim, resolution )

#
#  Return a Numpy Pickle zipped
#
def numpyZip ( imageargs, dbcfg, annoproj ):
  """Return a web readable Numpy Pickle zipped"""

  try:
    cube = cutout ( imageargs, dbcfg, annoproj )
  except restargs.RESTRangeError:
    return web.notfound()
  except restargs.RESTBadArgsError:
    return web.badrequest()

  try:
    # Create the compressed cube
    fileobj = StringIO.StringIO ()
    np.save ( fileobj, cube.data )
    cdz = zlib.compress (fileobj.getvalue())
  except:
    return web.notfound()

  # Package the object as a Web readable file handle
  fileobj = StringIO.StringIO ( cdz )
  fileobj.seek(0)
  web.header('Content-type', 'application/zip')
  return fileobj.read()

#
#  Return a HDF5 file
#
def HDF5 ( imageargs, dbcfg ):
  """Return a web readable HDF5 file"""

  try:
    cube = cutout ( imageargs, dbcfg, annoproj )
  except restargs.RESTRangeError:
    return web.notfound()
  except restargs.RESTBadArgsError:
    return web.badrequest()

  # Create an in-memory HDF5 file
  tmpfile = tempfile.NamedTemporaryFile ()
  fh5out = h5py.File ( tmpfile.name )
  ds = fh5out.create_dataset ( "cube", tuple(cube.data.shape), np.uint8,\
                                 compression='gzip', data=cube.data )
  fh5out.close()
  tmpfile.seek(0)
  return tmpfile.read()

#
#  **Image return a readable png object
#    where ** is xy, xz, yz
#
def xyImage ( imageargs, dbcfg, annoproj ):
  """Return an xy plane fileobj.read()"""

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  # Don't love this, but must either parse or set resolution before xyArgs call
  args.setResolution ( annoproj.getResolution() )
  args.xyArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

#RBRM reinstate try/catch block
#  try:
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  cb = annodb.cutout ( corner, dim, resolution )
  fileobj = StringIO.StringIO ( )
  cb.xySlice ( fileobj )
#  except:
#    print "Exception"
#    return web.notfound()

  web.header('Content-type', 'image/png')
  fileobj.seek(0)
  return fileobj.read()

def xzImage ( imageargs, dbcfg, annoproj ):
  """Return an xz plane fileobj.read()"""

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  args.setResolution ( annoproj.getResolution() )
  args.xzArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

#RBRM reinstate try/catch block
#  try:
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  cb = annodb.cutout ( corner, dim, resolution )
  fileobj = StringIO.StringIO ( )
  cb.xzSlice ( fileobj )
#  except:
#    print "Exception"
#    return web.notfound()

  web.header('Content-type', 'image/png')
  fileobj.seek(0)
  return fileobj.read()

def yzImage ( imageargs, dbcfg, annoproj ):
  """Return an yz plane fileobj.read()"""

  # Perform argument processing
  args = restargs.BrainRestArgs ();
  args.setResolution ( annoproj.getResolution() )
  args.yzArgs ( imageargs, dbcfg )

  # Extract the relevant values
  corner = args.getCorner()
  dim = args.getDim()
  resolution = args.getResolution()

#RBRM reinstate try/catch block
#  try:
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  cb = annodb.cutout ( corner, dim, resolution )
  fileobj = StringIO.StringIO ( )
  cb.yzSlice ( fileobj )
#  except:
#    print "Exception"
#    return web.notfound()

  web.header('Content-type', 'image/png')
  fileobj.seek(0)
  return fileobj.read()

#
#  annId
#    return the annotation identifier of a pixel
#
def annId ( imageargs, dbcfg, annoproj ):
  """Return the annotation identifier of a voxel"""

  # Perform argument processing
  voxel = restargs.voxel ( imageargs, dbcfg, annoproj.getResolution() )

  # Get the identifier
  annodb = anndb.AnnotateDB ( dbcfg, annoproj )
  return annodb.getVoxel ( voxel )

#
#  Select the service that you want.
#  Truncate this from the arguments and past
#  the rest of the RESTful arguments to the
#  appropriate function.  At this point, we have a
#  data set and a service.
#
def selectService ( webargs, dbcfg, annoproj ):
  """Parse the first arg and call service, HDF5, mpz, etc."""

  [ service, sym, rangeargs ] = webargs.partition ('/')

  if service == 'xy':
    return xyImage ( rangeargs, dbcfg, annoproj )

  elif service == 'xz':
    return xzImage ( rangeargs, dbcfg, annoproj)

  elif service == 'yz':
    return yzImage ( rangeargs, dbcfg, annoproj )

  elif service == 'hdf5':
    return HDF5 ( rangeargs, dbcfg, annoproj )

  elif service == 'npz':
    return  numpyZip ( rangeargs, dbcfg, annoproj )

  elif service == 'annid':
    return annId ( rangeargs, dbcfg, annoproj )

  else:
    return web.badrequest()

#
#  Select the service that you want.
#  Truncate this from the arguments and past
#  the rest of the RESTful arguments to the
#  appropriate function.  At this point, we have a
#  data set and a service.
#
def selectPost ( webargs, dbcfg, annoproj ):
  """Parse the first arg and call the right post service"""

  [ service, sym, postargs ] = webargs.partition ('/')

  # choose to overwrite (default), preserve, or make exception lists
  #  when voxels conflict
  # Perform argument processing

  if service == 'npnew':

    conflictopt = restargs.conflictOption ( postargs )

    try:
      # Grab the voxel list
      fileobj = cStringIO.StringIO ( web.data() )
      voxlist = np.load ( fileobj )

      # Make the annotation to the database
      annoDB = anndb.AnnotateDB ( dbcfg, annoproj )
      entityid = annoDB.newEntity ( voxlist, conflictopt )

    except:
      return web.BadRequest()

    return str(entityid)

  elif service == 'npextend':

    [ entity, sym, conflictargs ] = postargs.partition ('/')
    conflictopt = restargs.conflictOption ( conflictargs )

    try:
      # Grab the voxel list
      fileobj = cStringIO.StringIO ( web.data() )
      voxlist = np.load ( fileobj )

      # Make the annotation to the database
      annoDB = anndb.AnnotateDB ( dbcfg, annoproj )
      entityid = annoDB.extendEntity ( int(entity), voxlist, conflictopt )

    except:
      return web.BadRequest()

    return str(entityid)

  elif service == 'npadd':

    [ entity, sym, conflictargs ] = postargs.partition ('/')
    conflictopt = restargs.conflictOption ( conflictargs )

    try:
      # Grab the voxel list
      fileobj = cStringIO.StringIO ( web.data() )
      voxlist = np.load ( fileobj )

      # Make the annotation to the database
      annoDB = anndb.AnnotateDB ( dbcfg, annoproj )
      entityid = annoDB.addEntity ( int(entity), voxlist, conflictopt )

    except:
      return web.BadRequest()

    return str(entityid)

  #RBTODO HDF5 for matlab users?
  elif service == 'HDF5':
    return "Not yet"
    pass

  elif service == 'csv':
    return "No csv format specified yet"

  else:
    return web.notfound()

def switchDataset ( annoproj ):
  """Load the appropriate dbconfig project based on the dataset name"""

  # Switch on the dataset
  if annoproj.getDataset() == 'hayworth5nm':
    import dbconfighayworth5nm
    return dbconfighayworth5nm.dbConfigHayworth5nm()
  elif annoproj.getDataset() == 'bock11':
    import dbconfigbock11
    return dbconfigbock11.dbConfigBock11()
  elif annoproj.getDataset() == 'kasthuri11':
    import dbconfigkasthuri11
    return dbconfigkasthuri11.dbConfigKasthuri11()
  else:
    return web.badrequest()

#
#  Interface to annotation by project.
#   Lookup the project token in the database and figure out the
#   right database to load.
#
def annoget ( webargs ):
  """Interface to the cutout service for annotations.
      Load the annotation project and invoke the appropriate
      dataset."""

  [ token, sym, rangeargs ] = webargs.partition ('/')
  annprojdb = annproj.AnnotateProjectsDB()
  annoproj = annprojdb.getAnnoProj ( token )
  dbcfg = switchDataset ( annoproj )
  return selectService ( rangeargs, dbcfg, annoproj )

def annopost ( webargs ):
  """Interface to the annotation write service
      Load the annotation project and invoke the appropriate
      dataset."""

  [ token, sym, rangeargs ] = webargs.partition ('/')
  annprojdb = annproj.AnnotateProjectsDB()
  annoproj = annprojdb.getAnnoProj ( token )
  dbcfg = switchDataset ( annoproj )
  return selectPost ( rangeargs, dbcfg, annoproj )

<|endoftext|>


<commit_before>import os
import shutil
import storage
import scanner
import utility

# Copies headers in module to build/headers/name, dropping one
# directory layer (i.e. name/sources/header.h => name/header.h)
# TODO: minimize headers
def prepare_headers(name):
	root = storage.module_path(name)
	baseheaders = root + '/build/headers/kit'
	headers = baseheaders + '/' + name
	for path in utility.headers_under(root):
		if path.find('build') == 0:
			continue
		dest = path.replace(root + '/', '')
		dest = '/'.join(dest.split('/')[1:])
		if not os.path.exists(headers):
			os.makedirs(headers)
		shutil.copy(path, headers + '/' + dest)

		if path.replace(root + '/sources/', '') == name + '.h':
			shutil.copy(path, baseheaders)
	print ' - prepared headers'

# Ensure that module is compiled.
def ready_indexed_module(name):
	if storage.module_compiled(name):
		print utility.color(' - already compiled', 'green')
	else:
		path = storage.module_path(name)
		build_directory(path)
		prepare_headers(name)
		print utility.color(' - compiled module: ' + name, 'green')

# Generates CMakeLists file for project. This should be removed after
# compilation.
def generate_cmake(path, deps):
	name = os.path.abspath(path).split('/')[-1]
	with open(path + '/CMakeLists.txt', 'w') as f:
		f.write('project(KitModule C)\n')
		f.write('cmake_minimum_required(VERSION 2.6)\n')
		f.write('cmake_policy(VERSION 2.6)\n')
		f.write('set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n')
		f.write('set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n')
		f.write('set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n')
		f.write('SET(CMAKE_C_FLAGS  "' + scanner.metadata()['cflags'] + '")\n')
		f.write('file(GLOB_RECURSE sources "sources/*.h" "sources/*.c")\n')
		f.write('file(GLOB_RECURSE test_sources "tests/*.h" "tests/*.c")\n')
		f.write('set(headers "")\n')
		for dep in deps:
			f.write('file(GLOB_RECURSE t1 "' + storage.module_header_path(dep) + '/*.h")\n')
			f.write('set(t2 ${headers})\n')
			f.write('set(headers ${t1} ${t2})\n')

		if os.path.exists(path + '/sources/main.c'):
			f.write('add_executable(' + name + ' ${sources} ${headers})\n')
			f.write('set(test_sources2 ${sources} ${test_sources})\n')
			f.write('list(REMOVE_ITEM test_sources2 "${PROJECT_SOURCE_DIR}/sources/main.c")\n')
			f.write('add_executable(tests ${test_sources2} ${headers})\n')
		else:
			f.write('add_library(' + name + ' STATIC ${sources} ${headers})\n')
			f.write('add_executable(tests ${sources} ${test_sources} ${headers})\n')
		for dep in deps:
			f.write('add_library(' + dep + ' SHARED IMPORTED)\n')
			f.write('set_target_properties(' + dep + ' PROPERTIES IMPORTED_LOCATION "' + storage.module_library_path(dep) + '")\n')
			f.write('TARGET_LINK_LIBRARIES(tests ' + dep + ')\n')
			f.write('TARGET_LINK_LIBRARIES(' + name + ' ' + dep + ')\n')
			f.write('include_directories(' + storage.module_header_path(dep) + ')\n')
	print ' - generated CMakeLists.txt'

# Compiles executables and libraries for given project, assuming
# that all dependencies have been resolved a priori.
def make(path):
	print ' - running `make`...'
	wd = os.getcwd()
	os.chdir(path)
	os.system('mkdir -p build')
	os.chdir('build')
	c1 = os.system('cmake -Wno-dev .. > /dev/null')
	c2 = os.system('make > /dev/null')
	os.system('rm ../CMakeLists.txt')
	os.chdir(wd)
	if c1 == 0 and c2 == 0:
		print utility.color(' - build successfull', 'green')
	else:
		print utility.color(' - build failed', 'red')
		exit(1)

# Compiles libraries and headers for given directory.
def build_directory(path):
	deps = scanner.directory_dependencies(path)
	for dep in deps:
		print ' - resolving dependency:', dep
		if not storage.contains_module(dep):
			if storage.remote_contains_module(dep):
				storage.fetch_module(dep)
			else:
				print utility.color(' - module not found: ' + dep, 'red')
				print utility.color(' - build failed', 'red')
				exit(1)
		ready_indexed_module(dep)
	generate_cmake(path, deps)
	make(path)
<commit_msg>Changed header consolidation to include api.h as module-name.h in preprocessing.<commit_after>import os
import shutil
import storage
import scanner
import utility

# Copies headers in module to build/headers/name, dropping one
# directory layer (i.e. name/sources/header.h => name/header.h)
# TODO: minimize headers
def prepare_headers(name):
	root = storage.module_path(name)
	baseheaders = root + '/build/headers/kit'
	headers = baseheaders + '/' + name
	for path in utility.headers_under(root):
		# don't duplicate if already prepared
		if path.find('build') == 0:
			continue

		# move file
		dest = path.replace(root + '/', '')
		dest = '/'.join(dest.split('/')[1:])
		if not os.path.exists(headers):
			os.makedirs(headers)
		shutil.copy(path, headers + '/' + dest)

		# make `#include <kit/name.h>` function as `#include <kit/name/api.h>`
		short = path.replace(root + '/sources/', '')
		if short == 'api.h':
			shutil.copy(path, baseheaders + '/' + name + '.h')
	print ' - prepared headers'

# Ensure that module is compiled.
def ready_indexed_module(name):
	if storage.module_compiled(name):
		print utility.color(' - already compiled', 'green')
	else:
		path = storage.module_path(name)
		build_directory(path)
		prepare_headers(name)
		print utility.color(' - compiled module: ' + name, 'green')

# Generates CMakeLists file for project. This should be removed after
# compilation.
def generate_cmake(path, deps):
	name = os.path.abspath(path).split('/')[-1]
	with open(path + '/CMakeLists.txt', 'w') as f:
		f.write('project(KitModule C)\n')
		f.write('cmake_minimum_required(VERSION 2.6)\n')
		f.write('cmake_policy(VERSION 2.6)\n')
		f.write('set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n')
		f.write('set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n')
		f.write('set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n')
		f.write('SET(CMAKE_C_FLAGS  "' + scanner.metadata()['cflags'] + '")\n')
		f.write('file(GLOB_RECURSE sources "sources/*.h" "sources/*.c")\n')
		f.write('file(GLOB_RECURSE test_sources "tests/*.h" "tests/*.c")\n')
		f.write('set(headers "")\n')
		for dep in deps:
			f.write('file(GLOB_RECURSE t1 "' + storage.module_header_path(dep) + '/*.h")\n')
			f.write('set(t2 ${headers})\n')
			f.write('set(headers ${t1} ${t2})\n')

		if os.path.exists(path + '/sources/main.c'):
			f.write('add_executable(' + name + ' ${sources} ${headers})\n')
			f.write('set(test_sources2 ${sources} ${test_sources})\n')
			f.write('list(REMOVE_ITEM test_sources2 "${PROJECT_SOURCE_DIR}/sources/main.c")\n')
			f.write('add_executable(tests ${test_sources2} ${headers})\n')
		else:
			f.write('add_library(' + name + ' STATIC ${sources} ${headers})\n')
			f.write('add_executable(tests ${sources} ${test_sources} ${headers})\n')
		for dep in deps:
			f.write('add_library(' + dep + ' SHARED IMPORTED)\n')
			f.write('set_target_properties(' + dep + ' PROPERTIES IMPORTED_LOCATION "' + storage.module_library_path(dep) + '")\n')
			f.write('TARGET_LINK_LIBRARIES(tests ' + dep + ')\n')
			f.write('TARGET_LINK_LIBRARIES(' + name + ' ' + dep + ')\n')
			f.write('include_directories(' + storage.module_header_path(dep) + ')\n')
	print ' - generated CMakeLists.txt'

# Compiles executables and libraries for given project, assuming
# that all dependencies have been resolved a priori.
def make(path):
	print ' - running `make`...'
	wd = os.getcwd()
	os.chdir(path)
	os.system('mkdir -p build')
	os.chdir('build')
	c1 = os.system('cmake -Wno-dev .. > /dev/null')
	c2 = os.system('make > /dev/null')
	os.system('rm ../CMakeLists.txt')
	os.chdir(wd)
	if c1 == 0 and c2 == 0:
		print utility.color(' - build successfull', 'green')
	else:
		print utility.color(' - build failed', 'red')
		exit(1)

# Compiles libraries and headers for given directory.
def build_directory(path):
	deps = scanner.directory_dependencies(path)
	for dep in deps:
		print ' - resolving dependency:', dep
		if not storage.contains_module(dep):
			if storage.remote_contains_module(dep):
				storage.fetch_module(dep)
			else:
				print utility.color(' - module not found: ' + dep, 'red')
				print utility.color(' - build failed', 'red')
				exit(1)
		ready_indexed_module(dep)
	generate_cmake(path, deps)
	make(path)
<|endoftext|>


<commit_before>"""
Django settings for iosblog project.

Generated by 'django-admin startproject' using Django 1.9.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '(3$68-dhb79a%j9fsr+@8g(fbb*3-e1-b6d1voogql&7&0#8mv'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'x123.apps.X123Config',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
    'oauth2_provider',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'iosblog.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'iosblog.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.mysql',
        'OPTIONS': {
            'read_default_file': os.path.join(BASE_DIR, 'db.cnf')
        }
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'zh-cn'

TIME_ZONE = 'Asia/Shanghai'

USE_I18N = True

USE_L10N = True

USE_TZ = False

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

REST_FRAMEWORK = {
    # Use Django's standard `django.contrib.auth` permissions,
    # or allow read-only access for unauthenticated users.
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ],
    'PAGE_SIZE': 10,
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'oauth2_provider.ext.rest_framework.OAuth2Authentication',
    )
}

STATIC_ROOT= '/var/www/iosblog/static'
MEDIA_ROOT = '/var/www/iosblog/media'

if DEBUG:
    MEDIA_URL = '/media/'
else:
    MEDIA_URL = 'http://iosblog.cc/media/'
<commit_msg>setting production<commit_after>"""
Django settings for iosblog project.

Generated by 'django-admin startproject' using Django 1.9.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

RUNMODE = os.environ['IOSBLOGRUNMODE']

if RUNMODE == 'production':
    DEBUG = False

    with open('/etc/iosblog/secret_key.txt') as f:
        SECRET_KEY = f.read().strip()
else:
    # Quick-start development settings - unsuitable for production
    # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

    # SECURITY WARNING: keep the secret key used in production secret!
    SECRET_KEY = '(3$68-dhb79a%j9fsr+@8g(fbb*3-e1-b6d1voogql&7&0#8mv'

    # SECURITY WARNING: don't run with debug turned on in production!
    DEBUG = True

ALLOWED_HOSTS = ['.iosblog.cc']

# Application definition

INSTALLED_APPS = [
    'x123.apps.X123Config',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
    'oauth2_provider',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'iosblog.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'iosblog.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.mysql',
        'OPTIONS': {
            'read_default_file': os.path.join(BASE_DIR, 'db.cnf')
        }
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'zh-cn'

TIME_ZONE = 'Asia/Shanghai'

USE_I18N = True

USE_L10N = True

USE_TZ = False

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

REST_FRAMEWORK = {
    # Use Django's standard `django.contrib.auth` permissions,
    # or allow read-only access for unauthenticated users.
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ],
    'PAGE_SIZE': 10,
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'oauth2_provider.ext.rest_framework.OAuth2Authentication',
    )
}

STATIC_ROOT= '/var/www/iosblog/static'
MEDIA_ROOT = '/var/www/iosblog/media'

if DEBUG:
    MEDIA_URL = '/media/'
else:
    MEDIA_URL = 'http://iosblog.cc/media/'
<|endoftext|>


<commit_before>'''
Logger object
=============

Differents logging levels are available : trace, debug, info, warning, error
and critical.

Examples of usage::

    from kivy.logger import Logger

    Logger.info('title: This is a info message.')
    Logger.debug('title: This is a debug message.')

    try:
        raise Exception('bleh')
    except Exception:
        Logger.exception('Something happened!')

The message passed to the logger is split into two parts, separated by a colon
(:). The first part is used as a title, and the second part is used as the
message. This way, you can "categorize" your message easily. ::

    Logger.info('Application: This is a test')

    # will appear as

    [INFO   ] [Application ] This is a test

Logger configuration
--------------------

The Logger can be controlled via the Kivy configuration file::

    [kivy]
    log_level = info
    log_enable = 1
    log_dir = logs
    log_name = kivy_%y-%m-%d_%_.txt
    log_maxfiles = 100

More information about the allowed values are described in the
:mod:`kivy.config` module.

Logger history
--------------

Even if the logger is not enabled, you still have access to the last 100
messages::

    from kivy.logger import LoggerHistory

    print(LoggerHistory.history)

'''

import logging
import os
import sys
import kivy
from kivy.compat import PY2
from random import randint
from functools import partial

__all__ = (
    'Logger', 'LOG_LEVELS', 'COLORS', 'LoggerHistory', 'file_log_handler')

Logger = None

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(8))

# These are the sequences need to get colored ouput
RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"

previous_stderr = sys.stderr

def formatter_message(message, use_color=True):
    if use_color:
        message = message.replace("$RESET", RESET_SEQ)
        message = message.replace("$BOLD", BOLD_SEQ)
    else:
        message = message.replace("$RESET", "").replace("$BOLD", "")
    return message

COLORS = {
    'TRACE': MAGENTA,
    'WARNING': YELLOW,
    'INFO': GREEN,
    'DEBUG': CYAN,
    'CRITICAL': RED,
    'ERROR': RED}

logging.TRACE = 9
LOG_LEVELS = {
    'trace': logging.TRACE,
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL}

class FileHandler(logging.Handler):
    history = []
    filename = 'log.txt'
    fd = None
    log_dir = ''

    def purge_logs(self):
        '''Purge log is called randomly to prevent the log directory from being
        filled by lots and lots of log files.
        You've a chance of 1 in 20 that purge log will be fired.
        '''
        if randint(0, 20) != 0:
            return
        if not self.log_dir:
            raise ValueError('Log directory not yet set')

        from kivy.config import Config
        maxfiles = Config.getint('kivy', 'log_maxfiles')

        if maxfiles < 0:
            return

        Logger.info('Logger: Purge log fired. Analysing...')
        join = os.path.join
        unlink = os.unlink

        # search all log files
        lst = [join(self.log_dir, x) for x in os.listdir(self.log_dir)]
        if len(lst) > maxfiles:
            # get creation time on every files
            lst = [{'fn': x, 'ctime': os.path.getctime(x)} for x in lst]

            # sort by date
            lst = sorted(lst, key=lambda x: x['ctime'])

            # get the oldest (keep last maxfiles)
            lst = lst[:-maxfiles] if maxfiles else lst
            Logger.info('Logger: Purge %d log files' % len(lst))

            # now, unlink every file in the list
            for filename in lst:
                try:
                    unlink(filename['fn'])
                except PermissionError as e:
                    Logger.info('Logger: Skipped file {0}, {1}'.
                                format(filename['fn'], e))

        Logger.info('Logger: Purge finished!')

    def _configure(self, *largs, **kwargs):
        from time import strftime
        from kivy.config import Config
        log_dir = Config.get('kivy', 'log_dir')
        log_name = Config.get('kivy', 'log_name')

        _dir = kivy.kivy_home_dir
        if log_dir and os.path.isabs(log_dir):
            _dir = log_dir
        else:
            _dir = os.path.join(_dir, log_dir)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
        self.log_dir = _dir

        pattern = log_name.replace('%_', '@@NUMBER@@')
        pattern = os.path.join(_dir, strftime(pattern))
        n = 0
        while True:
            filename = pattern.replace('@@NUMBER@@', str(n))
            if not os.path.exists(filename):
                break
            n += 1
            if n > 10000:  # prevent maybe flooding ?
                raise Exception('Too many logfile, remove them')

        if FileHandler.filename == filename and FileHandler.fd is not None:
            return
        FileHandler.filename = filename
        if FileHandler.fd is not None:
            FileHandler.fd.close()
        FileHandler.fd = open(filename, 'w')

        Logger.info('Logger: Record log in %s' % filename)

    def _write_message(self, record):
        if FileHandler.fd in (None, False):
            return

        msg = self.format(record)
        stream = FileHandler.fd
        fs = "%s\n"
        stream.write('[%-7s] ' % record.levelname)
        if PY2:
            try:
                if (isinstance(msg, unicode) and
                        getattr(stream, 'encoding', None)):
                    ufs = u'%s\n'
                    try:
                        stream.write(ufs % msg)
                    except UnicodeEncodeError:
                        stream.write((ufs % msg).encode(stream.encoding))
                else:
                    stream.write(fs % msg)
            except UnicodeError:
                stream.write(fs % msg.encode("UTF-8"))
        else:
            stream.write(fs % msg)
        stream.flush()

    def emit(self, message):
        # during the startup, store the message in the history
        if Logger.logfile_activated is None:
            FileHandler.history += [message]
            return

        # startup done, if the logfile is not activated, avoid history.
        if Logger.logfile_activated is False:
            FileHandler.history = []
            return

        if FileHandler.fd is None:
            try:
                self._configure()
                from kivy.config import Config
                Config.add_callback(self._configure, 'kivy', 'log_dir')
                Config.add_callback(self._configure, 'kivy', 'log_name')
            except Exception:
                # deactivate filehandler...
                FileHandler.fd = False
                Logger.exception('Error while activating FileHandler logger')
                return
            while FileHandler.history:
                _message = FileHandler.history.pop()
                self._write_message(_message)

        self._write_message(message)

class LoggerHistory(logging.Handler):

    history = []

    def emit(self, message):
        LoggerHistory.history = [message] + LoggerHistory.history[:100]

class ColoredFormatter(logging.Formatter):

    def __init__(self, msg, use_color=True):
        logging.Formatter.__init__(self, msg)
        self.use_color = use_color

    def format(self, record):
        try:
            msg = record.msg.split(':', 1)
            if len(msg) == 2:
                record.msg = '[%-12s]%s' % (msg[0], msg[1])
        except:
            pass
        levelname = record.levelname
        if record.levelno == logging.TRACE:
            levelname = 'TRACE'
            record.levelname = levelname
        if self.use_color and levelname in COLORS:
            levelname_color = (
                COLOR_SEQ % (30 + COLORS[levelname]) + levelname + RESET_SEQ)
            record.levelname = levelname_color
        return logging.Formatter.format(self, record)

class ConsoleHandler(logging.StreamHandler):

    def filter(self, record):
        try:
            msg = record.msg
            k = msg.split(':', 1)
            if k[0] == 'stderr' and len(k) == 2:
                previous_stderr.write(k[1] + '\n')
                return False
        except:
            pass
        return True

class LogFile(object):

    def __init__(self, channel, func):
        self.buffer = ''
        self.func = func
        self.channel = channel
        self.errors = ''

    def write(self, s):
        s = self.buffer + s
        self.flush()
        f = self.func
        channel = self.channel
        lines = s.split('\n')
        for l in lines[:-1]:
            f('%s: %s' % (channel, l))
        self.buffer = lines[-1]

    def flush(self):
        return

    def isatty(self):
        return False

def logger_config_update(section, key, value):
    if LOG_LEVELS.get(value) is None:
        raise AttributeError('Loglevel {0!r} doesn\'t exists'.format(value))
    Logger.setLevel(level=LOG_LEVELS.get(value))

#: Kivy default logger instance
Logger = logging.getLogger('kivy')
Logger.logfile_activated = None
Logger.trace = partial(Logger.log, logging.TRACE)

# set the Kivy logger as the default
logging.root = Logger

# add default kivy logger
Logger.addHandler(LoggerHistory())
file_log_handler = None
if 'KIVY_NO_FILELOG' not in os.environ:
    file_log_handler = FileHandler()
    Logger.addHandler(file_log_handler)

# Use the custom handler instead of streaming one.
if 'KIVY_NO_CONSOLELOG' not in os.environ:
    if hasattr(sys, '_kivy_logging_handler'):
        Logger.addHandler(getattr(sys, '_kivy_logging_handler'))
    else:
        use_color = (
            os.name != 'nt' and
            os.environ.get('KIVY_BUILD') not in ('android', 'ios') and
            os.environ.get('TERM') in (
                'rxvt',
                'rxvt-256color',
                'rxvt-unicode',
                'rxvt-unicode-256color',
                'xterm',
                'xterm-256color',
            )
        )
        if not use_color:
            # No additional control characters will be inserted inside the
            # levelname field, 7 chars will fit "WARNING"
            color_fmt = formatter_message(
                '[%(levelname)-7s] %(message)s', use_color)
        else:
            # levelname field width need to take into account the length of the
            # color control codes (7+4 chars for bold+color, and reset)
            color_fmt = formatter_message(
                '[%(levelname)-18s] %(message)s', use_color)
        formatter = ColoredFormatter(color_fmt, use_color=use_color)
        console = ConsoleHandler()
        console.setFormatter(formatter)
        Logger.addHandler(console)

# install stderr handlers
sys.stderr = LogFile('stderr', Logger.warning)

#: Kivy history handler
LoggerHistory = LoggerHistory
<commit_msg>Don't error if log firectory is not set<commit_after>'''
Logger object
=============

Differents logging levels are available : trace, debug, info, warning, error
and critical.

Examples of usage::

    from kivy.logger import Logger

    Logger.info('title: This is a info message.')
    Logger.debug('title: This is a debug message.')

    try:
        raise Exception('bleh')
    except Exception:
        Logger.exception('Something happened!')

The message passed to the logger is split into two parts, separated by a colon
(:). The first part is used as a title, and the second part is used as the
message. This way, you can "categorize" your message easily. ::

    Logger.info('Application: This is a test')

    # will appear as

    [INFO   ] [Application ] This is a test

Logger configuration
--------------------

The Logger can be controlled via the Kivy configuration file::

    [kivy]
    log_level = info
    log_enable = 1
    log_dir = logs
    log_name = kivy_%y-%m-%d_%_.txt
    log_maxfiles = 100

More information about the allowed values are described in the
:mod:`kivy.config` module.

Logger history
--------------

Even if the logger is not enabled, you still have access to the last 100
messages::

    from kivy.logger import LoggerHistory

    print(LoggerHistory.history)

'''

import logging
import os
import sys
import kivy
from kivy.compat import PY2
from random import randint
from functools import partial

__all__ = (
    'Logger', 'LOG_LEVELS', 'COLORS', 'LoggerHistory', 'file_log_handler')

Logger = None

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(8))

# These are the sequences need to get colored ouput
RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"

previous_stderr = sys.stderr

def formatter_message(message, use_color=True):
    if use_color:
        message = message.replace("$RESET", RESET_SEQ)
        message = message.replace("$BOLD", BOLD_SEQ)
    else:
        message = message.replace("$RESET", "").replace("$BOLD", "")
    return message

COLORS = {
    'TRACE': MAGENTA,
    'WARNING': YELLOW,
    'INFO': GREEN,
    'DEBUG': CYAN,
    'CRITICAL': RED,
    'ERROR': RED}

logging.TRACE = 9
LOG_LEVELS = {
    'trace': logging.TRACE,
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL}

class FileHandler(logging.Handler):
    history = []
    filename = 'log.txt'
    fd = None
    log_dir = ''

    def purge_logs(self):
        '''Purge log is called randomly to prevent the log directory from being
        filled by lots and lots of log files.
        You've a chance of 1 in 20 that purge log will be fired.
        '''
        if randint(0, 20) != 0:
            return
        if not self.log_dir:
            return

        from kivy.config import Config
        maxfiles = Config.getint('kivy', 'log_maxfiles')

        if maxfiles < 0:
            return

        Logger.info('Logger: Purge log fired. Analysing...')
        join = os.path.join
        unlink = os.unlink

        # search all log files
        lst = [join(self.log_dir, x) for x in os.listdir(self.log_dir)]
        if len(lst) > maxfiles:
            # get creation time on every files
            lst = [{'fn': x, 'ctime': os.path.getctime(x)} for x in lst]

            # sort by date
            lst = sorted(lst, key=lambda x: x['ctime'])

            # get the oldest (keep last maxfiles)
            lst = lst[:-maxfiles] if maxfiles else lst
            Logger.info('Logger: Purge %d log files' % len(lst))

            # now, unlink every file in the list
            for filename in lst:
                try:
                    unlink(filename['fn'])
                except PermissionError as e:
                    Logger.info('Logger: Skipped file {0}, {1}'.
                                format(filename['fn'], e))

        Logger.info('Logger: Purge finished!')

    def _configure(self, *largs, **kwargs):
        from time import strftime
        from kivy.config import Config
        log_dir = Config.get('kivy', 'log_dir')
        log_name = Config.get('kivy', 'log_name')

        _dir = kivy.kivy_home_dir
        if log_dir and os.path.isabs(log_dir):
            _dir = log_dir
        else:
            _dir = os.path.join(_dir, log_dir)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
        self.log_dir = _dir

        pattern = log_name.replace('%_', '@@NUMBER@@')
        pattern = os.path.join(_dir, strftime(pattern))
        n = 0
        while True:
            filename = pattern.replace('@@NUMBER@@', str(n))
            if not os.path.exists(filename):
                break
            n += 1
            if n > 10000:  # prevent maybe flooding ?
                raise Exception('Too many logfile, remove them')

        if FileHandler.filename == filename and FileHandler.fd is not None:
            return
        FileHandler.filename = filename
        if FileHandler.fd is not None:
            FileHandler.fd.close()
        FileHandler.fd = open(filename, 'w')

        Logger.info('Logger: Record log in %s' % filename)

    def _write_message(self, record):
        if FileHandler.fd in (None, False):
            return

        msg = self.format(record)
        stream = FileHandler.fd
        fs = "%s\n"
        stream.write('[%-7s] ' % record.levelname)
        if PY2:
            try:
                if (isinstance(msg, unicode) and
                        getattr(stream, 'encoding', None)):
                    ufs = u'%s\n'
                    try:
                        stream.write(ufs % msg)
                    except UnicodeEncodeError:
                        stream.write((ufs % msg).encode(stream.encoding))
                else:
                    stream.write(fs % msg)
            except UnicodeError:
                stream.write(fs % msg.encode("UTF-8"))
        else:
            stream.write(fs % msg)
        stream.flush()

    def emit(self, message):
        # during the startup, store the message in the history
        if Logger.logfile_activated is None:
            FileHandler.history += [message]
            return

        # startup done, if the logfile is not activated, avoid history.
        if Logger.logfile_activated is False:
            FileHandler.history = []
            return

        if FileHandler.fd is None:
            try:
                self._configure()
                from kivy.config import Config
                Config.add_callback(self._configure, 'kivy', 'log_dir')
                Config.add_callback(self._configure, 'kivy', 'log_name')
            except Exception:
                # deactivate filehandler...
                FileHandler.fd = False
                Logger.exception('Error while activating FileHandler logger')
                return
            while FileHandler.history:
                _message = FileHandler.history.pop()
                self._write_message(_message)

        self._write_message(message)

class LoggerHistory(logging.Handler):

    history = []

    def emit(self, message):
        LoggerHistory.history = [message] + LoggerHistory.history[:100]

class ColoredFormatter(logging.Formatter):

    def __init__(self, msg, use_color=True):
        logging.Formatter.__init__(self, msg)
        self.use_color = use_color

    def format(self, record):
        try:
            msg = record.msg.split(':', 1)
            if len(msg) == 2:
                record.msg = '[%-12s]%s' % (msg[0], msg[1])
        except:
            pass
        levelname = record.levelname
        if record.levelno == logging.TRACE:
            levelname = 'TRACE'
            record.levelname = levelname
        if self.use_color and levelname in COLORS:
            levelname_color = (
                COLOR_SEQ % (30 + COLORS[levelname]) + levelname + RESET_SEQ)
            record.levelname = levelname_color
        return logging.Formatter.format(self, record)

class ConsoleHandler(logging.StreamHandler):

    def filter(self, record):
        try:
            msg = record.msg
            k = msg.split(':', 1)
            if k[0] == 'stderr' and len(k) == 2:
                previous_stderr.write(k[1] + '\n')
                return False
        except:
            pass
        return True

class LogFile(object):

    def __init__(self, channel, func):
        self.buffer = ''
        self.func = func
        self.channel = channel
        self.errors = ''

    def write(self, s):
        s = self.buffer + s
        self.flush()
        f = self.func
        channel = self.channel
        lines = s.split('\n')
        for l in lines[:-1]:
            f('%s: %s' % (channel, l))
        self.buffer = lines[-1]

    def flush(self):
        return

    def isatty(self):
        return False

def logger_config_update(section, key, value):
    if LOG_LEVELS.get(value) is None:
        raise AttributeError('Loglevel {0!r} doesn\'t exists'.format(value))
    Logger.setLevel(level=LOG_LEVELS.get(value))

#: Kivy default logger instance
Logger = logging.getLogger('kivy')
Logger.logfile_activated = None
Logger.trace = partial(Logger.log, logging.TRACE)

# set the Kivy logger as the default
logging.root = Logger

# add default kivy logger
Logger.addHandler(LoggerHistory())
file_log_handler = None
if 'KIVY_NO_FILELOG' not in os.environ:
    file_log_handler = FileHandler()
    Logger.addHandler(file_log_handler)

# Use the custom handler instead of streaming one.
if 'KIVY_NO_CONSOLELOG' not in os.environ:
    if hasattr(sys, '_kivy_logging_handler'):
        Logger.addHandler(getattr(sys, '_kivy_logging_handler'))
    else:
        use_color = (
            os.name != 'nt' and
            os.environ.get('KIVY_BUILD') not in ('android', 'ios') and
            os.environ.get('TERM') in (
                'rxvt',
                'rxvt-256color',
                'rxvt-unicode',
                'rxvt-unicode-256color',
                'xterm',
                'xterm-256color',
            )
        )
        if not use_color:
            # No additional control characters will be inserted inside the
            # levelname field, 7 chars will fit "WARNING"
            color_fmt = formatter_message(
                '[%(levelname)-7s] %(message)s', use_color)
        else:
            # levelname field width need to take into account the length of the
            # color control codes (7+4 chars for bold+color, and reset)
            color_fmt = formatter_message(
                '[%(levelname)-18s] %(message)s', use_color)
        formatter = ColoredFormatter(color_fmt, use_color=use_color)
        console = ConsoleHandler()
        console.setFormatter(formatter)
        Logger.addHandler(console)

# install stderr handlers
sys.stderr = LogFile('stderr', Logger.warning)

#: Kivy history handler
LoggerHistory = LoggerHistory
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np

import sys

sys.path.append('../')
import tfutil as t

np.random.seed(777)
tf.set_random_seed(777)  # reproducibility

class BigGAN:

    def __init__(self, s, batch_size=64, height=128, width=128, channel=3,
                 sample_num=10 * 10, sample_size=10,
                 df_dim=64, gf_dim=64, fc_unit=512, z_dim=128, lr=1e-4):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 64
        :param height: image height, default 64
        :param width: image width, default 64
        :param channel: image channel, default 3

        # Output Settings
        :param sample_num: the number of output images, default 100
        :param sample_size: sample image size, default 10

        # For Model
        :param df_dim: discriminator conv filter, default 64
        :param gf_dim: generator conv filter, default 64
        :param fc_unit: the number of fully connected layer units, default 512

        # Training Option
        :param z_dim: z dimension (kinda noise), default 128
        :param lr: learning rate, default 1e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.height = height
        self.width = width
        self.channel = channel
        self.image_shape = [self.batch_size, self.height, self.width, self.channel]

        assert self.height == self.width
        assert self.height in [128, 256, 512]

        self.sample_num = sample_num
        self.sample_size = sample_size

        self.df_dim = df_dim
        self.gf_dim = gf_dim
        self.fc_unit = fc_unit

        self.up_sampling = True

        self.z_dim = z_dim
        self.beta1 = 0.9
        self.beta2 = .999
        self.lr = lr

        self.res_block_disc = None
        self.res_block_gen = None
        if self.height == 128:
            self.res_block_disc = ([16, 8, 4, 2], [1])
            self.res_block_gen = ([1], [2, 4, 8, 16, 16])
        elif self.height == 256:
            self.res_block_disc = ([16, 8, 8, 4, 2], [1])
            self.res_block_gen = ([1, 2], [4, 8, 8, 16, 16])
        elif self.height == 512:
            self.res_block_disc = ([16, 8, 8, 4], [2, 1, 1])
            self.res_block_gen = ([1, 1, 2], [4, 8, 8, 16, 16])
        else:
            raise NotImplementedError

        # pre-defined
        self.g_loss = 0.
        self.d_loss = 0.
        self.c_loss = 0.

        self.g = None

        self.d_op = None
        self.g_op = None

        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x = tf.placeholder(tf.float32,
                                shape=[None, self.height, self.width, self.channel],
                                name="x-image")  # (64, 64, 64, 3)
        self.z = tf.placeholder(tf.float32, shape=[None, self.z_dim], name="z-noise")  # (-1, 128)

        self.build_sagan()  # build SAGAN model

    @staticmethod
    def res_block_up(x, c, f, name):
        with tf.variable_scope("res_block_up-%s" % name):
            ssc = x

            x = t.batch_norm(x, name="bn-1")  # <- noise
            x = tf.nn.relu(x)
            x = t.conv2d(x, f, name="conv2d-1")

            x = t.batch_norm(x, name="bn-2")  # <- noise
            x = tf.nn.relu(x)
            x = t.conv2d(x, f, name="conv2d-2")
            return x + ssc

    def res_block_down(self, x, c, f, name):
        pass

    @staticmethod
    def non_local_block(x, f, sub_sampling=False, name="nonlocal"):
        """ non-local block, https://arxiv.org/pdf/1711.07971.pdf """
        with tf.variable_scope("non_local_block-%s" % name):
            theta = t.conv2d(x, f=f, k=1, s=1, name="theta")
            if sub_sampling:
                theta = tf.layers.max_pooling2d(theta, pool_size=(2, 2), name="max_pool-theta")
            theta = tf.reshape(theta, (-1, theta.get_shape().as_list()[-1]))

            phi = t.conv2d(x, f=f, k=1, s=1, name="phi")
            if sub_sampling:
                phi = tf.layers.max_pooling2d(theta, pool_size=(2, 2), name="max_pool-phi")
            phi = tf.reshape(phi, (-1, phi.get_shape().as_list()[-1]))
            phi = tf.transpose(phi, [1, 0])

            g = t.conv2d(x, f=f, k=1, s=1, name="g")
            if sub_sampling:
                g = tf.layers.max_pooling2d(theta, pool_size=(2, 2), name="max_pool-g")
            g = tf.reshape(g, (-1, g.get_shape().as_list()[-1]))

            theta_phi = tf.multiply(theta, phi)
            theta_phi = tf.nn.softmax(theta_phi)

            theta_phi_g = tf.multiply(theta_phi, g)

            theta_phi_g = t.conv2d(theta_phi_g, f=f, k=1, s=1, name="theta_phi_g")
            return x + theta_phi_g

    def discriminator(self, x, reuse=None):
        """
        :param x: images
        :param y: labels
        :param reuse: re-usable
        :return: classification, probability (fake or real), network
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            f = self.gf_dim

            x = t.conv2d_alt(x, f, 4, 2, pad=1, sn=True, name='disc-conv2d-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            for i in range(self.n_layer // 2):
                x = t.conv2d_alt(x, f * 2, 4, 2, pad=1, sn=True, name='disc-conv2d-%d' % (i + 2))
                x = tf.nn.leaky_relu(x, alpha=0.1)

                f *= 2

            # Self-Attention Layer
            x = self.attention(x, f, reuse=reuse)

            for i in range(self.n_layer // 2, self.n_layer):
                x = t.conv2d_alt(x, f * 2, 4, 2, pad=1, sn=True, name='disc-conv2d-%d' % (i + 2))
                x = tf.nn.leaky_relu(x, alpha=0.1)

                f *= 2

            x = t.flatten(x)

            x = t.dense_alt(x, 1, sn=True, name='disc-fc-1')
            return x

    def generator(self, z, c, reuse=None):
        """
        :param z: noise
        :param c: image label
        :param reuse: re-usable
        :return: prob
        """
        with tf.variable_scope("generator", reuse=reuse):
            # split
            z = tf.split(z, num_or_size_splits=4, axis=-1)  # expected [None, 32] * 4

            # linear projection
            x = t.dense(z, f=4 * 4 * 16 * self.channel, name="disc-dense-1")
            x = tf.nn.relu(x)

            res = x
            for i in range(4):
                res = self.res_block_up(res, f=(16 // (2 ** i)) * self.channel, name="res%d" % (i + 1))
                res = tf.concat([res, z], axis=-1)

            x = res  # To-Do : non-local block

            x = self.res_block_up(x, f=self.channel, name="res4")

            x = t.batch_norm(x, name="bn-last")  # <- noise
            x = tf.nn.relu(x)
            x = t.conv2d(x, f=3, k=3, name="conv2d-last")

            x = tf.nn.tanh(x)
            return x

    def build_sagan(self):
        # Generator
        self.g = self.generator(self.z)

        # Discriminator
        d_real = self.discriminator(self.x)
        d_fake = self.discriminator(self.g, reuse=True)

        # Losses
        d_real_loss = t.sce_loss(d_real, tf.ones_like(d_real))
        d_fake_loss = t.sce_loss(d_fake, tf.zeros_like(d_fake))
        self.d_loss = d_real_loss + d_fake_loss
        self.g_loss = t.sce_loss(d_fake, tf.ones_like(d_fake))

        # Summary
        tf.summary.scalar("loss/d_real_loss", d_real_loss)
        tf.summary.scalar("loss/d_fake_loss", d_fake_loss)
        tf.summary.scalar("loss/d_loss", self.d_loss)
        tf.summary.scalar("loss/g_loss", self.g_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]

        self.d_op = tf.train.AdamOptimizer(self.lr * 4,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(self.lr * 1,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.g_loss, var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<commit_msg>fix: multiply to dot product (referred in the paper)<commit_after>import tensorflow as tf
import numpy as np

import sys

sys.path.append('../')
import tfutil as t

np.random.seed(777)
tf.set_random_seed(777)  # reproducibility

class BigGAN:

    def __init__(self, s, batch_size=64, height=128, width=128, channel=3,
                 sample_num=10 * 10, sample_size=10,
                 df_dim=64, gf_dim=64, fc_unit=512, z_dim=128, lr=1e-4):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 64
        :param height: image height, default 64
        :param width: image width, default 64
        :param channel: image channel, default 3

        # Output Settings
        :param sample_num: the number of output images, default 100
        :param sample_size: sample image size, default 10

        # For Model
        :param df_dim: discriminator conv filter, default 64
        :param gf_dim: generator conv filter, default 64
        :param fc_unit: the number of fully connected layer units, default 512

        # Training Option
        :param z_dim: z dimension (kinda noise), default 128
        :param lr: learning rate, default 1e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.height = height
        self.width = width
        self.channel = channel
        self.image_shape = [self.batch_size, self.height, self.width, self.channel]

        assert self.height == self.width
        assert self.height in [128, 256, 512]

        self.sample_num = sample_num
        self.sample_size = sample_size

        self.df_dim = df_dim
        self.gf_dim = gf_dim
        self.fc_unit = fc_unit

        self.up_sampling = True

        self.z_dim = z_dim
        self.beta1 = 0.9
        self.beta2 = .999
        self.lr = lr

        self.res_block_disc = None
        self.res_block_gen = None
        if self.height == 128:
            self.res_block_disc = ([16, 8, 4, 2], [1])
            self.res_block_gen = ([1], [2, 4, 8, 16, 16])
        elif self.height == 256:
            self.res_block_disc = ([16, 8, 8, 4, 2], [1])
            self.res_block_gen = ([1, 2], [4, 8, 8, 16, 16])
        elif self.height == 512:
            self.res_block_disc = ([16, 8, 8, 4], [2, 1, 1])
            self.res_block_gen = ([1, 1, 2], [4, 8, 8, 16, 16])
        else:
            raise NotImplementedError

        # pre-defined
        self.g_loss = 0.
        self.d_loss = 0.
        self.c_loss = 0.

        self.g = None

        self.d_op = None
        self.g_op = None

        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x = tf.placeholder(tf.float32,
                                shape=[None, self.height, self.width, self.channel],
                                name="x-image")  # (64, 64, 64, 3)
        self.z = tf.placeholder(tf.float32, shape=[None, self.z_dim], name="z-noise")  # (-1, 128)

        self.build_sagan()  # build SAGAN model

    @staticmethod
    def res_block_up(x, c, f, name):
        with tf.variable_scope("res_block_up-%s" % name):
            ssc = x

            x = t.batch_norm(x, name="bn-1")  # <- noise
            x = tf.nn.relu(x)
            x = t.conv2d(x, f, name="conv2d-1")

            x = t.batch_norm(x, name="bn-2")  # <- noise
            x = tf.nn.relu(x)
            x = t.conv2d(x, f, name="conv2d-2")
            return x + ssc

    def res_block_down(self, x, c, f, name):
        pass

    @staticmethod
    def non_local_block(x, f, sub_sampling=False, name="nonlocal"):
        """ non-local block, https://arxiv.org/pdf/1711.07971.pdf """
        with tf.variable_scope("non_local_block-%s" % name):
            with tf.name_scope("theta"):
                theta = t.conv2d(x, f=f, k=1, s=1, name="theta")
                if sub_sampling:
                    theta = tf.layers.max_pooling2d(theta, pool_size=(2, 2), name="max_pool-theta")
                theta = tf.reshape(theta, (-1, theta.get_shape().as_list()[-1]))

            with tf.name_scope("phi"):
                phi = t.conv2d(x, f=f, k=1, s=1, name="phi")
                if sub_sampling:
                    phi = tf.layers.max_pooling2d(theta, pool_size=(2, 2), name="max_pool-phi")
                phi = tf.reshape(phi, (-1, phi.get_shape().as_list()[-1]))
                phi = tf.transpose(phi, [1, 0])

            with tf.name_scope("g"):
                g = t.conv2d(x, f=f, k=1, s=1, name="g")
                if sub_sampling:
                    g = tf.layers.max_pooling2d(theta, pool_size=(2, 2), name="max_pool-g")
                g = tf.reshape(g, (-1, g.get_shape().as_list()[-1]))

            with tf.name_scope("self-attention"):
                theta_phi = tf.tensordot(theta, phi, axis=-1)
                theta_phi = tf.nn.softmax(theta_phi)

                theta_phi_g = tf.tensordot(theta_phi, g, axis=-1)

            theta_phi_g = t.conv2d(theta_phi_g, f=f, k=1, s=1, name="theta_phi_g")
            return x + theta_phi_g

    def discriminator(self, x, reuse=None):
        """
        :param x: images
        :param y: labels
        :param reuse: re-usable
        :return: classification, probability (fake or real), network
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            f = self.gf_dim

            x = t.conv2d_alt(x, f, 4, 2, pad=1, sn=True, name='disc-conv2d-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            for i in range(self.n_layer // 2):
                x = t.conv2d_alt(x, f * 2, 4, 2, pad=1, sn=True, name='disc-conv2d-%d' % (i + 2))
                x = tf.nn.leaky_relu(x, alpha=0.1)

                f *= 2

            # Self-Attention Layer
            x = self.attention(x, f, reuse=reuse)

            for i in range(self.n_layer // 2, self.n_layer):
                x = t.conv2d_alt(x, f * 2, 4, 2, pad=1, sn=True, name='disc-conv2d-%d' % (i + 2))
                x = tf.nn.leaky_relu(x, alpha=0.1)

                f *= 2

            x = t.flatten(x)

            x = t.dense_alt(x, 1, sn=True, name='disc-fc-1')
            return x

    def generator(self, z, c, reuse=None):
        """
        :param z: noise
        :param c: image label
        :param reuse: re-usable
        :return: prob
        """
        with tf.variable_scope("generator", reuse=reuse):
            # split
            z = tf.split(z, num_or_size_splits=4, axis=-1)  # expected [None, 32] * 4

            # linear projection
            x = t.dense(z, f=4 * 4 * 16 * self.channel, name="disc-dense-1")
            x = tf.nn.relu(x)

            res = x
            for i in range(4):
                res = self.res_block_up(res, f=(16 // (2 ** i)) * self.channel, name="res%d" % (i + 1))
                res = tf.concat([res, z], axis=-1)

            x = res  # To-Do : non-local block

            x = self.res_block_up(x, f=self.channel, name="res4")

            x = t.batch_norm(x, name="bn-last")  # <- noise
            x = tf.nn.relu(x)
            x = t.conv2d(x, f=3, k=3, name="conv2d-last")

            x = tf.nn.tanh(x)
            return x

    def build_sagan(self):
        # Generator
        self.g = self.generator(self.z)

        # Discriminator
        d_real = self.discriminator(self.x)
        d_fake = self.discriminator(self.g, reuse=True)

        # Losses
        d_real_loss = t.sce_loss(d_real, tf.ones_like(d_real))
        d_fake_loss = t.sce_loss(d_fake, tf.zeros_like(d_fake))
        self.d_loss = d_real_loss + d_fake_loss
        self.g_loss = t.sce_loss(d_fake, tf.ones_like(d_fake))

        # Summary
        tf.summary.scalar("loss/d_real_loss", d_real_loss)
        tf.summary.scalar("loss/d_fake_loss", d_fake_loss)
        tf.summary.scalar("loss/d_loss", self.d_loss)
        tf.summary.scalar("loss/g_loss", self.g_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]

        self.d_op = tf.train.AdamOptimizer(self.lr * 4,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(self.lr * 1,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.g_loss, var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<|endoftext|>


<commit_before>import committee_meetings
import fixtures
import lxml.etree
import os
import unittest
import utils

# Parsing the House hearing info
class HearingInfo(unittest.TestCase):

    def test_hearing(self):
        os.system("git clone -q --depth 1 https://github.com/unitedstates/congress-legislators congress-legislators")
        committees = {}
        for c in utils.yaml_load("congress-legislators/committees-current.yaml"):
            committees[c["thomas_id"]] = c
            if "house_committee_id" in c:
                committees[c["house_committee_id"] + "00"] = c
            c["subcommittees"] = dict((s["thomas_id"], s) for s in c.get("subcommittees", []))

        hearing_xml = "test/fixtures/hearings/sample_hearing.xml"
        file_xml = open(hearing_xml, "r")
        dom = lxml.etree.parse(file_xml)
        test_output = committee_meetings.parse_house_committee_meeting('102252', dom, [], committees, {"debug": False}, None)

        self.assertEqual(test_output['bills'], ['hr4435-113'])
        self.assertEqual(test_output['chamber'], 'house')
        self.assertEqual(test_output['committee'], 'HSRU')
        self.assertEqual(test_output['congress'], 113)
        self.assertEqual(test_output['house_meeting_type'], 'HMTG')
        self.assertEqual(test_output['meeting_documents'][0]['description'], 'H.R. 4435 (as introduced)')
        self.assertEqual(test_output['meeting_documents'][0]['legislation_number'], 'H.R. 4435')
        self.assertEqual(test_output['meeting_documents'][0]['legislation_stage'], 'ih' )
        self.assertEqual(test_output['meeting_documents'][0]['type'], 'BR' )
        self.assertEqual(test_output['meeting_documents'][0]['urls'], [ 'http://beta.congress.gov/113/bills/hr4435/BILLS-113hr4435ih.pdf',
                                     'http://beta.congress.gov/113/bills/hr4435/BILLS-113hr4435ih.xml'])
        self.assertEqual(test_output['occurs_at'], '2014-05-19T17:00:00')
        self.assertEqual(test_output['room'], 'CAPITOL H-313')
        self.assertEqual(test_output['subcommittee'], None)
        self.assertEqual(test_output['topic'], u'H.R. 4435\u2014National Defense Authorization Act for Fiscal Year 2015 [General Debate]; H.R. 4660\u2014Commerce, Justice, Science, and Related Agencies Appropriations Act, 2015')
        self.assertEqual(test_output['url'], 'http://docs.house.gov/Committee/Calendar/ByEvent.aspx?EventID=102252')

    def test_witnesses(self):
        witness_xml = "test/fixtures/hearings/sample_witness.xml"
        file_xml = open(witness_xml, "r")
        witness_tree = lxml.etree.parse(file_xml)

        test_output = committee_meetings.parse_witness_list(witness_tree)[0]

        self.assertEqual(test_output['documents'][0]['type'], 'WB')
        self.assertEqual(test_output['documents'][0]['description'], 'Cochrane Bio')
        self.assertEqual(test_output['documents'][0]['urls'], ['http://docs.house.gov/meetings/GO/GO25/20140522/102266/HHRG-113-GO25-Bio-CochraneJ-20140522.pdf'])
        self.assertEqual(test_output['house_event_id'], '102266')
        self.assertEqual(test_output['firstname'], 'James')
        self.assertEqual(test_output['lastname'], 'Cochrane')
        self.assertEqual(test_output['position'], 'Chief Information Officer and Executive Vice President')
        self.assertEqual(test_output['organization'], 'U.S. Postal Service')
<commit_msg>pep8/pyflakes<commit_after>import committee_meetings
import lxml.etree
import os
import unittest
import utils

# Parsing the House hearing info

class HearingInfo(unittest.TestCase):

    def test_hearing(self):
        os.system("git clone -q --depth 1 https://github.com/unitedstates/congress-legislators congress-legislators")
        committees = {}
        for c in utils.yaml_load(
                "congress-legislators/committees-current.yaml"):
            committees[c["thomas_id"]] = c
            if "house_committee_id" in c:
                committees[c["house_committee_id"] + "00"] = c
            c["subcommittees"] = dict((s["thomas_id"], s)
                                      for s in c.get("subcommittees", []))

        hearing_xml = "test/fixtures/hearings/sample_hearing.xml"
        file_xml = open(hearing_xml, "r")
        dom = lxml.etree.parse(file_xml)
        test_output = committee_meetings.parse_house_committee_meeting(
            '102252', dom, [], committees, {"debug": False}, None)

        self.assertEqual(test_output['bills'], ['hr4435-113'])
        self.assertEqual(test_output['chamber'], 'house')
        self.assertEqual(test_output['committee'], 'HSRU')
        self.assertEqual(test_output['congress'], 113)
        self.assertEqual(test_output['house_meeting_type'], 'HMTG')
        self.assertEqual(test_output['meeting_documents'][0][
                         'description'], 'H.R. 4435 (as introduced)')
        self.assertEqual(
            test_output['meeting_documents'][0]['legislation_number'],
            'H.R. 4435')
        self.assertEqual(
            test_output['meeting_documents'][0]['legislation_stage'], 'ih')
        self.assertEqual(test_output['meeting_documents'][0]['type'], 'BR')
        self.assertEqual(
            test_output['meeting_documents'][0]['urls'], [
                'http://beta.congress.gov/113/bills/hr4435/BILLS-113hr4435ih.pdf',
                'http://beta.congress.gov/113/bills/hr4435/BILLS-113hr4435ih.xml'])
        self.assertEqual(test_output['occurs_at'], '2014-05-19T17:00:00')
        self.assertEqual(test_output['room'], 'CAPITOL H-313')
        self.assertEqual(test_output['subcommittee'], None)
        self.assertEqual(test_output[
                         'topic'], u'H.R. 4435\u2014National Defense Authorization Act for Fiscal Year 2015 [General Debate]; H.R. 4660\u2014Commerce, Justice, Science, and Related Agencies Appropriations Act, 2015')
        self.assertEqual(test_output[
                         'url'], 'http://docs.house.gov/Committee/Calendar/ByEvent.aspx?EventID=102252')

    def test_witnesses(self):
        witness_xml = "test/fixtures/hearings/sample_witness.xml"
        file_xml = open(witness_xml, "r")
        witness_tree = lxml.etree.parse(file_xml)

        test_output = committee_meetings.parse_witness_list(witness_tree)[0]

        self.assertEqual(test_output['documents'][0]['type'], 'WB')
        self.assertEqual(
            test_output['documents'][0]['description'], 'Cochrane Bio')
        self.assertEqual(test_output['documents'][0]['urls'], [
                         'http://docs.house.gov/meetings/GO/GO25/20140522/102266/HHRG-113-GO25-Bio-CochraneJ-20140522.pdf'])
        self.assertEqual(test_output['house_event_id'], '102266')
        self.assertEqual(test_output['firstname'], 'James')
        self.assertEqual(test_output['lastname'], 'Cochrane')
        self.assertEqual(
            test_output['position'],
            'Chief Information Officer and Executive Vice President')
        self.assertEqual(test_output['organization'], 'U.S. Postal Service')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Oct 19 14:19:53 2015

@author: burger
"""

import os
import cPickle as pkl

import numpy as np

from deeplearning.tools.serialization import loads
from deeplearning.workflow import Workflow

def main(dir_):
    from matplotlib import pyplot as plt
    config_file = os.path.join(dir_, "config.json")
    with open(config_file, 'r') as f:
        config = loads(f.read())

    workflow = Workflow.build(config)

    classifier_file = os.path.join(dir_,
                                   "classifierCache", "OpPickleCache.pkl")
    with open(classifier_file, 'r') as f:
        classifier = pkl.load(f)

    workflow.set_classifier(classifier)

    target_shape = workflow.Target.meta.shape
    assert target_shape[0] == np.prod(target_shape),\
        "expected 1d for regression"
    target = workflow.Target[:].wait().squeeze()
    prediction = workflow.Prediction[:].wait().squeeze()

    plt.plot(target, label="ground truth")
    plt.plot(prediction, label="prediction")
    plt.show()

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser()
    parser.add_argument("experiment_dir",
                        help="directory containing the experiment")

    args = parser.parse_args()

    main(args.experiment_dir)
<commit_msg>added visualization of convergence<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Oct 19 14:19:53 2015

@author: burger
"""

import os
import cPickle as pkl
import warnings

import numpy as np

from deeplearning.tools.serialization import loads
from deeplearning.workflow import Workflow

try:
    from matplotlib import pyplot as plt
except ImportError:
    warnings.warn("can't plot without matplotlib")

def plot_regression(workflow, dir_):
    classifier_file = os.path.join(dir_,
                                   "classifierCache", "OpPickleCache.pkl")
    with open(classifier_file, 'r') as f:
        classifier = pkl.load(f)

    workflow.set_classifier(classifier)

    target_shape = workflow.Target.meta.shape
    assert target_shape[0] == np.prod(target_shape),\
        "expected 1d for regression"
    target = workflow.Target[:].wait().squeeze()
    prediction = workflow.Prediction[:].wait().squeeze()

    plt.figure()
    plt.plot(target, label="ground truth")
    plt.plot(prediction, label="prediction")
    plt.xlabel("time")
    plt.ylabel("target")
    plt.legend()

def plot_convergence(dir_):
    progress_file = os.path.join(dir_, "train", "progressmonitor.pkl")
    if not os.path.exists(progress_file):
        warnings.warn("no progress file found")
        return

    with open(progress_file, "r") as f:
        progress = np.asarray(pkl.load(f))

    plt.figure()
    plt.semilogy(progress, label="objective (validation dataset)")
    plt.xlabel("epochs")
    plt.ylabel("objective")
    plt.legend()

def main(dir_, show_regression=True, show_convergence=True):
    config_file = os.path.join(dir_, "config.json")
    with open(config_file, 'r') as f:
        config = loads(f.read())

    workflow = Workflow.build(config)

    if show_regression:
        plot_regression(workflow, dir_)

    if show_convergence:
        plot_convergence(dir_)

    plt.show()

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser()
    parser.add_argument("experiment_dir",
                        help="directory containing the experiment")
    parser.add_argument("-r", "--no_regression", action="store_true",
                        default=False,
                        help="don't show regression results")
    parser.add_argument("-p", "--no_progress", action="store_true",
                        default=False,
                        help="don't show progress report")

    args = parser.parse_args()

    main(args.experiment_dir, show_regression=not args.no_regression,
         show_convergence=not args.no_progress)
<|endoftext|>