<commit_before>#!/usr/bin/env python
import time

import argparse
import datetime
import dateutil.parser
import dateutil.relativedelta
import logging
import os
import sys

from ece2cmor3 import ece2cmorlib, taskloader, components

# Logger object
log = logging.getLogger(__name__)

def main(args=None):
    if args is None:
        pass

    formatter = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog,max_help_position=30)

    parser = argparse.ArgumentParser(description="Post-processing and cmorization of EC-Earth output",
                                     formatter_class=formatter)

    parser.add_argument("datadir", metavar="DIR", type=str, help="EC-Earth data directory, i.e. for a given component, "
                                                                 "for a given leg")
    parser.add_argument("--vars", metavar="FILE", type=str, default=None,
                        help="File (json|f90 namelist|xlsx) containing cmor variables")
    parser.add_argument("--conf", metavar="FILE.json", type=str, default=ece2cmorlib.conf_path_default,
                        help="Input metadata file")
    parser.add_argument("--exp", metavar="EXPID", type=str, default="ECE3", help="Experiment prefix")
    parser.add_argument("--odir", metavar="DIR", type=str, default=None, help="Output directory, by default the "
                                                                              "metadata \'outpath\' entry")
    parser.add_argument("--refd", metavar="YYYY-mm-dd", type=str, default="1850-01-01",
                        help="Reference date for output time axes")
    parser.add_argument("--npp", metavar="N", type=int, default=8, help="Number of parallel tasks (only relevant for IFS cmorization")

    model_attributes = {}
    for c in components.models:
            flag1, flag2 = components.get_script_options(c)
            if flag2 is None:
                parser.add_argument("--" + flag1, action="store_true", default=False,
                                    help="Run ece2cmor3 exclusively for %s data" % c)
            else:
                parser.add_argument("--" + flag1, '-' + flag2, action="store_true", default=False,
                                    help="Run ece2cmor3 exclusively for %s data" % c)
            model_attributes[c] = flag1

    parser.add_argument("--log", action="store_true", default=False, help="Write to log file")
    parser.add_argument("--flatdir", action="store_true", default=False, help="Do not create sub-directories in "
                                                                                "output folder")
    parser.add_argument("--tabledir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default,
                        help="Cmorization table directory")
    parser.add_argument("--tableprefix", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default,
                        help="Cmorization table prefix string")
    parser.add_argument("--tmpdir", metavar="DIR", type=str, default="/tmp/ece2cmor",
                        help="Temporary working directory")
    parser.add_argument("--overwritemode", metavar="MODE", type=str, default="preserve", help="MODE:preserve|replace|append, CMOR netcdf overwrite mode",
                        choices=["preserve", "replace", "append"])
    # Deprecated arguments, only for backward compatibility
    parser.add_argument("--ncdo", metavar="N", type=int, default=4, help=argparse.SUPPRESS)
    parser.add_argument("--nomask", action="store_true", default=False, help=argparse.SUPPRESS)
    parser.add_argument("--nofilter", action="store_true", default=False, help=argparse.SUPPRESS)
    parser.add_argument("--freq", metavar="N", type=int, default=3, help=argparse.SUPPRESS)

    model_tabfile_attributes = {}
    for c in components.models:
        tabfile = components.models[c].get(components.table_file, "")
        if tabfile:
            option = os.path.basename(tabfile)
            model_tabfile_attributes[c] = option
            parser.add_argument("--" + option, metavar="FILE.json", type=str, default=tabfile,
                                help="%s variable table (optional)" % c)

    args = parser.parse_args()

    logfile = None
    if getattr(args, "log", False):
        dirs = os.path.abspath(args.datadir).split(os.sep)
        fname = '-'.join([args.exp] + dirs[-2:] + [time.strftime("%Y%m%d%H%M%S", time.gmtime())])
        logfile = '.'.join([fname, "log"])
        logging.basicConfig(filename=logfile, level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.DEBUG)

    if not os.path.isdir(args.datadir):
        log.fatal("Your data directory argument %s cannot be found." % args.datadir)
        sys.exit(' Exiting ece2cmor.')

    if not os.path.isfile(args.vars):
        log.fatal("Your data request file %s cannot be found." % args.vars)
        sys.exit(' Exiting ece2cmor.')

    if not os.path.isfile(args.conf):
        log.fatal("Your metadata file %s cannot be found." % args.conf)
        sys.exit(' Exiting ece2cmor.')

    modedict = {"preserve": ece2cmorlib.PRESERVE, "append": ece2cmorlib.APPEND, "replace": ece2cmorlib.REPLACE}

    # Initialize ece2cmor:
    ece2cmorlib.initialize(args.conf, mode=modedict[args.mode], tabledir=args.tabdir, tableprefix=args.tabid,
                           outputdir=args.odir, logfile=logfile, create_subdirs=(not args.flatdir))
    ece2cmorlib.enable_masks = not args.nomask
    ece2cmorlib.auto_filter = not args.nofilter

    # Fix exclusive run flags: if none are used, we cmorize for all components
    model_active_flags = dict.fromkeys(components.models, False)
    for model in model_attributes:
        model_active_flags[model] = getattr(args, model_attributes[model], False)
    if not any(model_active_flags.values()):
        model_active_flags = dict.fromkeys(model_active_flags, True)

    # Load the variables as task targets:
    for model in model_tabfile_attributes:
        tabfile_attribute = model_tabfile_attributes[model]
        attribute_value = getattr(args, tabfile_attribute, None)
        if attribute_value is not None:
            components.models[model][components.table_file] = attribute_value

    taskloader.load_targets(args.vars, model_active_flags)

    refdate = datetime.datetime.combine(dateutil.parser.parse(args.refd), datetime.datetime.min.time())

    if model_active_flags["ifs"]:
        # Execute the atmosphere cmorization:
        ece2cmorlib.perform_ifs_tasks(args.datadir, args.exp,
                                      refdate=refdate,
                                      outputfreq=args.freq,
                                      tempdir=args.tmpdir,
                                      taskthreads=args.npp,
                                      cdothreads=args.ncdo)
    if model_active_flags["nemo"]:
        ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, refdate)

    if model_active_flags["lpjg"]:
        ece2cmorlib.perform_lpjg_tasks(args.datadir, args.tmpdir, args.exp, refdate)
    if model_active_flags["tm5"]:
        refdate = dateutil.parser.parse(args.refd) if args.refd else None
        ece2cmorlib.perform_tm5_tasks(args.datadir, args.tmpdir, args.exp, refdate)

#   if procNEWCOMPONENT:
#       ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, refdate)

    ece2cmorlib.finalize()

if __name__ == "__main__":
    main()
<commit_msg>Fix bug in previous commit: adjust tableprefix and overwritemode<commit_after>#!/usr/bin/env python
import time

import argparse
import datetime
import dateutil.parser
import dateutil.relativedelta
import logging
import os
import sys

from ece2cmor3 import ece2cmorlib, taskloader, components

# Logger object
log = logging.getLogger(__name__)

def main(args=None):
    if args is None:
        pass

    formatter = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog,max_help_position=30)

    parser = argparse.ArgumentParser(description="Post-processing and cmorization of EC-Earth output",
                                     formatter_class=formatter)

    parser.add_argument("datadir", metavar="DIR", type=str, help="EC-Earth data directory, i.e. for a given component, "
                                                                 "for a given leg")
    parser.add_argument("--vars", metavar="FILE", type=str, default=None,
                        help="File (json|f90 namelist|xlsx) containing cmor variables")
    parser.add_argument("--conf", metavar="FILE.json", type=str, default=ece2cmorlib.conf_path_default,
                        help="Input metadata file")
    parser.add_argument("--exp", metavar="EXPID", type=str, default="ECE3", help="Experiment prefix")
    parser.add_argument("--odir", metavar="DIR", type=str, default=None, help="Output directory, by default the "
                                                                              "metadata \'outpath\' entry")
    parser.add_argument("--refd", metavar="YYYY-mm-dd", type=str, default="1850-01-01",
                        help="Reference date for output time axes")
    parser.add_argument("--npp", metavar="N", type=int, default=8, help="Number of parallel tasks (only relevant for IFS cmorization")

    model_attributes = {}
    for c in components.models:
            flag1, flag2 = components.get_script_options(c)
            if flag2 is None:
                parser.add_argument("--" + flag1, action="store_true", default=False,
                                    help="Run ece2cmor3 exclusively for %s data" % c)
            else:
                parser.add_argument("--" + flag1, '-' + flag2, action="store_true", default=False,
                                    help="Run ece2cmor3 exclusively for %s data" % c)
            model_attributes[c] = flag1

    parser.add_argument("--log", action="store_true", default=False, help="Write to log file")
    parser.add_argument("--flatdir", action="store_true", default=False, help="Do not create sub-directories in "
                                                                                "output folder")
    parser.add_argument("--tabledir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default,
                        help="Cmorization table directory")
    parser.add_argument("--tableprefix", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default,
                        help="Cmorization table prefix string")
    parser.add_argument("--tmpdir", metavar="DIR", type=str, default="/tmp/ece2cmor",
                        help="Temporary working directory")
    parser.add_argument("--overwritemode", metavar="MODE", type=str, default="preserve", help="MODE:preserve|replace|append, CMOR netcdf overwrite mode",
                        choices=["preserve", "replace", "append"])
    # Deprecated arguments, only for backward compatibility
    parser.add_argument("--ncdo", metavar="N", type=int, default=4, help=argparse.SUPPRESS)
    parser.add_argument("--nomask", action="store_true", default=False, help=argparse.SUPPRESS)
    parser.add_argument("--nofilter", action="store_true", default=False, help=argparse.SUPPRESS)
    parser.add_argument("--freq", metavar="N", type=int, default=3, help=argparse.SUPPRESS)

    model_tabfile_attributes = {}
    for c in components.models:
        tabfile = components.models[c].get(components.table_file, "")
        if tabfile:
            option = os.path.basename(tabfile)
            model_tabfile_attributes[c] = option
            parser.add_argument("--" + option, metavar="FILE.json", type=str, default=tabfile,
                                help="%s variable table (optional)" % c)

    args = parser.parse_args()

    logfile = None
    if getattr(args, "log", False):
        dirs = os.path.abspath(args.datadir).split(os.sep)
        fname = '-'.join([args.exp] + dirs[-2:] + [time.strftime("%Y%m%d%H%M%S", time.gmtime())])
        logfile = '.'.join([fname, "log"])
        logging.basicConfig(filename=logfile, level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.DEBUG)

    if not os.path.isdir(args.datadir):
        log.fatal("Your data directory argument %s cannot be found." % args.datadir)
        sys.exit(' Exiting ece2cmor.')

    if not os.path.isfile(args.vars):
        log.fatal("Your data request file %s cannot be found." % args.vars)
        sys.exit(' Exiting ece2cmor.')

    if not os.path.isfile(args.conf):
        log.fatal("Your metadata file %s cannot be found." % args.conf)
        sys.exit(' Exiting ece2cmor.')

    modedict = {"preserve": ece2cmorlib.PRESERVE, "append": ece2cmorlib.APPEND, "replace": ece2cmorlib.REPLACE}

    # Initialize ece2cmor:
    ece2cmorlib.initialize(args.conf, mode=modedict[args.overwritemode], tabledir=args.tabledir, tableprefix=args.tableprefix,
                           outputdir=args.odir, logfile=logfile, create_subdirs=(not args.flatdir))
    ece2cmorlib.enable_masks = not args.nomask
    ece2cmorlib.auto_filter = not args.nofilter

    # Fix exclusive run flags: if none are used, we cmorize for all components
    model_active_flags = dict.fromkeys(components.models, False)
    for model in model_attributes:
        model_active_flags[model] = getattr(args, model_attributes[model], False)
    if not any(model_active_flags.values()):
        model_active_flags = dict.fromkeys(model_active_flags, True)

    # Load the variables as task targets:
    for model in model_tabfile_attributes:
        tabfile_attribute = model_tabfile_attributes[model]
        attribute_value = getattr(args, tabfile_attribute, None)
        if attribute_value is not None:
            components.models[model][components.table_file] = attribute_value

    taskloader.load_targets(args.vars, model_active_flags)

    refdate = datetime.datetime.combine(dateutil.parser.parse(args.refd), datetime.datetime.min.time())

    if model_active_flags["ifs"]:
        # Execute the atmosphere cmorization:
        ece2cmorlib.perform_ifs_tasks(args.datadir, args.exp,
                                      refdate=refdate,
                                      outputfreq=args.freq,
                                      tempdir=args.tmpdir,
                                      taskthreads=args.npp,
                                      cdothreads=args.ncdo)
    if model_active_flags["nemo"]:
        ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, refdate)

    if model_active_flags["lpjg"]:
        ece2cmorlib.perform_lpjg_tasks(args.datadir, args.tmpdir, args.exp, refdate)
    if model_active_flags["tm5"]:
        refdate = dateutil.parser.parse(args.refd) if args.refd else None
        ece2cmorlib.perform_tm5_tasks(args.datadir, args.tmpdir, args.exp, refdate)

#   if procNEWCOMPONENT:
#       ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, refdate)

    ece2cmorlib.finalize()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># The MIT License (MIT)
# Copyright (c) 2016 by the ECT Development Team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Description
===========

This plugin module adds the ESA CCI Data Portal's FTP data source to
the data store registry and makes it the default data store.

Verification
============

The module's unit-tests are located in `test/ds/test_esa_cci_ftp.py <https://github.com/CCI-Tools/ect-core/blob/master/test/ds/test_esa_cci_ftp.py>`_
and may be executed using ``$ py.test test/ds/test_esa_cci_ftp.py --cov=ect/ds/esa_cci_ftp.py`` for extra code coverage information.

Components
==========
"""

import os
import os.path
import pkgutil

from ect.core.io import FileSetDataStore, DATA_STORE_REGISTRY

DEFAULT_DATA_SOURCES_DIR = os.path.expanduser(os.path.join('~', '.ect', 'data_stores'))
DEFAULT_DATA_ROOT = os.path.join(DEFAULT_DATA_SOURCES_DIR, 'esa_cci_portal_ftp')

def set_default_data_store():
    """
    Defines the ESA CCI Data Portal's FTP data source and makes it the default data store.

    All data sources of the FTP data store are read from a JSON file ``esa_cci_ftp.json`` contained in this package.
    This JSON file has been generated from a scan of the entire FTP tree.
    """
    ect_data_root_dir = os.environ.get('ECT_DATA_ROOT', DEFAULT_DATA_ROOT)
    json_data = pkgutil.get_data('ect.ds', 'esa_cci_ftp.json')
    data_store = FileSetDataStore.from_json(ect_data_root_dir, json_data.decode('utf-8'))
    DATA_STORE_REGISTRY.add_data_store('default', data_store)
<commit_msg>made globals private<commit_after># The MIT License (MIT)
# Copyright (c) 2016 by the ECT Development Team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Description
===========

This plugin module adds the ESA CCI Data Portal's FTP data source to
the data store registry and makes it the default data store.

Verification
============

The module's unit-tests are located in `test/ds/test_esa_cci_ftp.py <https://github.com/CCI-Tools/ect-core/blob/master/test/ds/test_esa_cci_ftp.py>`_
and may be executed using ``$ py.test test/ds/test_esa_cci_ftp.py --cov=ect/ds/esa_cci_ftp.py`` for extra code coverage information.

Components
==========
"""

import os
import os.path
import pkgutil

from ect.core.io import FileSetDataStore, DATA_STORE_REGISTRY

_DATA_SOURCES_DIR = os.path.expanduser(os.path.join('~', '.ect', 'data_stores'))
_DATA_ROOT = os.path.join(_DATA_SOURCES_DIR, 'esa_cci_ftp')

def set_default_data_store():
    """
    Defines the ESA CCI Data Portal's FTP data store and makes it the default data store.

    All data sources of the FTP data store are read from a JSON file ``esa_cci_ftp.json`` contained in this package.
    This JSON file has been generated from a scan of the entire FTP tree.
    """
    ect_data_root_dir = os.environ.get('ECT_DATA_ROOT', _DATA_ROOT)
    json_data = pkgutil.get_data('ect.ds', 'esa_cci_ftp.json')
    data_store = FileSetDataStore.from_json(ect_data_root_dir, json_data.decode('utf-8'))
    DATA_STORE_REGISTRY.add_data_store('default', data_store)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Module to handle the areas in the game.
The world is divided in a grid of areas.
An area can have up to 4 neighbours (one for each cardinal point).
"""

from models.Model import Model
from models import item

import json
from core.localisation import _
from core import config
import core.exception

"""
Available directions
"""
directions = {
	_('DIRECTION_KEY_NORTH'): (1, (0, 1)),
	_('DIRECTION_KEY_EAST'): (2, (1, 0)),
	_('DIRECTION_KEY_SOUTH'): (4, (0, -1)),
	_('DIRECTION_KEY_WEST'): (8, (-1, 0))
}

class area:
	"""
	Class to interact with the areas in the game.
	"""

	"""
	Available items of the area.

	@param dict
	"""
	items = dict()
	types = (_('AREA_TYPE_LAND'), _('AREA_TYPE_DUNGEON'))

	@staticmethod
	def getNeighbourFromDirection(idArea, direction):
		"""
		area.area.getNeighbourFromDirection(idArea, direction) -> area.area

		Return the neighbour area of the given area according to the given
		direction.

		@param idArea integer id of the reference area
		@param direction string direction of the area to return, must be in
			area.directions.

		@return area.area if an area is found, None else.
		"""
		a = model.loadById(idArea)
		m = model.getFromDirection(
			(a['x'] + directions[direction][1][0], a['y'] + directions[direction][1][1])
		)

		if len(m) == 0:
			return None
		else:
			a = area()
			a._model = m
			return a

	@staticmethod
	def getItems(idArea):
		"""
		area.area.getItems(idArea) -> list()

		Method to get the items available in an area.

		@param idArea id of the desired area.

		@return list
		"""
		if config.memoization_enabled is False or idArea not in area.items.keys():
			area.items[idArea] = item.inventory.fromStr(model.loadById(idArea, ['items'])['items'])
		return area.items[idArea]

	@staticmethod
	def removeItems(idArea, items):
		"""
		area.area.removeItems(idArea, items)

		Method to remove some items in an area.
		Items not in the area will be ignored.

		@param idArea id of the area the items must be removed from.
		@param items list of items to remove
		"""
		area.items[idArea] = item.inventory.removeItems(area.getItems(idArea), items)

	@staticmethod
	def addItems(idArea, items):
		"""
		area.area.addItems(idArea, items)

		Method to add some items in an area.

		@param idArea id of the area the items must be added to.
		@param items list of items to add
		"""
		area.items[idArea] = item.inventory.addItems(area.getItems(idArea), items)

	@staticmethod
	def getDirections():
		"""
		Return the list of possible directions
		"""

		return directions.keys()

	@staticmethod
	def getValidDirections(directionsBits):
		"""
		From a integer containing, for each bits, a direction, a list will
		be returned containing the directions being in the integer.
		For example, if directionBits == 6 (0b0110), the returned directions
		will be ("east", "south").
		Uses the package variable directions.
		"""
		return list(filter(lambda k: area.canGoTo(directionsBits, k), directions))

	@staticmethod
	def canGoTo(directionsBits, direction):
		"""
		Check if a direction match a provided integer.
		For example, if directionBits == 6 (0b0110), the method will return true
		if direction == "east" or "south".
		Uses the package variable directions.
		"""
		return (directions[direction][0] & directionsBits) == directions[direction][0]

	@staticmethod
	def getRegionNameFromAreaId(idArea):
		return model.getRegionNameFromAreaId(idArea)

	@classmethod
	def saveChangedAreas(cls):
		for idArea in cls.items:
			model.saveAvailableItems(idArea, cls.items[idArea])

class model(Model):
	"""
	Class to interact with the values in the database.
	"""

	fields = (
		'id_area', 'id_area_type', 'id_region',
		'x', 'y',
		'directions',
		'container',
		'items'
	)

	@staticmethod
	def getRegionNameFromAreaId(idArea):
		"""
		area.model.getRegionNameFromAreaId(idArea) -> string

		Returns the name of the current area's region.

		@param idArea integer id of the reference area

		@return string name of the region
		"""

		query = "\
			SELECT\
				r.region_name\
			FROM\
				area AS a\
				JOIN region AS r\
			WHERE\
				a.id_area = ?\
		"

		return Model.fetchOneRow(query, [idArea])['region_name']

	@staticmethod
	def getFromDirection(direction):
		"""
		area.model.getFromDirection(direction) -> dict()

		Returns the neighbour of the area given in arguments from a given
		direction.

		@direction tuple of the area to return, represented by its relative
			values of x and y from idArea ((-1, 0) for example)

		@return dict informations of the found area, empty dict if not found.
		"""

		query = "\
			SELECT\
				%s\
			FROM\
				area\
			WHERE\
				x = ?\
				AND y = ?\
		" % (', '.join(model.fields))

		return Model.fetchOneRow(query, direction)

	@staticmethod
	def getSurroundingAreas(idArea):
		"""
		area.model.getSurroundingAreas(idArea) -> dict()

		Return the available neighbour areas of the area given in argument.

		@param idArea integer id of the reference area

		@return dict a list of directions, with for each direction, True if
			there is an area in this direction, False else.
		"""
		query = "\
			SELECT\
				orig.directions\
			FROM\
				area AS orig\
				JOIN area AS dest ON (dest.x = orig.x - 1 OR dest.x = orig.x + 1 OR dest.x = orig.x)\
					AND (dest.y = orig.y - 1 OR dest.y = orig.y + 1 OR dest.y = orig.y)\
					AND orig.id_area <> dest.id_area\
			WHERE\
				orig.id_area = ?\
		"

		return Model.fetchOneRow(query, [idArea])

	@staticmethod
	def saveAvailableItems(idArea, items):
		"""
		area.model.saveAvailableItems(idArea, items)

		Update an area's items list.

		@param idArea id of the area the items must be removed from.
		@param items list of items to remove
		"""
		model.update(
			{'items': json.dumps(items)},
			('id_area = ?', [idArea])
		)

class exception(core.exception.exception):
	"""
	Class for the exceptions concerning areas.
	"""
	pass
<commit_msg>The memoized items in the areas are reset after being saved to save some memory<commit_after># -*- coding: utf-8 -*-

"""
Module to handle the areas in the game.
The world is divided in a grid of areas.
An area can have up to 4 neighbours (one for each cardinal point).
"""

from models.Model import Model
from models import item

import json
from core.localisation import _
from core import config
import core.exception

"""
Available directions
"""
directions = {
	_('DIRECTION_KEY_NORTH'): (1, (0, 1)),
	_('DIRECTION_KEY_EAST'): (2, (1, 0)),
	_('DIRECTION_KEY_SOUTH'): (4, (0, -1)),
	_('DIRECTION_KEY_WEST'): (8, (-1, 0))
}

class area:
	"""
	Class to interact with the areas in the game.
	"""

	"""
	Available items of the area.

	@param dict
	"""
	items = dict()
	types = (_('AREA_TYPE_LAND'), _('AREA_TYPE_DUNGEON'))

	@staticmethod
	def getNeighbourFromDirection(idArea, direction):
		"""
		area.area.getNeighbourFromDirection(idArea, direction) -> area.area

		Return the neighbour area of the given area according to the given
		direction.

		@param idArea integer id of the reference area
		@param direction string direction of the area to return, must be in
			area.directions.

		@return area.area if an area is found, None else.
		"""
		a = model.loadById(idArea)
		m = model.getFromDirection(
			(a['x'] + directions[direction][1][0], a['y'] + directions[direction][1][1])
		)

		if len(m) == 0:
			return None
		else:
			a = area()
			a._model = m
			return a

	@staticmethod
	def getItems(idArea):
		"""
		area.area.getItems(idArea) -> list()

		Method to get the items available in an area.

		@param idArea id of the desired area.

		@return list
		"""
		if config.memoization_enabled is False or idArea not in area.items.keys():
			area.items[idArea] = item.inventory.fromStr(model.loadById(idArea, ['items'])['items'])
		return area.items[idArea]

	@staticmethod
	def removeItems(idArea, items):
		"""
		area.area.removeItems(idArea, items)

		Method to remove some items in an area.
		Items not in the area will be ignored.

		@param idArea id of the area the items must be removed from.
		@param items list of items to remove
		"""
		area.items[idArea] = item.inventory.removeItems(area.getItems(idArea), items)

	@staticmethod
	def addItems(idArea, items):
		"""
		area.area.addItems(idArea, items)

		Method to add some items in an area.

		@param idArea id of the area the items must be added to.
		@param items list of items to add
		"""
		area.items[idArea] = item.inventory.addItems(area.getItems(idArea), items)

	@staticmethod
	def getDirections():
		"""
		Return the list of possible directions
		"""

		return directions.keys()

	@staticmethod
	def getValidDirections(directionsBits):
		"""
		From a integer containing, for each bits, a direction, a list will
		be returned containing the directions being in the integer.
		For example, if directionBits == 6 (0b0110), the returned directions
		will be ("east", "south").
		Uses the package variable directions.
		"""
		return list(filter(lambda k: area.canGoTo(directionsBits, k), directions))

	@staticmethod
	def canGoTo(directionsBits, direction):
		"""
		Check if a direction match a provided integer.
		For example, if directionBits == 6 (0b0110), the method will return true
		if direction == "east" or "south".
		Uses the package variable directions.
		"""
		return (directions[direction][0] & directionsBits) == directions[direction][0]

	@staticmethod
	def getRegionNameFromAreaId(idArea):
		return model.getRegionNameFromAreaId(idArea)

	@classmethod
	def saveChangedAreas(cls):
		for idArea in cls.items:
			model.saveAvailableItems(idArea, cls.items[idArea])

		cls.items = dict()

class model(Model):
	"""
	Class to interact with the values in the database.
	"""

	fields = (
		'id_area', 'id_area_type', 'id_region',
		'x', 'y',
		'directions',
		'container',
		'items'
	)

	@staticmethod
	def getRegionNameFromAreaId(idArea):
		"""
		area.model.getRegionNameFromAreaId(idArea) -> string

		Returns the name of the current area's region.

		@param idArea integer id of the reference area

		@return string name of the region
		"""

		query = "\
			SELECT\
				r.region_name\
			FROM\
				area AS a\
				JOIN region AS r\
			WHERE\
				a.id_area = ?\
		"

		return Model.fetchOneRow(query, [idArea])['region_name']

	@staticmethod
	def getFromDirection(direction):
		"""
		area.model.getFromDirection(direction) -> dict()

		Returns the neighbour of the area given in arguments from a given
		direction.

		@direction tuple of the area to return, represented by its relative
			values of x and y from idArea ((-1, 0) for example)

		@return dict informations of the found area, empty dict if not found.
		"""

		query = "\
			SELECT\
				%s\
			FROM\
				area\
			WHERE\
				x = ?\
				AND y = ?\
		" % (', '.join(model.fields))

		return Model.fetchOneRow(query, direction)

	@staticmethod
	def getSurroundingAreas(idArea):
		"""
		area.model.getSurroundingAreas(idArea) -> dict()

		Return the available neighbour areas of the area given in argument.

		@param idArea integer id of the reference area

		@return dict a list of directions, with for each direction, True if
			there is an area in this direction, False else.
		"""
		query = "\
			SELECT\
				orig.directions\
			FROM\
				area AS orig\
				JOIN area AS dest ON (dest.x = orig.x - 1 OR dest.x = orig.x + 1 OR dest.x = orig.x)\
					AND (dest.y = orig.y - 1 OR dest.y = orig.y + 1 OR dest.y = orig.y)\
					AND orig.id_area <> dest.id_area\
			WHERE\
				orig.id_area = ?\
		"

		return Model.fetchOneRow(query, [idArea])

	@staticmethod
	def saveAvailableItems(idArea, items):
		"""
		area.model.saveAvailableItems(idArea, items)

		Update an area's items list.

		@param idArea id of the area the items must be removed from.
		@param items list of items to remove
		"""
		model.update(
			{'items': json.dumps(items)},
			('id_area = ?', [idArea])
		)

class exception(core.exception.exception):
	"""
	Class for the exceptions concerning areas.
	"""
	pass
<|endoftext|>


<commit_before>from datetime import datetime

from extensions import db

posts_to_tags = db.Table('posts_to_tags',
                         db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')),
                         db.Column('post_id', db.Integer, db.ForeignKey('post.id')))

class Post(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.Text)
    content = db.Column(db.Text)
    author_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    url = db.Column(db.Text)
    created_at = db.Column(db.DateTime)
    published_at = db.Column(db.DateTime)
    edited_at = db.Column(db.DateTime)
    view_count = db.Column(db.Integer, default=0)
    state = db.Column(db.Integer, db.ForeignKey('state.id'))
    tags = db.relationship('Tag', secondary=posts_to_tags, backref=db.backref('posts', lazy='dynamic'))

    def __init__(self, title, content, author_id):
        self.title = title
        self.set_url_from_title()
        self.content = content
        self.author_id = author_id
        self.created_at = datetime.utcnow()
        self.state = 1

    def set_url_from_title(self):
        self.url = self.title.lower().replace(' ', '_')

class Tag(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Text, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

class State(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Text, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

<commit_msg>generate random url for new post<commit_after>from datetime import datetime

from extensions import db

posts_to_tags = db.Table('posts_to_tags',
                         db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')),
                         db.Column('post_id', db.Integer, db.ForeignKey('post.id')))

class Post(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.Text)
    content = db.Column(db.Text)
    author_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    url = db.Column(db.Text)
    created_at = db.Column(db.DateTime)
    published_at = db.Column(db.DateTime)
    edited_at = db.Column(db.DateTime)
    view_count = db.Column(db.Integer, default=0)
    state = db.Column(db.Integer, db.ForeignKey('state.id'))
    tags = db.relationship('Tag', secondary=posts_to_tags, backref=db.backref('posts', lazy='dynamic'))

    def __init__(self, title, content, author_id):
        self.title = title
        self.url = 'random_url_' + generate_random_string()
        self.content = content
        self.author_id = author_id
        self.created_at = datetime.utcnow()
        self.state = 1

    def set_url_from_title(self):
        self.url = self.title.lower().replace(' ', '_')

def generate_random_string(size=32):
    import random
    import string
    choices = string.ascii_letters + string.digits
    random_characters = [random.choice(choices) for _ in range(size)]
    return ''.join(random_characters)

class Tag(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Text, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

class State(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Text, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

<|endoftext|>


<commit_before>import logging
from collections import UserDict
import pymysql
import datetime

log = logging.getLogger('tyggbot')

class User:
    def __init__(self):
        self.needs_sync = False
        self.ban_immune = False

    def remove_ban_immunity(self):
        self.ban_immune = False

    """
    Update the capitalization of a users username.
    """
    def update_username(self, new_username):
        if self.username_raw != new_username:
            # The capitalization has changed!
            self.username_raw = new_username
            self.needs_sync = True

    def __eq__(self, other):
        return self.username == other.username

    @classmethod
    def load(cls, cursor, username):
        user = cls()

        cursor.execute('SELECT * FROM `tb_user` WHERE `username`=%s', (username.lower()))
        row = cursor.fetchone()
        if row:
            # We found a user in the database!
            user.id = row['id']
            user.username = row['username']
            user.username_raw = row['username_raw']
            user.level = row['level']
            user.num_lines = row['num_lines']
            user.subscriber = row['subscriber'] == 1
            user.points = row['points']
            user.last_seen = row['last_seen']
            user.last_active = row['last_active']
            user.minutes_in_chat_online = row['minutes_in_chat_online']
            user.minutes_in_chat_offline = row['minutes_in_chat_offline']
        else:
            # No user was found with this username, create a new one!
            user.id = -1  # An ID of -1 means it will be inserted on sync
            user.username = username.lower()
            user.username_raw = user.username
            user.level = 100
            user.num_lines = 0
            user.needs_sync = True
            user.subscriber = False
            user.points = 0
            user.last_seen = None
            user.last_active = None
            user.minutes_in_chat_online = 0
            user.minutes_in_chat_offline = 0

        return user

    def spend(self, points_to_spend):
        if points_to_spend <= self.points:
            self.points -= points_to_spend
            self.needs_sync = True
            return True

        return False

    def sync(self, cursor):
        _last_seen = None if not self.last_seen else self.last_seen.strftime('%Y-%m-%d %H:%M:%S')
        _last_active = None if not self.last_active else self.last_active.strftime('%Y-%m-%d %H:%M:%S')
        if self.id == -1:
            cursor.execute('INSERT INTO `tb_user` (`username`, `username_raw`, `level`, `num_lines`, `subscriber`, `points`, `last_seen`, `last_active`, `minutes_in_chat_online`, `minutes_in_chat_offline`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)',
                    (self.username, self.username_raw, self.level, self.num_lines, self.subscriber, self.points, _last_seen, _last_active, self.minutes_in_chat_online, self.minutes_in_chat_offline))
            self.id = cursor.lastrowid
        else:
            # TODO: What values should we sync? For now, we only sync level and num_lines
            cursor.execute('UPDATE `tb_user` SET `level`=%s, `num_lines`=%s, `subscriber`=%s, `points`=%s, `last_seen`=%s, `last_active`=%s, `minutes_in_chat_online`=%s, `minutes_in_chat_offline`=%s, `username_raw`=%s WHERE `id`=%s',
                    (self.level, self.num_lines, self.subscriber, self.points, _last_seen, _last_active, self.minutes_in_chat_online, self.minutes_in_chat_offline, self.username_raw, self.id))
        self.needs_sync = False

    def touch(self, add_points=0):
        self.last_seen = datetime.datetime.now()
        self.points += add_points
        self.needs_sync = True

    def wrote_message(self, add_line=True):
        self.last_active = datetime.datetime.now()
        self.last_seen = datetime.datetime.now()
        if add_line:
            self.num_lines += 1
        self.needs_sync = True

class UserManager(UserDict):
    def __init__(self, sqlconn):
        UserDict.__init__(self)
        self.sqlconn = sqlconn

    def get_cursor(self):
        self.sqlconn.ping()
        return self.sqlconn.cursor(pymysql.cursors.DictCursor)

    def get_normal_cursor(self):
        self.sqlconn.ping()
        return self.sqlconn.cursor()

    def sync(self):
        self.sqlconn.autocommit(False)
        cursor = self.get_normal_cursor()
        for user in [user for k, user in self.data.items() if user.needs_sync]:
            user.sync(cursor)

        cursor.close()
        self.sqlconn.autocommit(True)

    def find(self, username):
        user = self[username]
        if user.id == -1:
            del self[username]
            return None
        return user

    def __getitem__(self, key):
        if key not in self.data:
            self.data[key] = User.load(self.get_cursor(), key)

        return self.data[key]
<commit_msg>Fixed a bug where new users could be deleted if .find was called on their username before they had their proper ID.<commit_after>import logging
from collections import UserDict
import pymysql
import datetime

log = logging.getLogger('tyggbot')

class User:
    def __init__(self):
        self.needs_sync = False
        self.ban_immune = False

    def remove_ban_immunity(self):
        self.ban_immune = False

    """
    Update the capitalization of a users username.
    """
    def update_username(self, new_username):
        if self.username_raw != new_username:
            # The capitalization has changed!
            self.username_raw = new_username
            self.needs_sync = True

    def __eq__(self, other):
        return self.username == other.username

    @classmethod
    def load(cls, cursor, username):
        user = cls()

        cursor.execute('SELECT * FROM `tb_user` WHERE `username`=%s', (username.lower()))
        row = cursor.fetchone()
        if row:
            # We found a user in the database!
            user.id = row['id']
            user.username = row['username']
            user.username_raw = row['username_raw']
            user.level = row['level']
            user.num_lines = row['num_lines']
            user.subscriber = row['subscriber'] == 1
            user.points = row['points']
            user.last_seen = row['last_seen']
            user.last_active = row['last_active']
            user.minutes_in_chat_online = row['minutes_in_chat_online']
            user.minutes_in_chat_offline = row['minutes_in_chat_offline']
        else:
            # No user was found with this username, create a new one!
            user.id = -1  # An ID of -1 means it will be inserted on sync
            user.username = username.lower()
            user.username_raw = user.username
            user.level = 100
            user.num_lines = 0
            user.subscriber = False
            user.points = 0
            user.last_seen = None
            user.last_active = None
            user.minutes_in_chat_online = 0
            user.minutes_in_chat_offline = 0

        return user

    def spend(self, points_to_spend):
        if points_to_spend <= self.points:
            self.points -= points_to_spend
            self.needs_sync = True
            return True

        return False

    def sync(self, cursor):
        _last_seen = None if not self.last_seen else self.last_seen.strftime('%Y-%m-%d %H:%M:%S')
        _last_active = None if not self.last_active else self.last_active.strftime('%Y-%m-%d %H:%M:%S')
        if self.id == -1:
            cursor.execute('INSERT INTO `tb_user` (`username`, `username_raw`, `level`, `num_lines`, `subscriber`, `points`, `last_seen`, `last_active`, `minutes_in_chat_online`, `minutes_in_chat_offline`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)',
                    (self.username, self.username_raw, self.level, self.num_lines, self.subscriber, self.points, _last_seen, _last_active, self.minutes_in_chat_online, self.minutes_in_chat_offline))
            self.id = cursor.lastrowid
        else:
            # TODO: What values should we sync? For now, we only sync level and num_lines
            cursor.execute('UPDATE `tb_user` SET `level`=%s, `num_lines`=%s, `subscriber`=%s, `points`=%s, `last_seen`=%s, `last_active`=%s, `minutes_in_chat_online`=%s, `minutes_in_chat_offline`=%s, `username_raw`=%s WHERE `id`=%s',
                    (self.level, self.num_lines, self.subscriber, self.points, _last_seen, _last_active, self.minutes_in_chat_online, self.minutes_in_chat_offline, self.username_raw, self.id))
        self.needs_sync = False

    def touch(self, add_points=0):
        self.last_seen = datetime.datetime.now()
        self.points += add_points
        self.needs_sync = True

    def wrote_message(self, add_line=True):
        self.last_active = datetime.datetime.now()
        self.last_seen = datetime.datetime.now()
        if add_line:
            self.num_lines += 1
        self.needs_sync = True

class UserManager(UserDict):
    def __init__(self, sqlconn):
        UserDict.__init__(self)
        self.sqlconn = sqlconn

    def get_cursor(self):
        self.sqlconn.ping()
        return self.sqlconn.cursor(pymysql.cursors.DictCursor)

    def get_normal_cursor(self):
        self.sqlconn.ping()
        return self.sqlconn.cursor()

    def sync(self):
        self.sqlconn.autocommit(False)
        cursor = self.get_normal_cursor()
        for user in [user for k, user in self.data.items() if user.needs_sync]:
            user.sync(cursor)

        cursor.close()
        self.sqlconn.autocommit(True)

    def find(self, username):
        user = self[username]
        if user.id == -1 and user.needs_sync is False:
            del self[username]
            return None
        return user

    def __getitem__(self, key):
        if key not in self.data:
            self.data[key] = User.load(self.get_cursor(), key)

        return self.data[key]
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import division, print_function

import os
import shlex
import subprocess
import tempfile
from glob import glob

import magic
import psutil

import dxpy

@dxpy.entry_point('main')
def main(tumor_bams=None, normal_bams=None, vcfs=None, cn_reference=None,
         baits=None, fasta=None, annotation=None,
         method='hybrid', is_male_normal=True, drop_low_coverage=False,
         antitarget_avg_size=None, target_avg_size=None,
         purity=None, ploidy=None, do_parallel=True):
    if not tumor_bams and not normal_bams:
        raise dxpy.AppError("Must provide tumor_bams or normal_bams (or both)")
    if cn_reference and any((normal_bams, baits, fasta, annotation)):
        raise dxpy.AppError("Reference profile (cn_reference) cannot be used "
                            "alongside normal_bams, baits, fasta, "
                            "or annotation")
    if method != "wgs" and tumor_bams and not any((baits, cn_reference)):
        raise dxpy.AppError("Need cn_reference or baits to process tumor_bams")
    if tumor_bams:
        purities = validate_per_tumor(purity, len(tumor_bams), "purity values",
                                      lambda p: 0 < p <= 1)
        ploidies = validate_per_tumor(ploidy, len(tumor_bams), "ploidy values",
                                      lambda p: p > 0)
        vcfs = validate_per_tumor(vcfs, len(tumor_bams), "VCF files")

    print("Downloading file inputs to the local file system")
    cn_reference = download_link(cn_reference)
    baits = download_link(baits)
    fasta = download_link(fasta)
    annotation = download_link(annotation)
    if tumor_bams is not None:
        tumor_bams = map(download_link, tumor_bams)
    if normal_bams is not None:
        normal_bams = map(download_link, normal_bams)
    if vcfs is not None:
        vcfs = map(download_link, vcfs)

    # If these input files are gzipped, decompress them
    fasta = maybe_gunzip(fasta, "ref", "fa")
    annotation = maybe_gunzip(annotation, "annot", "txt")

    out_fnames = run_cnvkit(tumor_bams, normal_bams, vcfs, cn_reference, baits,
                            fasta, annotation, method, is_male_normal,
                            drop_low_coverage, antitarget_avg_size,
                            target_avg_size, purities, ploidies, do_parallel)

    print("Uploading local file outputs to the DNAnexus platform")
    output = {}
    for filekey in ("cn_reference", "seg", "metrics", "sexes", "scatter_pdf",
                    "diagram_pdf"):
        if filekey in out_fnames:
            output[filekey] = dxpy.dxlink(
                dxpy.upload_local_file(out_fnames[filekey]))
    for listkey in ("copy_ratios", "copy_segments", "gainloss", "breaks"):
        if listkey in out_fnames:
            output[listkey] = [dxpy.dxlink(dxpy.upload_local_file(fname))
                               for fname in out_fnames[listkey]]
    return output

def validate_per_tumor(values, n_expected, title, criterion=None):
    """Ensure a per-tumor input array matches the number of tumor BAMs, etc.

    Also allow a value of None to skip checks & downstream processing, or a
    single value to apply to every tumor sample.

    Returns: list of values the same length as `n_expected`.
    """
    if values is None:
        out_vals = [None] * n_expected
    else:
        if criterion is not None and not all(map(criterion, values)):
            raise dxpy.AppError(
                """Tumor {} must all be between 0 and 1; got: {}"""
                .format(title, values))
        if len(values) == n_expected:
            out_vals = values
        elif len(values) == 1 and n_expected > 1:
            out_vals = [values] * n_expected
        else:
            raise dxpy.AppError(
                """Number of tumor {} specified ({}) does not
                match the number of tumor BAM files given ({})"""
                .format(title, len(values), n_expected))
    return out_vals

def run_cnvkit(tumor_bams, normal_bams, vcfs, reference, baits, fasta,
               annotation, method, is_male_normal, drop_low_coverage,
               antitarget_avg_size, target_avg_size, purities, ploidies,
               do_parallel):
    """Run the CNVkit pipeline.

    Returns a dict of the generated file names.
    """
    sh("mkdir -p /workdir")

    print("Running the main CNVkit pipeline")
    command = ["batch", "-m", method]
    if tumor_bams:
        command.extend(tumor_bams)
        command.extend(["--scatter", "--diagram"])
    if reference:
        # Use the given reference
        command.extend(["-r", reference])
    else:
        # Build a new reference
        reference = safe_fname("cnv-reference", "cnn")
        command.extend(["--output-reference", reference,
                        "-n"])
        if normal_bams:
            command.extend(normal_bams)
        if baits:
            command.extend(["-t", baits, "--short-names"])
        if fasta:
            command.extend(["-f", fasta])
        if annotation:
            command.extend(["--annotate", annotation])
        if antitarget_avg_size:
            command.extend(["--antitarget-avg-size", antitarget_avg_size])
        if target_avg_size:
            command.extend(["--target-avg-size", target_avg_size])
    if drop_low_coverage:
        command.append("--drop-low-coverage")
    if do_parallel:
        command.extend(["-p", str(psutil.cpu_count(logical=True))])
    yflag = "-y" if is_male_normal else ""
    command.append(yflag)

    cnvkit_docker(*command)
    sh("ls -Altr")  # Show the generated files in the DNAnexus log

    # Collect the outputs
    if not tumor_bams:
        return {"cn_reference": reference}

    all_cnr = glob("*.cnr")
    all_cns = glob("*.cns")
    if not len(all_cnr) == len(all_cns):
        raise dxpy.AppError(
            "Segmentation failed silently, try running again with "
            "do_parallel=false and see logs")

    all_gainloss = []
    all_breaks = []
    all_segmetrics = []
    all_call = []
    for acnr, acns, vcf, purity, ploidy in zip(all_cnr, all_cns, vcfs,
                                               purities, ploidies):
        name = acnr.split('.')[0]

        gainloss = name + ".gainloss.csv"
        cnvkit_docker("gainloss", acnr, "-s", acns, "-m 3 -t 0.3", yflag,
                      "-o", gainloss)
        all_gainloss.append(gainloss)

        breaks = name + ".breaks.csv"
        cnvkit_docker("breaks", acnr, acns, "-m 3", "-o", breaks)
        all_breaks.append(breaks)

        segmetrics = name + ".segmetrics.cns"
        cnvkit_docker("segmetrics", acnr, "-s", acns, "--ci -a .1",
                      "-o", segmetrics)
        all_segmetrics.append(segmetrics)

        call_fname = name + ".call.cns"
        call_opts = ["call", segmetrics, "-o", call_fname,
                     yflag, "--filter", "ci", "--center"]
        if purity or ploidy:
            call_opts.append("-m clonal")
            if purity:
                call_opts.extend(["--purity", purity])
            if ploidy:
                call_opts.extend(["--ploidy", ploidy])
        else:
            call_opts.append("-m threshold")
        if vcf:
            call_opts.extend(["--vcf", vcf])
        cnvkit_docker(*call_opts)
        all_call.append(call_fname)

    seg = safe_fname("cn_segments", "seg")
    cnvkit_docker("export seg", " ".join(all_cns), "-o", seg)

    sexes = safe_fname("sex", "csv")
    cnvkit_docker("sex", yflag, "-o", sexes, *all_cnr)

    metrics = safe_fname("metrics", "csv")
    cnvkit_docker("metrics", " ".join(all_cnr), "-s", " ".join(all_cns), "-o", metrics)

    outputs = {
        "cn_reference": reference,
        "copy_ratios": all_cnr,
        "copy_segments": all_segmetrics,
        "gainloss": all_gainloss,
        "breaks": all_breaks,
        "seg": seg,
        "sexes": sexes,
        "metrics": metrics,
    }

    all_scatters = glob("*-scatter.pdf")
    if all_scatters:
        if len(all_scatters) == 1:
            scatter_pdf = all_scatters[0]
        else:
            scatter_pdf = safe_fname("cnv-scatters", "pdf")
            sh("pdfunite", " ".join(all_scatters), scatter_pdf)
        outputs["scatter_pdf"] = scatter_pdf

    all_diagrams = glob("*-diagram.pdf")
    if all_diagrams:
        if len(all_diagrams) == 1:
            diagram_pdf = all_diagrams[0]
        else:
            diagram_pdf = safe_fname("cnv-diagrams", "pdf")
            sh("pdfunite", " ".join(all_diagrams), diagram_pdf)
        outputs["diagram_pdf"] = diagram_pdf

    return outputs

# _____________________________________________________________________________

def download_link(dxlink):
    """Download a DNAnexus object reference to the original filename.

    If the object is actually None, return None.
    """
    if dxlink is not None:
        dxf = dxpy.DXFile(dxlink)
        fname = dxf.describe()['name']
        if ' ' in fname:
            fname_nospace = fname.replace(' ', '_')
            print("Filename", repr(fname), "contains spaces; renaming to",
                  repr(fname_nospace))
            fname = fname_nospace
        dxpy.download_dxfile(dxf.get_id(), fname)
        return fname

def maybe_gunzip(fname, base, ext):
    if fname and 'gzip' in magic.from_file(fname):
        newf = safe_fname(base, ext)
        sh("gunzip", fname, "-c >", newf)
        fname = newf
    return fname

def safe_fname(base, ext):
    """Ensure the chosen filename will not overwrite an existing file.

    If it would, insert some garbage in the middle of the chosen filename to
    avoid the naming conflict.
    """
    fname = base + "." + ext
    if os.path.exists(fname):
        fname = tempfile.mktemp(prefix=base + "-", suffix="." + ext, dir=".")
    return fname

def sh(*command):
    """Run a shell command."""
    cmd = " ".join(map(str, command))
    print("$", cmd)
    try:
        subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError as exc:
        check_files(command[1:])
        raise exc
    print()

def check_files(maybe_filenames):
    fnames = []
    for fname in maybe_filenames:
        if isinstance(fname, basestring):
            fnames.extend(shlex.split(fname))
    for fname in fnames:
        if '.' in fname:
            # It might be a filename
            if os.path.exists(fname):
                print("File:", os.path.abspath(fname))
            else:
                print("Not file:", os.path.abspath(fname))

def cnvkit_docker(*args):
    """Run a CNVkit sub-command."""
    docker_prefix = ["dx-docker", "run",
                     "-v", "/home/dnanexus:/workdir", "-w", "/workdir",
                     "etal/cnvkit:0.9.1", "cnvkit.py"]
    sh(*(docker_prefix + list(args)))

# _____________________________________________________________________________

dxpy.run()
<commit_msg>Make all-sample output filenames consistent ("cnv-*.csv")<commit_after>#!/usr/bin/env python
from __future__ import division, print_function

import os
import shlex
import subprocess
import tempfile
from glob import glob

import magic
import psutil

import dxpy

@dxpy.entry_point('main')
def main(tumor_bams=None, normal_bams=None, vcfs=None, cn_reference=None,
         baits=None, fasta=None, annotation=None,
         method='hybrid', is_male_normal=True, drop_low_coverage=False,
         antitarget_avg_size=None, target_avg_size=None,
         purity=None, ploidy=None, do_parallel=True):
    if not tumor_bams and not normal_bams:
        raise dxpy.AppError("Must provide tumor_bams or normal_bams (or both)")
    if cn_reference and any((normal_bams, baits, fasta, annotation)):
        raise dxpy.AppError("Reference profile (cn_reference) cannot be used "
                            "alongside normal_bams, baits, fasta, "
                            "or annotation")
    if method != "wgs" and tumor_bams and not any((baits, cn_reference)):
        raise dxpy.AppError("Need cn_reference or baits to process tumor_bams")
    if tumor_bams:
        purities = validate_per_tumor(purity, len(tumor_bams), "purity values",
                                      lambda p: 0 < p <= 1)
        ploidies = validate_per_tumor(ploidy, len(tumor_bams), "ploidy values",
                                      lambda p: p > 0)
        vcfs = validate_per_tumor(vcfs, len(tumor_bams), "VCF files")

    print("Downloading file inputs to the local file system")
    cn_reference = download_link(cn_reference)
    baits = download_link(baits)
    fasta = download_link(fasta)
    annotation = download_link(annotation)
    if tumor_bams is not None:
        tumor_bams = map(download_link, tumor_bams)
    if normal_bams is not None:
        normal_bams = map(download_link, normal_bams)
    if vcfs is not None:
        vcfs = map(download_link, vcfs)

    # If these input files are gzipped, decompress them
    fasta = maybe_gunzip(fasta, "ref", "fa")
    annotation = maybe_gunzip(annotation, "annot", "txt")

    out_fnames = run_cnvkit(tumor_bams, normal_bams, vcfs, cn_reference, baits,
                            fasta, annotation, method, is_male_normal,
                            drop_low_coverage, antitarget_avg_size,
                            target_avg_size, purities, ploidies, do_parallel)

    print("Uploading local file outputs to the DNAnexus platform")
    output = {}
    for filekey in ("cn_reference", "seg", "metrics", "sexes", "scatter_pdf",
                    "diagram_pdf"):
        if filekey in out_fnames:
            output[filekey] = dxpy.dxlink(
                dxpy.upload_local_file(out_fnames[filekey]))
    for listkey in ("copy_ratios", "copy_segments", "gainloss", "breaks"):
        if listkey in out_fnames:
            output[listkey] = [dxpy.dxlink(dxpy.upload_local_file(fname))
                               for fname in out_fnames[listkey]]
    return output

def validate_per_tumor(values, n_expected, title, criterion=None):
    """Ensure a per-tumor input array matches the number of tumor BAMs, etc.

    Also allow a value of None to skip checks & downstream processing, or a
    single value to apply to every tumor sample.

    Returns: list of values the same length as `n_expected`.
    """
    if values is None:
        out_vals = [None] * n_expected
    else:
        if criterion is not None and not all(map(criterion, values)):
            raise dxpy.AppError(
                """Tumor {} must all be between 0 and 1; got: {}"""
                .format(title, values))
        if len(values) == n_expected:
            out_vals = values
        elif len(values) == 1 and n_expected > 1:
            out_vals = [values] * n_expected
        else:
            raise dxpy.AppError(
                """Number of tumor {} specified ({}) does not
                match the number of tumor BAM files given ({})"""
                .format(title, len(values), n_expected))
    return out_vals

def run_cnvkit(tumor_bams, normal_bams, vcfs, reference, baits, fasta,
               annotation, method, is_male_normal, drop_low_coverage,
               antitarget_avg_size, target_avg_size, purities, ploidies,
               do_parallel):
    """Run the CNVkit pipeline.

    Returns a dict of the generated file names.
    """
    sh("mkdir -p /workdir")

    print("Running the main CNVkit pipeline")
    command = ["batch", "-m", method]
    if tumor_bams:
        command.extend(tumor_bams)
        command.extend(["--scatter", "--diagram"])
    if reference:
        # Use the given reference
        command.extend(["-r", reference])
    else:
        # Build a new reference
        reference = safe_fname("cnv-reference", "cnn")
        command.extend(["--output-reference", reference,
                        "-n"])
        if normal_bams:
            command.extend(normal_bams)
        if baits:
            command.extend(["-t", baits, "--short-names"])
        if fasta:
            command.extend(["-f", fasta])
        if annotation:
            command.extend(["--annotate", annotation])
        if antitarget_avg_size:
            command.extend(["--antitarget-avg-size", antitarget_avg_size])
        if target_avg_size:
            command.extend(["--target-avg-size", target_avg_size])
    if drop_low_coverage:
        command.append("--drop-low-coverage")
    if do_parallel:
        command.extend(["-p", str(psutil.cpu_count(logical=True))])
    yflag = "-y" if is_male_normal else ""
    command.append(yflag)

    cnvkit_docker(*command)
    sh("ls -Altr")  # Show the generated files in the DNAnexus log

    # Collect the outputs
    if not tumor_bams:
        return {"cn_reference": reference}

    all_cnr = glob("*.cnr")
    all_cns = glob("*.cns")
    if not len(all_cnr) == len(all_cns):
        raise dxpy.AppError(
            "Segmentation failed silently, try running again with "
            "do_parallel=false and see logs")

    all_gainloss = []
    all_breaks = []
    all_segmetrics = []
    all_call = []
    for acnr, acns, vcf, purity, ploidy in zip(all_cnr, all_cns, vcfs,
                                               purities, ploidies):
        name = acnr.split('.')[0]

        gainloss = name + ".gainloss.csv"
        cnvkit_docker("gainloss", acnr, "-s", acns, "-m 3 -t 0.3", yflag,
                      "-o", gainloss)
        all_gainloss.append(gainloss)

        breaks = name + ".breaks.csv"
        cnvkit_docker("breaks", acnr, acns, "-m 3", "-o", breaks)
        all_breaks.append(breaks)

        segmetrics = name + ".segmetrics.cns"
        cnvkit_docker("segmetrics", acnr, "-s", acns, "--ci -a .1",
                      "-o", segmetrics)
        all_segmetrics.append(segmetrics)

        call_fname = name + ".call.cns"
        call_opts = ["call", segmetrics, "-o", call_fname,
                     yflag, "--filter", "ci", "--center"]
        if purity or ploidy:
            call_opts.append("-m clonal")
            if purity:
                call_opts.extend(["--purity", purity])
            if ploidy:
                call_opts.extend(["--ploidy", ploidy])
        else:
            call_opts.append("-m threshold")
        if vcf:
            call_opts.extend(["--vcf", vcf])
        cnvkit_docker(*call_opts)
        all_call.append(call_fname)

    seg = safe_fname("cnv-segments", "seg")
    cnvkit_docker("export seg", " ".join(all_cns), "-o", seg)

    sexes = safe_fname("cnv-sex", "csv")
    cnvkit_docker("sex", yflag, "-o", sexes, *all_cnr)

    metrics = safe_fname("cnv-metrics", "csv")
    cnvkit_docker("metrics", " ".join(all_cnr), "-s", " ".join(all_cns), "-o", metrics)

    outputs = {
        "cn_reference": reference,
        "copy_ratios": all_cnr,
        "copy_segments": all_segmetrics,
        "gainloss": all_gainloss,
        "breaks": all_breaks,
        "seg": seg,
        "sexes": sexes,
        "metrics": metrics,
    }

    all_scatters = glob("*-scatter.pdf")
    if all_scatters:
        if len(all_scatters) == 1:
            scatter_pdf = all_scatters[0]
        else:
            scatter_pdf = safe_fname("cnv-scatters", "pdf")
            sh("pdfunite", " ".join(all_scatters), scatter_pdf)
        outputs["scatter_pdf"] = scatter_pdf

    all_diagrams = glob("*-diagram.pdf")
    if all_diagrams:
        if len(all_diagrams) == 1:
            diagram_pdf = all_diagrams[0]
        else:
            diagram_pdf = safe_fname("cnv-diagrams", "pdf")
            sh("pdfunite", " ".join(all_diagrams), diagram_pdf)
        outputs["diagram_pdf"] = diagram_pdf

    return outputs

# _____________________________________________________________________________

def download_link(dxlink):
    """Download a DNAnexus object reference to the original filename.

    If the object is actually None, return None.
    """
    if dxlink is not None:
        dxf = dxpy.DXFile(dxlink)
        fname = dxf.describe()['name']
        if ' ' in fname:
            fname_nospace = fname.replace(' ', '_')
            print("Filename", repr(fname), "contains spaces; renaming to",
                  repr(fname_nospace))
            fname = fname_nospace
        dxpy.download_dxfile(dxf.get_id(), fname)
        return fname

def maybe_gunzip(fname, base, ext):
    if fname and 'gzip' in magic.from_file(fname):
        newf = safe_fname(base, ext)
        sh("gunzip", fname, "-c >", newf)
        fname = newf
    return fname

def safe_fname(base, ext):
    """Ensure the chosen filename will not overwrite an existing file.

    If it would, insert some garbage in the middle of the chosen filename to
    avoid the naming conflict.
    """
    fname = base + "." + ext
    if os.path.exists(fname):
        fname = tempfile.mktemp(prefix=base + "-", suffix="." + ext, dir=".")
    return fname

def sh(*command):
    """Run a shell command."""
    cmd = " ".join(map(str, command))
    print("$", cmd)
    try:
        subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError as exc:
        check_files(command[1:])
        raise exc
    print()

def check_files(maybe_filenames):
    fnames = []
    for fname in maybe_filenames:
        if isinstance(fname, basestring):
            fnames.extend(shlex.split(fname))
    for fname in fnames:
        if '.' in fname:
            # It might be a filename
            if os.path.exists(fname):
                print("File:", os.path.abspath(fname))
            else:
                print("Not file:", os.path.abspath(fname))

def cnvkit_docker(*args):
    """Run a CNVkit sub-command."""
    docker_prefix = ["dx-docker", "run",
                     "-v", "/home/dnanexus:/workdir", "-w", "/workdir",
                     "etal/cnvkit:0.9.1", "cnvkit.py"]
    sh(*(docker_prefix + list(args)))

# _____________________________________________________________________________

dxpy.run()
<|endoftext|>


<commit_before>from time import strftime
import codecs
import os
import shutil
import re

class Migration(object):

    MIGRATION_FILES_EXTENSION = ".migration"
    MIGRATION_FILES_MASK = r"[0-9]{14}\w+%s$" % MIGRATION_FILES_EXTENSION
    TEMPLATE = 'SQL_UP = u"""\n\n"""\n\nSQL_DOWN = u"""\n\n"""'

    def __init__(self, file):
        if not os.path.exists(file):
            raise Exception('migration file does not exist (%s)' % file)

        file_name = os.path.split(file)[1]
        if not Migration.is_file_name_valid(file_name):
            raise Exception('invalid migration file name (%s)' % file_name)

        self.abspath = os.path.abspath(file)
        self.file_name = file_name
        self.version = file_name[0:file_name.find("_")]
        self.sql_up, self.sql_down = self._get_commands()

    def _get_commands(self):
        f = codecs.open(self.abspath, "rU", "utf-8")
        exec(f.read())
        f.close()

        try:
            (SQL_UP, SQL_DOWN)
        except NameError:
            raise Exception("migration file is incorrect; it does not define 'SQL_UP' or 'SQL_DOWN' (%s)" % self.abspath)

        if SQL_UP is None or SQL_UP == "":
            raise Exception("migration command 'SQL_UP' is empty (%s)" % self.abspath)

        if SQL_DOWN is None or SQL_DOWN == "":
            raise Exception("migration command 'SQL_DOWN' is empty (%s)" % self.abspath)

        return SQL_UP, SQL_DOWN

    def compare_to(self, another_migration):
        if self.version < another_migration.version:
            return -1
        elif self.version > another_migration.version:
            return 1
        return 0

    @staticmethod
    def sort_migrations_list(migrations, reverse=False):
        return sorted(migrations, cmp=lambda x,y: x.compare_to(y), reverse=reverse)

    @staticmethod
    def is_file_name_valid(file_name):
        match = re.match(Migration.MIGRATION_FILES_MASK, file_name, re.IGNORECASE)
        return match != None

    @staticmethod
    def create(migration_name, migration_dir='.'):
        timestamp = strftime("%Y%m%d%H%M%S")
        file_name = "%s_%s%s" % (timestamp, migration_name, Migration.MIGRATION_FILES_EXTENSION)

        if not Migration.is_file_name_valid(file_name):
            raise Exception("invalid migration name; it should contain only letters, numbers and/or underscores ('%s')" % migration_name)

        new_file_name = "%s/%s" % (migration_dir, file_name)

        try:
            f = codecs.open(new_file_name, "w", "utf-8")
            f.write(Migration.TEMPLATE)
            f.close()
        except IOError:
            raise Exception("could not create file ('%s')" % new_file_name)

        return new_file_name

class SimpleDBMigrate(object):

    def __init__(self, config=None):
        self._migrations_dir = config.get("migrations_dir")

    def get_all_migrations(self):
        migrations = []

        for dir in self._migrations_dir:
            path = os.path.abspath(dir)

            dir_list = None
            try:
                dir_list = os.listdir(path)
            except OSError:
                raise Exception("directory not found ('%s')" % path)

            for dir_file in dir_list:
                if dir_file.endswith(Migration.MIGRATION_FILES_EXTENSION) and Migration.is_file_name_valid(dir_file):
                    migration = Migration('%s/%s' % (path, dir_file))
                    migrations.append(migration)

        if len(migrations) == 0:
            raise Exception("no migration files found")

        return Migration.sort_migrations_list(migrations)

    def get_all_migration_versions(self):
        return [migration.version for migration in self.get_all_migrations()]

    def get_all_migration_versions_up_to(self, limit_version):
        return [version for version in self.get_all_migration_versions() if version < limit_version]

    def check_if_version_exists(self, version):
        return version in self.get_all_migration_versions()

    def latest_version_available(self):
        all_migrations = self.get_all_migrations()
        all_migrations = Migration.sort_migrations_list(all_migrations, reverse=True)
        return all_migrations[0].version

    def get_migration_from_version_number(self, version):
        migrations = [migration for migration in self.get_all_migrations() if migration.version == version]
        if len(migrations) > 0:
            return migrations[0]
        return None
<commit_msg>Cosmetic change.<commit_after>from time import strftime
import codecs
import os
import shutil
import re

class Migration(object):

    MIGRATION_FILES_EXTENSION = ".migration"
    MIGRATION_FILES_MASK = r"[0-9]{14}\w+%s$" % MIGRATION_FILES_EXTENSION
    TEMPLATE = 'SQL_UP = u"""\n\n"""\n\nSQL_DOWN = u"""\n\n"""'

    def __init__(self, file):
        file_name = os.path.split(file)[1]
        if not Migration.is_file_name_valid(file_name):
            raise Exception('invalid migration file name (%s)' % file_name)

        if not os.path.exists(file):
            raise Exception('migration file does not exist (%s)' % file)

        self.abspath = os.path.abspath(file)
        self.file_name = file_name
        self.version = file_name[0:file_name.find("_")]
        self.sql_up, self.sql_down = self._get_commands()

    def _get_commands(self):
        f = codecs.open(self.abspath, "rU", "utf-8")
        exec(f.read())
        f.close()

        try:
            (SQL_UP, SQL_DOWN)
        except NameError:
            raise Exception("migration file is incorrect; it does not define 'SQL_UP' or 'SQL_DOWN' (%s)" % self.abspath)

        if SQL_UP is None or SQL_UP == "":
            raise Exception("migration command 'SQL_UP' is empty (%s)" % self.abspath)

        if SQL_DOWN is None or SQL_DOWN == "":
            raise Exception("migration command 'SQL_DOWN' is empty (%s)" % self.abspath)

        return SQL_UP, SQL_DOWN

    def compare_to(self, another_migration):
        if self.version < another_migration.version:
            return -1
        elif self.version > another_migration.version:
            return 1
        return 0

    @staticmethod
    def sort_migrations_list(migrations, reverse=False):
        return sorted(migrations, cmp=lambda x,y: x.compare_to(y), reverse=reverse)

    @staticmethod
    def is_file_name_valid(file_name):
        match = re.match(Migration.MIGRATION_FILES_MASK, file_name, re.IGNORECASE)
        return match != None

    @staticmethod
    def create(migration_name, migration_dir='.'):
        timestamp = strftime("%Y%m%d%H%M%S")
        file_name = "%s_%s%s" % (timestamp, migration_name, Migration.MIGRATION_FILES_EXTENSION)

        if not Migration.is_file_name_valid(file_name):
            raise Exception("invalid migration name; it should contain only letters, numbers and/or underscores ('%s')" % migration_name)

        new_file_name = "%s/%s" % (migration_dir, file_name)

        try:
            f = codecs.open(new_file_name, "w", "utf-8")
            f.write(Migration.TEMPLATE)
            f.close()
        except IOError:
            raise Exception("could not create file ('%s')" % new_file_name)

        return new_file_name

class SimpleDBMigrate(object):

    def __init__(self, config=None):
        self._migrations_dir = config.get("migrations_dir")

    def get_all_migrations(self):
        migrations = []

        for dir in self._migrations_dir:
            path = os.path.abspath(dir)

            dir_list = None
            try:
                dir_list = os.listdir(path)
            except OSError:
                raise Exception("directory not found ('%s')" % path)

            for dir_file in dir_list:
                if dir_file.endswith(Migration.MIGRATION_FILES_EXTENSION) and Migration.is_file_name_valid(dir_file):
                    migration = Migration('%s/%s' % (path, dir_file))
                    migrations.append(migration)

        if len(migrations) == 0:
            raise Exception("no migration files found")

        return Migration.sort_migrations_list(migrations)

    def get_all_migration_versions(self):
        return [migration.version for migration in self.get_all_migrations()]

    def get_all_migration_versions_up_to(self, limit_version):
        return [version for version in self.get_all_migration_versions() if version < limit_version]

    def check_if_version_exists(self, version):
        return version in self.get_all_migration_versions()

    def latest_version_available(self):
        all_migrations = self.get_all_migrations()
        all_migrations = Migration.sort_migrations_list(all_migrations, reverse=True)
        return all_migrations[0].version

    def get_migration_from_version_number(self, version):
        migrations = [migration for migration in self.get_all_migrations() if migration.version == version]
        if len(migrations) > 0:
            return migrations[0]
        return None
<|endoftext|>


<commit_before>from subprocess import check_output, CalledProcessError
import json

class Exif(object):
    def __init__(self, file):
        self.file = file

    def data(self):
        try:
            data = check_output('exiftool -time:all -mimetype -j ' + self.file, shell=True).decode('UTF-8')
            exif = json.loads(data)[0]
        except (CalledProcessError, UnicodeDecodeError):
            return None

        return exif
<commit_msg>Update exif.py<commit_after>from subprocess import check_output, CalledProcessError
import json

class Exif(object):
    def __init__(self, file):
        self.file = file

    def data(self):
        try:
            data = check_output('exiftool -time:all -mimetype -j "%s"' % self.file, shell=True).decode('UTF-8')
            exif = json.loads(data)[0]
        except (CalledProcessError, UnicodeDecodeError):
            return None

        return exif
<|endoftext|>


<commit_before>"""
Fast exponentiation by squaring
"""

def fpow0(x, n):
    """
    Perform recursive fast exponentiation by squaring.

    :param x: the base number
    :param n: the target power value, an integer
    :return: x to the power of n
    """

    if x == 0:
        if n == 0:
            return 1
        if n > 0:
            return 0

        raise ZeroDivisionError("Cannot raise 0.0 to a negative power")

    if n == 0:
        return 1

    if n == 1:
        return x

    if n < 0:
        return fpow0(1 / x, -n)

    assert n == int(n), "n must be an integer"

    return fpow0(x * x, n // 2) if n % 2 == 0 else x * fpow0(x * x, n // 2)

def fpow1(x, n):
    """
    Perform iterative fast exponentiation by squaring.

    :param x: the base number
    :param n: the target power value, an integer
    :return: x to the power of n
    """

    if x == 0:
        if n == 0:
            return 1
        if n > 0:
            return 0

        raise ZeroDivisionError("Cannot raise 0.0 to a negative power")

    if n == 0:
        return 1

    if n < 0:
        x = 1 / x
        n = -n

    y = 1

    while n != 1:
        if n % 2 == 0:
            x = x * x
            n = n // 2
        else:
            y = x * y
            x = x * x
            n = n // 2

    return x * y

def fpow2(x, n):
    """
    Perform iterative fast exponentiation by squaring.

    :param x: the base number
    :param n: the target power value, an integer
    :return: x to the power of n
    """

    if x == 0:
        if n == 0:
            return 1
        if n > 0:
            return 0

        raise ZeroDivisionError('Cannot raise 0.0 to a negative power')

    if n == 0:
        return 1

    if n < 0:
        x = 1 / x
        n = -n

    y = 1

    while n != 1:
        if n % 2 == 1:
            y *= x

        x *= x
        n = n // 2

    return y * x
<commit_msg>Remove explicit ZeroDivisionError from fpow<commit_after>"""
Fast exponentiation by squaring
"""

def fpow0(x, n):
    """
    Perform recursive fast exponentiation by squaring.

    :param x: the base number
    :param n: the target power value, an integer
    :return: x to the power of n
    """
    if n < 0:
        return fpow0(1 / x, -n)
    if n == 0:
        return 1
    if x == 0:
        return 0

    assert n == int(n), "n must be an integer"

    return fpow0(x * x, n // 2) if n % 2 == 0 else x * fpow0(x * x, n // 2)

def fpow1(x, n):
    """
    Perform iterative fast exponentiation by squaring.

    :param x: the base number
    :param n: the target power value, an integer
    :return: x to the power of n
    """
    if n < 0:
        return fpow1(1 / x, -n)
    if n == 0:
        return 1
    if x == 0:
        return 0

    y = 1

    while n != 1:
        if n % 2 == 0:
            x = x * x
            n = n // 2
        else:
            y = x * y
            x = x * x
            n = n // 2

    return x * y

def fpow2(x, n):
    """
    Perform iterative fast exponentiation by squaring.

    :param x: the base number
    :param n: the target power value, an integer
    :return: x to the power of n
    """
    if n < 0:
        return fpow2(1 / x, -n)
    if n == 0:
        return 1
    if x == 0:
        return 0

    y = 1

    while n != 1:
        if n % 2 == 1:
            y *= x

        x *= x
        n = n // 2

    return y * x
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import numpy as np
from random import randint, random

class Grid:
    def __init__(self, dimensions, fill):
        self.dimensions = dimensions
        self.num_dimensions = len(dimensions)
        self.grid = np.ndarray(dimensions, dtype=object)
        self.shape = self.grid.shape
        if fill:
            self.fill()

    def get_cell(self, coordinates):
        coordinates = tuple(coordinates)
        if not isinstance(self.grid[coordinates], Cell):
        # if not self.fill and not self.grid[coordinates] is Cell:
            print "add cell"
            self.grid[coordinates] = Cell(self, coordinates)
        return self.grid[tuple(coordinates)]

    def fill(self):
        iterator = np.nditer(self.grid,
                             flags=['refs_ok', 'multi_index'],
                             op_flags=['readwrite'])
        for cell in iterator:
            cell[...] = Cell(self, iterator.multi_index)

    def random_coordinate(self):
        return self.get_cell([randint(0, x - 1) for x in self.dimensions])

    def reset(self):
        iterator = np.nditer(self.grid,
                             flags=['refs_ok', 'multi_index'],
                             op_flags=['readwrite'])
        for cell in iterator:
            new_cell = Cell(self, iterator.multi_index)
            if not self.fill and (cell is None or str(cell) == "None"):
                pass
            else:
                new_cell.cost = float(str(cell))
            cell[...] = new_cell

class Cell:
    def __init__(self, grid, coordinates):
        self.grid = grid
        self.coordinates = coordinates

        self.cost = 1 + random()
        self.visited = False
        self.closed = False
        self.previous = None

        self.known_cost = 100000000.0
        self.predicted_cost = 100000000.0
        self.total_cost = 200000000.0

    def __str__(self):
        return str(self.cost)

    def __eq__(self, other):
        return self.coordinates == other.coordinates

    def __lt__(self, other):
        return self.total_cost < other.total_cost

    def get_neighbors(self):
        dims = self.grid.num_dimensions

        offsets = np.take(np.r_[-1, 0, 1], np.indices((3,) * dims).reshape(dims, -1).T)
        offsets = offsets[np.any(offsets, 1)]

        neighbours = self.coordinates + offsets
        valid = np.all((neighbours < np.array(self.grid.shape)) & (neighbours >= 0), axis=1)
        neighbours = neighbours[valid]

        return [self.grid.get_cell(coordinates) for coordinates in neighbours]
<commit_msg>Minor Tweak<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import numpy as np
from random import randint, random

class Grid:
    def __init__(self, dimensions, fill):
        self.dimensions = dimensions
        self.num_dimensions = len(dimensions)
        self.grid = np.ndarray(dimensions, dtype=object)
        self.shape = self.grid.shape
        if fill:
            self.fill()

    def get_cell(self, coordinates):
        coordinates = tuple(coordinates)
        if not self.fill and not isinstance(self.grid[coordinates], Cell):
            self.grid[coordinates] = Cell(self, coordinates)
        return self.grid[tuple(coordinates)]

    def fill(self):
        iterator = np.nditer(self.grid,
                             flags=['refs_ok', 'multi_index'],
                             op_flags=['readwrite'])
        for cell in iterator:
            cell[...] = Cell(self, iterator.multi_index)

    def random_coordinate(self):
        return self.get_cell([randint(0, x - 1) for x in self.dimensions])

    def reset(self):
        iterator = np.nditer(self.grid,
                             flags=['refs_ok', 'multi_index'],
                             op_flags=['readwrite'])
        for cell in iterator:
            new_cell = Cell(self, iterator.multi_index)
            if not self.fill and (cell is None or str(cell) == "None"):
                pass
            else:
                new_cell.cost = float(str(cell))
            cell[...] = new_cell

class Cell:
    def __init__(self, grid, coordinates):
        self.grid = grid
        self.coordinates = coordinates

        self.cost = 1 + random()
        self.visited = False
        self.closed = False
        self.previous = None

        self.known_cost = 100000000.0
        self.predicted_cost = 100000000.0
        self.total_cost = 200000000.0

    def __str__(self):
        return str(self.cost)

    def __eq__(self, other):
        return self.coordinates == other.coordinates

    def __lt__(self, other):
        return self.total_cost < other.total_cost

    def get_neighbors(self):
        dims = self.grid.num_dimensions

        offsets = np.take(np.r_[-1, 0, 1], np.indices((3,) * dims).reshape(dims, -1).T)
        offsets = offsets[np.any(offsets, 1)]

        neighbours = self.coordinates + offsets
        valid = np.all((neighbours < np.array(self.grid.shape)) & (neighbours >= 0), axis=1)
        neighbours = neighbours[valid]

        return [self.grid.get_cell(coordinates) for coordinates in neighbours]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import sys
import random
from errorslog import *
import framenetallreader
from framestructure import *
from stats import *
from errorslog import *
from bootstrap import bootstrap_algorithm
import options
import argguesser
import roleextractor
import verbnetreader
import framematcher
import rolematcher
import probabilitymodel
import headwordextractor
import paths
import dumper

def init_verbnet(path):
    print("Loading VerbNet data...", file=sys.stderr)
    reader = verbnetreader.VerbnetReader(path)
    errors["vn_parsing"] = reader.unhandled
    return reader.verbs, reader.classes

if __name__ == "__main__":
    verbnet_predicates, verbnet_classes = init_verbnet(paths.VERBNET_PATH)

    print("Loading FrameNet and VerbNet roles associations...", file=sys.stderr)
    role_matcher = rolematcher.VnFnRoleMatcher(paths.VNFN_MATCHING)
    model = probabilitymodel.ProbabilityModel()

    annotated_frames = []
    vn_frames = []

    if options.gold_args:
        print("Loading frames...", file=sys.stderr)
        fn_reader = framenetallreader.FNAllReader(
            paths.FRAMENET_FULLTEXT, paths.FRAMENET_PARSED,
            core_args_only=options.core_args_only)

        for frame in fn_reader.frames:
            stats_data["args"] += len(frame.args)
            stats_data["args_instanciated"] += len(
                [x for x in frame.args if x.instanciated])
            stats_data["frames"] += 1

            if not frame.predicate.lemma in verbnet_predicates:
                log_vn_missing(frame)
                continue

            stats_data["frames_with_predicate_in_verbnet"] += 1

            annotated_frames.append(frame)

            converted_frame = VerbnetFrame.build_from_frame(frame)

            vn_frames.append(converted_frame)
        stats_data["files"] += fn_reader.stats["files"]
    else:
        arg_guesser = argguesser.ArgGuesser(paths.FRAMENET_PARSED, verbnet_classes)
        extracted_frame = [x for x in arg_guesser.handle_corpus()]
        annotated_frames = roleextractor.fill_roles(
            extracted_frame, verbnet_classes, role_matcher)
        for frame in annotated_frames:
            converted_frame = VerbnetFrame.build_from_frame(frame)
            vn_frames.append(converted_frame)

    print("Frame matching...", file=sys.stderr)
    for good_frame, frame in zip(annotated_frames, vn_frames):
        num_instanciated = len([x for x in good_frame.args if x.instanciated])
        predicate = good_frame.predicate.lemma

        if good_frame.arg_annotated or good_frame.frame_name == "":
            stats_data["args_kept"] += num_instanciated

        stats_ambiguous_roles(good_frame, num_instanciated,
            role_matcher, verbnet_classes)

        # Find FrameNet frame <-> VerbNet class mapping
        try:
            matcher = framematcher.FrameMatcher(frame, options.matching_algorithm)
        except framematcher.EmptyFrameError:
            log_frame_without_slot(good_frame, frame)
            continue

        stats_data["frames_mapped"] += 1

        # Actual frame matching
        for test_frame in verbnet_predicates[predicate]:
            if options.passive and good_frame.passive:
                try:
                    for passivized_frame in test_frame.passivize():
                        matcher.new_match(passivized_frame)
                except:
                    #print(good_frame.sentence)
                    pass
            else:
                matcher.new_match(test_frame)
        frame.roles = matcher.possible_distribs()

        # Update probability model
        if not options.bootstrap:
            for roles, slot_type, prep in zip(
                frame.roles, frame.slot_types, frame.slot_preps
            ):
                if len(roles) == 1:
                    model.add_data(slot_type, next(iter(roles)), prep, predicate)

        if options.debug and set() in frame.roles:
            log_debug_data(good_frame, frame, matcher, frame.roles, verbnet_classes)

    if options.dump:
        dumper.add_data_frame_matching(annotated_frames, vn_frames,
            role_matcher, verbnet_classes,
            verbnet_predicates, options.matching_algorithm)
    else:
        print("Frame matching stats...", file=sys.stderr)
        stats_quality(annotated_frames, vn_frames, role_matcher, verbnet_classes, options.gold_args)
        display_stats(options.gold_args)

    if options.bootstrap:
        hw_extractor = headwordextractor.HeadWordExtractor(paths.FRAMENET_PARSED)

        print("Extracting arguments headwords...", file=sys.stderr)
        hw_extractor.compute_all_headwords(annotated_frames, vn_frames)

        print("Computing headwords classes...", file=sys.stderr);
        hw_extractor.compute_word_classes()

        print("Bootstrap algorithm...", file=sys.stderr)
        bootstrap_algorithm(vn_frames, model, hw_extractor, verbnet_classes)
    else:
        print("Applying probabilistic model...", file=sys.stderr)
        for frame in vn_frames:
            for i in range(0, len(frame.roles)):
                if len(frame.roles[i]) > 1:
                    new_role = model.best_role(
                        frame.roles[i], frame.slot_types[i], frame.slot_preps[i],
                        frame.predicate, options.probability_model)
                    if new_role != None:
                        frame.roles[i] = set([new_role])

    if options.dump:
        dumper.add_data_prob_model(annotated_frames, vn_frames, role_matcher, verbnet_classes)
        dumper.dump(options.dump_file)
    else:
        print("Final stats...", file=sys.stderr)

        stats_quality(annotated_frames, vn_frames, role_matcher, verbnet_classes, options.gold_args)
        display_stats(options.gold_args)

    if options.debug: display_debug(n_debug)

<commit_msg>Fix value of stats_data["arg_kept"] with --no-gold-args options<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import sys
import random
from errorslog import *
import framenetallreader
from framestructure import *
from stats import *
from errorslog import *
from bootstrap import bootstrap_algorithm
import options
import argguesser
import roleextractor
import verbnetreader
import framematcher
import rolematcher
import probabilitymodel
import headwordextractor
import paths
import dumper

def init_verbnet(path):
    print("Loading VerbNet data...", file=sys.stderr)
    reader = verbnetreader.VerbnetReader(path)
    errors["vn_parsing"] = reader.unhandled
    return reader.verbs, reader.classes

if __name__ == "__main__":
    verbnet_predicates, verbnet_classes = init_verbnet(paths.VERBNET_PATH)

    print("Loading FrameNet and VerbNet roles associations...", file=sys.stderr)
    role_matcher = rolematcher.VnFnRoleMatcher(paths.VNFN_MATCHING)
    model = probabilitymodel.ProbabilityModel()

    annotated_frames = []
    vn_frames = []

    if options.gold_args:
        print("Loading frames...", file=sys.stderr)
        fn_reader = framenetallreader.FNAllReader(
            paths.FRAMENET_FULLTEXT, paths.FRAMENET_PARSED,
            core_args_only=options.core_args_only)

        for frame in fn_reader.frames:
            stats_data["args"] += len(frame.args)
            stats_data["args_instanciated"] += len(
                [x for x in frame.args if x.instanciated])
            stats_data["frames"] += 1

            if not frame.predicate.lemma in verbnet_predicates:
                log_vn_missing(frame)
                continue

            stats_data["frames_with_predicate_in_verbnet"] += 1

            annotated_frames.append(frame)

            converted_frame = VerbnetFrame.build_from_frame(frame)

            vn_frames.append(converted_frame)
        stats_data["files"] += fn_reader.stats["files"]
    else:
        arg_guesser = argguesser.ArgGuesser(paths.FRAMENET_PARSED, verbnet_classes)
        extracted_frame = [x for x in arg_guesser.handle_corpus()]
        annotated_frames = roleextractor.fill_roles(
            extracted_frame, verbnet_classes, role_matcher)
        for frame in annotated_frames:
            converted_frame = VerbnetFrame.build_from_frame(frame)
            vn_frames.append(converted_frame)

    print("Frame matching...", file=sys.stderr)
    for good_frame, frame in zip(annotated_frames, vn_frames):
        num_instanciated = len([x for x in good_frame.args if x.instanciated])
        predicate = good_frame.predicate.lemma

        if good_frame.arg_annotated:
            stats_data["args_kept"] += num_instanciated

        stats_ambiguous_roles(good_frame, num_instanciated,
            role_matcher, verbnet_classes)

        # Find FrameNet frame <-> VerbNet class mapping
        try:
            matcher = framematcher.FrameMatcher(frame, options.matching_algorithm)
        except framematcher.EmptyFrameError:
            log_frame_without_slot(good_frame, frame)
            continue

        stats_data["frames_mapped"] += 1

        # Actual frame matching
        for test_frame in verbnet_predicates[predicate]:
            if options.passive and good_frame.passive:
                try:
                    for passivized_frame in test_frame.passivize():
                        matcher.new_match(passivized_frame)
                except:
                    #print(good_frame.sentence)
                    pass
            else:
                matcher.new_match(test_frame)
        frame.roles = matcher.possible_distribs()

        # Update probability model
        if not options.bootstrap:
            for roles, slot_type, prep in zip(
                frame.roles, frame.slot_types, frame.slot_preps
            ):
                if len(roles) == 1:
                    model.add_data(slot_type, next(iter(roles)), prep, predicate)

        if options.debug and set() in frame.roles:
            log_debug_data(good_frame, frame, matcher, frame.roles, verbnet_classes)

    if options.dump:
        dumper.add_data_frame_matching(annotated_frames, vn_frames,
            role_matcher, verbnet_classes,
            verbnet_predicates, options.matching_algorithm)
    else:
        print("Frame matching stats...", file=sys.stderr)
        stats_quality(annotated_frames, vn_frames, role_matcher, verbnet_classes, options.gold_args)
        display_stats(options.gold_args)

    if options.bootstrap:
        hw_extractor = headwordextractor.HeadWordExtractor(paths.FRAMENET_PARSED)

        print("Extracting arguments headwords...", file=sys.stderr)
        hw_extractor.compute_all_headwords(annotated_frames, vn_frames)

        print("Computing headwords classes...", file=sys.stderr);
        hw_extractor.compute_word_classes()

        print("Bootstrap algorithm...", file=sys.stderr)
        bootstrap_algorithm(vn_frames, model, hw_extractor, verbnet_classes)
    else:
        print("Applying probabilistic model...", file=sys.stderr)
        for frame in vn_frames:
            for i in range(0, len(frame.roles)):
                if len(frame.roles[i]) > 1:
                    new_role = model.best_role(
                        frame.roles[i], frame.slot_types[i], frame.slot_preps[i],
                        frame.predicate, options.probability_model)
                    if new_role != None:
                        frame.roles[i] = set([new_role])

    if options.dump:
        dumper.add_data_prob_model(annotated_frames, vn_frames, role_matcher, verbnet_classes)
        dumper.dump(options.dump_file)
    else:
        print("Final stats...", file=sys.stderr)

        stats_quality(annotated_frames, vn_frames, role_matcher, verbnet_classes, options.gold_args)
        display_stats(options.gold_args)

    if options.debug: display_debug(n_debug)

<|endoftext|>