<commit_before>import datetime
import gzip
import logging
import os
import shutil
import time

import fasteners

from django.core import management
from django.conf import settings
from django.core.paginator import Paginator
from django_cron import CronJobBase, Schedule
from django.db import transaction

from document.models import Submitter
from government.models import GovernmentMember
from parliament.models import PartyMember
from parliament.models import ParliamentMember
from person.models import Person
from person.views import PersonTKIDCheckView

import openkamer.dossier
import openkamer.gift
import openkamer.kamervraag
import openkamer.parliament
import openkamer.travel
import openkamer.verslagao

import stats.models

from website import settings

logger = logging.getLogger(__name__)

def years_from_str_list(year_start):
    start_date = datetime.date(year=int(year_start), month=1, day=1)
    current_date = start_date
    today = datetime.date.today()
    years = []
    while current_date.year <= today.year:
        years.append(str(current_date.year))
        current_date = datetime.date(year=current_date.year + 1, month=1, day=1)
    years.reverse()
    return years

class LockJob(CronJobBase):
    """
    django-cron does provide a file lock backend,
    but this is not working due to issue https://github.com/Tivix/django-cron/issues/74
    """
    code = None

    def do(self):
        logger.info('BEGIN: {}'.format(self.code))
        lockfilepath = os.path.join(settings.CRON_LOCK_DIR, 'tmp_{}_lockfile'.format(self.code))
        a_lock = fasteners.InterProcessLock(lockfilepath)
        gotten = a_lock.acquire(timeout=1.0)
        try:
            if gotten:
                self.do_imp()
            else:
                logger.info('Not able to acquire lock, job is already running')
        finally:
            if gotten:
                a_lock.release()
        logger.info('END: {}'.format(self.code))

    def do_imp(self):
        raise NotImplementedError

class TestJob(LockJob):
    RUN_EVERY_MINS = 0
    schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
    code = 'website.cron.TestJob'

    def do_imp(self):
        logger.info('BEGIN')
        time.sleep(10)
        logger.info('END')

class UpdateParliamentAndGovernment(LockJob):
    RUN_AT_TIMES = ['20:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateParliamentAndGovernment'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            openkamer.parliament.create_parliament_and_government()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateActiveDossiers(LockJob):
    RUN_AT_TIMES = ['21:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateActiveDossiers'

    def do_imp(self):
        # TODO: also update dossiers that have closed since last update
        logger.info('update active dossiers cronjob')
        failed_dossiers = openkamer.dossier.create_wetsvoorstellen_active()
        if failed_dossiers:
            logger.error('the following dossiers failed: ' + str(failed_dossiers))

class UpdateInactiveDossiers(LockJob):
    RUN_AT_TIMES = ['12:00']  # note that while it runs every day, it only updates a set depending on the week day
    DAYS_PER_WEEK = 7
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateInactiveDossiers'

    def do_imp(self):
        logger.info('update inactive dossiers cronjob')
        week_day = datetime.date.today().weekday()
        try:
            years = years_from_str_list(2008)
            for year in years:
                year = int(year)
                if year % self.DAYS_PER_WEEK == week_day:
                    logger.info('year: {}'.format(year))
                    failed_dossiers = openkamer.dossier.create_wetsvoorstellen_inactive(year=year)
                    if failed_dossiers:
                        logger.error('the following dossiers failed: {}'.format(failed_dossiers))
        except:
            logger.exception('something failed')

class UpdateVerslagenAlgemeenOverleg(LockJob):
    RUN_AT_TIMES = ['23:30']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateVerslagenAlgemeenOverleg'

    def do_imp(self):
        logger.info('update verslagen algemeen overleg')
        skip_if_exists = datetime.date.today().day % 7 == 0
        years = years_from_str_list(2008)
        for year in years:
            openkamer.verslagao.create_verslagen_algemeen_overleg(year, max_n=None, skip_if_exists=skip_if_exists)

class UpdateKamervragenRecent(LockJob):
    RUN_AT_TIMES = ['00:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateKamervragenRecent'

    def do_imp(self):
        logger.info('update kamervragen and kamerantwoorden')
        years = years_from_str_list(2019)
        for year in years:
            openkamer.kamervraag.create_kamervragen(year, skip_if_exists=False)

class UpdateKamervragenAll(LockJob):
    RUN_EVERY_MINS = 60 * 24 * 7  # 7 days
    schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
    code = 'website.cron.UpdateKamervragenAll'

    def do_imp(self):
        logger.info('update kamervragen and kamerantwoorden')
        years = years_from_str_list(2010)
        for year in years:
            openkamer.kamervraag.create_kamervragen(year, skip_if_exists=False)

class UpdateSubmitters(LockJob):
    RUN_AT_TIMES = ['05:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateSubmitters'

    def do_imp(self):
        logger.info('BEGIN')
        BATCH_SIZE = 1000
        try:
            submitters = Submitter.objects.all().order_by('party_slug')
            paginator = Paginator(submitters, BATCH_SIZE)
            logger.info('{} submitters to update'.format(paginator.count))
            for page in paginator.page_range:
                progress_percent = int(page / paginator.num_pages * 100)
                logger.info('{}%'.format(progress_percent))
                submitters = paginator.get_page(page)
                self.update_batch(submitters)
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

    @transaction.atomic
    def update_batch(self, submitters):
        for submitter in submitters:
            submitter.update_submitter_party_slug()

class UpdateSearchIndex(LockJob):
    RUN_AT_TIMES = ['06:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateSearchIndex'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            management.call_command('update_index', remove=True)
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateStatsData(LockJob):
    RUN_AT_TIMES = ['07:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateStatsData'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            stats.models.update_all()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateGifts(LockJob):
    RUN_AT_TIMES = ['12:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateGifts'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            openkamer.gift.create_gifts()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateTravels(LockJob):
    RUN_AT_TIMES = ['14:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateTravels'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            openkamer.travel.create_travels()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class CleanUnusedPersons(CronJobBase):
    RUN_AT_TIMES = ['08:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.CleanUnusedPersons'

    def do(self):
        logger.info('run unused persons cleanup')
        persons = Person.objects.all()
        persons_to_delete_ids = []
        for person in persons:
            members = PartyMember.objects.filter(person=person)
            if members:
                continue
            members = ParliamentMember.objects.filter(person=person)
            if members:
                continue
            members = GovernmentMember.objects.filter(person=person)
            if members:
                continue
            submitters = Submitter.objects.filter(person=person)
            if submitters:
                continue
            persons_to_delete_ids.append(person.id)
        Person.objects.filter(id__in=persons_to_delete_ids).delete()
        logger.info('deleted persons: ' + str(persons_to_delete_ids))
        logger.info('END')

class MergeDuplicatePersons(CronJobBase):
    RUN_AT_TIMES = ['08:15']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.MergeDuplicatePersons'

    def do(self):
        logger.info('run merge duplicate persons')
        tk_ids = PersonTKIDCheckView.get_duplicate_person_tk_ids()
        logger.info('duplicate ids: {}'.format(tk_ids))
        try:
            for tk_id in tk_ids:
                persons = Person.objects.filter(tk_id=tk_id)
                for person in persons:
                    logger.info('===============')
                    logger.info('person: {}'.format(person))
                    if not person.surname or not person.initials:
                        continue
                    person_best = Person.find_surname_initials(person.surname, person.initials, persons)
                    persons_delete = persons.exclude(id=person_best.id)
                    logger.info('best {}, delete: {}'.format(person_best, persons_delete))
                    submitters = Submitter.objects.filter(person__in=persons_delete)
                    for submitter in submitters:
                        if Submitter.objects.filter(person=person, document=submitter.document).exists():
                            continue
                        submitter.person = person
                        submitter.update_submitter_party_slug()
                        submitter.save()
                    persons_delete.delete()
                    logger.info('deleted persons: {}'.format(persons_delete))
                    break
        except Exception as e:
            logger.exception('Error merging duplicated persons')
            raise
        logger.info('END')

class BackupDaily(LockJob):
    RUN_AT_TIMES = ['18:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.BackupDaily'

    def do_imp(self):
        logger.info('run daily backup cronjob')
        management.call_command('dbbackup', '--clean')
        try:
            BackupDaily.create_json_dump()
        except Exception as e:
            logger.exception(e)
            raise

    @staticmethod
    def create_json_dump():
        filepath = os.path.join(settings.DBBACKUP_STORAGE_OPTIONS['location'], 'openkamer-' + str(datetime.date.today()) + '.json')
        filepath_compressed = filepath + '.gz'
        with open(filepath, 'w') as fileout:
            management.call_command(
                'dumpdata',
                '--all',
                '--natural-foreign',
                '--exclude', 'auth.permission',
                '--exclude', 'contenttypes',
                'person',
                'parliament',
                'government',
                'document',
                'stats',
                'website',
                stdout=fileout
            )
        with open(filepath, 'rb') as f_in:
            with gzip.open(filepath_compressed, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
        os.remove(filepath)
        BackupDaily.remove_old_json_dumps(days_old=30)

    @staticmethod
    def remove_old_json_dumps(days_old):
        for (dirpath, dirnames, filenames) in os.walk(settings.DBBACKUP_STORAGE_OPTIONS['location']):
            for file in filenames:
                if '.json.gz' not in file:
                    continue
                filepath = os.path.join(dirpath, file)
                datetime_created = datetime.datetime.fromtimestamp(os.path.getctime(filepath))
                if datetime_created < datetime.datetime.now() - datetime.timedelta(days=days_old):
                    os.remove(filepath)
<commit_msg>Use best person to update submitters<commit_after>import datetime
import gzip
import logging
import os
import shutil
import time

import fasteners

from django.core import management
from django.conf import settings
from django.core.paginator import Paginator
from django_cron import CronJobBase, Schedule
from django.db import transaction

from document.models import Submitter
from government.models import GovernmentMember
from parliament.models import PartyMember
from parliament.models import ParliamentMember
from person.models import Person
from person.views import PersonTKIDCheckView

import openkamer.dossier
import openkamer.gift
import openkamer.kamervraag
import openkamer.parliament
import openkamer.travel
import openkamer.verslagao

import stats.models

from website import settings

logger = logging.getLogger(__name__)

def years_from_str_list(year_start):
    start_date = datetime.date(year=int(year_start), month=1, day=1)
    current_date = start_date
    today = datetime.date.today()
    years = []
    while current_date.year <= today.year:
        years.append(str(current_date.year))
        current_date = datetime.date(year=current_date.year + 1, month=1, day=1)
    years.reverse()
    return years

class LockJob(CronJobBase):
    """
    django-cron does provide a file lock backend,
    but this is not working due to issue https://github.com/Tivix/django-cron/issues/74
    """
    code = None

    def do(self):
        logger.info('BEGIN: {}'.format(self.code))
        lockfilepath = os.path.join(settings.CRON_LOCK_DIR, 'tmp_{}_lockfile'.format(self.code))
        a_lock = fasteners.InterProcessLock(lockfilepath)
        gotten = a_lock.acquire(timeout=1.0)
        try:
            if gotten:
                self.do_imp()
            else:
                logger.info('Not able to acquire lock, job is already running')
        finally:
            if gotten:
                a_lock.release()
        logger.info('END: {}'.format(self.code))

    def do_imp(self):
        raise NotImplementedError

class TestJob(LockJob):
    RUN_EVERY_MINS = 0
    schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
    code = 'website.cron.TestJob'

    def do_imp(self):
        logger.info('BEGIN')
        time.sleep(10)
        logger.info('END')

class UpdateParliamentAndGovernment(LockJob):
    RUN_AT_TIMES = ['20:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateParliamentAndGovernment'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            openkamer.parliament.create_parliament_and_government()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateActiveDossiers(LockJob):
    RUN_AT_TIMES = ['21:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateActiveDossiers'

    def do_imp(self):
        # TODO: also update dossiers that have closed since last update
        logger.info('update active dossiers cronjob')
        failed_dossiers = openkamer.dossier.create_wetsvoorstellen_active()
        if failed_dossiers:
            logger.error('the following dossiers failed: ' + str(failed_dossiers))

class UpdateInactiveDossiers(LockJob):
    RUN_AT_TIMES = ['12:00']  # note that while it runs every day, it only updates a set depending on the week day
    DAYS_PER_WEEK = 7
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateInactiveDossiers'

    def do_imp(self):
        logger.info('update inactive dossiers cronjob')
        week_day = datetime.date.today().weekday()
        try:
            years = years_from_str_list(2008)
            for year in years:
                year = int(year)
                if year % self.DAYS_PER_WEEK == week_day:
                    logger.info('year: {}'.format(year))
                    failed_dossiers = openkamer.dossier.create_wetsvoorstellen_inactive(year=year)
                    if failed_dossiers:
                        logger.error('the following dossiers failed: {}'.format(failed_dossiers))
        except:
            logger.exception('something failed')

class UpdateVerslagenAlgemeenOverleg(LockJob):
    RUN_AT_TIMES = ['23:30']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateVerslagenAlgemeenOverleg'

    def do_imp(self):
        logger.info('update verslagen algemeen overleg')
        skip_if_exists = datetime.date.today().day % 7 == 0
        years = years_from_str_list(2008)
        for year in years:
            openkamer.verslagao.create_verslagen_algemeen_overleg(year, max_n=None, skip_if_exists=skip_if_exists)

class UpdateKamervragenRecent(LockJob):
    RUN_AT_TIMES = ['00:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateKamervragenRecent'

    def do_imp(self):
        logger.info('update kamervragen and kamerantwoorden')
        years = years_from_str_list(2019)
        for year in years:
            openkamer.kamervraag.create_kamervragen(year, skip_if_exists=False)

class UpdateKamervragenAll(LockJob):
    RUN_EVERY_MINS = 60 * 24 * 7  # 7 days
    schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
    code = 'website.cron.UpdateKamervragenAll'

    def do_imp(self):
        logger.info('update kamervragen and kamerantwoorden')
        years = years_from_str_list(2010)
        for year in years:
            openkamer.kamervraag.create_kamervragen(year, skip_if_exists=False)

class UpdateSubmitters(LockJob):
    RUN_AT_TIMES = ['05:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateSubmitters'

    def do_imp(self):
        logger.info('BEGIN')
        BATCH_SIZE = 1000
        try:
            submitters = Submitter.objects.all().order_by('party_slug')
            paginator = Paginator(submitters, BATCH_SIZE)
            logger.info('{} submitters to update'.format(paginator.count))
            for page in paginator.page_range:
                progress_percent = int(page / paginator.num_pages * 100)
                logger.info('{}%'.format(progress_percent))
                submitters = paginator.get_page(page)
                self.update_batch(submitters)
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

    @transaction.atomic
    def update_batch(self, submitters):
        for submitter in submitters:
            submitter.update_submitter_party_slug()

class UpdateSearchIndex(LockJob):
    RUN_AT_TIMES = ['06:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateSearchIndex'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            management.call_command('update_index', remove=True)
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateStatsData(LockJob):
    RUN_AT_TIMES = ['07:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateStatsData'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            stats.models.update_all()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateGifts(LockJob):
    RUN_AT_TIMES = ['12:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateGifts'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            openkamer.gift.create_gifts()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class UpdateTravels(LockJob):
    RUN_AT_TIMES = ['14:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.UpdateTravels'

    def do_imp(self):
        logger.info('BEGIN')
        try:
            openkamer.travel.create_travels()
        except Exception as error:
            logger.exception(error)
            raise
        logger.info('END')

class CleanUnusedPersons(CronJobBase):
    RUN_AT_TIMES = ['08:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.CleanUnusedPersons'

    def do(self):
        logger.info('run unused persons cleanup')
        persons = Person.objects.all()
        persons_to_delete_ids = []
        for person in persons:
            members = PartyMember.objects.filter(person=person)
            if members:
                continue
            members = ParliamentMember.objects.filter(person=person)
            if members:
                continue
            members = GovernmentMember.objects.filter(person=person)
            if members:
                continue
            submitters = Submitter.objects.filter(person=person)
            if submitters:
                continue
            persons_to_delete_ids.append(person.id)
        Person.objects.filter(id__in=persons_to_delete_ids).delete()
        logger.info('deleted persons: ' + str(persons_to_delete_ids))
        logger.info('END')

class MergeDuplicatePersons(CronJobBase):
    RUN_AT_TIMES = ['08:15']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.MergeDuplicatePersons'

    def do(self):
        logger.info('run merge duplicate persons')
        tk_ids = PersonTKIDCheckView.get_duplicate_person_tk_ids()
        logger.info('duplicate ids: {}'.format(tk_ids))
        try:
            for tk_id in tk_ids:
                persons = Person.objects.filter(tk_id=tk_id)
                for person in persons:
                    logger.info('===============')
                    logger.info('person: {}'.format(person))
                    if not person.surname or not person.initials:
                        continue
                    person_best = Person.find_surname_initials(person.surname, person.initials, persons)
                    persons_delete = persons.exclude(id=person_best.id)
                    logger.info('best {}, delete: {}'.format(person_best, persons_delete))
                    submitters = Submitter.objects.filter(person__in=persons_delete)
                    for submitter in submitters:
                        if Submitter.objects.filter(person=person_best, document=submitter.document).exists():
                            continue
                        submitter.person = person_best
                        submitter.update_submitter_party_slug()
                        submitter.save()
                    persons_delete.delete()
                    break
        except Exception as e:
            logger.exception('Error merging duplicated persons')
            raise
        logger.info('END')

class BackupDaily(LockJob):
    RUN_AT_TIMES = ['18:00']
    schedule = Schedule(run_at_times=RUN_AT_TIMES)
    code = 'website.cron.BackupDaily'

    def do_imp(self):
        logger.info('run daily backup cronjob')
        management.call_command('dbbackup', '--clean')
        try:
            BackupDaily.create_json_dump()
        except Exception as e:
            logger.exception(e)
            raise

    @staticmethod
    def create_json_dump():
        filepath = os.path.join(settings.DBBACKUP_STORAGE_OPTIONS['location'], 'openkamer-' + str(datetime.date.today()) + '.json')
        filepath_compressed = filepath + '.gz'
        with open(filepath, 'w') as fileout:
            management.call_command(
                'dumpdata',
                '--all',
                '--natural-foreign',
                '--exclude', 'auth.permission',
                '--exclude', 'contenttypes',
                'person',
                'parliament',
                'government',
                'document',
                'stats',
                'website',
                stdout=fileout
            )
        with open(filepath, 'rb') as f_in:
            with gzip.open(filepath_compressed, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
        os.remove(filepath)
        BackupDaily.remove_old_json_dumps(days_old=30)

    @staticmethod
    def remove_old_json_dumps(days_old):
        for (dirpath, dirnames, filenames) in os.walk(settings.DBBACKUP_STORAGE_OPTIONS['location']):
            for file in filenames:
                if '.json.gz' not in file:
                    continue
                filepath = os.path.join(dirpath, file)
                datetime_created = datetime.datetime.fromtimestamp(os.path.getctime(filepath))
                if datetime_created < datetime.datetime.now() - datetime.timedelta(days=days_old):
                    os.remove(filepath)
<|endoftext|>


<commit_before>from functools import wraps
from .. import h5py as patched_h5py

import zarr

# -------------------------------------------------------------------------------
# Type conversion
# -------------------------------------------------------------------------------

def is_float(string):
    """Check whether string is float.

    See also
    --------
    http://stackoverflow.com/questions/736043/checking-if-a-string-can-be-converted-to-float-in-python
    """
    try:
        float(string)
        return True
    except ValueError:
        return False

def is_int(string):
    """Check whether string is integer.
    """
    try:
        int(string)
        return True
    except ValueError:
        return False

def convert_bool(string):
    """Check whether string is boolean.
    """
    if string == 'True':
        return True, True
    elif string == 'False':
        return True, False
    else:
        return False, False

def convert_string(string):
    """Convert string to int, float or bool.
    """
    if is_int(string):
        return int(string)
    elif is_float(string):
        return float(string)
    elif convert_bool(string)[0]:
        return convert_bool(string)[1]
    elif string == 'None':
        return None
    else:
        return string

class AnnDataReadError(OSError):
    """Error caused while trying to read in AnnData."""
    pass

def report_key_on_error(func):
    """
    A decorator for zarr element reading which makes keys involved in errors get reported.

    Example
    -------
    >>> import zarr
    >>> @report_key_on_error
        def read_arr(group):
            raise NotImplementedError()
    >>> z = zarr.open("tmp.zarr")
    >>> z["X"] = [1, 2, 3]
    >>> read_arr(z["X"])
    """
    @wraps(func)
    def func_wrapper(elem, *args, **kwargs):
        try:
            return func(elem, *args, **kwargs)
        except Exception as e:
            if isinstance(e, AnnDataReadError):
                raise e
            else:
                if isinstance(elem, (zarr.Group, zarr.Array)):
                    parent = elem.store  # Not sure how to always get a name out of this
                else:
                    if isinstance(elem, patched_h5py.Group):
                        parent = elem.h5py_group.file.name
                    else:
                        parent = elem.file.name
                raise AnnDataReadError(
                    f"Above error raised while reading key '{elem.name}' of type {type(elem)} from {parent}."
                )
    return func_wrapper
<commit_msg>Conditionally import zarr<commit_after>from functools import wraps
from .. import h5py as patched_h5py

# -------------------------------------------------------------------------------
# Type conversion
# -------------------------------------------------------------------------------

def is_float(string):
    """Check whether string is float.

    See also
    --------
    http://stackoverflow.com/questions/736043/checking-if-a-string-can-be-converted-to-float-in-python
    """
    try:
        float(string)
        return True
    except ValueError:
        return False

def is_int(string):
    """Check whether string is integer.
    """
    try:
        int(string)
        return True
    except ValueError:
        return False

def convert_bool(string):
    """Check whether string is boolean.
    """
    if string == 'True':
        return True, True
    elif string == 'False':
        return True, False
    else:
        return False, False

def convert_string(string):
    """Convert string to int, float or bool.
    """
    if is_int(string):
        return int(string)
    elif is_float(string):
        return float(string)
    elif convert_bool(string)[0]:
        return convert_bool(string)[1]
    elif string == 'None':
        return None
    else:
        return string

class AnnDataReadError(OSError):
    """Error caused while trying to read in AnnData."""
    pass

def report_key_on_error(func):
    """
    A decorator for zarr element reading which makes keys involved in errors get reported.

    Example
    -------
    >>> import zarr
    >>> @report_key_on_error
    ... def read_arr(group):
    ...     raise NotImplementedError()
    >>> z = zarr.open("tmp.zarr")
    >>> z["X"] = [1, 2, 3]
    >>> read_arr(z["X"])
    """
    @wraps(func)
    def func_wrapper(elem, *args, **kwargs):
        try:
            return func(elem, *args, **kwargs)
        except Exception as e:
            if isinstance(e, AnnDataReadError):
                raise e
            else:
                try:
                    import zarr
                except ImportError:
                    zarr = None
                if zarr and isinstance(elem, (zarr.Group, zarr.Array)):
                    parent = elem.store  # Not sure how to always get a name out of this
                elif isinstance(elem, patched_h5py.Group):
                    parent = elem.h5py_group.file.name
                else:
                    parent = elem.file.name
                raise AnnDataReadError(
                    f"Above error raised while reading key '{elem.name}' of type {type(elem)} from {parent}."
                )
    return func_wrapper
<|endoftext|>


<commit_before>from django.conf.urls import url

from . import views

app_name = "website"
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^login$', views.login_user, name='login'),
    url(r'^logout$', views.user_logout, name='logout'),
    url(r'^register$', views.register, name='register'),
    url(r'^profile$', views.profile, name='profile'),
    url(r'^sell$', views.sell_product, name='sell'),
    url(r'^products$', views.list_products, name='list_products'),
    url(r'^single_product/(?P<product_id>[0-9]+)/$', views.single_product, name='single_product'),
    url(r'^product_types$', views.list_product_types, name='product_types'),
    url(r'^product_type_products/(?P<type_id>[0-9]+)/$', views.get_product_types, name='get_product_types'),
    url(r'^add_payment_type$', views.add_payment_type, name='add_payment_type'),
    url(r'^user_payment_types$', views.user_payment_types, name='user_payment_types'),
    url(r'^delete_payment_type$', views.delete_payment_type, name='delete_payment_type'),
    url(r'^add_to_cart/(?P<product_id>[0-9]+)/$', views.add_product_to_order, name='add_product_to_order'),
    url(r'^cart$', views.view_cart, name='cart'),
    url(r'^checkout/(?P<order_id>[0-9]+)/$', views.complete_order_add_payment, name='checkout'),
    url(r'^order_confirmation$', views.order_confirmation, name='order_confirmation'),
    url(r'^delete_product_from_cart$', views.delete_product_from_cart, name='delete_product_from_cart'),
    url(r'^final_order_view$', views.view_cancel_order, name='final_order_view'),
]<commit_msg>added url for user's list of products<commit_after>from django.conf.urls import url

from . import views

app_name = "website"
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^login$', views.login_user, name='login'),
    url(r'^logout$', views.user_logout, name='logout'),
    url(r'^register$', views.register, name='register'),
    url(r'^profile$', views.profile, name='profile'),
    url(r'^sell$', views.sell_product, name='sell'),
    url(r'^products$', views.list_products, name='list_products'),
    url(r'^single_product/(?P<product_id>[0-9]+)/$', views.single_product, name='single_product'),
    url(r'^product_types$', views.list_product_types, name='product_types'),
    url(r'^product_type_products/(?P<type_id>[0-9]+)/$', views.get_product_types, name='get_product_types'),
    url(r'^add_payment_type$', views.add_payment_type, name='add_payment_type'),
    url(r'^user_payment_types$', views.user_payment_types, name='user_payment_types'),
    url(r'^user_products$', views.user_products, name='user_products'),
    url(r'^delete_payment_type$', views.delete_payment_type, name='delete_payment_type'),
    url(r'^add_to_cart/(?P<product_id>[0-9]+)/$', views.add_product_to_order, name='add_product_to_order'),
    url(r'^cart$', views.view_cart, name='cart'),
    url(r'^checkout/(?P<order_id>[0-9]+)/$', views.complete_order_add_payment, name='checkout'),
    url(r'^order_confirmation$', views.order_confirmation, name='order_confirmation'),
    url(r'^delete_product_from_cart$', views.delete_product_from_cart, name='delete_product_from_cart'),
    url(r'^final_order_view$', views.view_cancel_order, name='final_order_view'),
]<|endoftext|>


<commit_before>from functools import wraps, singledispatch
import sys
from ..core.sparsedataset import SparseDataset

# -------------------------------------------------------------------------------
# Type conversion
# -------------------------------------------------------------------------------

# Could be numba'd if it returned tuples instead of slices
def idx_chunks_along_axis(shape: tuple, axis: int, chunk_size: int):
    """
    Gives indexer tuples chunked along an axis.

    Params
    ------
    shape
        Shape of array to be chunked
    axis
        Axis to chunk along
    chunk_size
        Size of chunk along axis

    Returns
    -------
    An iterator of tuples for indexing into an array of passed shape.
    """
    total = shape[axis]
    cur = 0
    mutable_idx = [slice(None) for i in range(len(shape))]
    while cur + chunk_size < total:
        mutable_idx[axis] = slice(cur, cur + chunk_size)
        yield tuple(mutable_idx)
        cur += chunk_size
    mutable_idx[axis] = slice(cur, None)
    yield tuple(mutable_idx)

def is_float(string):
    """Check whether string is float.

    See also
    --------
    http://stackoverflow.com/questions/736043/checking-if-a-string-can-be-converted-to-float-in-python
    """
    try:
        float(string)
        return True
    except ValueError:
        return False

def is_int(string):
    """Check whether string is integer.
    """
    try:
        int(string)
        return True
    except ValueError:
        return False

def convert_bool(string):
    """Check whether string is boolean.
    """
    if string == 'True':
        return True, True
    elif string == 'False':
        return True, False
    else:
        return False, False

def convert_string(string):
    """Convert string to int, float or bool.
    """
    if is_int(string):
        return int(string)
    elif is_float(string):
        return float(string)
    elif convert_bool(string)[0]:
        return convert_bool(string)[1]
    elif string == 'None':
        return None
    else:
        return string

# -------------------------------------------------------------------------------
# Generic functions
# -------------------------------------------------------------------------------

@singledispatch
def write_attribute(*args, **kwargs):
    raise NotImplementedError(
        "Unrecognized argument types for `write_attribute`."
    )

@singledispatch
def read_attribute(*args, **kwargs):
    raise NotImplementedError(
        "Unrecognized argument types for `read_attribute`."
    )

@read_attribute.register(type(None))
def read_attribute_none(value) -> None:
    return None

# -------------------------------------------------------------------------------
# Errors handling
# -------------------------------------------------------------------------------

class AnnDataReadError(OSError):
    """Error caused while trying to read in AnnData."""

    pass

def report_read_key_on_error(func):
    """
    A decorator for zarr element reading which makes keys involved in errors get reported.

    Example
    -------
    >>> import zarr
    >>> @report_read_key_on_error
    ... def read_arr(group):
    ...     raise NotImplementedError()
    >>> z = zarr.open("tmp.zarr")
    >>> z["X"] = [1, 2, 3]
    >>> read_arr(z["X"])
    """

    @wraps(func)
    def func_wrapper(elem, *args, **kwargs):
        try:
            return func(elem, *args, **kwargs)
        except Exception as e:
            if isinstance(e, AnnDataReadError):
                raise e
            else:
                try:
                    import zarr
                except ImportError:
                    zarr = None
                if zarr and isinstance(elem, (zarr.Group, zarr.Array)):
                    # Not sure how to always get a name out of this
                    parent = elem.store
                elif isinstance(elem, SparseDataset):
                    parent = elem.group.file.name
                else:
                    parent = elem.file.name
                raise AnnDataReadError(
                    f"Above error raised while reading key '{elem.name}' of type {type(elem)} from {parent}."
                )

    return func_wrapper

def report_write_key_on_error(func):
    """
    A decorator for zarr element reading which makes keys involved in errors get reported.

    Example
    -------
    >>> import zarr
    >>> @report_write_key_on_error
    ... def write_arr(group, key, val):
    ...     raise NotImplementedError()
    >>> z = zarr.open("tmp.zarr")
    >>> X = [1, 2, 3]
    >>> write_arr(z, "X", X)
    """

    @wraps(func)
    def func_wrapper(elem, key, val, *args, **kwargs):
        try:
            return func(elem, key, val, *args, **kwargs)
        except Exception as e:
            if isinstance(e, AnnDataReadError):
                raise e
            else:
                try:
                    import zarr
                except ImportError:
                    zarr = None
                if zarr and isinstance(elem, (zarr.Group, zarr.Array)):
                    parent = (
                        elem.store
                    )  # Not sure how to always get a name out of this
                elif isinstance(elem, SparseDataset):
                    parent = elem.group.file.name
                else:
                    parent = elem.file.name
                raise type(e)(
                    "\n\n".join(
                        (
                            str(e),
                            f"Above error raised while writing key '{key}' of type {type(val)} from {parent}.",
                        )
                    )
                )
                # raise AnnDataReadError(
                #     f"Above error raised while writing key '{key}' of type {type(val)} from {parent}."
                # )

    return func_wrapper
<commit_msg>Fix write error being thrown twice<commit_after>from functools import wraps, singledispatch
import sys
from ..core.sparsedataset import SparseDataset

# -------------------------------------------------------------------------------
# Type conversion
# -------------------------------------------------------------------------------

# Could be numba'd if it returned tuples instead of slices
def idx_chunks_along_axis(shape: tuple, axis: int, chunk_size: int):
    """
    Gives indexer tuples chunked along an axis.

    Params
    ------
    shape
        Shape of array to be chunked
    axis
        Axis to chunk along
    chunk_size
        Size of chunk along axis

    Returns
    -------
    An iterator of tuples for indexing into an array of passed shape.
    """
    total = shape[axis]
    cur = 0
    mutable_idx = [slice(None) for i in range(len(shape))]
    while cur + chunk_size < total:
        mutable_idx[axis] = slice(cur, cur + chunk_size)
        yield tuple(mutable_idx)
        cur += chunk_size
    mutable_idx[axis] = slice(cur, None)
    yield tuple(mutable_idx)

def is_float(string):
    """Check whether string is float.

    See also
    --------
    http://stackoverflow.com/questions/736043/checking-if-a-string-can-be-converted-to-float-in-python
    """
    try:
        float(string)
        return True
    except ValueError:
        return False

def is_int(string):
    """Check whether string is integer.
    """
    try:
        int(string)
        return True
    except ValueError:
        return False

def convert_bool(string):
    """Check whether string is boolean.
    """
    if string == 'True':
        return True, True
    elif string == 'False':
        return True, False
    else:
        return False, False

def convert_string(string):
    """Convert string to int, float or bool.
    """
    if is_int(string):
        return int(string)
    elif is_float(string):
        return float(string)
    elif convert_bool(string)[0]:
        return convert_bool(string)[1]
    elif string == 'None':
        return None
    else:
        return string

# -------------------------------------------------------------------------------
# Generic functions
# -------------------------------------------------------------------------------

@singledispatch
def write_attribute(*args, **kwargs):
    raise NotImplementedError(
        "Unrecognized argument types for `write_attribute`."
    )

@singledispatch
def read_attribute(*args, **kwargs):
    raise NotImplementedError(
        "Unrecognized argument types for `read_attribute`."
    )

@read_attribute.register(type(None))
def read_attribute_none(value) -> None:
    return None

# -------------------------------------------------------------------------------
# Errors handling
# -------------------------------------------------------------------------------
# TODO: Is there a consistent way to do this which just modifies the previously
# thrown error? Could do a warning?

class AnnDataReadError(OSError):
    """Error caused while trying to read in AnnData."""

    pass

def report_read_key_on_error(func):
    """
    A decorator for zarr element reading which makes keys involved in errors get reported.

    Example
    -------
    >>> import zarr
    >>> @report_read_key_on_error
    ... def read_arr(group):
    ...     raise NotImplementedError()
    >>> z = zarr.open("tmp.zarr")
    >>> z["X"] = [1, 2, 3]
    >>> read_arr(z["X"])
    """

    @wraps(func)
    def func_wrapper(elem, *args, **kwargs):
        try:
            return func(elem, *args, **kwargs)
        except Exception as e:
            if isinstance(e, AnnDataReadError):
                raise e
            else:
                try:
                    import zarr
                except ImportError:
                    zarr = None
                if zarr and isinstance(elem, (zarr.Group, zarr.Array)):
                    # Not sure how to always get a name out of this
                    parent = elem.store
                elif isinstance(elem, SparseDataset):
                    parent = elem.group.file.name
                else:
                    parent = elem.file.name
                raise AnnDataReadError(
                    f"Above error raised while reading key '{elem.name}' of type {type(elem)} from {parent}."
                )

    return func_wrapper

def report_write_key_on_error(func):
    """
    A decorator for zarr element reading which makes keys involved in errors get reported.

    Example
    -------
    >>> import zarr
    >>> @report_write_key_on_error
    ... def write_arr(group, key, val):
    ...     raise NotImplementedError()
    >>> z = zarr.open("tmp.zarr")
    >>> X = [1, 2, 3]
    >>> write_arr(z, "X", X)
    """

    @wraps(func)
    def func_wrapper(elem, key, val, *args, **kwargs):
        try:
            return func(elem, key, val, *args, **kwargs)
        except Exception as e:
            if isinstance(e, AnnDataReadError):
                raise e
            else:
                try:
                    import zarr
                except ImportError:
                    zarr = None
                if zarr and isinstance(elem, (zarr.Group, zarr.Array)):
                    parent = (
                        elem.store
                    )  # Not sure how to always get a name out of this
                elif isinstance(elem, SparseDataset):
                    parent = elem.group.file.name
                else:
                    parent = elem.file.name
                raise type(e)(
                    "\n\n".join(
                        (
                            str(e),
                            f"Above error raised while writing key '{key}' of "
                            f"type {type(val)} from {parent}.",
                        )
                    )
                ).with_traceback(sys.exc_info()[2])

    return func_wrapper
<|endoftext|>


<commit_before>import numpy
import six

from chainer import cuda
from chainer import function
from chainer.utils import type_check

def _extract_gates(x):
    r = x.reshape((x.shape[0], x.shape[1] // 4, 4) + x.shape[2:])
    return (r[:, :, i] for i in six.moves.range(4))

def _sigmoid(x):
    half = x.dtype.type(0.5)
    return numpy.tanh(x * half) * half + half

def _grad_sigmoid(x):
    return x * (1 - x)

def _grad_tanh(x):
    return 1 - x * x

_preamble = '''
template <typename T> __device__ T sigmoid(T x) {
    const T half = 0.5;
    return tanh(x * half) * half + half;
}
template <typename T> __device__ T grad_sigmoid(T y) { return y * (1 - y); }
template <typename T> __device__ T grad_tanh(T y) { return 1 - y * y; }

#define COMMON_ROUTINE \
    T aa1 = tanh(a1); \
    T ai1 = sigmoid(i1); \
    T af1 = sigmoid(f1); \
    T aa2 = tanh(a2); \
    T ai2 = sigmoid(i2); \
    T af2 = sigmoid(f2); \
    T ao = sigmoid(o1 + o2);
'''

class SLSTM(function.Function):

    """S-LSTM unit.

    It has four inputs (c1, c2, x1, x2) and two outputs (c, h), where c
    indicates the cell state. x1 and x2 must have four times channels compared
    to the number of units.

    """

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 4)
        c1_type, c2_type, x1_type, x2_type = in_types

        type_check.expect(
            c1_type.dtype.kind == 'f',
            c2_type.dtype == c1_type.dtype,
            x1_type.dtype == c1_type.dtype,
            x2_type.dtype == c1_type.dtype,

            c1_type.ndim >= 2,
            c2_type.ndim >= 2,
            x1_type.ndim >= 2,
            x2_type.ndim >= 2,
            c1_type.ndim == x1_type.ndim,
            c1_type.ndim == c2_type.ndim,
            c1_type.ndim == x2_type.ndim,

            c1_type.shape[0] == x1_type.shape[0],
            c1_type.shape[0] == c2_type.shape[0],
            c1_type.shape[0] == x2_type.shape[0],
            x1_type.shape[1] == 4 * c1_type.shape[1],
            x2_type.shape[1] == 4 * c2_type.shape[1],
        )
        for i in range(2, type_check.eval(c1_type.ndim)):
            type_check.expect(x1_type.shape[i] == c1_type.shape[i])
            type_check.expect(x2_type.shape[i] == c2_type.shape[i])
            type_check.expect(x1_type.shape[i] == x2_type.shape[i])

    def forward(self, inputs):
        c_prev1, c_prev2, x1, x2 = inputs
        a1, i1, f1, o1 = _extract_gates(x1)
        a2, i2, f2, o2 = _extract_gates(x2)

        if isinstance(x1, numpy.ndarray):
            self.a1 = numpy.tanh(a1)
            self.i1 = _sigmoid(i1)
            self.f1 = _sigmoid(f1)

            self.a2 = numpy.tanh(a2)
            self.i2 = _sigmoid(i2)
            self.f2 = _sigmoid(f2)

            self.o = _sigmoid(o1 + o2)
            self.c = self.a1 * self.i1 + self.a2 * self.i2 + \
                self.f1 * c_prev1 + self.f2 * c_prev2

            h = self.o * numpy.tanh(self.c)
        else:
            self.c, h = cuda.elementwise(
                '''T c_prev1, T a1, T i1, T f1, T o1,
                   T c_prev2, T a2, T i2, T f2, T o2''',
                'T c, T h',
                '''
                    COMMON_ROUTINE;
                    c = aa1 * ai1 + af1 * c_prev1 + aa2 * ai2 + af2 * c_prev2;
                    h = ao * tanh(c);
                ''',
                'slstm_fwd', preamble=_preamble)(
                    c_prev1, a1, i1, f1, o1, c_prev2, a2, i2, f2, o2)

        return self.c, h

    def backward(self, inputs, grad_outputs):
        xp = cuda.get_array_module(*inputs)
        c_prev1, c_prev2, x1, x2 = inputs
        gc, gh = grad_outputs

        gx1 = xp.empty_like(x1)
        gx2 = xp.empty_like(x2)
        ga1, gi1, gf1, go1 = _extract_gates(gx1)
        ga2, gi2, gf2, go2 = _extract_gates(gx2)

        # Consider the case that either gradient is not given
        if gc is None:
            gc = 0
        if gh is None:
            gh = 0

        if xp is numpy:
            co = numpy.tanh(self.c)
            # multiply f later
            gc_prev = gh * self.o * _grad_tanh(co) + gc
            ga1[:] = gc_prev * self.i1 * _grad_tanh(self.a1)
            gi1[:] = gc_prev * self.a1 * _grad_sigmoid(self.i1)
            gf1[:] = gc_prev * c_prev1 * _grad_sigmoid(self.f1)
            go1[:] = gh * co * _grad_sigmoid(self.o)
            ga2[:] = gc_prev * self.i2 * _grad_tanh(self.a2)
            gi2[:] = gc_prev * self.a2 * _grad_sigmoid(self.i2)
            gf2[:] = gc_prev * c_prev2 * _grad_sigmoid(self.f2)
            go2[:] = gh * co * _grad_sigmoid(self.o)
            # multiply f here
            gc_prev1 = gc_prev * self.f1
            gc_prev2 = gc_prev * self.f2
        else:
            a1, i1, f1, o1 = _extract_gates(x1)
            a2, i2, f2, o2 = _extract_gates(x2)
            gc_prev1 = xp.empty_like(c_prev1)
            gc_prev2 = xp.empty_like(c_prev2)
            cuda.elementwise(
                '''T c_prev1, T a1, T i1, T f1, T o1,
                T c_prev2, T a2, T i2, T f2, T o2,
                T c, T gc, T gh''',
                '''T gc_prev1, T ga1, T gi1, T gf1, T go1,
                T gc_prev2, T ga2, T gi2, T gf2, T go2''',
                '''
                    COMMON_ROUTINE;
                    T co = tanh(c);
                    T temp = gh * ao * grad_tanh(co) + gc;
                    ga1 = temp * ai1 * grad_tanh(aa1);
                    gi1 = temp * aa1 * grad_sigmoid(ai1);
                    gf1 = temp * c_prev1 * grad_sigmoid(af1);
                    go1 = gh * co * grad_sigmoid(ao);
                    gc_prev1 = temp * af1;
                    ga2 = temp * ai2 * grad_tanh(aa2);
                    gi2 = temp * aa2 * grad_sigmoid(ai2);
                    gf2 = temp * c_prev2 * grad_sigmoid(af2);
                    go2 = gh * co * grad_sigmoid(ao);
                    gc_prev2 = temp * af2;
                ''',
                'lstm_bwd', preamble=_preamble)(
                    c_prev1, a1, i1, f1, o1,
                    c_prev2, a2, i2, f2, o2,
                    self.c, gc, gh,
                    gc_prev1, ga1, gi1, gf1, go1,
                    gc_prev2, ga2, gi2, gf2, go2)

        return gc_prev1, gc_prev2, gx1, gx2

def slstm(c_prev1, c_prev2, x1, x2):
    """S-LSTM units as an activation function.

    This function implements S-LSTM unit. It is an extension of LSTM unit
    applied to tree structures.
    The function is applied to binary trees. Each node has two child nodes.
    It gets four arguments, previous cell states ``c_prev1`` and ``c_prev2``,
    and input arrays ``x1`` and ``x2``.

    First both input arrays ``x1`` and ``x2`` are split into eight arrays
    :math:`a_1, i_1, f_1, o_1`, and :math:`a_2, i_2, f_2, o_2`. They have the
    same shape along the second axis.
    It means that ``x1`` and ``x2`` 's second axis must have 4 times
    the length of ``c_prev1`` and ``c_prev2``.

    The split input arrays are corresponding to:

        - :math:`a_i` : sources of cell input
        - :math:`i_i` : sources of input gate
        - :math:`f_i` : sources of forget gate
        - :math:`o_i` : sources of output gate

    It computes the updated cell state ``c`` and the outgoing signal
    ``h`` as:

    .. math::

        c &= \\tanh(a_1 + a_2) \\sigma(i_1 + i_2)
           + c_{\\text{prev}1} \\sigma(f_1)
           + c_{\\text{prev}2} \\sigma(f_2), \\\\
        h &= \\tanh(c) \\sigma(o_1 + o_2),

    where :math:`\\sigma` is the elementwise sigmoid function.
    The function returns ``c`` and ``h`` as a tuple.

    Args:
        c_prev1 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the previous cell state of the first child
            node. The cell state should be a zero array or the output of
            the previous call of LSTM.
        c_prev2 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the previous cell state of the second child
            node.
        x1 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the sources of cell input, input gate, forget
            gate and output gate from the first child node. It must have the
            second dimension whose size is four times of that of the cell
            state.
        x2 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the input sources from the second child node.

    Returns:
        tuple: Two :class:`~chainer.Variable` objects ``c`` and ``h``. ``c`` is
        the cell state. ``h`` indicates the outgoing signal.

    See detail in paper: `Long Short-Term Memory Over Tree Structures \
    <https://arxiv.org/abs/1503.04881>`_.

    .. admonition:: Example

        Assuming ``c1``, ``c2`` is the previous cell state of children,
        and ``h1``, ``h2`` is the previous outgoing signal from children.
        Each of ``c1``, ``c2``, ``h1`` and ``h2`` has ``n_units`` channels.
        Most typical preparation of ``x1``, ``x2`` is:

        >>> n_units = 100
        >>> h1 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> h2 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> c1 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> c2 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> model1 = chainer.Chain(w=L.Linear(n_units, 4 * n_units),
        ...                        v=L.Linear(n_units, 4 * n_units))
        >>> model2 = chainer.Chain(w=L.Linear(n_units, 4 * n_units),
        ...                        v=L.Linear(n_units, 4 * n_units))
        >>> x1 = model1.w(c1) + model1.v(h1)
        >>> x2 = model2.w(c2) + model2.v(h2)
        >>> c, h = F.slstm(c1, c2, x1, x2)

        It corresponds to calculate the input array ``x1``, or the input
        sources :math:`a_1, i_1, f_1, o_1` from the previous cell state of
        first child node ``c1``, and the previous outgoing signal from first
        child node ``h1``. Different parameters are used for different kind of
        input sources.

    """
    return SLSTM()(c_prev1, c_prev2, x1, x2)
<commit_msg>Fix slstm docstring<commit_after>import numpy
import six

from chainer import cuda
from chainer import function
from chainer.utils import type_check

def _extract_gates(x):
    r = x.reshape((x.shape[0], x.shape[1] // 4, 4) + x.shape[2:])
    return (r[:, :, i] for i in six.moves.range(4))

def _sigmoid(x):
    half = x.dtype.type(0.5)
    return numpy.tanh(x * half) * half + half

def _grad_sigmoid(x):
    return x * (1 - x)

def _grad_tanh(x):
    return 1 - x * x

_preamble = '''
template <typename T> __device__ T sigmoid(T x) {
    const T half = 0.5;
    return tanh(x * half) * half + half;
}
template <typename T> __device__ T grad_sigmoid(T y) { return y * (1 - y); }
template <typename T> __device__ T grad_tanh(T y) { return 1 - y * y; }

#define COMMON_ROUTINE \
    T aa1 = tanh(a1); \
    T ai1 = sigmoid(i1); \
    T af1 = sigmoid(f1); \
    T aa2 = tanh(a2); \
    T ai2 = sigmoid(i2); \
    T af2 = sigmoid(f2); \
    T ao = sigmoid(o1 + o2);
'''

class SLSTM(function.Function):

    """S-LSTM unit.

    It has four inputs (c1, c2, x1, x2) and two outputs (c, h), where c
    indicates the cell state. x1 and x2 must have four times channels compared
    to the number of units.

    """

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 4)
        c1_type, c2_type, x1_type, x2_type = in_types

        type_check.expect(
            c1_type.dtype.kind == 'f',
            c2_type.dtype == c1_type.dtype,
            x1_type.dtype == c1_type.dtype,
            x2_type.dtype == c1_type.dtype,

            c1_type.ndim >= 2,
            c2_type.ndim >= 2,
            x1_type.ndim >= 2,
            x2_type.ndim >= 2,
            c1_type.ndim == x1_type.ndim,
            c1_type.ndim == c2_type.ndim,
            c1_type.ndim == x2_type.ndim,

            c1_type.shape[0] == x1_type.shape[0],
            c1_type.shape[0] == c2_type.shape[0],
            c1_type.shape[0] == x2_type.shape[0],
            x1_type.shape[1] == 4 * c1_type.shape[1],
            x2_type.shape[1] == 4 * c2_type.shape[1],
        )
        for i in range(2, type_check.eval(c1_type.ndim)):
            type_check.expect(x1_type.shape[i] == c1_type.shape[i])
            type_check.expect(x2_type.shape[i] == c2_type.shape[i])
            type_check.expect(x1_type.shape[i] == x2_type.shape[i])

    def forward(self, inputs):
        c_prev1, c_prev2, x1, x2 = inputs
        a1, i1, f1, o1 = _extract_gates(x1)
        a2, i2, f2, o2 = _extract_gates(x2)

        if isinstance(x1, numpy.ndarray):
            self.a1 = numpy.tanh(a1)
            self.i1 = _sigmoid(i1)
            self.f1 = _sigmoid(f1)

            self.a2 = numpy.tanh(a2)
            self.i2 = _sigmoid(i2)
            self.f2 = _sigmoid(f2)

            self.o = _sigmoid(o1 + o2)
            self.c = self.a1 * self.i1 + self.a2 * self.i2 + \
                self.f1 * c_prev1 + self.f2 * c_prev2

            h = self.o * numpy.tanh(self.c)
        else:
            self.c, h = cuda.elementwise(
                '''T c_prev1, T a1, T i1, T f1, T o1,
                   T c_prev2, T a2, T i2, T f2, T o2''',
                'T c, T h',
                '''
                    COMMON_ROUTINE;
                    c = aa1 * ai1 + af1 * c_prev1 + aa2 * ai2 + af2 * c_prev2;
                    h = ao * tanh(c);
                ''',
                'slstm_fwd', preamble=_preamble)(
                    c_prev1, a1, i1, f1, o1, c_prev2, a2, i2, f2, o2)

        return self.c, h

    def backward(self, inputs, grad_outputs):
        xp = cuda.get_array_module(*inputs)
        c_prev1, c_prev2, x1, x2 = inputs
        gc, gh = grad_outputs

        gx1 = xp.empty_like(x1)
        gx2 = xp.empty_like(x2)
        ga1, gi1, gf1, go1 = _extract_gates(gx1)
        ga2, gi2, gf2, go2 = _extract_gates(gx2)

        # Consider the case that either gradient is not given
        if gc is None:
            gc = 0
        if gh is None:
            gh = 0

        if xp is numpy:
            co = numpy.tanh(self.c)
            # multiply f later
            gc_prev = gh * self.o * _grad_tanh(co) + gc
            ga1[:] = gc_prev * self.i1 * _grad_tanh(self.a1)
            gi1[:] = gc_prev * self.a1 * _grad_sigmoid(self.i1)
            gf1[:] = gc_prev * c_prev1 * _grad_sigmoid(self.f1)
            go1[:] = gh * co * _grad_sigmoid(self.o)
            ga2[:] = gc_prev * self.i2 * _grad_tanh(self.a2)
            gi2[:] = gc_prev * self.a2 * _grad_sigmoid(self.i2)
            gf2[:] = gc_prev * c_prev2 * _grad_sigmoid(self.f2)
            go2[:] = gh * co * _grad_sigmoid(self.o)
            # multiply f here
            gc_prev1 = gc_prev * self.f1
            gc_prev2 = gc_prev * self.f2
        else:
            a1, i1, f1, o1 = _extract_gates(x1)
            a2, i2, f2, o2 = _extract_gates(x2)
            gc_prev1 = xp.empty_like(c_prev1)
            gc_prev2 = xp.empty_like(c_prev2)
            cuda.elementwise(
                '''T c_prev1, T a1, T i1, T f1, T o1,
                T c_prev2, T a2, T i2, T f2, T o2,
                T c, T gc, T gh''',
                '''T gc_prev1, T ga1, T gi1, T gf1, T go1,
                T gc_prev2, T ga2, T gi2, T gf2, T go2''',
                '''
                    COMMON_ROUTINE;
                    T co = tanh(c);
                    T temp = gh * ao * grad_tanh(co) + gc;
                    ga1 = temp * ai1 * grad_tanh(aa1);
                    gi1 = temp * aa1 * grad_sigmoid(ai1);
                    gf1 = temp * c_prev1 * grad_sigmoid(af1);
                    go1 = gh * co * grad_sigmoid(ao);
                    gc_prev1 = temp * af1;
                    ga2 = temp * ai2 * grad_tanh(aa2);
                    gi2 = temp * aa2 * grad_sigmoid(ai2);
                    gf2 = temp * c_prev2 * grad_sigmoid(af2);
                    go2 = gh * co * grad_sigmoid(ao);
                    gc_prev2 = temp * af2;
                ''',
                'lstm_bwd', preamble=_preamble)(
                    c_prev1, a1, i1, f1, o1,
                    c_prev2, a2, i2, f2, o2,
                    self.c, gc, gh,
                    gc_prev1, ga1, gi1, gf1, go1,
                    gc_prev2, ga2, gi2, gf2, go2)

        return gc_prev1, gc_prev2, gx1, gx2

def slstm(c_prev1, c_prev2, x1, x2):
    """S-LSTM units as an activation function.

    This function implements S-LSTM unit. It is an extension of LSTM unit
    applied to tree structures.
    The function is applied to binary trees. Each node has two child nodes.
    It gets four arguments, previous cell states ``c_prev1`` and ``c_prev2``,
    and input arrays ``x1`` and ``x2``.

    First both input arrays ``x1`` and ``x2`` are split into eight arrays
    :math:`a_1, i_1, f_1, o_1`, and :math:`a_2, i_2, f_2, o_2`. They have the
    same shape along the second axis.
    It means that ``x1`` and ``x2`` 's second axis must have 4 times
    the length of ``c_prev1`` and ``c_prev2``.

    The split input arrays are corresponding to:

        - :math:`a_i` : sources of cell input
        - :math:`i_i` : sources of input gate
        - :math:`f_i` : sources of forget gate
        - :math:`o_i` : sources of output gate

    It computes the updated cell state ``c`` and the outgoing signal
    ``h`` as:

    .. math::

        c &= \\tanh(a_1 + a_2) \\sigma(i_1 + i_2)
           + c_{\\text{prev}1} \\sigma(f_1)
           + c_{\\text{prev}2} \\sigma(f_2), \\\\
        h &= \\tanh(c) \\sigma(o_1 + o_2),

    where :math:`\\sigma` is the elementwise sigmoid function.
    The function returns ``c`` and ``h`` as a tuple.

    Args:
        c_prev1 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the previous cell state of the first child
            node. The cell state should be a zero array or the output of
            the previous call of LSTM.
        c_prev2 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the previous cell state of the second child
            node.
        x1 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the sources of cell input, input gate, forget
            gate and output gate from the first child node. It must have the
            second dimension whose size is four times of that of the cell
            state.
        x2 (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Variable that holds the input sources from the second child node.

    Returns:
        tuple: Two :class:`~chainer.Variable` objects ``c`` and ``h``. ``c`` is
        the cell state. ``h`` indicates the outgoing signal.

    See detail in paper: `Long Short-Term Memory Over Tree Structures \
    <https://arxiv.org/abs/1503.04881>`_.

    .. admonition:: Example

        Assuming ``c1``, ``c2`` is the previous cell state of children,
        and ``h1``, ``h2`` is the previous outgoing signal from children.
        Each of ``c1``, ``c2``, ``h1`` and ``h2`` has ``n_units`` channels.
        Most typical preparation of ``x1``, ``x2`` is:

        >>> n_units = 100
        >>> h1 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> h2 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> c1 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> c2 = chainer.Variable(np.zeros((1, n_units), 'f'))
        >>> model1 = chainer.Chain()
        >>> with model1.init_scope():
        ...   model1.w = L.Linear(n_units, 4 * n_units)
        ...   model1.v = L.Linear(n_units, 4 * n_units)
        >>> model2 = chainer.Chain()
        >>> with model2.init_scope():
        ...   model2.w = L.Linear(n_units, 4 * n_units)
        ...   model2.v = L.Linear(n_units, 4 * n_units)
        >>> x1 = model1.w(c1) + model1.v(h1)
        >>> x2 = model2.w(c2) + model2.v(h2)
        >>> c, h = F.slstm(c1, c2, x1, x2)

        It corresponds to calculate the input array ``x1``, or the input
        sources :math:`a_1, i_1, f_1, o_1` from the previous cell state of
        first child node ``c1``, and the previous outgoing signal from first
        child node ``h1``. Different parameters are used for different kind of
        input sources.

    """
    return SLSTM()(c_prev1, c_prev2, x1, x2)
<|endoftext|>


<commit_before># Copyright (c) Charl P. Botha, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import config
from install_package import InstallPackage
import os
import shutil
import utils

BASENAME = "wrapitk"
# use the "maint" branche, trunk is now meant for ITK 4 (in development)
SVN_REPO = \
        "http://wrapitk.googlecode.com/svn/branches/maint"
SVN_REL = 527

BO_PATCH = "wrapitk_maint_bigobj_cl64.diff"

dependencies = ['CMake', 'ITK', 'CableSwig', 'SWIG']

class WrapITK(InstallPackage):

    def __init__(self):
        self.source_dir = os.path.join(config.archive_dir, BASENAME)
        self.build_dir = os.path.join(config.build_dir, '%s-build' %
                                      (BASENAME,))
        self.inst_dir = os.path.join(config.inst_dir, BASENAME)

    def get(self):
        if os.path.exists(self.source_dir):
            utils.output("wrapitk already checked out, skipping checkout.")

        else:
            os.chdir(config.archive_dir)
            # checkout trunk into directory vtktudoss
            ret = os.system("%s co %s %s -r%s" % (config.SVN,
                SVN_REPO, BASENAME, SVN_REL))
            if ret != 0:
                utils.error("Could not SVN checkout.  Fix and try again.")

        # now apply patch if necessary
        patch_dst = os.path.join(config.archive_dir, BO_PATCH)
        if not os.path.exists(patch_dst):
            utils.output("Applying PATCH: wrapitk maint /bigobj cl64")
            patch_src = os.path.join(config.patches_dir,BO_PATCH)
            shutil.copy(patch_src, patch_dst)
            os.chdir(self.source_dir)
            ret = os.system("%s -p0 < %s" % (config.PATCH, patch_dst))

    def unpack(self):
        # no unpack step
        pass

    def configure(self):
        if os.path.exists(
            os.path.join(self.build_dir, 'CMakeFiles/cmake.check_cache')):
            utils.output("wrapitk build already configured.")
            return

        if not os.path.exists(self.build_dir):
            os.mkdir(self.build_dir)

        # need unsigned short for itkPyImageFilter
        # with the switches below, I need /bigobj on win64 for the
        # following projects: ITKPyBasePython
        cmake_params = \
                "-DBUILD_TESTING=OFF " \
                "-DCMAKE_BUILD_TYPE=RelWithDebInfo " \
                "-DCMAKE_INSTALL_PREFIX=%s " \
                "-DINSTALL_WRAP_ITK_COMPATIBILITY=NO " \
                "-DITK_DIR=%s " \
                "-DITK_TEST_DRIVER=%s " \
                "-DCableSwig_DIR=%s " \
                "-DSWIG_DIR=%s " \
                "-DSWIG_EXECUTABLE=%s " \
                "-DVTK_DIR=%s " \
                "-DWRAP_ITK_PYTHON=ON " \
                "-DWRAP_complex_double=OFF " \
                "-DWRAP_complex_float=ON " \
                "-DWRAP_covariant_vector_double=OFF " \
                "-DWRAP_covariant_vector_float=ON " \
                "-DWRAP_double=OFF " \
                "-DWRAP_float=ON " \
                "-DWRAP_rgb_unsigned_char=OFF " \
                "-DWRAP_rgb_unsigned_short=ON " \
                "-DWRAP_rgba_unsigned_char=OFF " \
                "-DWRAP_rgba_unsigned_short=OFF " \
                "-DWRAP_signed_char=OFF " \
                "-DWRAP_signed_long=OFF " \
                "-DWRAP_signed_short=ON " \
                "-DWRAP_unsigned_char=OFF " \
                "-DWRAP_unsigned_long=OFF " \
                "-DWRAP_unsigned_short=ON " \
                "-DWRAP_vector_double=OFF " \
                "-DWRAP_vector_float=ON " \
                "-DPYTHON_EXECUTABLE=%s " \
                "-DPYTHON_LIBRARY=%s " \
                "-DPYTHON_INCLUDE_PATH=%s " % \
                (self.inst_dir, config.ITK_DIR,
                 config.ITK_TEST_DRIVER,
                 config.CABLESWIG_DIR,
                 config.SWIG_DIR,
                 config.SWIG_EXECUTABLE, config.VTK_DIR,
                 config.PYTHON_EXECUTABLE,
                 config.PYTHON_LIBRARY,
                 config.PYTHON_INCLUDE_PATH)

        ret = utils.cmake_command(self.build_dir, self.source_dir,
                cmake_params)

        if ret != 0:
            utils.error("Could not configure WrapITK.  Fix and try again.")

    def build(self):
        # NB: if you run "make" in wrapitk-build yourself, make sure
        # the johannes python is in the path (by having ran source
        # jpython_setup.sh), as the build process relies on the
        # correct python being available.

        posix_file = os.path.join(self.build_dir,
                'lib', '_RegistrationPython.so')
        nt_file = os.path.join(self.build_dir,
                'lib', config.BUILD_TARGET,
                '_RegistrationPython' + config.PYE_EXT)

        if utils.file_exists(posix_file, nt_file):
            utils.output("WrapITK already built.  Skipping build step.")

        else:
            os.chdir(self.build_dir)
            ret = utils.make_command('WrapITK.sln')

            if ret != 0:
                utils.error("Could not build WrapITK.  Fix and try again.")

    def install(self):
        config.WRAPITK_TOPLEVEL = self.inst_dir
        # this dir contains the WrapITK cmake config (WrapITKConfig.cmake)
        config.WRAPITK_DIR = os.path.join(
                self.inst_dir, 'lib', 'InsightToolkit', 'WrapITK')
        # contains all WrapITK shared objects / libraries
        config.WRAPITK_LIB = os.path.join(config.WRAPITK_DIR, 'lib')
        # contains itk.py
        config.WRAPITK_PYTHON = os.path.join(config.WRAPITK_DIR, 'Python')
        # subsequent wrapitk components will need this
        config.WRAPITK_SOURCE_DIR = self.source_dir

        posix_file = os.path.join(
                config.WRAPITK_LIB, '_RegistrationPython.so')
        nt_file = os.path.join(
                config.WRAPITK_LIB, '_RegistrationPython' + config.PYE_EXT)

        if utils.file_exists(posix_file, nt_file):
            utils.output("WrapITK already installed, skipping step.")
        else:
            os.chdir(self.build_dir)
            ret = utils.make_command('WrapITK.sln', install=True)

            if ret != 0:
                utils.error(
                "Could not install WrapITK.  Fix and try again.")

    def clean_build(self):
        utils.output("Removing build and installation directories.")
        if os.path.exists(self.inst_dir):
            shutil.rmtree(self.inst_dir)

        if os.path.exists(self.build_dir):
            shutil.rmtree(self.build_dir)

    def clean_install(self):
        utils.output("Removing installation directory.")
        if os.path.exists(self.inst_dir):
            shutil.rmtree(self.inst_dir)

    def get_installed_version(self):
        import itk
        return itk.GetVersion()

<commit_msg>Integrated itk320 / wrapitk / vtk58 patch.<commit_after># Copyright (c) Charl P. Botha, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import config
from install_package import InstallPackage
import os
import shutil
import utils

BASENAME = "wrapitk"
# use the "maint" branche, trunk is now meant for ITK 4 (in development)
SVN_REPO = \
        "http://wrapitk.googlecode.com/svn/branches/maint"
SVN_REL = 527

BO_PATCH = "wrapitk_maint_bigobj_cl64.diff"
VTK58_PATCH = "itk320-vtk58-wrapitk.diff"

dependencies = ['CMake', 'ITK', 'CableSwig', 'SWIG']

class WrapITK(InstallPackage):

    def __init__(self):
        self.source_dir = os.path.join(config.archive_dir, BASENAME)
        self.build_dir = os.path.join(config.build_dir, '%s-build' %
                                      (BASENAME,))
        self.inst_dir = os.path.join(config.inst_dir, BASENAME)

    def get(self):
        if os.path.exists(self.source_dir):
            utils.output("wrapitk already checked out, skipping checkout.")

        else:
            os.chdir(config.archive_dir)
            # checkout trunk into directory vtktudoss
            ret = os.system("%s co %s %s -r%s" % (config.SVN,
                SVN_REPO, BASENAME, SVN_REL))
            if ret != 0:
                utils.error("Could not SVN checkout.  Fix and try again.")

        # now apply patch if necessary
        patch_dst = os.path.join(config.archive_dir, BO_PATCH)
        if not os.path.exists(patch_dst):
            utils.output("Applying PATCH: wrapitk maint /bigobj cl64")
            patch_src = os.path.join(config.patches_dir,BO_PATCH)
            shutil.copy(patch_src, patch_dst)
            os.chdir(self.source_dir)
            ret = os.system("%s -p0 < %s" % (config.PATCH, patch_dst))

        patch_dst = os.path.join(config.archive_dir, VTK58_PATCH)
        if not os.path.exists(patch_dst):
            utils.output("Applying PATCH: wrapitk / itk320 VTK58 patch")
            patch_src = os.path.join(config.patches_dir, VTK58_PATCH)
            shutil.copy(patch_src, patch_dst)
            os.chdir(self.source_dir)
            ret = os.system("%s -p0 < %s" % (config.PATCH, patch_dst))

    def unpack(self):
        # no unpack step
        pass

    def configure(self):
        if os.path.exists(
            os.path.join(self.build_dir, 'CMakeFiles/cmake.check_cache')):
            utils.output("wrapitk build already configured.")
            return

        if not os.path.exists(self.build_dir):
            os.mkdir(self.build_dir)

        # need unsigned short for itkPyImageFilter
        # with the switches below, I need /bigobj on win64 for the
        # following projects: ITKPyBasePython
        cmake_params = \
                "-DBUILD_TESTING=OFF " \
                "-DCMAKE_BUILD_TYPE=RelWithDebInfo " \
                "-DCMAKE_INSTALL_PREFIX=%s " \
                "-DINSTALL_WRAP_ITK_COMPATIBILITY=NO " \
                "-DITK_DIR=%s " \
                "-DITK_TEST_DRIVER=%s " \
                "-DCableSwig_DIR=%s " \
                "-DSWIG_DIR=%s " \
                "-DSWIG_EXECUTABLE=%s " \
                "-DVTK_DIR=%s " \
                "-DWRAP_ITK_PYTHON=ON " \
                "-DWRAP_complex_double=OFF " \
                "-DWRAP_complex_float=ON " \
                "-DWRAP_covariant_vector_double=OFF " \
                "-DWRAP_covariant_vector_float=ON " \
                "-DWRAP_double=OFF " \
                "-DWRAP_float=ON " \
                "-DWRAP_rgb_unsigned_char=OFF " \
                "-DWRAP_rgb_unsigned_short=ON " \
                "-DWRAP_rgba_unsigned_char=OFF " \
                "-DWRAP_rgba_unsigned_short=OFF " \
                "-DWRAP_signed_char=OFF " \
                "-DWRAP_signed_long=OFF " \
                "-DWRAP_signed_short=ON " \
                "-DWRAP_unsigned_char=OFF " \
                "-DWRAP_unsigned_long=OFF " \
                "-DWRAP_unsigned_short=ON " \
                "-DWRAP_vector_double=OFF " \
                "-DWRAP_vector_float=ON " \
                "-DPYTHON_EXECUTABLE=%s " \
                "-DPYTHON_LIBRARY=%s " \
                "-DPYTHON_INCLUDE_PATH=%s " % \
                (self.inst_dir, config.ITK_DIR,
                 config.ITK_TEST_DRIVER,
                 config.CABLESWIG_DIR,
                 config.SWIG_DIR,
                 config.SWIG_EXECUTABLE, config.VTK_DIR,
                 config.PYTHON_EXECUTABLE,
                 config.PYTHON_LIBRARY,
                 config.PYTHON_INCLUDE_PATH)

        ret = utils.cmake_command(self.build_dir, self.source_dir,
                cmake_params)

        if ret != 0:
            utils.error("Could not configure WrapITK.  Fix and try again.")

    def build(self):
        # NB: if you run "make" in wrapitk-build yourself, make sure
        # the johannes python is in the path (by having ran source
        # jpython_setup.sh), as the build process relies on the
        # correct python being available.

        posix_file = os.path.join(self.build_dir,
                'lib', '_RegistrationPython.so')
        nt_file = os.path.join(self.build_dir,
                'lib', config.BUILD_TARGET,
                '_RegistrationPython' + config.PYE_EXT)

        if utils.file_exists(posix_file, nt_file):
            utils.output("WrapITK already built.  Skipping build step.")

        else:
            os.chdir(self.build_dir)
            ret = utils.make_command('WrapITK.sln')

            if ret != 0:
                utils.error("Could not build WrapITK.  Fix and try again.")

    def install(self):
        config.WRAPITK_TOPLEVEL = self.inst_dir
        # this dir contains the WrapITK cmake config (WrapITKConfig.cmake)
        config.WRAPITK_DIR = os.path.join(
                self.inst_dir, 'lib', 'InsightToolkit', 'WrapITK')
        # contains all WrapITK shared objects / libraries
        config.WRAPITK_LIB = os.path.join(config.WRAPITK_DIR, 'lib')
        # contains itk.py
        config.WRAPITK_PYTHON = os.path.join(config.WRAPITK_DIR, 'Python')
        # subsequent wrapitk components will need this
        config.WRAPITK_SOURCE_DIR = self.source_dir

        posix_file = os.path.join(
                config.WRAPITK_LIB, '_RegistrationPython.so')
        nt_file = os.path.join(
                config.WRAPITK_LIB, '_RegistrationPython' + config.PYE_EXT)

        if utils.file_exists(posix_file, nt_file):
            utils.output("WrapITK already installed, skipping step.")
        else:
            os.chdir(self.build_dir)
            ret = utils.make_command('WrapITK.sln', install=True)

            if ret != 0:
                utils.error(
                "Could not install WrapITK.  Fix and try again.")

    def clean_build(self):
        utils.output("Removing build and installation directories.")
        if os.path.exists(self.inst_dir):
            shutil.rmtree(self.inst_dir)

        if os.path.exists(self.build_dir):
            shutil.rmtree(self.build_dir)

    def clean_install(self):
        utils.output("Removing installation directory.")
        if os.path.exists(self.inst_dir):
            shutil.rmtree(self.inst_dir)

    def get_installed_version(self):
        import itk
        return itk.GetVersion()

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from ds_binary_search_tree import TreeNode, BinarySearchTree

class AVLTreeNode(TreeNode):
    def __init__(self, *args, **kwargs):
        super(AVLTreeNode, self).__init__(*args, **kwargs)
        self.balance_factor = 0

class AVLTree(BinarySearchTree):
    TreeNode = AVLTreeNode

    def update_balance(self, node):
        if self.balance_factor < -1 or self.balance_factor > 1:
            self.rebalance(node)
            return None
        if node.parent is not None:
            if node.is_left_child():
                node.parent.balance_factor += 1
            elif node.is_right_child():
                node.parent.balance_factor -= 1
            else:
                pass

            if node.parent.balance_factor != 0:
                self.update_balance(node.parent)

    def _put(self, key, value, current_node):
        if key == current_node.key:
            self.replace_node_data(
                key, value, current_node.left_child, current_node.right_child)
        elif key < current_node.key:
            if current_node.has_left_child():
                self._put(key, value, current_node.left_child)
            else:
                current_node.left_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.left_child)
        else:
            if current_node.has_right_child():
                self._put(key, value, current_node.right_child)
            else:
                current_node.right_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.right_child)

    pass

def main():
    avlt = AVLTree()
    avlt[3] = 'red'
    avlt[4] = 'blue'
    avlt[6] = 'yellow'
    avlt[2] = 'black'

    print('len(avlt): {}'.format(len(avlt)))
    print('avlt.size: {}'.format(avlt.size))

    print(avlt[6])
    print(avlt[2])

    print('Iterate by the inorder traversal algorithm:')
    for x in avlt:
        print(x)

    print('Remove avlt[2] and then put it back')
    del avlt[2]
    for x in avlt:
        print(x)
    avlt[2] = 'black'

    print('Remove avlt[6] and then put it back')
    del avlt[6]
    for x in avlt:
        print(x)
    avlt[6] = 'yellow'

    print('Remove avlt[4] and then put it back')
    del avlt[4]
    for x in avlt:
        print(x)
    avlt[4] = 'blue'

    print('Iterate after del avlt[3]:')
    del avlt[3]
    for x in avlt:
        print(x)

if __name__ == '__main__':
    main()
<commit_msg>Add rotate_left()<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from ds_binary_search_tree import TreeNode, BinarySearchTree

class AVLTreeNode(TreeNode):
    def __init__(self, *args, **kwargs):
        super(AVLTreeNode, self).__init__(*args, **kwargs)
        self.balance_factor = 0

class AVLTree(BinarySearchTree):
    TreeNode = AVLTreeNode

    def rotate_left(self, rotate_root):
        pass

    def update_balance(self, node):
        if self.balance_factor < -1 or self.balance_factor > 1:
            self.rebalance(node)
            return None
        if node.parent is not None:
            if node.is_left_child():
                node.parent.balance_factor += 1
            elif node.is_right_child():
                node.parent.balance_factor -= 1
            else:
                pass

            if node.parent.balance_factor != 0:
                self.update_balance(node.parent)

    def _put(self, key, value, current_node):
        if key == current_node.key:
            self.replace_node_data(
                key, value, current_node.left_child, current_node.right_child)
        elif key < current_node.key:
            if current_node.has_left_child():
                self._put(key, value, current_node.left_child)
            else:
                current_node.left_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.left_child)
        else:
            if current_node.has_right_child():
                self._put(key, value, current_node.right_child)
            else:
                current_node.right_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.right_child)

    pass

def main():
    avlt = AVLTree()
    avlt[3] = 'red'
    avlt[4] = 'blue'
    avlt[6] = 'yellow'
    avlt[2] = 'black'

    print('len(avlt): {}'.format(len(avlt)))
    print('avlt.size: {}'.format(avlt.size))

    print(avlt[6])
    print(avlt[2])

    print('Iterate by the inorder traversal algorithm:')
    for x in avlt:
        print(x)

    print('Remove avlt[2] and then put it back')
    del avlt[2]
    for x in avlt:
        print(x)
    avlt[2] = 'black'

    print('Remove avlt[6] and then put it back')
    del avlt[6]
    for x in avlt:
        print(x)
    avlt[6] = 'yellow'

    print('Remove avlt[4] and then put it back')
    del avlt[4]
    for x in avlt:
        print(x)
    avlt[4] = 'blue'

    print('Iterate after del avlt[3]:')
    del avlt[3]
    for x in avlt:
        print(x)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># $Id$

# this exception get overriden by the platform specific modules if necessary

class Error(Exception):
    pass

class NoSuchProcess(Error):
    """No process was found for the given parameters."""

    def __init__(self, pid=None, msg=None):
        self.pid = pid
        self.msg = msg

    def __str__(self):
        if self.msg:
            return self.msg
        elif self.pid:
            return "no such process with pid %d" % self.pid
        else:
            return ""

class AccessDenied(Error):
    """Exception raised when permission to perform an action is denied."""

    def __init__(self, pid=None, msg=None):
        self.pid = pid
        self.msg = msg

    def __str__(self):
        return self.msg or ""

<commit_msg>set svn:propset<commit_after># $Id

# this exception get overriden by the platform specific modules if necessary

class Error(Exception):
    pass

class NoSuchProcess(Error):
    """No process was found for the given parameters."""

    def __init__(self, pid=None, msg=None):
        self.pid = pid
        self.msg = msg

    def __str__(self):
        if self.msg:
            return self.msg
        elif self.pid:
            return "no such process with pid %d" % self.pid
        else:
            return ""

class AccessDenied(Error):
    """Exception raised when permission to perform an action is denied."""

    def __init__(self, pid=None, msg=None):
        self.pid = pid
        self.msg = msg

    def __str__(self):
        return self.msg or ""

<|endoftext|>


<commit_before># Django settings for bloggato project.

# Initialize App Engine and import the default settings (DB backend, etc.).
# If you want to use a different backend you have to remove all occurences
# of "djangoappengine" from this file.
from djangoappengine.settings_base import *
import os

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# Activate django-dbindexer for the default database
DATABASES['default'] = {'ENGINE': 'dbindexer', 'TARGET': DATABASES['default']}

AUTOLOAD_SITECONF = 'indexes'

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['blogga-to.appspot.com']

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = False

PROJECT_PATH = os.path.realpath(os.path.dirname(__file__))

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (os.path.join(PROJECT_PATH, "static"),)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'n_&=b_d-i721-!l@gmmxe#be%ge$9nunbnebb5ttz@i9se@3ho'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    # This loads the index definitions, so it has to come first
    'autoload.middleware.AutoloadMiddleware',

    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'djangotoolbox',
    'autoload',
    'dbindexer',
    'blog',

    # djangoappengine should come last, so it can override a few manage.py commands
    'djangoappengine',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

LOGIN_URL = '/login/'
LOGOUT_URL = '/logout/'
LOGIN_REDIRECT_URL = '/'
<commit_msg>Change the LOGIN_REDIRECT_URL<commit_after># Django settings for bloggato project.

# Initialize App Engine and import the default settings (DB backend, etc.).
# If you want to use a different backend you have to remove all occurences
# of "djangoappengine" from this file.
from djangoappengine.settings_base import *
import os

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# Activate django-dbindexer for the default database
DATABASES['default'] = {'ENGINE': 'dbindexer', 'TARGET': DATABASES['default']}

AUTOLOAD_SITECONF = 'indexes'

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['blogga-to.appspot.com']

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = False

PROJECT_PATH = os.path.realpath(os.path.dirname(__file__))

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (os.path.join(PROJECT_PATH, "static"),)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'n_&=b_d-i721-!l@gmmxe#be%ge$9nunbnebb5ttz@i9se@3ho'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    # This loads the index definitions, so it has to come first
    'autoload.middleware.AutoloadMiddleware',

    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'djangotoolbox',
    'autoload',
    'dbindexer',
    'blog',

    # djangoappengine should come last, so it can override a few manage.py commands
    'djangoappengine',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

LOGIN_URL = '/login/'
LOGOUT_URL = '/logout/'
LOGIN_REDIRECT_URL = '/blog'
<|endoftext|>