<commit_before>"""build script for our lesson repositories"""
import json
import os
import itertools
import logging
import subprocess
import collections
from . import github, config, misc

LOGGER = logging.getLogger(__name__)
TEX_OUTPUT = {
    'htlatex': 'html',
    'pdflatex': 'pdf',
    'xelatex': 'pdf',
    'hevea': 'html'
}
PATHS = {
    'app': config.BASE_DIRECTORY
}

def partition(condition, iterable, output_class=tuple):
    """
    split an iterable into two according to a function evaluating to either
     true or false on each element

    :param condition: boolean function
    :param iterable: iterable to split
    :param output_class: type of the returned iterables
    :return: two iterables
    """
    true = []
    false = []

    for i in iterable:
        if condition(i):
            true.append(i)
        else:
            false.append(i)

    return output_class(true), output_class(false)

class Build:
    """
    Abstracts builds
    """
    def __init__(self, name, command, target_dir, source_dir, files, base_dir=''):
        self.name = name
        if command not in TEX_OUTPUT:
            raise ValueError('Unrecognized TeX command {}'.format(command))
        self.command = command
        self.target_dir = os.path.join(base_dir, target_dir)
        self.source_dir = os.path.join(base_dir, source_dir)
        self.files = files

    def abuild_file(self, file, cwd='.', env=os.environ):
        """
        Start a build of a single file and return the running process wrapper

        :param file: file to build
        :return:
        """
        source = os.path.join(self.source_dir, file)

        if not os.path.exists(self.target_dir):
            os.makedirs(self.target_dir)

        return file, misc.Popen(
            (
                self.command,
                '-halt-on-error',
                '-interaction=nonstopmode',
                '-output-directory', self.target_dir,
                source
            ),
            env=env,
            cwd=cwd
        )

    def abuild(self, cwd='.', env=os.environ):
        """
        Start this build and return a tuple of the running processes

        :return:
        """
        if not os.path.exists(self.target_dir):
            os.makedirs(self.target_dir)
        return tuple(self.abuild_file(file, cwd=cwd) for file in self.files)

class Include:
    """
    Abstract include
    """
    def __init__(self, repository=None, directory=None, target_dir=None):
        self.repository = repository
        self.directory = directory
        self.target_dir = target_dir

    @classmethod
    def from_config(cls, git_url, directory=None, target_dir=None):
        """
        Alternative constructor from the config dict:

        :param git_url: url
        :param directory:
        :param target_dir:
        :return:
        """
        return cls(
            github.GitRepository.from_url(git_url),
            directory,
            target_dir
        )

    @classmethod
    def relative_to(
            cls,
            base_dir,
            git_url=None,
            directory=None,
            target_dir=None
    ):
        """
        Alternative constructor from the config arguments
        relative to a directory

        :param base_dir:
        :param git_url:
        :param directory:
        :param target_dir:
        :return:
        """
        return cls.from_config(
            git_url,
            os.path.join(base_dir, directory),
            os.path.join(base_dir, target_dir)
        )

def refresh_includes(includes):
    """
    Pull/clone the include repositories and return those that succeeded

    :param includes:
    :return:
    """
    # calculating pull results
    pull_results = map(
        lambda a: (a[0], a[1].wait()),
        # starting pulls
        tuple((i, i.repository.apull(i.directory)) for i in includes)
    )
    pull_success, pull_failed = partition(lambda a: a[1] == 0, pull_results)
    clone_results = map(
        lambda a: (a[0], a[1].wait()),
        tuple(
            (include, include.repository.aclone(include.directory))
            for include, status in pull_failed)
    )

    clone_success, clone_failed = partition(
        lambda a: a[1] == 0,
        clone_results
    )

    for include, code in clone_failed:
        LOGGER.error(
            'Pull and clone failed for {} with code {}'.format(
                include.repository.name, code
            )
        )

    first_only = lambda a: a[0]

    return tuple(
        itertools.chain(
            map(first_only, pull_success), map(first_only, clone_success)
        )
    )

def read_conf(wd):
    """
    Open the config file and return the json stuff within

    :param wd:
    :return:
    """
    with open(os.path.join(wd, config.CONFIG_NAME)) as file:
        return json.load(file)

def refresh_tex_includes(repo: github.GitRepository, directory):
    repo.refresh(directory)

def build_includes(conf: dict, wd: str, env: dict) -> ((Build, subprocess.Popen), ...):
    includes_folder = os.path.join(wd, conf.get('includes_directory', ''))

    includes = tuple(
        Include.relative_to(includes_folder, **i) for i in conf.get('include', ())
    )

    building = tuple(itertools.chain.from_iterable(
        abuild_directory(include.directory, env=env)
        for include in refresh_includes(includes)
    ))

    finished = finish_builds(building)
    return finished

def handle_env_conf(conf: dict, env: dict) -> dict:
    if 'tex_include' in conf:
        template_dir = conf['tex_include']['directory'].format(**PATHS)
        repo = github.GitRepository.from_url(conf['tex_includes']['clone_url'])
        refresh_tex_includes(repo, template_dir)
        return collections.ChainMap(
            {
                'TEXINPUTS': '.:{}//:'.format(template_dir)
            },
            env
        )
    else:
        return env

def abuild_directory(wd: str, env: dict=os.environ) -> ((Build, subprocess.Popen), ...):
    """
    Asynchronously build whats defined in the build_conf in that directory
    returns an empty tuple if something fails

    :param wd: working directory
    :return: executing builds
    """
    try:
        conf = read_conf(wd)
    except ValueError as e:
        LOGGER.error(
            'Could not parse config {} due to error {}'.format(
                os.path.join(wd, config.CONFIG_NAME), e)
        )
        return ()
    except FileNotFoundError as e:
        LOGGER.error(e)
        return ()

    env = handle_env_conf(conf, env)

    finished_includes = build_includes(conf, wd, env)

    builds = (
        Build(name, base_dir=wd, **b_conf)
        for name, b_conf in conf.get('builds', {}).items()
    )

    # this catches any FileNotFoundError's and logs them so we still build the good configs
    building_own = tuple(catch_abuilds(builds, env=env))

    return finished_includes + building_own

def catch_abuilds(builds, env=os.environ):
    for build in builds:
        try:
            yield build, build.abuild(cwd=build.source_dir, env=env)
        except FileNotFoundError as e:
            LOGGER.critical(
                'Build {} could not execute due to {}'.format(build.name, e)
            )

def output_from_command(file, command):
    """
    Construct the default file name for the output of an executed command

    :param file: name of input file
    :param command: executed command
    :return: string name
    """
    return file.rsplit('.', 1)[0] + '.' + TEX_OUTPUT[command]

def finish_builds(builds):
    """
    Wait for the executing builds or kill them

    :param builds:
    :return:
    """

    for file, process in builds:
        if process.poll() is None:
            try:
                process.wait(config.BUILD_TIMEOUT)
            except subprocess.TimeoutExpired:
                process.kill()
    return builds

def build_and_report(wd):
    """
    Build a directory and return some printable information on how it went

    :param wd: working directory
    :return:
    """
    building = abuild_directory(wd)

    def format_status(process):
        code = process.returncode
        if code == 0:
            return code
        else:
            return str(code) + misc.error_capture_format(
                ('stdout', process.stdout.read().decode() if process.stdout else ''),
                ('stderr', process.stderr.read().decode() if process.stderr else '')
            )

    def print_finished(builds):
        """
        Transform the finished builds into some useful printable information
        about their status

        :param builds:
        :return:
        """
        return '\n'.join(
            'Build {} with {} files:\n      {}'.format(
                build.name, len(building_files), '\n      '.join(
                    '{}   ->   {}    with code {}'.format(
                        file,
                        output_from_command(file, build.command),
                        format_status(process)
                    )
                    for file, process in (
                        finish_builds(building_files)
                    )
                )
            )
            for build, building_files in builds
        )

    return print_finished(building)
<commit_msg>fixed one minor logger formatting error<commit_after>"""build script for our lesson repositories"""
import json
import os
import itertools
import logging
import subprocess
import collections
from . import github, config, misc

LOGGER = logging.getLogger(__name__)
TEX_OUTPUT = {
    'htlatex': 'html',
    'pdflatex': 'pdf',
    'xelatex': 'pdf',
    'hevea': 'html'
}
PATHS = {
    'app': config.BASE_DIRECTORY
}

def partition(condition, iterable, output_class=tuple):
    """
    split an iterable into two according to a function evaluating to either
     true or false on each element

    :param condition: boolean function
    :param iterable: iterable to split
    :param output_class: type of the returned iterables
    :return: two iterables
    """
    true = []
    false = []

    for i in iterable:
        if condition(i):
            true.append(i)
        else:
            false.append(i)

    return output_class(true), output_class(false)

class Build:
    """
    Abstracts builds
    """
    def __init__(self, name, command, target_dir, source_dir, files, base_dir=''):
        self.name = name
        if command not in TEX_OUTPUT:
            raise ValueError('Unrecognized TeX command {}'.format(command))
        self.command = command
        self.target_dir = os.path.join(base_dir, target_dir)
        self.source_dir = os.path.join(base_dir, source_dir)
        self.files = files

    def abuild_file(self, file, cwd='.', env=os.environ):
        """
        Start a build of a single file and return the running process wrapper

        :param file: file to build
        :return:
        """
        source = os.path.join(self.source_dir, file)

        if not os.path.exists(self.target_dir):
            os.makedirs(self.target_dir)

        return file, misc.Popen(
            (
                self.command,
                '-halt-on-error',
                '-interaction=nonstopmode',
                '-output-directory', self.target_dir,
                source
            ),
            env=env,
            cwd=cwd
        )

    def abuild(self, cwd='.', env=os.environ):
        """
        Start this build and return a tuple of the running processes

        :return:
        """
        if not os.path.exists(self.target_dir):
            os.makedirs(self.target_dir)
        return tuple(self.abuild_file(file, cwd=cwd) for file in self.files)

class Include:
    """
    Abstract include
    """
    def __init__(self, repository=None, directory=None, target_dir=None):
        self.repository = repository
        self.directory = directory
        self.target_dir = target_dir

    @classmethod
    def from_config(cls, git_url, directory=None, target_dir=None):
        """
        Alternative constructor from the config dict:

        :param git_url: url
        :param directory:
        :param target_dir:
        :return:
        """
        return cls(
            github.GitRepository.from_url(git_url),
            directory,
            target_dir
        )

    @classmethod
    def relative_to(
            cls,
            base_dir,
            git_url=None,
            directory=None,
            target_dir=None
    ):
        """
        Alternative constructor from the config arguments
        relative to a directory

        :param base_dir:
        :param git_url:
        :param directory:
        :param target_dir:
        :return:
        """
        return cls.from_config(
            git_url,
            os.path.join(base_dir, directory),
            os.path.join(base_dir, target_dir)
        )

def refresh_includes(includes):
    """
    Pull/clone the include repositories and return those that succeeded

    :param includes:
    :return:
    """

    wait = lambda a: (a[0], a[1].wait())

    code_is_0 = lambda a: a[1] == 0

    # calculating pull results
    pull_results = map(
        wait,
        # starting pulls
        tuple((i, i.repository.apull(i.directory)) for i in includes)
    )
    pull_success, pull_failed = partition(code_is_0, pull_results)
    clone_results = map(
        wait,
        tuple(
            (include, include.repository.aclone(include.directory))
            for include, status in pull_failed
        )
    )

    clone_success, clone_failed = partition(
        code_is_0,
        clone_results
    )

    for include, code in clone_failed:
        LOGGER.error(
            'Pull and clone failed for {} with code {}'.format(
                include.repository.name, code
            )
        )

    first_only = lambda a: a[0]

    return tuple(
        itertools.chain(
            map(first_only, pull_success), map(first_only, clone_success)
        )
    )

def read_conf(wd):
    """
    Open the config file and return the json stuff within

    :param wd:
    :return:
    """
    with open(os.path.join(wd, config.CONFIG_NAME)) as file:
        return json.load(file)

def refresh_tex_includes(repo: github.GitRepository, directory):
    repo.refresh(directory)

def build_includes(conf: dict, wd: str, env: dict) -> ((Build, subprocess.Popen), ...):
    includes_folder = os.path.join(wd, conf.get('includes_directory', ''))

    includes = tuple(
        Include.relative_to(includes_folder, **i) for i in conf.get('include', ())
    )

    building = tuple(itertools.chain.from_iterable(
        abuild_directory(include.directory, env=env)
        for include in refresh_includes(includes)
    ))

    finished = finish_builds(building)
    return finished

def handle_env_conf(conf: dict, env: dict) -> dict:
    if 'tex_include' in conf:
        template_dir = conf['tex_include']['directory'].format(**PATHS)
        repo = github.GitRepository.from_url(conf['tex_includes']['clone_url'])
        refresh_tex_includes(repo, template_dir)
        return collections.ChainMap(
            {
                'TEXINPUTS': '.:{}//:'.format(template_dir)
            },
            env
        )
    else:
        return env

def abuild_directory(wd: str, env: dict=os.environ) -> ((Build, subprocess.Popen), ...):
    """
    Asynchronously build whats defined in the build_conf in that directory
    returns an empty tuple if something fails

    :param wd: working directory
    :return: executing builds
    """
    try:
        conf = read_conf(wd)
    except ValueError as e:
        LOGGER.error(
            'Could not parse config {} due to error {}'.format(
                os.path.join(wd, config.CONFIG_NAME), e)
        )
        return ()
    except FileNotFoundError as e:
        LOGGER.error(e)
        return ()

    env = handle_env_conf(conf, env)

    finished_includes = build_includes(conf, wd, env)

    builds = (
        Build(name, base_dir=wd, **b_conf)
        for name, b_conf in conf.get('builds', {}).items()
    )

    # this catches any FileNotFoundError's and logs them so we still build the good configs
    building_own = tuple(catch_abuilds(builds, env=env))

    return finished_includes + building_own

def catch_abuilds(builds, env=os.environ):
    for build in builds:
        try:
            yield build, build.abuild(cwd=build.source_dir, env=env)
        except FileNotFoundError as e:
            LOGGER.critical(
                'Build {} could not execute due to {}'.format(build.name, e)
            )

def output_from_command(file, command):
    """
    Construct the default file name for the output of an executed command

    :param file: name of input file
    :param command: executed command
    :return: string name
    """
    return file.rsplit('.', 1)[0] + '.' + TEX_OUTPUT[command]

def finish_builds(builds):
    """
    Wait for the executing builds or kill them

    :param builds:
    :return:
    """

    for file, process in builds:
        if process.poll() is None:
            try:
                process.wait(config.BUILD_TIMEOUT)
            except subprocess.TimeoutExpired:
                process.kill()
    return builds

def build_and_report(wd):
    """
    Build a directory and return some printable information on how it went

    :param wd: working directory
    :return:
    """
    building = abuild_directory(wd)

    def format_status(process):
        code = process.returncode
        if code == 0:
            return code
        else:
            return '{}\n{}'.format(
                code, misc.error_capture_format(
                    ('stdout', process.stdout.read().decode() if process.stdout else ''),
                    ('stderr', process.stderr.read().decode() if process.stderr else '')
                )
            )

    def print_finished(builds):
        """
        Transform the finished builds into some useful printable information
        about their status

        :param builds:
        :return:
        """
        return '\n'.join(
            'Build {} with {} files:\n      {}'.format(
                build.name, len(building_files), '\n      '.join(
                    '{}   ->   {}    with code {}'.format(
                        file,
                        output_from_command(file, build.command),
                        format_status(process)
                    )
                    for file, process in (
                        finish_builds(building_files)
                    )
                )
            )
            for build, building_files in builds
        )

    return print_finished(building)
<|endoftext|>


<commit_before>from flask import session, render_template, Blueprint
from lexos.helpers import constants
from lexos.views.base_view import detect_active_docs
from lexos.models.filemanager_model import FileManagerModel

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
bct_analysis_blueprint = Blueprint('bct_analysis', __name__)

@bct_analysis_blueprint.route("/bct_analysis", methods=['GET'])
def bct_analysis():
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    # Get labels with their ids.
    id_label_map = \
        FileManagerModel().load_file_manager().get_active_labels_with_id()

    # Fill in default options.
    if 'analyoption' not in session:
        session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
    if 'hierarchyoption' not in session:
        session['bctoption'] = constants.DEFAULT_BCT_OPTIONS

    # Render the HTML template.
    return render_template(
        'bct_analysis.html',
        itm="bct-analysis",
        labels=id_label_map,
        numActiveDocs=num_active_docs
    )

@bct_analysis_blueprint.route("/bct_analysis", methods=['POST'])
def get_bct_result():
<commit_msg>Get the ajax call<commit_after>from flask import session, render_template, Blueprint
from lexos.helpers import constants
from lexos.managers import session_manager
from lexos.models.bct_model import BCTModel
from lexos.views.base_view import detect_active_docs
from lexos.models.filemanager_model import FileManagerModel

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
bct_analysis_blueprint = Blueprint('bct_analysis', __name__)

@bct_analysis_blueprint.route("/bct_analysis", methods=['GET'])
def bct_analysis():
    """Display the web page when first got to bootstrap consensus analysis.

    :return: The rendered template.
    """
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    # Get labels with their ids.
    id_label_map = \
        FileManagerModel().load_file_manager().get_active_labels_with_id()

    # Fill in default options.
    if 'analyoption' not in session:
        session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
    if 'bctoption' not in session:
        session['bctoption'] = constants.DEFAULT_BCT_OPTIONS

    # Render the HTML template.
    return render_template(
        'bct_analysis.html',
        itm="bct-analysis",
        labels=id_label_map,
        numActiveDocs=num_active_docs
    )

@bct_analysis_blueprint.route("/bct_analysis_result", methods=['POST'])
def get_bct_result():
    """Send the BCT result to frontend

    :return: I don't know yet.
    """
    # Cache all the options.
    session_manager.cache_bct_option()
    session_manager.cache_analysis_option()
    # Get the bootstrap consensus tree result.
    return BCTModel().get_bootstrap_consensus_tree()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8; -*-
#
# Copyright 2012 Juliano Martinez
# Copyright 2012 Diego Souza
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# @author: Juliano Martinez
# @author: Diego Souza

import time
from datetime import datetime
from datetime import timedelta
from leela import funcs
from leela import config
from leela.lasergun.data import Lasergun

def dump_day3(config, cassandra, hostname, service, date):
    laser = Lasergun(config, cassandra)
    data  = laser.retrieve(hostname, service, funcs.datetime_timestamp(date))
    f     = lambda slot: funcs.slot_to_timestamp(date.year, date.month, date.day, slot)
    return(funcs.service_map_slot(f, data))

def dump_day(config, cassandra, hostname, service):
    return(dump_day3(config, cassandra, hostname, service, datetime.today()))

def dump_last24(config, cassandra, hostname, service):
    today          = datetime.today()
    yesterday      = today - timedelta(days=1)
    today_data     = dump_day3(config, cassandra, hostname, service, today)
    yesterday_data = dump_day3(config, cassandra, hostname, service, yesterday)
    mintime        = time.time() - (24.0 * 60 * 60)
    f = lambda k, v, acc: funcs.dict_update(funcs.dict_merge, acc, k, v)
    g = lambda kv: dict([(k,v) for (k,v) in kv.iteritems() if (k > mintime)])
    return(funcs.service_reduce(f, g, yesterday_data, dict(today_data)))
<commit_msg>bugfix: missing carbon argument<commit_after>#!/usr/bin/python
# -*- coding: utf-8; -*-
#
# Copyright 2012 Juliano Martinez
# Copyright 2012 Diego Souza
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# @author: Juliano Martinez
# @author: Diego Souza

import time
from datetime import datetime
from datetime import timedelta
from leela import funcs
from leela import config
from leela.lasergun.data import Lasergun

def dump_day3(config, cassandra, hostname, service, date):
    laser = Lasergun(config, cassandra=cassandra, carbon=None)
    data  = laser.retrieve(hostname, service, funcs.datetime_timestamp(date))
    f     = lambda slot: funcs.slot_to_timestamp(date.year, date.month, date.day, slot)
    return(funcs.service_map_slot(f, data))

def dump_day(config, cassandra, hostname, service):
    return(dump_day3(config, cassandra, hostname, service, datetime.today()))

def dump_last24(config, cassandra, hostname, service):
    today          = datetime.today()
    yesterday      = today - timedelta(days=1)
    today_data     = dump_day3(config, cassandra, hostname, service, today)
    yesterday_data = dump_day3(config, cassandra, hostname, service, yesterday)
    mintime        = time.time() - (24.0 * 60 * 60)
    f = lambda k, v, acc: funcs.dict_update(funcs.dict_merge, acc, k, v)
    g = lambda kv: dict([(k,v) for (k,v) in kv.iteritems() if (k > mintime)])
    return(funcs.service_reduce(f, g, yesterday_data, dict(today_data)))
<|endoftext|>


<commit_before>import scrapy
from grouch.spiders.oscar_spider import OscarSpider
import unittest
import CS1332

class TestOscarSpider(unittest.TestCase):
    def setUp(self):
        self.response = scrapy.http.TextResponse("",body=CS1332.resp)
        self.spider = OscarSpider()

    def test_parse_detail(self):
        item = self.spider.parse_detail(self.response)
        self.assertEqual(item['fields'], ["Grade Basis", "Restrictions", "Prerequisites"])
        self.assertEqual(item['fullname'][0], "CS 1332 - Data Struct & Algorithms")
        self.assertEqual(item['name'][0], "Data Struct & Algorithms")
        self.assertEqual(item['school'][0], "CS")
        self.assertEqual(item['number'][0], "1332")
<commit_msg>Fixed a version 2/3 error in the test file<commit_after>import scrapy
import unittest
from grouch.spiders.oscar_spider import OscarSpider
from grouch.test import CS1332

class TestOscarSpider(unittest.TestCase):
    def setUp(self):
        self.response = scrapy.http.TextResponse("",body=CS1332.resp)
        self.spider = OscarSpider()

    def test_parse_detail(self):
        item = self.spider.parse_detail(self.response)
        self.assertEqual(item['fields'], ["Grade Basis", "Restrictions", "Prerequisites"])
        self.assertEqual(item['fullname'][0], "CS 1332 - Data Struct & Algorithms")
        self.assertEqual(item['name'][0], "Data Struct & Algorithms")
        self.assertEqual(item['school'][0], "CS")
        self.assertEqual(item['number'][0], "1332")
<|endoftext|>


<commit_before># coding: utf-8
import copy
import json
from typing import Union, Iterator

from bson import json_util
from django.db.models.query import QuerySet
from django.db import transaction
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import PermissionDenied
from rest_framework import serializers
from rest_framework.pagination import _positive_int as positive_int

from kpi.constants import (
    INSTANCE_FORMAT_TYPE_XML,
    INSTANCE_FORMAT_TYPE_JSON,
    PERM_PARTIAL_SUBMISSIONS,
    PERM_VIEW_SUBMISSIONS,
)
from kpi.exceptions import AbstractMethodError
from kpi.interfaces.sync_backend_media import SyncBackendMediaInterface
from kpi.models.asset_file import AssetFile
from kpi.models.paired_data import PairedData
from kpi.utils.jsonbfield_helper import ReplaceValues

class BaseDeploymentBackend:

    INSTANCE_ID_FIELDNAME = '_id'
    STATUS_SYNCED = 'synced'
    STATUS_NOT_SYNCED = 'not-synced'

    def __init__(self, asset):
        self.asset = asset
        # Python-only attribute used by `kpi.views.v2.data.DataViewSet.list()`
        self.current_submissions_count = 0

    @property
    def active(self):
        return self.get_data('active', False)

    def connect(self, active=False):
        raise AbstractMethodError

    @property
    def backend(self):
        return self.get_data('backend', None)

    @property
    def backend_response(self):
        return self.get_data('backend_response', {})

    def bulk_assign_mapped_perms(self):
        raise AbstractMethodError

    def bulk_update_submissions(self):
        raise AbstractMethodError

    def calculated_submission_count(self, user: 'auth.User', **kwargs):
        raise AbstractMethodError

    def delete(self):
        self.asset._deployment_data.clear()  # noqa

    def get_data(self,
                 dotted_path: str = None,
                 default=None) -> Union[None, int, str, dict]:
        """
        Access `self.asset._deployment_data` and return corresponding value of
        `dotted_path` if it exists. Otherwise, it returns `default`.
        If `dotted_path` is not provided, it returns the whole
        dictionary.
        """
        if not dotted_path:
            # We do not want to return the mutable object whose could be altered
            # later. `self.asset._deployment_data` should never be accessed
            # directly
            return copy.deepcopy(self.asset._deployment_data) # noqa

        value = None
        nested_path = dotted_path.split('.')
        nested_dict = self.asset._deployment_data  # noqa
        for key in nested_path:
            try:
                value = nested_dict[key]
            except KeyError:
                value = None
                break

            nested_dict = value

        return value if value else default

    def delete_submission(self, pk: int, user: 'auth.User') -> dict:
        raise AbstractMethodError

    def duplicate_submission(
            self, user: 'auth.User', instance_id: int, **kwargs: dict
    ) -> dict:
        raise AbstractMethodError

    def get_data_download_links(self):
        raise AbstractMethodError

    def get_enketo_survey_links(self):
        raise AbstractMethodError

    def get_submission(self,
                       pk: int,
                       user: 'auth.User',
                       format_type: str = INSTANCE_FORMAT_TYPE_JSON,
                       **kwargs: dict) -> Union[dict, str, None]:
        """
        Returns the corresponding submission whose id equals `pk` and which
        `user` is allowed to access.
        Otherwise, it returns `None`.
        The format `format_type` can be either:
        - 'json' (See `kpi.constants.INSTANCE_FORMAT_TYPE_JSON)
        - 'xml' (See `kpi.constants.INSTANCE_FORMAT_TYPE_XML)

        MongoDB filters can be passed through `kwargs` to narrow down the
        result.
        """

        submissions = list(self.get_submissions(user,
                                                format_type, [int(pk)],
                                                **kwargs))
        try:
            return submissions[0]
        except IndexError:
            pass
        return None

    def get_submission_detail_url(self, submission_pk):
        raise AbstractMethodError

    def get_submission_edit_url(self, submission_pk, user, params=None):
        raise AbstractMethodError

    def get_submission_validation_status_url(self, submission_pk):
        # This doesn't really need to be implemented.
        # We keep it to stay close to `KobocatDeploymentBackend`
        url = '{detail_url}validation_status/'.format(
            detail_url=self.get_submission_detail_url(submission_pk)
        )
        return url

    def get_submissions(self,
                        user: 'auth.User',
                        format_type: str = INSTANCE_FORMAT_TYPE_JSON,
                        instance_ids: list = [],
                        **kwargs: dict) -> [Iterator[dict], Iterator[str]]:
        raise AbstractMethodError

    def get_validation_status(self, submission_pk, params, user):
        submission = self.get_submission(
            submission_pk,
            user=user,
            format_type=INSTANCE_FORMAT_TYPE_JSON,
        )
        return {
            "data": submission.get("_validation_status")
        }

    @property
    def identifier(self):
        return self.get_data('identifier', None)

    @property
    def last_submission_time(self):
        return self._last_submission_time()

    @property
    def mongo_userform_id(self):
        return None

    def set_active(self, active):
        raise AbstractMethodError

    def set_has_kpi_hooks(self):
        raise AbstractMethodError

    def set_status(self, status):

    def redeploy(self, active=None):
        raise AbstractMethodError

    def remove_from_kc_only_flag(self, *args, **kwargs):
        # TODO: This exists only to support KoBoCAT (see #1161) and should be
        # removed, along with all places where it is called, once we remove
        # KoBoCAT's ability to assign permissions (kobotoolbox/kobocat#642)

        # Do nothing, without complaint, so that callers don't have to worry
        # about whether the back end is KoBoCAT or something else
        pass

    def save_to_db(self, updates: dict):
        """
        Persist values from deployment data into the DB.
        `updates` is a dictionary of properties to update.
        E.g: `{"active": True, "status": "not-synced"}`
        """
        # Avoid circular imports
        # use `self.asset.__class__` instead of `from kpi.models import Asset`
        now = timezone.now()
        with transaction.atomic():
            self.asset.__class__.objects.select_for_update() \
                .filter(id=self.asset.pk).update(
                _deployment_data=ReplaceValues(
                    '_deployment_data',
                    updates=updates,
                ),
                date_modified=now,
            )
        self.store_data(updates)
        self.asset.date_modified = now

    def set_asset_uid(self, **kwargs) -> bool:
        raise AbstractMethodError

    def set_status(self, status):
        self.save_to_db({'status': status})

    def set_validation_status(self,
                              submission_pk: int,
                              data: dict,
                              user: 'auth.User',
                              method: str) -> dict:
        raise AbstractMethodError

    def set_validation_statuses(self, data: dict, user: 'auth.User') -> dict:
        raise AbstractMethodError

    @property
    def status(self):
        return self.get_data('status')

    def store_data(self, vals=None):
        self.asset._deployment_data.update(vals)  # noqa

    @property
    def submission_count(self):
        return self._submission_count()

    @property
    def submission_list_url(self):
        raise AbstractMethodError

    def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA):
        raise AbstractMethodError

    def validate_submission_list_params(
        self,
        user: 'auth.User',
        format_type: str = INSTANCE_FORMAT_TYPE_JSON,
        validate_count: bool = False,
        **kwargs: dict
    ) -> dict:
        """
        Validates parameters (`kwargs`) to be passed to Mongo.
        parameters can be:
            - start
            - limit
            - sort
            - fields
            - query
            - instance_ids
        If `validate_count` is True,`start`, `limit`, `fields` and `sort` are
        ignored.

        If `user` has partial permissions, conditions are
        applied to the query to narrow down results to what they are allowed
        to see.
        """

        if 'count' in kwargs:
            raise serializers.ValidationError(
                {
                    'count': _(
                        'This param is not implemented. Use `count` property '
                        'of the response instead.'
                    )
                }
            )

        if validate_count is False and format_type == INSTANCE_FORMAT_TYPE_XML:
            if 'sort' in kwargs:
                # FIXME. Use Mongo to sort data and ask PostgreSQL to follow the order  # noqa
                # See. https://stackoverflow.com/a/867578
                raise serializers.ValidationError({
                    'sort': _('This param is not supported in `XML` format')
                })

            if 'fields' in kwargs:
                raise serializers.ValidationError({
                    'fields': _('This is not supported in `XML` format')
                })

        start = kwargs.get('start', 0)
        limit = kwargs.get('limit')
        sort = kwargs.get('sort', {})
        fields = kwargs.get('fields', [])
        query = kwargs.get('query', {})
        instance_ids = kwargs.get('instance_ids', [])

        # TODO: Should this validation be in (or called directly by) the view
        # code? Does DRF have a validator for GET params?

        if isinstance(query, str):
            try:
                query = json.loads(query, object_hook=json_util.object_hook)
            except ValueError:
                raise serializers.ValidationError(
                    {'query': _('Value must be valid JSON.')}
                )

        if not isinstance(instance_ids, list):
            raise serializers.ValidationError(
                {'instance_ids': _('Value must be a list.')}
            )

        # This error should not be returned as `ValidationError` to user.
        # We want to return a 500.
        try:
            partial_perm = kwargs.pop('partial_perm', PERM_VIEW_SUBMISSIONS)
            permission_filters = self.asset.get_filters_for_partial_perm(
                user.pk, perm=partial_perm)
        except ValueError:
            raise ValueError(_('Invalid `user_id` param'))

        if validate_count:
            return {
                'query': query,
                'instance_ids': instance_ids,
                'permission_filters': permission_filters
            }

        if isinstance(sort, str):
            try:
                sort = json.loads(sort, object_hook=json_util.object_hook)
            except ValueError:
                raise serializers.ValidationError(
                    {'sort': _('Value must be valid JSON.')}
                )

        try:
            start = positive_int(start)
        except ValueError:
            raise serializers.ValidationError(
                {'start': _('A positive integer is required.')}
            )
        try:
            if limit is not None:
                limit = positive_int(limit, strict=True)
        except ValueError:
            raise serializers.ValidationError(
                {'limit': _('A positive integer is required.')}
            )

        if isinstance(fields, str):
            try:
                fields = json.loads(fields, object_hook=json_util.object_hook)
            except ValueError:
                raise serializers.ValidationError(
                    {'fields': _('Value must be valid JSON.')}
                )

        params = {
            'query': query,
            'start': start,
            'fields': fields,
            'sort': sort,
            'instance_ids': instance_ids,
            'permission_filters': permission_filters
        }

        if limit:
            params['limit'] = limit

        return params

    def validate_write_access_with_partial_perms(self,
                                                 user: 'auth.User',
                                                 perm: str,
                                                 instance_ids: list):
        """
        Validate whether `user` is allowed to perform write actions on
        submissions with the permission `perm`.
        It raises a `PermissionDenied` error if they cannot.

        No validation is made whether `user` is granted with other permissions
        than 'partial_submission' permission.
        """
        if PERM_PARTIAL_SUBMISSIONS not in self.get_perms(user):
            return

        results = self.get_submissions(
            requesting_user_id=user.pk,
            format_type=INSTANCE_FORMAT_TYPE_JSON,
            partial_perm=perm,
            fields=[self.INSTANCE_ID_FIELDNAME],
            instance_ids=instance_ids,
        )
        allowed_instance_ids = [r[self.INSTANCE_ID_FIELDNAME] for r in results]

        if sorted(allowed_instance_ids) != sorted(instance_ids):
            raise PermissionDenied

    @property
    def version(self):
        raise NotImplementedError('Use `asset.deployment.version_id`')

    @property
    def version_id(self):
        return self.get_data('version')

    def _get_metadata_queryset(self, file_type: str) -> Union[QuerySet, list]:
        """
        Returns a list of objects, or a QuerySet to pass to Celery to
        synchronize with the backend.
        Can be used inside the implementation of `sync_media_files()`
        """
        if file_type == AssetFile.FORM_MEDIA:
            # Order by `date_deleted` to process deleted files first in case
            # two entries contain the same file but one is flagged as deleted
            return self.asset.asset_files.filter(
                file_type=AssetFile.FORM_MEDIA
            ).order_by('date_deleted')
        else:
            queryset = PairedData.objects(self.asset).values()
            return queryset
<commit_msg>Fixed bad merge<commit_after># coding: utf-8
import copy
import json
from typing import Union, Iterator

from bson import json_util
from django.db.models.query import QuerySet
from django.db import transaction
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import PermissionDenied
from rest_framework import serializers
from rest_framework.pagination import _positive_int as positive_int

from kpi.constants import (
    INSTANCE_FORMAT_TYPE_XML,
    INSTANCE_FORMAT_TYPE_JSON,
    PERM_PARTIAL_SUBMISSIONS,
    PERM_VIEW_SUBMISSIONS,
)
from kpi.exceptions import AbstractMethodError
from kpi.models.asset_file import AssetFile
from kpi.models.paired_data import PairedData
from kpi.utils.jsonbfield_helper import ReplaceValues

class BaseDeploymentBackend:
    INSTANCE_ID_FIELDNAME = '_id'
    STATUS_SYNCED = 'synced'
    STATUS_NOT_SYNCED = 'not-synced'

    def __init__(self, asset):
        self.asset = asset
        # Python-only attribute used by `kpi.views.v2.data.DataViewSet.list()`
        self.current_submissions_count = 0

    @property
    def active(self):
        return self.get_data('active', False)

    def connect(self, active=False):
        raise AbstractMethodError

    @property
    def backend(self):
        return self.get_data('backend', None)

    @property
    def backend_response(self):
        return self.get_data('backend_response', {})

    def bulk_assign_mapped_perms(self):
        raise AbstractMethodError

    def bulk_update_submissions(self):
        raise AbstractMethodError

    def calculated_submission_count(self, user: 'auth.User', **kwargs):
        raise AbstractMethodError

    def delete(self):
        self.asset._deployment_data.clear()  # noqa

    def get_data(
        self, dotted_path: str = None, default=None
    ) -> Union[None, int, str, dict]:
        """
        Access `self.asset._deployment_data` and return corresponding value of
        `dotted_path` if it exists. Otherwise, it returns `default`.
        If `dotted_path` is not provided, it returns the whole
        dictionary.
        """
        if not dotted_path:
            # We do not want to return the mutable object whose could be altered
            # later. `self.asset._deployment_data` should never be accessed
            # directly
            return copy.deepcopy(self.asset._deployment_data)  # noqa

        value = None
        nested_path = dotted_path.split('.')
        nested_dict = self.asset._deployment_data  # noqa
        for key in nested_path:
            try:
                value = nested_dict[key]
            except KeyError:
                value = None
                break

            nested_dict = value

        return value if value else default

    def delete_submission(self, pk: int, user: 'auth.User') -> dict:
        raise AbstractMethodError

    def duplicate_submission(
            self, user: 'auth.User', instance_id: int, **kwargs: dict
    ) -> dict:
        raise AbstractMethodError

    def get_data_download_links(self):
        raise AbstractMethodError

    def get_enketo_survey_links(self):
        raise AbstractMethodError

    def get_submission(self,
                       pk: int,
                       user: 'auth.User',
                       format_type: str = INSTANCE_FORMAT_TYPE_JSON,
                       **kwargs: dict) -> Union[dict, str, None]:
        """
        Returns the corresponding submission whose id equals `pk` and which
        `user` is allowed to access.
        Otherwise, it returns `None`.
        The format `format_type` can be either:
        - 'json' (See `kpi.constants.INSTANCE_FORMAT_TYPE_JSON)
        - 'xml' (See `kpi.constants.INSTANCE_FORMAT_TYPE_XML)

        MongoDB filters can be passed through `kwargs` to narrow down the
        result.
        """

        submissions = list(self.get_submissions(user,
                                                format_type, [int(pk)],
                                                **kwargs))
        try:
            return submissions[0]
        except IndexError:
            pass
        return None

    def get_submission_detail_url(self, submission_pk):
        raise AbstractMethodError

    def get_submission_edit_url(self, submission_pk, user, params=None):
        raise AbstractMethodError

    def get_submission_validation_status_url(self, submission_pk):
        # This doesn't really need to be implemented.
        # We keep it to stay close to `KobocatDeploymentBackend`
        url = '{detail_url}validation_status/'.format(
            detail_url=self.get_submission_detail_url(submission_pk)
        )
        return url

    def get_submissions(self,
                        user: 'auth.User',
                        format_type: str = INSTANCE_FORMAT_TYPE_JSON,
                        instance_ids: list = [],
                        **kwargs: dict) -> [Iterator[dict], Iterator[str]]:
        raise AbstractMethodError

    def get_validation_status(self, submission_pk, params, user):
        submission = self.get_submission(
            submission_pk,
            user=user,
            format_type=INSTANCE_FORMAT_TYPE_JSON,
        )
        return {
            "data": submission.get("_validation_status")
        }

    @property
    def identifier(self):
        return self.get_data('identifier', None)

    @property
    def last_submission_time(self):
        return self._last_submission_time()

    @property
    def mongo_userform_id(self):
        return None

    def redeploy(self, active=None):
        raise AbstractMethodError

    def remove_from_kc_only_flag(self, *args, **kwargs):
        # TODO: This exists only to support KoBoCAT (see #1161) and should be
        # removed, along with all places where it is called, once we remove
        # KoBoCAT's ability to assign permissions (kobotoolbox/kobocat#642)

        # Do nothing, without complaint, so that callers don't have to worry
        # about whether the back end is KoBoCAT or something else
        pass

    def save_to_db(self, updates: dict):
        """
        Persist values from deployment data into the DB.
        `updates` is a dictionary of properties to update.
        E.g: `{"active": True, "status": "not-synced"}`
        """
        # Avoid circular imports
        # use `self.asset.__class__` instead of `from kpi.models import Asset`
        now = timezone.now()
        with transaction.atomic():
            self.asset.__class__.objects.select_for_update() \
                .filter(id=self.asset.pk).update(
                _deployment_data=ReplaceValues(
                    '_deployment_data',
                    updates=updates,
                ),
                date_modified=now,
            )
        self.store_data(updates)
        self.asset.date_modified = now

    def set_active(self, active):
        raise AbstractMethodError

    def set_asset_uid(self, **kwargs) -> bool:
        raise AbstractMethodError

    def set_has_kpi_hooks(self):
        raise AbstractMethodError

    def set_status(self, status):
        self.save_to_db({'status': status})

    def set_validation_status(self,
                              submission_pk: int,
                              data: dict,
                              user: 'auth.User',
                              method: str) -> dict:
        raise AbstractMethodError

    def set_validation_statuses(self, data: dict, user: 'auth.User') -> dict:
        raise AbstractMethodError

    @property
    def status(self):
        return self.get_data('status')

    def store_data(self, vals=None):
        self.asset._deployment_data.update(vals)  # noqa

    @property
    def submission_count(self):
        return self._submission_count()

    @property
    def submission_list_url(self):
        raise AbstractMethodError

    def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA):
        raise AbstractMethodError

    def validate_submission_list_params(
        self,
        user: 'auth.User',
        format_type: str = INSTANCE_FORMAT_TYPE_JSON,
        validate_count: bool = False,
        **kwargs: dict
    ) -> dict:
        """
        Validates parameters (`kwargs`) to be passed to Mongo.
        parameters can be:
            - start
            - limit
            - sort
            - fields
            - query
            - instance_ids
        If `validate_count` is True,`start`, `limit`, `fields` and `sort` are
        ignored.

        If `user` has partial permissions, conditions are
        applied to the query to narrow down results to what they are allowed
        to see.
        """

        if 'count' in kwargs:
            raise serializers.ValidationError(
                {
                    'count': _(
                        'This param is not implemented. Use `count` property '
                        'of the response instead.'
                    )
                }
            )

        if validate_count is False and format_type == INSTANCE_FORMAT_TYPE_XML:
            if 'sort' in kwargs:
                # FIXME. Use Mongo to sort data and ask PostgreSQL to follow the order  # noqa
                # See. https://stackoverflow.com/a/867578
                raise serializers.ValidationError({
                    'sort': _('This param is not supported in `XML` format')
                })

            if 'fields' in kwargs:
                raise serializers.ValidationError({
                    'fields': _('This is not supported in `XML` format')
                })

        start = kwargs.get('start', 0)
        limit = kwargs.get('limit')
        sort = kwargs.get('sort', {})
        fields = kwargs.get('fields', [])
        query = kwargs.get('query', {})
        instance_ids = kwargs.get('instance_ids', [])

        # TODO: Should this validation be in (or called directly by) the view
        # code? Does DRF have a validator for GET params?

        if isinstance(query, str):
            try:
                query = json.loads(query, object_hook=json_util.object_hook)
            except ValueError:
                raise serializers.ValidationError(
                    {'query': _('Value must be valid JSON.')}
                )

        if not isinstance(instance_ids, list):
            raise serializers.ValidationError(
                {'instance_ids': _('Value must be a list.')}
            )

        # This error should not be returned as `ValidationError` to user.
        # We want to return a 500.
        try:
            partial_perm = kwargs.pop('partial_perm', PERM_VIEW_SUBMISSIONS)
            permission_filters = self.asset.get_filters_for_partial_perm(
                user.pk, perm=partial_perm)
        except ValueError:
            raise ValueError(_('Invalid `user_id` param'))

        if validate_count:
            return {
                'query': query,
                'instance_ids': instance_ids,
                'permission_filters': permission_filters
            }

        if isinstance(sort, str):
            try:
                sort = json.loads(sort, object_hook=json_util.object_hook)
            except ValueError:
                raise serializers.ValidationError(
                    {'sort': _('Value must be valid JSON.')}
                )

        try:
            start = positive_int(start)
        except ValueError:
            raise serializers.ValidationError(
                {'start': _('A positive integer is required.')}
            )
        try:
            if limit is not None:
                limit = positive_int(limit, strict=True)
        except ValueError:
            raise serializers.ValidationError(
                {'limit': _('A positive integer is required.')}
            )

        if isinstance(fields, str):
            try:
                fields = json.loads(fields, object_hook=json_util.object_hook)
            except ValueError:
                raise serializers.ValidationError(
                    {'fields': _('Value must be valid JSON.')}
                )

        params = {
            'query': query,
            'start': start,
            'fields': fields,
            'sort': sort,
            'instance_ids': instance_ids,
            'permission_filters': permission_filters
        }

        if limit:
            params['limit'] = limit

        return params

    def validate_write_access_with_partial_perms(self,
                                                 user: 'auth.User',
                                                 perm: str,
                                                 instance_ids: list):
        """
        Validate whether `user` is allowed to perform write actions on
        submissions with the permission `perm`.
        It raises a `PermissionDenied` error if they cannot.

        No validation is made whether `user` is granted with other permissions
        than 'partial_submission' permission.
        """
        if PERM_PARTIAL_SUBMISSIONS not in self.get_perms(user):
            return

        results = self.get_submissions(
            user=user,
            format_type=INSTANCE_FORMAT_TYPE_JSON,
            partial_perm=perm,
            fields=[self.INSTANCE_ID_FIELDNAME],
            instance_ids=instance_ids,
        )
        allowed_instance_ids = [r[self.INSTANCE_ID_FIELDNAME] for r in results]

        if sorted(allowed_instance_ids) != sorted(instance_ids):
            raise PermissionDenied

    @property
    def version(self):
        raise NotImplementedError('Use `asset.deployment.version_id`')

    @property
    def version_id(self):
        return self.get_data('version')

    def _get_metadata_queryset(self, file_type: str) -> Union[QuerySet, list]:
        """
        Returns a list of objects, or a QuerySet to pass to Celery to
        synchronize with the backend.
        Can be used inside the implementation of `sync_media_files()`
        """
        if file_type == AssetFile.FORM_MEDIA:
            # Order by `date_deleted` to process deleted files first in case
            # two entries contain the same file but one is flagged as deleted
            return self.asset.asset_files.filter(
                file_type=AssetFile.FORM_MEDIA
            ).order_by('date_deleted')
        else:
            queryset = PairedData.objects(self.asset).values()
            return queryset
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
Unit tests for landlab.model_grid
"""

import unittest
import numpy as np
from numpy.testing import assert_array_equal

from landlab import RasterModelGrid
from landlab.model_grid import BAD_INDEX_VALUE

_SPACING = 1.
(_NUM_ROWS, _NUM_COLS) = (4, 5)

class TestRasterModelGrid(unittest.TestCase):
    def setUp(self):
        """
        These tests use a grid that 4x5 nodes::

         15------16------17------18------19
          |       |       |       |       |
          |       |       |       |       |
          |       |       |       |       |
         10------11------12------13------14
          |       |       |       |       |
          |       |       |       |       |
          |       |       |       |       |
          5-------6-------7-------8-------9
          |       |       |       |       |
          |       |       |       |       |
          |       |       |       |       |
          0-------1-------2-------3-------4
        """
        self.grid = RasterModelGrid(num_rows=_NUM_ROWS, num_cols=_NUM_COLS,
                                    dx=_SPACING)

    def test_create_grid(self):
        """
        Create a raster grid.
        """
        grid = RasterModelGrid(num_rows=4, num_cols=5)

    def test_grid_dimensions(self):
        """
        Check the width of the grid. The x-dimension is the columns, and the
        y-dimension is the rows.
        """
        SPACING = 2.
        (NUM_ROWS, NUM_COLS) = (4, 5)

        grid = RasterModelGrid(num_rows=NUM_ROWS, num_cols=NUM_COLS,
                               dx=SPACING)

        self.assertEqual(grid.get_grid_ydimension(), NUM_ROWS * SPACING)
        self.assertEqual(grid.get_grid_xdimension(), NUM_COLS * SPACING)

    def test_grid_dimensions_default_spacing(self):
        """
        Use the default spacing of 1.
        """
        (NUM_ROWS, NUM_COLS) = (4, 5)

        grid = RasterModelGrid(num_rows=NUM_ROWS, num_cols=NUM_COLS)

        self.assertEqual(grid.get_grid_ydimension(), NUM_ROWS * 1.)
        self.assertEqual(grid.get_grid_xdimension(), NUM_COLS * 1.)

    def test_nodes_around_point(self):
        surrounding_ids = self.grid.get_nodes_around_point(2.1, 1.1)
        self.assertListEqual(
            list(surrounding_ids), [7, 8, 12, 13],
            'incorrect surrounding ids in test_nodes_around_point')

        surrounding_ids = self.grid.get_nodes_around_point(2.1, .9)
        self.assertListEqual(
            list(surrounding_ids), [2, 3, 7, 8],
            'incorrect surrounding ids in test_nodes_around_point')

    def test_neighbor_list(self):
        neighbors = self.grid.get_neighbor_list(id=6)
        self.assertEqual(list(neighbors), [7, 11, 5, 1],
                         'incorrect neighbors in test_neighbor_list')

    def test_neighbor_list_boundary(self):
        """
        All of the neighbor IDs for a boundary cell are -1.
        """
        neighbors = self.grid.get_neighbor_list(id=0)

        self.assertEqual(list(neighbors), [-1, -1, -1, -1])

    def test_cell_x(self):
        expected_x = [0., 1., 2., 3., 4.,
                      0., 1., 2., 3., 4.,
                      0., 1., 2., 3., 4.,
                      0., 1., 2., 3., 4.]

        for (cell_id, expected) in zip(xrange(12), expected_x):
            cell_x = self.grid.get_node_x(cell_id)
            self.assertEqual(cell_x, expected)

            cell_x = self.grid.node_x[cell_id]
            self.assertEqual(cell_x, expected)

    def test_cell_y(self):
        expected_y = [0., 0., 0., 0., 0.,
                      1., 1., 1., 1., 1.,
                      2., 2., 2., 2., 2.,
                      3., 3., 3., 3., 3.]

        for (cell_id, expected) in zip(xrange(12), expected_y):
            cell_y = self.grid.get_node_y(cell_id)
            self.assertEqual(cell_y, expected)

            cell_y = self.grid.node_y[cell_id]
            self.assertEqual(cell_y, expected)

    def test_node_x_coordinates(self):
        x_coords = self.grid.get_node_x_coords()

        self.assertEqual(list(x_coords), [0., 1., 2., 3., 4.,
                                          0., 1., 2., 3., 4.,
                                          0., 1., 2., 3., 4.,
                                          0., 1., 2., 3., 4.])

    def test_cell_y_coordinates(self):
        y_coords = self.grid.get_node_y_coords()

        self.assertEqual(list(y_coords), [0., 0., 0., 0., 0.,
                                          1., 1., 1., 1., 1.,
                                          2., 2., 2., 2., 2.,
                                          3., 3., 3., 3., 3.])

    def test_diagonal_list(self):
        diagonals = self.grid.get_diagonal_list(id=6)
        self.assertEqual(list(diagonals), [12, 10, 0, 2])

    def test_diagonal_list_boundary(self):
        diagonals = self.grid.get_diagonal_list(id=0)
        self.assertEqual(list(diagonals), [-1, -1, -1, -1])

    def test_is_interior(self):
        for cell_id in [0, 1, 2, 3, 4, 5, 9, 10, 14, 15, 16, 17, 18, 19]:
            self.assertFalse(self.grid.is_interior(cell_id))

        for cell_id in [6, 7, 8, 11, 12, 13]:
            self.assertTrue(self.grid.is_interior(cell_id))

    def test_get_interior_cells(self):
        interiors = self.grid.get_active_cell_node_ids()
        self.assertEqual(list(interiors), [6, 7, 8, 11, 12, 13])

    def test_active_links(self):
        self.assertTrue(isinstance(self.grid.active_links, np.ndarray))
        self.assertEqual(self.grid.num_active_links, 17)

        assert_array_equal(self.grid.active_links,
                           np.array([1, 2, 3, 6, 7, 8, 11, 12, 13,
                                     19, 20, 21, 22, 23, 24, 25, 26]))

    def test_active_link_fromnode(self):
        assert_array_equal(self.grid.activelink_fromnode,
                           np.array([1, 2, 3, 6, 7, 8, 11, 12, 13,
                                     5, 6, 7, 8, 10, 11, 12, 13]))

    def test_active_link_tonode(self):
        assert_array_equal(self.grid.activelink_tonode,
                           np.array([6, 7, 8, 11, 12, 13, 16, 17, 18,
                                     6, 7, 8, 9, 11, 12, 13, 14]))

    def test_active_link_num_inlink(self):
        assert_array_equal(self.grid.node_numactiveinlink,
                           np.array([0, 0, 0, 0, 0,
                                     0, 2, 2, 2, 1,
                                     0, 2, 2, 2, 1,
                                     0, 1, 1, 1, 0]))

    def test_active_link_num_outlink(self):
        assert_array_equal(self.grid.node_numactiveoutlink,
                           np.array([0, 1, 1, 1, 0,
                                     1, 2, 2, 2, 0,
                                     1, 2, 2, 2, 0,
                                     0, 0, 0, 0, 0]))

    def test_active_inlink_matrix(self):
        self.assertTupleEqual(self.grid.node_active_inlink_matrix.shape,
                              (2, 20))

        assert_array_equal(self.grid.node_active_inlink_matrix[0],
                           np.array([-1, -1, -1, -1, -1,
                                     -1,  0,  1,  2, 12,
                                     -1,  3,  4,  5, 16,
                                     -1,  6,  7,  8, -1]))
        assert_array_equal(self.grid.node_active_inlink_matrix[1],
                           np.array([-1, -1, -1, -1, -1,
                                     -1,  9, 10, 11, -1,
                                     -1, 13, 14, 15, -1,
                                     -1, -1, -1, -1, -1]))

    def test_active_outlink_matrix(self):
        self.assertTupleEqual(self.grid.node_active_outlink_matrix.shape,
                              (2, 20))

        assert_array_equal(self.grid.node_active_outlink_matrix[0],
                           np.array([-1,  0,  1,  2, -1,
                                      9,  3,  4,  5, -1,
                                     13,  6,  7,  8, -1,
                                     -1, -1, -1, -1, -1]))
        assert_array_equal(self.grid.node_active_outlink_matrix[1],
                           np.array([-1, -1, -1, -1, -1,
                                     -1, 10, 11, 12, -1,
                                     -1, 14, 15, 16, -1,
                                     -1, -1, -1, -1, -1]))

    def test_link_fromnode(self):
        assert_array_equal(self.grid.link_fromnode,
                           np.array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
                                     10, 11, 12, 13, 14,  0,  1,  2,  3,  5,
                                      6,  7,  8, 10, 11, 12, 13, 15, 16, 17,
                                     18]))

    def test_link_tonode(self):
        assert_array_equal(self.grid.link_tonode,
                           np.array([ 5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
                                     15, 16, 17, 18, 19,  1,  2,  3,  4,  6,
                                      7,  8,  9, 11, 12, 13, 14, 16, 17, 18,
                                     19]))

    def test_link_num_inlink(self):
        assert_array_equal(self.grid.node_numinlink,
                           np.array([0, 1, 1, 1, 1,
                                     1, 2, 2, 2, 2,
                                     1, 2, 2, 2, 2,
                                     1, 2, 2, 2, 2]))

    def test_link_num_outlink(self):
        assert_array_equal(self.grid.node_numoutlink,
                           np.array([2, 2, 2, 2, 1,
                                     2, 2, 2, 2, 1,
                                     2, 2, 2, 2, 1,
                                     1, 1, 1, 1, 0]))

    def test_node_inlink_matrix(self):
        self.assertTupleEqual(self.grid.node_inlink_matrix.shape,
                              (2, 20))
        assert_array_equal(self.grid.node_inlink_matrix[0],
                           np.array([-1, 15, 16, 17, 18,
                                      0,  1,  2,  3,  4,
                                      5,  6,  7,  8,  9,
                                     10, 11, 12, 13, 14]))
        assert_array_equal(self.grid.node_inlink_matrix[1],
                           np.array([-1, -1, -1, -1, -1,
                                     -1, 19, 20, 21, 22,
                                     -1, 23, 24, 25, 26,
                                     -1, 27, 28, 29, 30]))

    def test_node_outlink_matrix(self):
        self.assertTupleEqual(self.grid.node_outlink_matrix.shape,
                              (2, 20))
        assert_array_equal(self.grid.node_outlink_matrix[0],
                           np.array([ 0,  1,  2,  3,  4,
                                     5,  6,  7,  8,  9,
                                     10, 11, 12, 13, 14,
                                     27, 28, 29, 30, -1]))
        assert_array_equal(self.grid.node_outlink_matrix[1],
                           np.array([15, 16, 17, 18, -1,
                                     19, 20, 21, 22, -1,
                                     23, 24, 25, 26, -1,
                                     -1, -1, -1, -1, -1]))

    def test_link_face(self):
        from landlab.model_grid import _SLOW
        if _SLOW:
            BAD = None
        else:
            BAD = BAD_INDEX_VALUE

        assert_array_equal(self.grid.link_face,
                           np.array([BAD, 0, 1, 2, BAD,
                                     BAD, 3, 4, 5, BAD,
                                     BAD, 6, 7, 8, BAD,
                                     BAD, BAD, BAD, BAD,
                                     9, 10, 11, 12,
                                     13, 14, 15, 16,
                                     BAD, BAD, BAD, BAD]))

    def test_grid_coords_to_node_id(self):
        self.assertEqual(self.grid.grid_coords_to_node_id(3, 4), 19)
        assert_array_equal(self.grid.grid_coords_to_node_id((3, 2), (4, 1)),
                           np.array([19, 11]))

        with self.assertRaises(ValueError):
            self.grid.grid_coords_to_node_id(5, 0)

if __name__ == '__main__':
    unittest.main()
<commit_msg>* Added unit tests for new ModelGrid method, zeros.<commit_after>#! /usr/bin/env python
"""
Unit tests for landlab.model_grid
"""

import unittest
import numpy as np
from numpy.testing import assert_array_equal

from landlab import RasterModelGrid
from landlab.model_grid import BAD_INDEX_VALUE
from landlab.model_grid import BadCenteringString

_SPACING = 1.
(_NUM_ROWS, _NUM_COLS) = (4, 5)

class TestRasterModelGrid(unittest.TestCase):
    def setUp(self):
        """
        These tests use a grid that 4x5 nodes::

         15------16------17------18------19
          |       |       |       |       |
          |       |       |       |       |
          |       |       |       |       |
         10------11------12------13------14
          |       |       |       |       |
          |       |       |       |       |
          |       |       |       |       |
          5-------6-------7-------8-------9
          |       |       |       |       |
          |       |       |       |       |
          |       |       |       |       |
          0-------1-------2-------3-------4
        """
        self.grid = RasterModelGrid(num_rows=_NUM_ROWS, num_cols=_NUM_COLS,
                                    dx=_SPACING)

    def test_create_grid(self):
        """
        Create a raster grid.
        """
        grid = RasterModelGrid(num_rows=4, num_cols=5)

    def test_grid_dimensions(self):
        """
        Check the width of the grid. The x-dimension is the columns, and the
        y-dimension is the rows.
        """
        SPACING = 2.
        (NUM_ROWS, NUM_COLS) = (4, 5)

        grid = RasterModelGrid(num_rows=NUM_ROWS, num_cols=NUM_COLS,
                               dx=SPACING)

        self.assertEqual(grid.get_grid_ydimension(), NUM_ROWS * SPACING)
        self.assertEqual(grid.get_grid_xdimension(), NUM_COLS * SPACING)

    def test_grid_dimensions_default_spacing(self):
        """
        Use the default spacing of 1.
        """
        (NUM_ROWS, NUM_COLS) = (4, 5)

        grid = RasterModelGrid(num_rows=NUM_ROWS, num_cols=NUM_COLS)

        self.assertEqual(grid.get_grid_ydimension(), NUM_ROWS * 1.)
        self.assertEqual(grid.get_grid_xdimension(), NUM_COLS * 1.)

    def test_nodes_around_point(self):
        surrounding_ids = self.grid.get_nodes_around_point(2.1, 1.1)
        self.assertListEqual(
            list(surrounding_ids), [7, 8, 12, 13],
            'incorrect surrounding ids in test_nodes_around_point')

        surrounding_ids = self.grid.get_nodes_around_point(2.1, .9)
        self.assertListEqual(
            list(surrounding_ids), [2, 3, 7, 8],
            'incorrect surrounding ids in test_nodes_around_point')

    def test_neighbor_list(self):
        neighbors = self.grid.get_neighbor_list(id=6)
        self.assertEqual(list(neighbors), [7, 11, 5, 1],
                         'incorrect neighbors in test_neighbor_list')

    def test_neighbor_list_boundary(self):
        """
        All of the neighbor IDs for a boundary cell are -1.
        """
        neighbors = self.grid.get_neighbor_list(id=0)

        self.assertEqual(list(neighbors), [-1, -1, -1, -1])

    def test_cell_x(self):
        expected_x = [0., 1., 2., 3., 4.,
                      0., 1., 2., 3., 4.,
                      0., 1., 2., 3., 4.,
                      0., 1., 2., 3., 4.]

        for (cell_id, expected) in zip(xrange(12), expected_x):
            cell_x = self.grid.get_node_x(cell_id)
            self.assertEqual(cell_x, expected)

            cell_x = self.grid.node_x[cell_id]
            self.assertEqual(cell_x, expected)

    def test_cell_y(self):
        expected_y = [0., 0., 0., 0., 0.,
                      1., 1., 1., 1., 1.,
                      2., 2., 2., 2., 2.,
                      3., 3., 3., 3., 3.]

        for (cell_id, expected) in zip(xrange(12), expected_y):
            cell_y = self.grid.get_node_y(cell_id)
            self.assertEqual(cell_y, expected)

            cell_y = self.grid.node_y[cell_id]
            self.assertEqual(cell_y, expected)

    def test_node_x_coordinates(self):
        x_coords = self.grid.get_node_x_coords()

        self.assertEqual(list(x_coords), [0., 1., 2., 3., 4.,
                                          0., 1., 2., 3., 4.,
                                          0., 1., 2., 3., 4.,
                                          0., 1., 2., 3., 4.])

    def test_cell_y_coordinates(self):
        y_coords = self.grid.get_node_y_coords()

        self.assertEqual(list(y_coords), [0., 0., 0., 0., 0.,
                                          1., 1., 1., 1., 1.,
                                          2., 2., 2., 2., 2.,
                                          3., 3., 3., 3., 3.])

    def test_diagonal_list(self):
        diagonals = self.grid.get_diagonal_list(id=6)
        self.assertEqual(list(diagonals), [12, 10, 0, 2])

    def test_diagonal_list_boundary(self):
        diagonals = self.grid.get_diagonal_list(id=0)
        self.assertEqual(list(diagonals), [-1, -1, -1, -1])

    def test_is_interior(self):
        for cell_id in [0, 1, 2, 3, 4, 5, 9, 10, 14, 15, 16, 17, 18, 19]:
            self.assertFalse(self.grid.is_interior(cell_id))

        for cell_id in [6, 7, 8, 11, 12, 13]:
            self.assertTrue(self.grid.is_interior(cell_id))

    def test_get_interior_cells(self):
        interiors = self.grid.get_active_cell_node_ids()
        self.assertEqual(list(interiors), [6, 7, 8, 11, 12, 13])

    def test_active_links(self):
        self.assertTrue(isinstance(self.grid.active_links, np.ndarray))
        self.assertEqual(self.grid.num_active_links, 17)

        assert_array_equal(self.grid.active_links,
                           np.array([1, 2, 3, 6, 7, 8, 11, 12, 13,
                                     19, 20, 21, 22, 23, 24, 25, 26]))

    def test_active_link_fromnode(self):
        assert_array_equal(self.grid.activelink_fromnode,
                           np.array([1, 2, 3, 6, 7, 8, 11, 12, 13,
                                     5, 6, 7, 8, 10, 11, 12, 13]))

    def test_active_link_tonode(self):
        assert_array_equal(self.grid.activelink_tonode,
                           np.array([6, 7, 8, 11, 12, 13, 16, 17, 18,
                                     6, 7, 8, 9, 11, 12, 13, 14]))

    def test_active_link_num_inlink(self):
        assert_array_equal(self.grid.node_numactiveinlink,
                           np.array([0, 0, 0, 0, 0,
                                     0, 2, 2, 2, 1,
                                     0, 2, 2, 2, 1,
                                     0, 1, 1, 1, 0]))

    def test_active_link_num_outlink(self):
        assert_array_equal(self.grid.node_numactiveoutlink,
                           np.array([0, 1, 1, 1, 0,
                                     1, 2, 2, 2, 0,
                                     1, 2, 2, 2, 0,
                                     0, 0, 0, 0, 0]))

    def test_active_inlink_matrix(self):
        self.assertTupleEqual(self.grid.node_active_inlink_matrix.shape,
                              (2, 20))

        assert_array_equal(self.grid.node_active_inlink_matrix[0],
                           np.array([-1, -1, -1, -1, -1,
                                     -1,  0,  1,  2, 12,
                                     -1,  3,  4,  5, 16,
                                     -1,  6,  7,  8, -1]))
        assert_array_equal(self.grid.node_active_inlink_matrix[1],
                           np.array([-1, -1, -1, -1, -1,
                                     -1,  9, 10, 11, -1,
                                     -1, 13, 14, 15, -1,
                                     -1, -1, -1, -1, -1]))

    def test_active_outlink_matrix(self):
        self.assertTupleEqual(self.grid.node_active_outlink_matrix.shape,
                              (2, 20))

        assert_array_equal(self.grid.node_active_outlink_matrix[0],
                           np.array([-1,  0,  1,  2, -1,
                                      9,  3,  4,  5, -1,
                                     13,  6,  7,  8, -1,
                                     -1, -1, -1, -1, -1]))
        assert_array_equal(self.grid.node_active_outlink_matrix[1],
                           np.array([-1, -1, -1, -1, -1,
                                     -1, 10, 11, 12, -1,
                                     -1, 14, 15, 16, -1,
                                     -1, -1, -1, -1, -1]))

    def test_link_fromnode(self):
        assert_array_equal(self.grid.link_fromnode,
                           np.array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
                                     10, 11, 12, 13, 14,  0,  1,  2,  3,  5,
                                      6,  7,  8, 10, 11, 12, 13, 15, 16, 17,
                                     18]))

    def test_link_tonode(self):
        assert_array_equal(self.grid.link_tonode,
                           np.array([ 5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
                                     15, 16, 17, 18, 19,  1,  2,  3,  4,  6,
                                      7,  8,  9, 11, 12, 13, 14, 16, 17, 18,
                                     19]))

    def test_link_num_inlink(self):
        assert_array_equal(self.grid.node_numinlink,
                           np.array([0, 1, 1, 1, 1,
                                     1, 2, 2, 2, 2,
                                     1, 2, 2, 2, 2,
                                     1, 2, 2, 2, 2]))

    def test_link_num_outlink(self):
        assert_array_equal(self.grid.node_numoutlink,
                           np.array([2, 2, 2, 2, 1,
                                     2, 2, 2, 2, 1,
                                     2, 2, 2, 2, 1,
                                     1, 1, 1, 1, 0]))

    def test_node_inlink_matrix(self):
        self.assertTupleEqual(self.grid.node_inlink_matrix.shape,
                              (2, 20))
        assert_array_equal(self.grid.node_inlink_matrix[0],
                           np.array([-1, 15, 16, 17, 18,
                                      0,  1,  2,  3,  4,
                                      5,  6,  7,  8,  9,
                                     10, 11, 12, 13, 14]))
        assert_array_equal(self.grid.node_inlink_matrix[1],
                           np.array([-1, -1, -1, -1, -1,
                                     -1, 19, 20, 21, 22,
                                     -1, 23, 24, 25, 26,
                                     -1, 27, 28, 29, 30]))

    def test_node_outlink_matrix(self):
        self.assertTupleEqual(self.grid.node_outlink_matrix.shape,
                              (2, 20))
        assert_array_equal(self.grid.node_outlink_matrix[0],
                           np.array([ 0,  1,  2,  3,  4,
                                     5,  6,  7,  8,  9,
                                     10, 11, 12, 13, 14,
                                     27, 28, 29, 30, -1]))
        assert_array_equal(self.grid.node_outlink_matrix[1],
                           np.array([15, 16, 17, 18, -1,
                                     19, 20, 21, 22, -1,
                                     23, 24, 25, 26, -1,
                                     -1, -1, -1, -1, -1]))

    def test_link_face(self):
        from landlab.model_grid import _SLOW
        if _SLOW:
            BAD = None
        else:
            BAD = BAD_INDEX_VALUE

        assert_array_equal(self.grid.link_face,
                           np.array([BAD, 0, 1, 2, BAD,
                                     BAD, 3, 4, 5, BAD,
                                     BAD, 6, 7, 8, BAD,
                                     BAD, BAD, BAD, BAD,
                                     9, 10, 11, 12,
                                     13, 14, 15, 16,
                                     BAD, BAD, BAD, BAD]))

    def test_grid_coords_to_node_id(self):
        self.assertEqual(self.grid.grid_coords_to_node_id(3, 4), 19)
        assert_array_equal(self.grid.grid_coords_to_node_id((3, 2), (4, 1)),
                           np.array([19, 11]))

        with self.assertRaises(ValueError):
            self.grid.grid_coords_to_node_id(5, 0)

class TestRasterModelGridZeroArrays(unittest.TestCase):
    def test_default(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(), np.zeros((20, ), dtype=np.float))

    def test_at_nodes(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(centering='node'),
                           np.zeros((20, ), dtype=np.float))

    def test_at_cells(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(centering='cell'),
                           np.zeros((6, ), dtype=np.float))

    def test_at_links(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(centering='link'),
                           np.zeros((17, ), dtype=np.float))

    def test_at_faces(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(centering='face'),
                           np.zeros((17, ), dtype=np.float))

    def test_bad_centering(self):
        mg = RasterModelGrid(4, 5)
        with self.assertRaises(BadCenteringString):
            mg.zeros(centering='bad_centering_string')

    def test_int(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(dtype=np.int),
                           np.zeros((20, ), dtype=np.int))

    def test_int8(self):
        mg = RasterModelGrid(4, 5)
        assert_array_equal(mg.zeros(dtype=np.int8),
                           np.zeros((20, ), dtype=np.int8))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Lint as: python3
# Copyright 2020 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Entry of a PythonNode worker."""

import contextlib
import json
import os
import sys

from absl import app
from absl import flags
from absl import logging
import cloudpickle
from launchpad.launch import worker_manager
import six

FLAGS = flags.FLAGS

flags.DEFINE_integer(
    'lp_task_id', None, 'a list index deciding which '
    'worker to run. given a list of workers (obtained from the'
    ' data_file)')
flags.DEFINE_string('data_file', '', 'Pickle file location')
flags.DEFINE_string('flags_to_populate', '{}', '')

_FLAG_TYPE_MAPPING = {
    str: flags.DEFINE_string,
    six.text_type: flags.DEFINE_string,
    float: flags.DEFINE_float,
    int: flags.DEFINE_integer,
    bool: flags.DEFINE_boolean,
    list: flags.DEFINE_list,
}

def _populate_flags():
  """Populate flags that cannot be passed directly to this script."""
  FLAGS(sys.argv, known_only=True)

  flags_to_populate = json.loads(FLAGS.flags_to_populate)
  for name, value in flags_to_populate.items():
    value_type = type(value)
    if value_type in _FLAG_TYPE_MAPPING:
      flag_ctr = _FLAG_TYPE_MAPPING[value_type]
      logging.info('Defining flag %s with default value %s', name, value)
      flag_ctr(
          name,
          value,
          'This flag has been auto-generated.',
          allow_override=True)

def _get_task_id():
  """Returns current task's id."""
  if FLAGS.lp_task_id is None:
    # Running under CAIP...
    cluster_spec = os.environ.get('CLUSTER_SPEC', None)
    return json.loads(cluster_spec).get('task').get('index')

  return FLAGS.lp_task_id

def main(_):
  # Allow for importing modules from the current directory.
  sys.path.append(os.getcwd())
  data_file = FLAGS.data_file
  functions = cloudpickle.load(open(data_file, 'rb'))
  task_id = _get_task_id()

  # Worker manager is used here to handle termination signals and provide
  # preemption support.
  worker_manager.WorkerManager(
      register_in_thread=True,
      stop_main_thread=True)

  with contextlib.suppress():  # no-op context manager
    functions[task_id]()

if __name__ == '__main__':
  _populate_flags()
  app.run(main)
<commit_msg>In GCP runtime log to STDOUT so that logs are not reported as errors.<commit_after># Lint as: python3
# Copyright 2020 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Entry of a PythonNode worker."""

import contextlib
import json
import os
import sys

from absl import app
from absl import flags
from absl import logging
import cloudpickle
from launchpad.launch import worker_manager
import six

FLAGS = flags.FLAGS

flags.DEFINE_integer(
    'lp_task_id', None, 'a list index deciding which '
    'worker to run. given a list of workers (obtained from the'
    ' data_file)')
flags.DEFINE_string('data_file', '', 'Pickle file location')
flags.DEFINE_string('flags_to_populate', '{}', '')

_FLAG_TYPE_MAPPING = {
    str: flags.DEFINE_string,
    six.text_type: flags.DEFINE_string,
    float: flags.DEFINE_float,
    int: flags.DEFINE_integer,
    bool: flags.DEFINE_boolean,
    list: flags.DEFINE_list,
}

def _populate_flags():
  """Populate flags that cannot be passed directly to this script."""
  FLAGS(sys.argv, known_only=True)

  flags_to_populate = json.loads(FLAGS.flags_to_populate)
  for name, value in flags_to_populate.items():
    value_type = type(value)
    if value_type in _FLAG_TYPE_MAPPING:
      flag_ctr = _FLAG_TYPE_MAPPING[value_type]
      logging.info('Defining flag %s with default value %s', name, value)
      flag_ctr(
          name,
          value,
          'This flag has been auto-generated.',
          allow_override=True)

def _get_task_id():
  """Returns current task's id."""
  if FLAGS.lp_task_id is None:
    # Running under CAIP...
    cluster_spec = os.environ.get('CLUSTER_SPEC', None)
    return json.loads(cluster_spec).get('task').get('index')

  return FLAGS.lp_task_id

def main(_):
  # Allow for importing modules from the current directory.
  sys.path.append(os.getcwd())
  data_file = FLAGS.data_file

  if os.environ.get('TF_CONFIG', None):
    # For GCP runtime log to STDOUT so that logs are not reported as errors.
    logging.get_absl_handler().python_handler.stream = sys.stdout

  functions = cloudpickle.load(open(data_file, 'rb'))
  task_id = _get_task_id()

  # Worker manager is used here to handle termination signals and provide
  # preemption support.
  worker_manager.WorkerManager(
      register_in_thread=True,
      stop_main_thread=True)

  with contextlib.suppress():  # no-op context manager
    functions[task_id]()

if __name__ == '__main__':
  _populate_flags()
  app.run(main)
<|endoftext|>


<commit_before>import logging
import mimetypes
import os
import sys
import random

from appdirs import user_data_dir

from lbrynet.core.Error import InsufficientFundsError
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from lbrynet import reflector
from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS
from twisted.internet import threads, defer, reactor

if sys.platform != "darwin":
    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
    log_dir = user_data_dir("LBRY")

if not os.path.isdir(log_dir):
    os.mkdir(log_dir)

lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)

class Publisher(object):
    def __init__(self, session, lbry_file_manager, wallet):
        self.session = session
        self.lbry_file_manager = lbry_file_manager
        self.wallet = wallet
        self.received_file_name = False
        self.file_path = None
        self.file_name = None
        self.publish_name = None
        self.bid_amount = None
        self.verified = False
        self.lbry_file = None
        self.txid = None
        self.stream_hash = None
        reflector_server = random.choice(REFLECTOR_SERVERS)
        self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1]
        self.metadata = {}

    def start(self, name, file_path, bid, metadata):
        log.info('Starting publish for %s', name)
        def _show_result():
            log.info("Success! Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid)
            return defer.succeed(self.txid)

        self.publish_name = name
        self.file_path = file_path
        self.bid_amount = bid
        self.metadata = metadata

        d = self._check_file_path(self.file_path)
        d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
                                                 self.file_name, open(self.file_path)))
        d.addCallback(self.add_to_lbry_files)
        d.addCallback(lambda _: self._create_sd_blob())
        d.addCallback(lambda _: self._claim_name())
        d.addCallback(lambda _: self.set_status())
        d.addCallback(lambda _: self.start_reflector())
        d.addCallbacks(lambda _: _show_result(), self._show_publish_error)

        return d

    def start_reflector(self):
        reflector_server = random.choice(REFLECTOR_SERVERS)
        reflector_address, reflector_port = reflector_server[0], reflector_server[1]
        log.info("Reflecting new publication")
        factory = reflector.ClientFactory(
            self.session.blob_manager,
            self.lbry_file_manager.stream_info_manager,
            self.stream_hash
        )
        d = reactor.resolve(reflector_address)
        d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory))
        d.addCallback(lambda _: factory.finished_deferred)
        return d

    def _check_file_path(self, file_path):
        def check_file_threaded():
            f = open(file_path)
            f.close()
            self.file_name = os.path.basename(self.file_path)
            return True
        return threads.deferToThread(check_file_threaded)

    def set_lbry_file(self, lbry_file_downloader):
        self.lbry_file = lbry_file_downloader
        return defer.succeed(None)

    def add_to_lbry_files(self, stream_hash):
        self.stream_hash = stream_hash
        prm = PaymentRateManager(self.session.base_payment_rate_manager)
        d = self.lbry_file_manager.add_lbry_file(stream_hash, prm)
        d.addCallback(self.set_lbry_file)
        return d

    def _create_sd_blob(self):
        log.debug('Creating stream descriptor blob')
        d = publish_sd_blob(self.lbry_file_manager.stream_info_manager,
                            self.session.blob_manager,
                            self.lbry_file.stream_hash)

        def set_sd_hash(sd_hash):
            log.debug('stream descriptor hash: %s', sd_hash)
            if 'sources' not in self.metadata:
                self.metadata['sources'] = {}
            self.metadata['sources']['lbry_sd_hash'] = sd_hash

        d.addCallback(set_sd_hash)
        return d

    def set_status(self):
        log.debug('Setting status')
        d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
        d.addCallback(lambda _: self.lbry_file.restore())
        return d

    def _claim_name(self):
        log.debug('Claiming name')
        self._update_metadata()
        m = Metadata(self.metadata)

        def set_tx_hash(txid):
            log.debug('Name claimed using txid: %s', txid)
            self.txid = txid

        d = self.wallet.claim_name(self.publish_name, self.bid_amount, m)
        d.addCallback(set_tx_hash)
        return d

    def _update_metadata(self):
        filename = os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name)
        self.metadata['content-type'] = get_content_type(filename)
        self.metadata['ver'] = CURRENT_METADATA_VERSION

    def _show_publish_error(self, err):
        log.info(err.getTraceback())
        message = "An error occurred publishing %s to %s. Error: %s."
        if err.check(InsufficientFundsError):
            error_message = "Insufficient funds"
        else:
            error_message = err.getErrorMessage()

        log.error(error_message)
        log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback())

        return defer.fail(Exception("Publish failed"))

def get_content_type(filename):
    return mimetypes.guess_type(filename)[0]
<commit_msg>File handle for publishing on windows needs to handle binary data as 'rb'<commit_after>import logging
import mimetypes
import os
import sys
import random

from appdirs import user_data_dir

from lbrynet.core.Error import InsufficientFundsError
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from lbrynet import reflector
from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS
from twisted.internet import threads, defer, reactor

if sys.platform != "darwin":
    log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
    log_dir = user_data_dir("LBRY")

if not os.path.isdir(log_dir):
    os.mkdir(log_dir)

lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)

class Publisher(object):
    def __init__(self, session, lbry_file_manager, wallet):
        self.session = session
        self.lbry_file_manager = lbry_file_manager
        self.wallet = wallet
        self.received_file_name = False
        self.file_path = None
        self.file_name = None
        self.publish_name = None
        self.bid_amount = None
        self.verified = False
        self.lbry_file = None
        self.txid = None
        self.stream_hash = None
        reflector_server = random.choice(REFLECTOR_SERVERS)
        self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1]
        self.metadata = {}

    def start(self, name, file_path, bid, metadata):
        log.info('Starting publish for %s', name)
        def _show_result():
            log.info("Success! Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid)
            return defer.succeed(self.txid)

        self.publish_name = name
        self.file_path = file_path
        self.bid_amount = bid
        self.metadata = metadata

        d = self._check_file_path(self.file_path)
        d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
                                                 self.file_name, open(self.file_path, 'rb')))
        d.addCallback(self.add_to_lbry_files)
        d.addCallback(lambda _: self._create_sd_blob())
        d.addCallback(lambda _: self._claim_name())
        d.addCallback(lambda _: self.set_status())
        d.addCallback(lambda _: self.start_reflector())
        d.addCallbacks(lambda _: _show_result(), self._show_publish_error)

        return d

    def start_reflector(self):
        reflector_server = random.choice(REFLECTOR_SERVERS)
        reflector_address, reflector_port = reflector_server[0], reflector_server[1]
        log.info("Reflecting new publication")
        factory = reflector.ClientFactory(
            self.session.blob_manager,
            self.lbry_file_manager.stream_info_manager,
            self.stream_hash
        )
        d = reactor.resolve(reflector_address)
        d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory))
        d.addCallback(lambda _: factory.finished_deferred)
        return d

    def _check_file_path(self, file_path):
        def check_file_threaded():
            f = open(file_path)
            f.close()
            self.file_name = os.path.basename(self.file_path)
            return True
        return threads.deferToThread(check_file_threaded)

    def set_lbry_file(self, lbry_file_downloader):
        self.lbry_file = lbry_file_downloader
        return defer.succeed(None)

    def add_to_lbry_files(self, stream_hash):
        self.stream_hash = stream_hash
        prm = PaymentRateManager(self.session.base_payment_rate_manager)
        d = self.lbry_file_manager.add_lbry_file(stream_hash, prm)
        d.addCallback(self.set_lbry_file)
        return d

    def _create_sd_blob(self):
        log.debug('Creating stream descriptor blob')
        d = publish_sd_blob(self.lbry_file_manager.stream_info_manager,
                            self.session.blob_manager,
                            self.lbry_file.stream_hash)

        def set_sd_hash(sd_hash):
            log.debug('stream descriptor hash: %s', sd_hash)
            if 'sources' not in self.metadata:
                self.metadata['sources'] = {}
            self.metadata['sources']['lbry_sd_hash'] = sd_hash

        d.addCallback(set_sd_hash)
        return d

    def set_status(self):
        log.debug('Setting status')
        d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
        d.addCallback(lambda _: self.lbry_file.restore())
        return d

    def _claim_name(self):
        log.debug('Claiming name')
        self._update_metadata()
        m = Metadata(self.metadata)

        def set_tx_hash(txid):
            log.debug('Name claimed using txid: %s', txid)
            self.txid = txid

        d = self.wallet.claim_name(self.publish_name, self.bid_amount, m)
        d.addCallback(set_tx_hash)
        return d

    def _update_metadata(self):
        filename = os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name)
        self.metadata['content-type'] = get_content_type(filename)
        self.metadata['ver'] = CURRENT_METADATA_VERSION

    def _show_publish_error(self, err):
        log.info(err.getTraceback())
        message = "An error occurred publishing %s to %s. Error: %s."
        if err.check(InsufficientFundsError):
            error_message = "Insufficient funds"
        else:
            error_message = err.getErrorMessage()

        log.error(error_message)
        log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback())

        return defer.fail(Exception("Publish failed"))

def get_content_type(filename):
    return mimetypes.guess_type(filename)[0]
<|endoftext|>