<commit_before># Copyright 2017-2018 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re

import guild
import guild.run

from guild import config
from guild import util
from guild import var
from guild import view

from guild.commands import runs_impl

log = logging.getLogger("guild")

class ViewDataImpl(view.ViewData):

    def __init__(self, args):
        self._args = args

    def runs(self):
        return runs_impl.runs_for_args(self._args)

    def runs_data(self):
        return list(self._runs_data_iter(self.runs()))

    @staticmethod
    def one_run(run_id_prefix):
        try:
            id, path = next(var.find_runs(run_id_prefix))
        except StopIteration:
            return None
        else:
            run = guild.run.Run(id, path)
            return runs_impl.init_opref_attr(run)

    def one_run_data(self, run_id_prefix):
        run = self.one_run(run_id_prefix)
        if not run:
            return None
        return self._run_data(run)

    def _runs_data_iter(self, runs):
        for run in runs:
            try:
                yield self._run_data(run)
            except Exception as e:
                if log.getEffectiveLevel() <= logging.DEBUG:
                    log.exception("error processing run data for %s", run.id)
                else:
                    log.error("error processing run data for %s: %s", run.id, e)

    def _run_data(self, run):
        formatted = runs_impl.format_run(run)
        return {
            "id": run.id,
            "shortId": run.short_id,
            "path": run.path,
            "operation": formatted["operation"],
            "opModel": run.opref.model_name,
            "opName": run.opref.op_name,
            "started": formatted["started"],
            "stopped": formatted["stopped"],
            "label": formatted["label"],
            "status": run.status,
            "exitStatus": formatted["exit_status"] or None,
            "command": formatted["command"],
            "otherAttrs": self._other_attrs(run),
            "flags": run.get("flags", {}),
            "env": run.get("env", {}),
            "deps": self._format_deps(run.get("deps", {})),
            "files": self._format_files(run.iter_files(), run.path),
        }

    @staticmethod
    def _other_attrs(run):
        return {
            name: run.get(name)
            for name in runs_impl.other_attr_names(run)
        }

    def _format_deps(self, deps):
        runs_dir = var.runs_dir()
        runs = {}
        for paths in deps.values():
            for path in paths:
                if not path.startswith(runs_dir):
                    continue
                subdir = path[len(runs_dir)+1:]
                parts = subdir.split(os.path.sep)
                run_id = parts[0]
                try:
                    run, run_paths = runs[run_id]
                except KeyError:
                    run = self._run_for_id(run_id)
                    run_paths = []
                    runs[run_id] = run, run_paths
                run_paths.append(os.path.join(*parts[1:]))
        formatted = [
            self._format_dep(run, paths)
            for run, paths in runs.values()
        ]
        return sorted(formatted, key=lambda x: x["operation"])

    @staticmethod
    def _run_for_id(run_id):
        return runs_impl.init_opref_attr(var.get_run(run_id))

    @staticmethod
    def _format_dep(run, paths):
        return {
            "run": run.short_id,
            "operation": runs_impl.format_op_desc(run.opref, nowarn=True),
            "paths": paths
        }

    def _format_files(self, files, root):
        filtered = [
            path for path in files
            if os.path.islink(path) or os.path.isfile(path)
        ]
        formatted = [self._format_file(path, root) for path in filtered]
        return sorted(formatted, key=lambda x: x["path"])

    def _format_file(self, path, root):
        typeDesc, icon, iconTooltip, viewer = self._file_type_info(path)
        opDesc, opRun = self._op_source_info(path)
        relpath = path[len(root)+1:]
        return {
            "path": relpath,
            "size": util.safe_filesize(path),
            "type": typeDesc,
            "icon": icon,
            "iconTooltip": iconTooltip,
            "viewer": viewer,
            "operation": opDesc,
            "run": opRun,
        }

    def _file_type_info(self, path):
        typeDesc, icon, iconTooltip, viewer = self._base_file_type_info(path)
        if os.path.islink(path):
            target = os.path.realpath(path)
            link_type = "directory" if os.path.isdir(target) else "file"
            if target.startswith(var.runs_dir()):
                typeDesc = "Link to operation output"
            elif target.startswith(var.cache_dir()):
                typeDesc = "Link to resource {}".format(link_type)
            else:
                typeDesc = "Link"
            icon = "folder-move" if link_type == "directory" else "file-send"
            iconTooltip = "Link"
        return typeDesc, icon, iconTooltip, viewer

    @staticmethod
    def _base_file_type_info(path):
        path_lower = path.lower()
        if re.search(r"\.tfevents\.", path_lower):
            return "Event log", "file-chart", "File", None
        elif re.search(r"saved_model\.pb$", path_lower):
            return "SavedModel protocol buffer", "file", "File", None
        elif re.search(r"graph\.pb$", path_lower):
            return "GraphDef protocol buffer", "file", "File", None
        elif re.search(r"saved_model\.pbtxt$", path_lower):
            return (
                "SavedModel protocol buffer", "file-document", "Text file",
                "text")
        elif re.search(r"graph\.pbtxt$", path_lower):
            return (
                "GraphDef protocol buffer", "file-document", "Text file",
                "text")
        elif re.search(r"\.index$", path_lower):
            return "Checkpoint index", "file", "File", None
        elif re.search(r"\.meta$", path_lower):
            return "Checkpoint meta graph", "file", "File", None
        elif re.search(r"[/\\]checkpoint$", path_lower):
            return "Latest checkpoint marker", "file", "File", None
        elif re.search(r"data-\d+-of-\d+$", path_lower):
            return "Checkpoint values", "file", "File", None
        elif re.search(r"\.tfrecord$", path_lower):
            return "Dataset file", "file", "File", None
        elif re.search(r"\.(jpg|jpeg|gif|png|tiff)$", path_lower):
            return "Image", "file-image", "Image", "image"
        elif re.search(r"\.mid", path_lower):
            return "Audio", "file-music", "Audio", "midi"
        elif re.search(r"\.(wav|mp3)", path_lower):
            return "Audio", "file-music", "Audio", None
        else:
            if util.is_text_file(path):
                return "Text file", "file-document", "Text file", "text"
            else:
                return "File", "file", "File", None

    def _op_source_info(self, path):
        if not os.path.islink(path):
            return None, None
        path = os.path.realpath(path)
        runs_dir = var.runs_dir()
        if not path.startswith(runs_dir):
            return None, None
        subdir = path[len(runs_dir)+1:]
        parts = subdir.split(os.path.sep, 1)
        run = self._run_for_id(parts[0])
        operation = runs_impl.format_op_desc(run.opref, nowarn=True)
        return operation, run.short_id

    def config(self):
        cwd = self._formatted_cwd()
        return {
            "cwd": cwd,
            "titleLabel": self._title_label(cwd),
            "version": guild.version(),
        }

    @staticmethod
    def _formatted_cwd():
        cwd = config.cwd()
        abs_cwd = os.path.abspath(cwd)
        user_dir = os.path.expanduser("~")
        if abs_cwd.startswith(user_dir):
            return os.path.join("~", abs_cwd[len(user_dir)+1:])
        else:
            return abs_cwd

    def _title_label(self, cwd):
        if len(self._args.runs) == 1:
            return self._single_run_title_label(self._args.runs[0])
        elif self._args.all:
            return self._all_title_label(self._args)
        else:
            return self._cwd_title_label(cwd)

    @staticmethod
    def _single_run_title_label(run_id):
        return "[{}]".format(run_id)

    @staticmethod
    def _all_title_label(args):
        if args.ops:
            return "all {}".format(" ".join(args.ops))
        else:
            return "all"

    @staticmethod
    def _cwd_title_label(cwd):
        parts = cwd.split(os.path.sep)
        if len(parts) < 2:
            return cwd
        else:
            return os.path.join(*parts[-2:])

def main(args):
    data = ViewDataImpl(args)
    host = args.host or ""
    port = args.port or util.free_port()
    view.serve_forever(data, host, port, args.no_open, args.dev, args.logging)
<commit_msg>Use localhost as default hostname for view dev mode<commit_after># Copyright 2017-2018 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re

import guild
import guild.run

from guild import config
from guild import util
from guild import var
from guild import view

from guild.commands import runs_impl

log = logging.getLogger("guild")

class ViewDataImpl(view.ViewData):

    def __init__(self, args):
        self._args = args

    def runs(self):
        return runs_impl.runs_for_args(self._args)

    def runs_data(self):
        return list(self._runs_data_iter(self.runs()))

    @staticmethod
    def one_run(run_id_prefix):
        try:
            id, path = next(var.find_runs(run_id_prefix))
        except StopIteration:
            return None
        else:
            run = guild.run.Run(id, path)
            return runs_impl.init_opref_attr(run)

    def one_run_data(self, run_id_prefix):
        run = self.one_run(run_id_prefix)
        if not run:
            return None
        return self._run_data(run)

    def _runs_data_iter(self, runs):
        for run in runs:
            try:
                yield self._run_data(run)
            except Exception as e:
                if log.getEffectiveLevel() <= logging.DEBUG:
                    log.exception("error processing run data for %s", run.id)
                else:
                    log.error("error processing run data for %s: %s", run.id, e)

    def _run_data(self, run):
        formatted = runs_impl.format_run(run)
        return {
            "id": run.id,
            "shortId": run.short_id,
            "path": run.path,
            "operation": formatted["operation"],
            "opModel": run.opref.model_name,
            "opName": run.opref.op_name,
            "started": formatted["started"],
            "stopped": formatted["stopped"],
            "label": formatted["label"],
            "status": run.status,
            "exitStatus": formatted["exit_status"] or None,
            "command": formatted["command"],
            "otherAttrs": self._other_attrs(run),
            "flags": run.get("flags", {}),
            "env": run.get("env", {}),
            "deps": self._format_deps(run.get("deps", {})),
            "files": self._format_files(run.iter_files(), run.path),
        }

    @staticmethod
    def _other_attrs(run):
        return {
            name: run.get(name)
            for name in runs_impl.other_attr_names(run)
        }

    def _format_deps(self, deps):
        runs_dir = var.runs_dir()
        runs = {}
        for paths in deps.values():
            for path in paths:
                if not path.startswith(runs_dir):
                    continue
                subdir = path[len(runs_dir)+1:]
                parts = subdir.split(os.path.sep)
                run_id = parts[0]
                try:
                    run, run_paths = runs[run_id]
                except KeyError:
                    run = self._run_for_id(run_id)
                    run_paths = []
                    runs[run_id] = run, run_paths
                run_paths.append(os.path.join(*parts[1:]))
        formatted = [
            self._format_dep(run, paths)
            for run, paths in runs.values()
        ]
        return sorted(formatted, key=lambda x: x["operation"])

    @staticmethod
    def _run_for_id(run_id):
        return runs_impl.init_opref_attr(var.get_run(run_id))

    @staticmethod
    def _format_dep(run, paths):
        return {
            "run": run.short_id,
            "operation": runs_impl.format_op_desc(run.opref, nowarn=True),
            "paths": paths
        }

    def _format_files(self, files, root):
        filtered = [
            path for path in files
            if os.path.islink(path) or os.path.isfile(path)
        ]
        formatted = [self._format_file(path, root) for path in filtered]
        return sorted(formatted, key=lambda x: x["path"])

    def _format_file(self, path, root):
        typeDesc, icon, iconTooltip, viewer = self._file_type_info(path)
        opDesc, opRun = self._op_source_info(path)
        relpath = path[len(root)+1:]
        return {
            "path": relpath,
            "size": util.safe_filesize(path),
            "type": typeDesc,
            "icon": icon,
            "iconTooltip": iconTooltip,
            "viewer": viewer,
            "operation": opDesc,
            "run": opRun,
        }

    def _file_type_info(self, path):
        typeDesc, icon, iconTooltip, viewer = self._base_file_type_info(path)
        if os.path.islink(path):
            target = os.path.realpath(path)
            link_type = "directory" if os.path.isdir(target) else "file"
            if target.startswith(var.runs_dir()):
                typeDesc = "Link to operation output"
            elif target.startswith(var.cache_dir()):
                typeDesc = "Link to resource {}".format(link_type)
            else:
                typeDesc = "Link"
            icon = "folder-move" if link_type == "directory" else "file-send"
            iconTooltip = "Link"
        return typeDesc, icon, iconTooltip, viewer

    @staticmethod
    def _base_file_type_info(path):
        path_lower = path.lower()
        if re.search(r"\.tfevents\.", path_lower):
            return "Event log", "file-chart", "File", None
        elif re.search(r"saved_model\.pb$", path_lower):
            return "SavedModel protocol buffer", "file", "File", None
        elif re.search(r"graph\.pb$", path_lower):
            return "GraphDef protocol buffer", "file", "File", None
        elif re.search(r"saved_model\.pbtxt$", path_lower):
            return (
                "SavedModel protocol buffer", "file-document", "Text file",
                "text")
        elif re.search(r"graph\.pbtxt$", path_lower):
            return (
                "GraphDef protocol buffer", "file-document", "Text file",
                "text")
        elif re.search(r"\.index$", path_lower):
            return "Checkpoint index", "file", "File", None
        elif re.search(r"\.meta$", path_lower):
            return "Checkpoint meta graph", "file", "File", None
        elif re.search(r"[/\\]checkpoint$", path_lower):
            return "Latest checkpoint marker", "file", "File", None
        elif re.search(r"data-\d+-of-\d+$", path_lower):
            return "Checkpoint values", "file", "File", None
        elif re.search(r"\.tfrecord$", path_lower):
            return "Dataset file", "file", "File", None
        elif re.search(r"\.(jpg|jpeg|gif|png|tiff)$", path_lower):
            return "Image", "file-image", "Image", "image"
        elif re.search(r"\.mid", path_lower):
            return "Audio", "file-music", "Audio", "midi"
        elif re.search(r"\.(wav|mp3)", path_lower):
            return "Audio", "file-music", "Audio", None
        else:
            if util.is_text_file(path):
                return "Text file", "file-document", "Text file", "text"
            else:
                return "File", "file", "File", None

    def _op_source_info(self, path):
        if not os.path.islink(path):
            return None, None
        path = os.path.realpath(path)
        runs_dir = var.runs_dir()
        if not path.startswith(runs_dir):
            return None, None
        subdir = path[len(runs_dir)+1:]
        parts = subdir.split(os.path.sep, 1)
        run = self._run_for_id(parts[0])
        operation = runs_impl.format_op_desc(run.opref, nowarn=True)
        return operation, run.short_id

    def config(self):
        cwd = self._formatted_cwd()
        return {
            "cwd": cwd,
            "titleLabel": self._title_label(cwd),
            "version": guild.version(),
        }

    @staticmethod
    def _formatted_cwd():
        cwd = config.cwd()
        abs_cwd = os.path.abspath(cwd)
        user_dir = os.path.expanduser("~")
        if abs_cwd.startswith(user_dir):
            return os.path.join("~", abs_cwd[len(user_dir)+1:])
        else:
            return abs_cwd

    def _title_label(self, cwd):
        if len(self._args.runs) == 1:
            return self._single_run_title_label(self._args.runs[0])
        elif self._args.all:
            return self._all_title_label(self._args)
        else:
            return self._cwd_title_label(cwd)

    @staticmethod
    def _single_run_title_label(run_id):
        return "[{}]".format(run_id)

    @staticmethod
    def _all_title_label(args):
        if args.ops:
            return "all {}".format(" ".join(args.ops))
        else:
            return "all"

    @staticmethod
    def _cwd_title_label(cwd):
        parts = cwd.split(os.path.sep)
        if len(parts) < 2:
            return cwd
        else:
            return os.path.join(*parts[-2:])

def main(args):
    data = ViewDataImpl(args)
    host = _host(args)
    port = args.port or util.free_port()
    view.serve_forever(data, host, port, args.no_open, args.dev, args.logging)

def _host(args):
    if args.host:
        return host
    if args.dev:
        return "localhost"
    return ""
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# This script must run from its own directory
from __future__ import print_function
import gen_csharp
import gen_docs_json
import gen_java
import gen_python
import gen_R
import gen_thrift
import bindings
import sys
import os
sys.path.insert(0, "../../scripts")
import run
import atexit

bindings.check_requirements()

# Create results folder, where H2OCloud stores its logs
results_dir = "../build/logs"
if not os.path.exists(results_dir):
    os.makedirs(results_dir)

# Allow override of the H2O jarfile so we can use this with projects which extend h2o.jar
h2o_jarfile = os.getenv('H2O_JARFILE', '../../build/h2o.jar')
h2o_java_version = os.getenv('H2O_JAVA_VERSION', '1.8')
h2o_jvm_opts = "-Dsys.ai.h2o.debug.noJavaVersionCheck=true"
if h2o_java_version != '1.8':
    h2o_jvm_opts += " --add-opens=java.base/java.lang=ALL-UNNAMED"

# Start H2O cloud
print("Starting H2O cloud...")
cloud = run.H2OCloud(
    cloud_num=0,
    use_client=False,
    nodes_per_cloud=1,
    h2o_jar=os.path.abspath(h2o_jarfile),
    base_port=48000,
    xmx="4g",
    cp="",
    jvm_opts=h2o_jvm_opts,
    output_dir=results_dir,
    test_ssl=False,
    ldap_config_path=None,
    strict_port=False
)
atexit.register(lambda: cloud.stop())

cloud.start()
cloud.wait_for_cloud_to_be_up()

# Manipulate the command line arguments, so that bindings module would know which cloud to use
sys.argv.insert(1, "--usecloud")
sys.argv.insert(2, "%s:%s" % (cloud.get_ip(), cloud.get_port()))

# Actually generate all the bindings
print()
gen_R.main()
gen_python.main()
gen_docs_json.main()
gen_thrift.main()
gen_csharp.main()
gen_java.main()
bindings.done()
print()
<commit_msg>Fix bindings generator to work on Windows<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# This script must run from its own directory
from __future__ import print_function
import gen_csharp
import gen_docs_json
import gen_java
import gen_python
import gen_R
import gen_thrift
import bindings
import sys
import os
sys.path.insert(0, "../../scripts")
import run
import atexit

bindings.check_requirements()

# Create results folder, where H2OCloud stores its logs
results_dir = "../build/logs"
if not os.path.exists(results_dir):
    os.makedirs(results_dir)

# Allow override of the H2O jarfile so we can use this with projects which extend h2o.jar
h2o_jarfile = os.getenv('H2O_JARFILE', '../../build/h2o.jar')
h2o_java_version = os.getenv('H2O_JAVA_VERSION', '1.8')
h2o_jvm_opts = None if h2o_java_version == '1.8' else '--add-opens=java.base/java.lang=ALL-UNNAMED'

# Start H2O cloud
print("Starting H2O cloud...")
cloud = run.H2OCloud(
    cloud_num=0,
    use_client=False,
    nodes_per_cloud=1,
    h2o_jar=os.path.abspath(h2o_jarfile),
    base_port=48000,
    xmx="4g",
    cp="",
    jvm_opts=h2o_jvm_opts,
    output_dir=results_dir,
    test_ssl=False,
    ldap_config_path=None,
    strict_port=False
)
atexit.register(lambda: cloud.stop())

cloud.start()
cloud.wait_for_cloud_to_be_up()

# Manipulate the command line arguments, so that bindings module would know which cloud to use
sys.argv.insert(1, "--usecloud")
sys.argv.insert(2, "%s:%s" % (cloud.get_ip(), cloud.get_port()))

# Actually generate all the bindings
print()
gen_R.main()
gen_python.main()
gen_docs_json.main()
gen_thrift.main()
gen_csharp.main()
gen_java.main()
bindings.done()
print()
<|endoftext|>


<commit_before>
"""
    File object test module.

    Tests all aspects of File objects, including their creation.
"""

from __future__ import with_statement

import os

from .common import ut, TestCase
from h5py.highlevel import File

class TestFileOpen(TestCase):

    """
        Feature: Opening files with Python-style modes.
    """

    def test_create(self):
        """ Mode 'w' opens file in overwrite mode """
        fname = self.mktemp()
        fid = File(fname, 'w')
        self.assertTrue(fid)
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'w')
        self.assertNotIn('foo', fid)
        fid.close()

    def test_create_exclusive(self):
        """ Mode 'w-' opens file in exclusive mode """
        fname = self.mktemp()
        fid = File(fname, 'w-')
        self.assert_(fid)
        fid.close()
        with self.assertRaises(IOError):
            File(fname, 'w-')

    def test_append(self):
        """ Mode 'a' opens file in append/readwrite mode, creating if necessary """
        fname = self.mktemp()
        fid = File(fname, 'a')
        try:
            self.assert_(fid)
            fid.create_group('foo')
            self.assert_('foo' in fid)
        finally:
            fid.close()
        fid = File(fname, 'a')
        try:
            self.assert_('foo' in fid)
            fid.create_group('bar')
            self.assert_('bar' in fid)
        finally:
            fid.close()

    def test_readonly(self):
        """ Mode 'r' opens file in readonly mode """
        fname = self.mktemp()
        fid = File(fname, 'w')
        fid.close()
        self.assert_(not fid)
        fid = File(fname, 'r')
        self.assert_(fid)
        with self.assertRaises(ValueError):
            fid.create_group('foo')
        fid.close()

    def test_readwrite(self):
        """ Mode 'r+' opens existing file in readwrite mode """
        fname = self.mktemp()
        fid = File(fname, 'w')
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'r+')
        self.assert_('foo' in fid)
        fid.create_group('bar')
        self.assert_('bar' in fid)
        fid.close()

    def test_nonexistent_file(self):
        """ Modes 'r' and 'r+' do not create files """
        fname = self.mktemp()
        with self.assertRaises(IOError):
            File(fname, 'r')
        with self.assertRaises(IOError):
            File(fname, 'r+')

    def test_invalid_mode(self):
        """ Invalid modes raise ValueError """
        with self.assertRaises(ValueError):
            File(self.mktemp(), 'mongoose')

    def test_mode_attr(self):
        """ Mode equivalent can be retrieved via property """
        fname = self.mktemp()
        with File(fname, 'w') as f:
            self.assertEqual(f.mode, 'r+')
        with File(fname, 'r') as f:
            self.assertEqual(f.mode, 'r')

class TestDrivers(TestCase):

    """
        Feature: Files can be opened with low-level HDF5 drivers
    """

    @ut.skipUnless(os.name, 'posix')
    def test_stdio(self):
        """ Stdio driver is supported on posix """
        fid = File(self.mktemp(), 'w', driver='stdio')
        self.assertTrue(fid)
        self.assertEqual(fid.driver, 'stdio')
        fid.close()

    @ut.skipUnless(os.name, 'posix')
    def test_sec2(self):
        """ Sec2 driver is supported on posix """
        fid = File(self.mktemp(), 'w', driver='sec2')
        self.assert_(fid)
        self.assertEqual(fid.driver, 'sec2')
        fid.close()

    def test_core(self):
        """ Core driver is supported (no backing store) """
        fname = self.mktemp()
        fid = File(fname, 'w', driver='core', backing_store=False)
        self.assert_(fid)
        self.assertEqual(fid.driver, 'core')
        fid.close()
        self.assertFalse(os.path.exists(fname))

    def test_backing(self):
        """ Core driver saves to file when backing store used """
        fname = self.mktemp()
        fid = File(fname, 'w', driver='core', backing_store=True)
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'r')
        self.assert_('foo' in fid)
        fid.close()

    def test_readonly(self):
        """ Core driver can be used to open existing files """
        fname = self.mktemp()
        fid = File(fname, 'w')
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'r', driver='core')
        self.assert_(fid)
        self.assert_('foo' in fid)
        with self.assertRaises(ValueError):
            fid.create_group('bar')
        fid.close()

    def test_blocksize(self):
        """ Core driver supports variable block size """
        fname = self.mktemp()
        fid = File(fname, 'w', driver='core', block_size=1024,
                   backing_store=False)
        self.assert_(fid)
        fid.close()

    #TODO: family driver tests

class TestLibver(TestCase):

    """
        Feature: File format compatibility bounds can be specified when
        opening a file.
    """

    def test_single(self):
        """ Opening with single libver arg """
        f = File(self.mktemp(), 'w', libver='latest')
        self.assertEqual(f.libver, ('latest','latest'))
        f.close()

    def test_multiple(self):
        """ Opening with two libver args """
        f = File(self.mktemp(), 'w', libver=('earliest','latest'))
        self.assertEqual(f.libver, ('earliest', 'latest'))
        f.close()

    def test_none(self):
        """ Omitting libver arg results in maximum compatibility """
        f = File(self.mktemp(), 'w')
        self.assertEqual(f.libver, ('earliest', 'latest'))
        f.close()

class TestContextManager(TestCase):

    """
        Feature: File objects can be used as context managers
    """

    def test_context_manager(self):
        """ File objects can be used in with statements """
        with File(self.mktemp(), 'w') as fid:
            self.assertTrue(fid)
        self.assertTrue(not fid)

class TestUnicode(TestCase):

    """
        Feature: Unicode filenames are supported
    """

    def test_unicode(self):
        """ Unicode filenames can be used, and retrieved properly via .filename
        """
        fname = self.mktemp(prefix = u'\u201a')
        fid = File(fname, 'w')
        try:
            self.assertEqual(fid.filename, fname)
            self.assertIsInstance(fid.filename, unicode)
        finally:
            fid.close()

class TestFileProperty(TestCase):

    """
        Feature: A File object can be retrieved from any child object,
        via the .file property
    """

    def test_property(self):
        """ File object can be retrieved from subgroup """
        fname = self.mktemp()
        hfile = File(fname, 'w')
        try:
            hfile2 = hfile['/'].file
            self.assertEqual(hfile, hfile2)
        finally:
            hfile.close()

    def test_close(self):
        """ All retrieved File objects are closed at the same time """
        fname = self.mktemp()
        hfile = File(fname, 'w')
        grp = hfile.create_group('foo')
        hfile2 = grp.file
        hfile3 = hfile['/'].file
        hfile2.close()
        self.assertFalse(hfile)
        self.assertFalse(hfile2)
        self.assertFalse(hfile3)

    def test_mode(self):
        """ Retrieved File objects have a meaningful mode attribute """
        hfile = File(self.mktemp(),'w')
        try:
            grp = hfile.create_group('foo')
            self.assertEqual(grp.file.mode, hfile.mode)
        finally:
            hfile.close()

class TestClose(TestCase):

    """
        Feature: Files can be closed
    """

    def test_close(self):
        """ Close file via .close method """
        fid = File(self.mktemp())
        self.assert_(fid)
        fid.close()
        self.assert_(not fid)

    def test_closed_file(self):
        """ Trying to modify closed file raises ValueError """
        fid = File(self.mktemp(), 'w')
        fid.close()
        with self.assertRaises(ValueError):
            fid.create_group('foo')

class TestFlush(TestCase):

    """
        Feature: Files can be flushed
    """

    def test_flush(self):
        """ Flush via .flush method """
        fid = File(self.mktemp(), 'w')
        fid.flush()
        fid.close()

class TestRepr(TestCase):

    """
        Feature: File objects provide a helpful __repr__ string
    """

    def test_repr(self):
        """ __repr__ behaves itself when files are open and closed """
        fid = File(self.mktemp())
        self.assertIsInstance(repr(fid), basestring)
        fid.close()
        self.assertIsInstance(repr(fid), basestring)

class TestFilename(TestCase):

    """
        Feature: The name of a File object can be retrieved via .filename
    """

    def test_filename(self):
        """ .filename behaves properly for string data """
        fname = self.mktemp()
        fid = File(fname, 'w')
        try:
            self.assertEqual(fid.filename, fname)
            self.assertIsInstance(fid.filename, unicode)
        finally:
            fid.close()

class TestBackwardsCompat(TestCase):

    """
        Feauture: Deprecated attributes are included to support 1.3 code
    """

    def test_fid(self):
        """ File objects provide a .fid attribute aliased to the file ID """
        with File(self.mktemp(), 'w') as hfile:
            self.assertIs(hfile.fid, hfile.id)

<commit_msg>Add check for mode property with external links (issue 190)<commit_after>
"""
    File object test module.

    Tests all aspects of File objects, including their creation.
"""

from __future__ import with_statement

import os

from .common import ut, TestCase
from h5py.highlevel import File
import h5py

class TestFileOpen(TestCase):

    """
        Feature: Opening files with Python-style modes.
    """

    def test_create(self):
        """ Mode 'w' opens file in overwrite mode """
        fname = self.mktemp()
        fid = File(fname, 'w')
        self.assertTrue(fid)
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'w')
        self.assertNotIn('foo', fid)
        fid.close()

    def test_create_exclusive(self):
        """ Mode 'w-' opens file in exclusive mode """
        fname = self.mktemp()
        fid = File(fname, 'w-')
        self.assert_(fid)
        fid.close()
        with self.assertRaises(IOError):
            File(fname, 'w-')

    def test_append(self):
        """ Mode 'a' opens file in append/readwrite mode, creating if necessary """
        fname = self.mktemp()
        fid = File(fname, 'a')
        try:
            self.assert_(fid)
            fid.create_group('foo')
            self.assert_('foo' in fid)
        finally:
            fid.close()
        fid = File(fname, 'a')
        try:
            self.assert_('foo' in fid)
            fid.create_group('bar')
            self.assert_('bar' in fid)
        finally:
            fid.close()

    def test_readonly(self):
        """ Mode 'r' opens file in readonly mode """
        fname = self.mktemp()
        fid = File(fname, 'w')
        fid.close()
        self.assert_(not fid)
        fid = File(fname, 'r')
        self.assert_(fid)
        with self.assertRaises(ValueError):
            fid.create_group('foo')
        fid.close()

    def test_readwrite(self):
        """ Mode 'r+' opens existing file in readwrite mode """
        fname = self.mktemp()
        fid = File(fname, 'w')
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'r+')
        self.assert_('foo' in fid)
        fid.create_group('bar')
        self.assert_('bar' in fid)
        fid.close()

    def test_nonexistent_file(self):
        """ Modes 'r' and 'r+' do not create files """
        fname = self.mktemp()
        with self.assertRaises(IOError):
            File(fname, 'r')
        with self.assertRaises(IOError):
            File(fname, 'r+')

    def test_invalid_mode(self):
        """ Invalid modes raise ValueError """
        with self.assertRaises(ValueError):
            File(self.mktemp(), 'mongoose')

class TestModes(TestCase):

    """
        Feature: File mode can be retrieved via file.mode
    """

    def test_mode_attr(self):
        """ Mode equivalent can be retrieved via property """
        fname = self.mktemp()
        with File(fname, 'w') as f:
            self.assertEqual(f.mode, 'r+')
        with File(fname, 'r') as f:
            self.assertEqual(f.mode, 'r')

    def test_mode_external(self):
        """ Mode property works for files opened via external links

        Issue 190.
        """
        fname1 = self.mktemp()
        fname2 = self.mktemp()

        f1 = File(fname1,'w')
        f1.close()

        f2 = File(fname2,'w')
        try:
            f2['External'] = h5py.ExternalLink(fname1, '/')
            f3 = f2['External'].file
            self.assertEqual(f3.mode, 'r+')
        finally:
            f2.close()

        f2 = File(fname2,'r')
        try:
            f3 = f2['External'].file
            self.assertEqual(f3.mode, 'r')
        finally:
            f2.close()

class TestDrivers(TestCase):

    """
        Feature: Files can be opened with low-level HDF5 drivers
    """

    @ut.skipUnless(os.name, 'posix')
    def test_stdio(self):
        """ Stdio driver is supported on posix """
        fid = File(self.mktemp(), 'w', driver='stdio')
        self.assertTrue(fid)
        self.assertEqual(fid.driver, 'stdio')
        fid.close()

    @ut.skipUnless(os.name, 'posix')
    def test_sec2(self):
        """ Sec2 driver is supported on posix """
        fid = File(self.mktemp(), 'w', driver='sec2')
        self.assert_(fid)
        self.assertEqual(fid.driver, 'sec2')
        fid.close()

    def test_core(self):
        """ Core driver is supported (no backing store) """
        fname = self.mktemp()
        fid = File(fname, 'w', driver='core', backing_store=False)
        self.assert_(fid)
        self.assertEqual(fid.driver, 'core')
        fid.close()
        self.assertFalse(os.path.exists(fname))

    def test_backing(self):
        """ Core driver saves to file when backing store used """
        fname = self.mktemp()
        fid = File(fname, 'w', driver='core', backing_store=True)
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'r')
        self.assert_('foo' in fid)
        fid.close()

    def test_readonly(self):
        """ Core driver can be used to open existing files """
        fname = self.mktemp()
        fid = File(fname, 'w')
        fid.create_group('foo')
        fid.close()
        fid = File(fname, 'r', driver='core')
        self.assert_(fid)
        self.assert_('foo' in fid)
        with self.assertRaises(ValueError):
            fid.create_group('bar')
        fid.close()

    def test_blocksize(self):
        """ Core driver supports variable block size """
        fname = self.mktemp()
        fid = File(fname, 'w', driver='core', block_size=1024,
                   backing_store=False)
        self.assert_(fid)
        fid.close()

    #TODO: family driver tests

class TestLibver(TestCase):

    """
        Feature: File format compatibility bounds can be specified when
        opening a file.
    """

    def test_single(self):
        """ Opening with single libver arg """
        f = File(self.mktemp(), 'w', libver='latest')
        self.assertEqual(f.libver, ('latest','latest'))
        f.close()

    def test_multiple(self):
        """ Opening with two libver args """
        f = File(self.mktemp(), 'w', libver=('earliest','latest'))
        self.assertEqual(f.libver, ('earliest', 'latest'))
        f.close()

    def test_none(self):
        """ Omitting libver arg results in maximum compatibility """
        f = File(self.mktemp(), 'w')
        self.assertEqual(f.libver, ('earliest', 'latest'))
        f.close()

class TestContextManager(TestCase):

    """
        Feature: File objects can be used as context managers
    """

    def test_context_manager(self):
        """ File objects can be used in with statements """
        with File(self.mktemp(), 'w') as fid:
            self.assertTrue(fid)
        self.assertTrue(not fid)

class TestUnicode(TestCase):

    """
        Feature: Unicode filenames are supported
    """

    def test_unicode(self):
        """ Unicode filenames can be used, and retrieved properly via .filename
        """
        fname = self.mktemp(prefix = u'\u201a')
        fid = File(fname, 'w')
        try:
            self.assertEqual(fid.filename, fname)
            self.assertIsInstance(fid.filename, unicode)
        finally:
            fid.close()

class TestFileProperty(TestCase):

    """
        Feature: A File object can be retrieved from any child object,
        via the .file property
    """

    def test_property(self):
        """ File object can be retrieved from subgroup """
        fname = self.mktemp()
        hfile = File(fname, 'w')
        try:
            hfile2 = hfile['/'].file
            self.assertEqual(hfile, hfile2)
        finally:
            hfile.close()

    def test_close(self):
        """ All retrieved File objects are closed at the same time """
        fname = self.mktemp()
        hfile = File(fname, 'w')
        grp = hfile.create_group('foo')
        hfile2 = grp.file
        hfile3 = hfile['/'].file
        hfile2.close()
        self.assertFalse(hfile)
        self.assertFalse(hfile2)
        self.assertFalse(hfile3)

    def test_mode(self):
        """ Retrieved File objects have a meaningful mode attribute """
        hfile = File(self.mktemp(),'w')
        try:
            grp = hfile.create_group('foo')
            self.assertEqual(grp.file.mode, hfile.mode)
        finally:
            hfile.close()

class TestClose(TestCase):

    """
        Feature: Files can be closed
    """

    def test_close(self):
        """ Close file via .close method """
        fid = File(self.mktemp())
        self.assert_(fid)
        fid.close()
        self.assert_(not fid)

    def test_closed_file(self):
        """ Trying to modify closed file raises ValueError """
        fid = File(self.mktemp(), 'w')
        fid.close()
        with self.assertRaises(ValueError):
            fid.create_group('foo')

class TestFlush(TestCase):

    """
        Feature: Files can be flushed
    """

    def test_flush(self):
        """ Flush via .flush method """
        fid = File(self.mktemp(), 'w')
        fid.flush()
        fid.close()

class TestRepr(TestCase):

    """
        Feature: File objects provide a helpful __repr__ string
    """

    def test_repr(self):
        """ __repr__ behaves itself when files are open and closed """
        fid = File(self.mktemp())
        self.assertIsInstance(repr(fid), basestring)
        fid.close()
        self.assertIsInstance(repr(fid), basestring)

class TestFilename(TestCase):

    """
        Feature: The name of a File object can be retrieved via .filename
    """

    def test_filename(self):
        """ .filename behaves properly for string data """
        fname = self.mktemp()
        fid = File(fname, 'w')
        try:
            self.assertEqual(fid.filename, fname)
            self.assertIsInstance(fid.filename, unicode)
        finally:
            fid.close()

class TestBackwardsCompat(TestCase):

    """
        Feauture: Deprecated attributes are included to support 1.3 code
    """

    def test_fid(self):
        """ File objects provide a .fid attribute aliased to the file ID """
        with File(self.mktemp(), 'w') as hfile:
            self.assertIs(hfile.fid, hfile.id)

<|endoftext|>


<commit_before>"""Imports awardees / HPOs, organizations, and sites into the database.

Usage:
  tools/import_organizations.sh [--account <USER>@pmi-ops.org --project <PROJECT>] \
      --awardee_file <AWARDEE CSV> --organization_file <ORGANIZATION CSV> \
      --site_file <SITE CSV> [--dry_run]

  The CSV files originate from the Google Sheets found here:

  https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit

  Imports are idempotent; if you run the import multiple times, subsequent imports should
  have no effect.
"""
import os
import logging
import googlemaps
from dateutil.parser import parse
from tools.csv_importer import CsvImporter
from dao.hpo_dao import HPODao
from dao.organization_dao import OrganizationDao
from dao.site_dao import SiteDao
from model.hpo import HPO
from model.organization import Organization
from model.site import Site
from model.site_enums import SiteStatus
from participant_enums import OrganizationType
from main_util import get_parser, configure_logging

# Column headers from the Awardees sheet at:
# https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit#gid=1076878570
HPO_AWARDEE_ID_COLUMN = 'Awardee ID'
HPO_NAME_COLUMN = 'Name'
HPO_TYPE_COLUMN = 'Type'

# Column headers for the Organizations sheet at:
# https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit#gid=1098779958
ORGANIZATION_AWARDEE_ID_COLUMN = 'Awardee ID'
ORGANIZATION_ORGANIZATION_ID_COLUMN = 'Organization ID'
ORGANIZATION_NAME_COLUMN = 'Name'

# Column headers for the Sites sheet at:
# https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit#gid=0
SITE_ORGANIZATION_ID_COLUMN = 'Organization ID'
SITE_SITE_ID_COLUMN = 'Site ID / Google Group'
SITE_SITE_COLUMN = 'Site'
SITE_MAYOLINK_CLIENT_NUMBER_COLUMN = 'MayoLINK Client #'
SITE_NOTES_COLUMN = 'Notes'
SITE_STATUS_COLUMN = 'Status'
SITE_LAUNCH_DATE_COLUMN = 'Anticipated Launch Date'
SITE_DIRECTIONS_COLUMN = 'Directions'
SITE_PHYSICAL_LOCATION_NAME_COLUMN = 'Physical Location Name'
SITE_ADDRESS_1_COLUMN = 'Address 1'
SITE_ADDRESS_2_COLUMN = 'Address 2'
SITE_CITY_COLUMN = 'City'
SITE_STATE_COLUMN = 'State'
SITE_ZIP_COLUMN = 'Zip'
SITE_PHONE_COLUMN = 'Phone'
SITE_ADMIN_EMAIL_ADDRESSES_COLUMN = 'Admin Email Addresses'
SITE_LINK_COLUMN = 'Link'

class HPOImporter(CsvImporter):

  def __init__(self):
    super(HPOImporter, self).__init__('awardee', HPODao(), 'hpoId', 'name',
                                      [HPO_AWARDEE_ID_COLUMN, HPO_NAME_COLUMN, HPO_TYPE_COLUMN])
    self.new_count = 0

  def _entity_from_row(self, row):
    type_str = row[HPO_TYPE_COLUMN]
    try:
      organization_type = OrganizationType(type_str)
      if organization_type == OrganizationType.UNSET:
        organization_type = None
    except TypeError:
      logging.warn('Invalid organization type %s for awardee %s', type_str,
      row[HPO_AWARDEE_ID_COLUMN])
      return None
    return HPO(name=row[HPO_AWARDEE_ID_COLUMN],
               displayName=row[HPO_NAME_COLUMN],
               organizationType=organization_type)

  def _insert_entity(self, entity, existing_map, session, dry_run):
    # HPO IDs are not autoincremented by the database; manually set it here.
    entity.hpoId = len(existing_map) + self.new_count
    self.new_count += 1
    super(HPOImporter, self)._insert_entity(entity, existing_map, session, dry_run)

class OrganizationImporter(CsvImporter):

  def __init__(self):
    super(OrganizationImporter, self).__init__('organization', OrganizationDao(),
                                               'organizationId', 'externalId',
                                               [ORGANIZATION_AWARDEE_ID_COLUMN,
                                                ORGANIZATION_ORGANIZATION_ID_COLUMN,
                                                ORGANIZATION_NAME_COLUMN])
    self.hpo_dao = HPODao()

  def _entity_from_row(self, row):
    hpo = self.hpo_dao.get_by_name(row[ORGANIZATION_AWARDEE_ID_COLUMN])
    if hpo is None:
      logging.warn('Invalid awardee ID %s importing organization %s',
                   row[ORGANIZATION_AWARDEE_ID_COLUMN],
                   row[ORGANIZATION_ORGANIZATION_ID_COLUMN])
      return None
    return Organization(externalId=row[ORGANIZATION_ORGANIZATION_ID_COLUMN],
                        displayName=row[ORGANIZATION_NAME_COLUMN],
                        hpoId=hpo.hpoId)

class SiteImporter(CsvImporter):

  def __init__(self):
    super(SiteImporter, self).__init__('site', SiteDao(), 'siteId', 'googleGroup',
                                       [SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
                                       SITE_SITE_COLUMN, SITE_STATUS_COLUMN])

    self.organization_dao = OrganizationDao()
    args = parser.parse_args()
    self.geocode_flag = args.geocode_flag

  def _entity_from_row(self, row):
    google_group = row[SITE_SITE_ID_COLUMN].lower()
    organization = self.organization_dao.get_by_external_id(row[SITE_ORGANIZATION_ID_COLUMN])
    if organization is None:
      logging.warn('Invalid organization ID %s importing site %s', row[SITE_ORGANIZATION_ID_COLUMN],
                   google_group)
      return None

    launch_date = None
    launch_date_str = row.get(SITE_LAUNCH_DATE_COLUMN)
    if launch_date_str:
      try:
        launch_date = parse(launch_date_str).date()
      except ValueError:
        logging.warn('Invalid launch date %s for site %s', launch_date_str, google_group)
        return None
    name = row[SITE_SITE_COLUMN]
    mayolink_client_number = None
    mayolink_client_number_str = row.get(SITE_MAYOLINK_CLIENT_NUMBER_COLUMN)
    if mayolink_client_number_str:
      try:
        mayolink_client_number = int(mayolink_client_number_str)
      except ValueError:
        logging.warn('Invalid Mayolink Client # %s for site %s', mayolink_client_number_str,
                     google_group)
        return None
    notes = row.get(SITE_NOTES_COLUMN)
    try:
      site_status = SiteStatus(row[SITE_STATUS_COLUMN].upper())
    except TypeError:
      logging.warn('Invalid site status %s for site %s', row[SITE_STATUS_COLUMN], google_group)
      return None
    directions = row.get(SITE_DIRECTIONS_COLUMN)
    physical_location_name = row.get(SITE_PHYSICAL_LOCATION_NAME_COLUMN)
    address_1 = row.get(SITE_ADDRESS_1_COLUMN)
    address_2 = row.get(SITE_ADDRESS_2_COLUMN)
    city = row.get(SITE_CITY_COLUMN)
    state = row.get(SITE_STATE_COLUMN)
    zip_code = row.get(SITE_ZIP_COLUMN)
    phone = row.get(SITE_PHONE_COLUMN)
    admin_email_addresses = row.get(SITE_ADMIN_EMAIL_ADDRESSES_COLUMN)
    link = row.get(SITE_LINK_COLUMN)
    return Site(siteName=name,
                googleGroup=google_group,
                mayolinkClientNumber=mayolink_client_number,
                organizationId=organization.organizationId,
                hpoId=organization.hpoId,
                siteStatus=site_status,
                launchDate=launch_date,
                notes=notes,
                directions=directions,
                physicalLocationName=physical_location_name,
                address1=address_1,
                address2=address_2,
                city=city,
                state=state,
                zipCode=zip_code,
                phoneNumber=phone,
                adminEmails=admin_email_addresses,
                link=link)

  def _update_entity(self, entity, existing_entity, session, dry_run):
    self._populate_lat_lng_and_time_zone(entity, existing_entity)
    return super(SiteImporter, self)._update_entity(entity, existing_entity, session, dry_run)

  def _insert_entity(self, entity, existing_map, session, dry_run):
    self._populate_lat_lng_and_time_zone(entity, None)
    super(SiteImporter, self)._insert_entity(entity, existing_map, session, dry_run)

  def _populate_lat_lng_and_time_zone(self, site, existing_site):
    if site.address1 and site.city and site.state:
      if existing_site:
        if (existing_site.address1 == site.address1 and existing_site.city == site.city
            and existing_site.state == site.state and existing_site.latitude is not None
            and existing_site.longitude is not None and existing_site.timeZoneId is not None):
          # Address didn't change, use the existing lat/lng and time zone.
          site.latitude = existing_site.latitude
          site.longitude = existing_site.longitude
          site.timeZoneId = existing_site.timeZoneId
          return
      if self.geocode_flag:
        latitude, longitude = self._get_lat_long_for_site(site.address1, site.city, site.state)
        site.latitude = latitude
        site.longitude = longitude
        if latitude and longitude:
          site.timeZoneId = self._get_time_zone(latitude, longitude)

  def _get_lat_long_for_site(self, address_1, city, state):
    self.full_address = address_1 + ' ' +  city + ' ' + state
    try:
      self.api_key = os.environ.get('API_KEY')
      self.gmaps = googlemaps.Client(key=self.api_key)
      geocode_result = self.gmaps.geocode(address_1 + '' +  city + ' ' +  state)[0]
      if geocode_result:
        geometry = geocode_result.get('geometry')
        if geometry:
          location = geometry.get('location')
        if location:
          latitude = location.get('lat')
          longitude = location.get('lng')
          return latitude, longitude
        else:
          logging.warn('Can not find lat/long for %s', self.full_address)
          return None, None
      else:
        logging.warn('Geocode results failed for %s.', self.full_address)
        return None, None
    except ValueError as e:
      logging.exception('Invalid geocode key: %s. ERROR: %s', self.api_key, e)
      return None, None
    except IndexError as e:
      logging.exception('Geocoding failure Check that address is correct. ERROR: %s', e)
      return None, None

  def _get_time_zone(self, latitude, longitude):
    time_zone = self.gmaps.timezone(location=(latitude, longitude))
    if time_zone['status'] == 'OK':
      time_zone_id = time_zone['timeZoneId']
      return time_zone_id
    else:
      logging.info('can not retrieve time zone from %s', self.full_address)
      return None

def main(args):
  HPOImporter().run(args.awardee_file, args.dry_run)
  OrganizationImporter().run(args.organization_file, args.dry_run)
  SiteImporter().run(args.site_file, args.dry_run)

if __name__ == '__main__':
  configure_logging()
  parser = get_parser()
  parser.add_argument('--awardee_file', help='Filename containing awardee CSV to import',
                      required=True)
  parser.add_argument('--organization_file', help='Filename containing organization CSV to import',
                      required=True)
  parser.add_argument('--site_file', help='Filename containing site CSV to import',
                      required=True)
  parser.add_argument('--dry_run', help='Read CSV and check for diffs against database.',
                      action='store_true')
  parser.add_argument('--geocode_flag', help='If --account passed into import_organizations.sh, '
                                             'geocoding is performed.',
                      action='store_true')

  main(parser.parse_args())
<commit_msg>[DA-538] Switch import_organizations.sh to read from new PTC status column temporarily. (#616)<commit_after>"""Imports awardees / HPOs, organizations, and sites into the database.

Usage:
  tools/import_organizations.sh [--account <USER>@pmi-ops.org --project <PROJECT>] \
      --awardee_file <AWARDEE CSV> --organization_file <ORGANIZATION CSV> \
      --site_file <SITE CSV> [--dry_run]

  The CSV files originate from the Google Sheets found here:

  https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit

  Imports are idempotent; if you run the import multiple times, subsequent imports should
  have no effect.
"""
import os
import logging
import googlemaps
from dateutil.parser import parse
from tools.csv_importer import CsvImporter
from dao.hpo_dao import HPODao
from dao.organization_dao import OrganizationDao
from dao.site_dao import SiteDao
from model.hpo import HPO
from model.organization import Organization
from model.site import Site
from model.site_enums import SiteStatus
from participant_enums import OrganizationType
from main_util import get_parser, configure_logging

# Column headers from the Awardees sheet at:
# https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit#gid=1076878570
HPO_AWARDEE_ID_COLUMN = 'Awardee ID'
HPO_NAME_COLUMN = 'Name'
HPO_TYPE_COLUMN = 'Type'

# Column headers for the Organizations sheet at:
# https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit#gid=1098779958
ORGANIZATION_AWARDEE_ID_COLUMN = 'Awardee ID'
ORGANIZATION_ORGANIZATION_ID_COLUMN = 'Organization ID'
ORGANIZATION_NAME_COLUMN = 'Name'

# Column headers for the Sites sheet at:
# https://docs.google.com/spreadsheets/d/1CcIGRV0Bd6BIz7PeuvrV6QDGDRQkp83CJUkWAl-fG58/edit#gid=0
SITE_ORGANIZATION_ID_COLUMN = 'Organization ID'
SITE_SITE_ID_COLUMN = 'Site ID / Google Group'
SITE_SITE_COLUMN = 'Site'
SITE_MAYOLINK_CLIENT_NUMBER_COLUMN = 'MayoLINK Client #'
SITE_NOTES_COLUMN = 'Notes'
# TODO: switch this back to 'Status' [DA-538]
SITE_STATUS_COLUMN = 'PTSC Scheduling Status'
SITE_LAUNCH_DATE_COLUMN = 'Anticipated Launch Date'
SITE_DIRECTIONS_COLUMN = 'Directions'
SITE_PHYSICAL_LOCATION_NAME_COLUMN = 'Physical Location Name'
SITE_ADDRESS_1_COLUMN = 'Address 1'
SITE_ADDRESS_2_COLUMN = 'Address 2'
SITE_CITY_COLUMN = 'City'
SITE_STATE_COLUMN = 'State'
SITE_ZIP_COLUMN = 'Zip'
SITE_PHONE_COLUMN = 'Phone'
SITE_ADMIN_EMAIL_ADDRESSES_COLUMN = 'Admin Email Addresses'
SITE_LINK_COLUMN = 'Link'

class HPOImporter(CsvImporter):

  def __init__(self):
    super(HPOImporter, self).__init__('awardee', HPODao(), 'hpoId', 'name',
                                      [HPO_AWARDEE_ID_COLUMN, HPO_NAME_COLUMN, HPO_TYPE_COLUMN])
    self.new_count = 0

  def _entity_from_row(self, row):
    type_str = row[HPO_TYPE_COLUMN]
    try:
      organization_type = OrganizationType(type_str)
      if organization_type == OrganizationType.UNSET:
        organization_type = None
    except TypeError:
      logging.warn('Invalid organization type %s for awardee %s', type_str,
      row[HPO_AWARDEE_ID_COLUMN])
      return None
    return HPO(name=row[HPO_AWARDEE_ID_COLUMN],
               displayName=row[HPO_NAME_COLUMN],
               organizationType=organization_type)

  def _insert_entity(self, entity, existing_map, session, dry_run):
    # HPO IDs are not autoincremented by the database; manually set it here.
    entity.hpoId = len(existing_map) + self.new_count
    self.new_count += 1
    super(HPOImporter, self)._insert_entity(entity, existing_map, session, dry_run)

class OrganizationImporter(CsvImporter):

  def __init__(self):
    super(OrganizationImporter, self).__init__('organization', OrganizationDao(),
                                               'organizationId', 'externalId',
                                               [ORGANIZATION_AWARDEE_ID_COLUMN,
                                                ORGANIZATION_ORGANIZATION_ID_COLUMN,
                                                ORGANIZATION_NAME_COLUMN])
    self.hpo_dao = HPODao()

  def _entity_from_row(self, row):
    hpo = self.hpo_dao.get_by_name(row[ORGANIZATION_AWARDEE_ID_COLUMN])
    if hpo is None:
      logging.warn('Invalid awardee ID %s importing organization %s',
                   row[ORGANIZATION_AWARDEE_ID_COLUMN],
                   row[ORGANIZATION_ORGANIZATION_ID_COLUMN])
      return None
    return Organization(externalId=row[ORGANIZATION_ORGANIZATION_ID_COLUMN],
                        displayName=row[ORGANIZATION_NAME_COLUMN],
                        hpoId=hpo.hpoId)

class SiteImporter(CsvImporter):

  def __init__(self):
    super(SiteImporter, self).__init__('site', SiteDao(), 'siteId', 'googleGroup',
                                       [SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
                                       SITE_SITE_COLUMN, SITE_STATUS_COLUMN])

    self.organization_dao = OrganizationDao()
    args = parser.parse_args()
    self.geocode_flag = args.geocode_flag

  def _entity_from_row(self, row):
    google_group = row[SITE_SITE_ID_COLUMN].lower()
    organization = self.organization_dao.get_by_external_id(row[SITE_ORGANIZATION_ID_COLUMN])
    if organization is None:
      logging.warn('Invalid organization ID %s importing site %s', row[SITE_ORGANIZATION_ID_COLUMN],
                   google_group)
      return None

    launch_date = None
    launch_date_str = row.get(SITE_LAUNCH_DATE_COLUMN)
    if launch_date_str:
      try:
        launch_date = parse(launch_date_str).date()
      except ValueError:
        logging.warn('Invalid launch date %s for site %s', launch_date_str, google_group)
        return None
    name = row[SITE_SITE_COLUMN]
    mayolink_client_number = None
    mayolink_client_number_str = row.get(SITE_MAYOLINK_CLIENT_NUMBER_COLUMN)
    if mayolink_client_number_str:
      try:
        mayolink_client_number = int(mayolink_client_number_str)
      except ValueError:
        logging.warn('Invalid Mayolink Client # %s for site %s', mayolink_client_number_str,
                     google_group)
        return None
    notes = row.get(SITE_NOTES_COLUMN)
    try:
      site_status = SiteStatus(row[SITE_STATUS_COLUMN].upper())
    except TypeError:
      logging.warn('Invalid site status %s for site %s', row[SITE_STATUS_COLUMN], google_group)
      return None
    directions = row.get(SITE_DIRECTIONS_COLUMN)
    physical_location_name = row.get(SITE_PHYSICAL_LOCATION_NAME_COLUMN)
    address_1 = row.get(SITE_ADDRESS_1_COLUMN)
    address_2 = row.get(SITE_ADDRESS_2_COLUMN)
    city = row.get(SITE_CITY_COLUMN)
    state = row.get(SITE_STATE_COLUMN)
    zip_code = row.get(SITE_ZIP_COLUMN)
    phone = row.get(SITE_PHONE_COLUMN)
    admin_email_addresses = row.get(SITE_ADMIN_EMAIL_ADDRESSES_COLUMN)
    link = row.get(SITE_LINK_COLUMN)
    return Site(siteName=name,
                googleGroup=google_group,
                mayolinkClientNumber=mayolink_client_number,
                organizationId=organization.organizationId,
                hpoId=organization.hpoId,
                siteStatus=site_status,
                launchDate=launch_date,
                notes=notes,
                directions=directions,
                physicalLocationName=physical_location_name,
                address1=address_1,
                address2=address_2,
                city=city,
                state=state,
                zipCode=zip_code,
                phoneNumber=phone,
                adminEmails=admin_email_addresses,
                link=link)

  def _update_entity(self, entity, existing_entity, session, dry_run):
    self._populate_lat_lng_and_time_zone(entity, existing_entity)
    return super(SiteImporter, self)._update_entity(entity, existing_entity, session, dry_run)

  def _insert_entity(self, entity, existing_map, session, dry_run):
    self._populate_lat_lng_and_time_zone(entity, None)
    super(SiteImporter, self)._insert_entity(entity, existing_map, session, dry_run)

  def _populate_lat_lng_and_time_zone(self, site, existing_site):
    if site.address1 and site.city and site.state:
      if existing_site:
        if (existing_site.address1 == site.address1 and existing_site.city == site.city
            and existing_site.state == site.state and existing_site.latitude is not None
            and existing_site.longitude is not None and existing_site.timeZoneId is not None):
          # Address didn't change, use the existing lat/lng and time zone.
          site.latitude = existing_site.latitude
          site.longitude = existing_site.longitude
          site.timeZoneId = existing_site.timeZoneId
          return
      if self.geocode_flag:
        latitude, longitude = self._get_lat_long_for_site(site.address1, site.city, site.state)
        site.latitude = latitude
        site.longitude = longitude
        if latitude and longitude:
          site.timeZoneId = self._get_time_zone(latitude, longitude)

  def _get_lat_long_for_site(self, address_1, city, state):
    self.full_address = address_1 + ' ' +  city + ' ' + state
    try:
      self.api_key = os.environ.get('API_KEY')
      self.gmaps = googlemaps.Client(key=self.api_key)
      geocode_result = self.gmaps.geocode(address_1 + '' +  city + ' ' +  state)[0]
      if geocode_result:
        geometry = geocode_result.get('geometry')
        if geometry:
          location = geometry.get('location')
        if location:
          latitude = location.get('lat')
          longitude = location.get('lng')
          return latitude, longitude
        else:
          logging.warn('Can not find lat/long for %s', self.full_address)
          return None, None
      else:
        logging.warn('Geocode results failed for %s.', self.full_address)
        return None, None
    except ValueError as e:
      logging.exception('Invalid geocode key: %s. ERROR: %s', self.api_key, e)
      return None, None
    except IndexError as e:
      logging.exception('Geocoding failure Check that address is correct. ERROR: %s', e)
      return None, None

  def _get_time_zone(self, latitude, longitude):
    time_zone = self.gmaps.timezone(location=(latitude, longitude))
    if time_zone['status'] == 'OK':
      time_zone_id = time_zone['timeZoneId']
      return time_zone_id
    else:
      logging.info('can not retrieve time zone from %s', self.full_address)
      return None

def main(args):
  HPOImporter().run(args.awardee_file, args.dry_run)
  OrganizationImporter().run(args.organization_file, args.dry_run)
  SiteImporter().run(args.site_file, args.dry_run)

if __name__ == '__main__':
  configure_logging()
  parser = get_parser()
  parser.add_argument('--awardee_file', help='Filename containing awardee CSV to import',
                      required=True)
  parser.add_argument('--organization_file', help='Filename containing organization CSV to import',
                      required=True)
  parser.add_argument('--site_file', help='Filename containing site CSV to import',
                      required=True)
  parser.add_argument('--dry_run', help='Read CSV and check for diffs against database.',
                      action='store_true')
  parser.add_argument('--geocode_flag', help='If --account passed into import_organizations.sh, '
                                             'geocoding is performed.',
                      action='store_true')

  main(parser.parse_args())
<|endoftext|>


<commit_before>"""Generates API documentation by introspection."""
from django.http import HttpRequest

from rest_framework import viewsets
from rest_framework.serializers import BaseSerializer

from .introspectors import APIViewIntrospector, \
    ViewSetIntrospector, BaseMethodIntrospector, IntrospectorHelper, \
    get_resolved_value, YAMLDocstringParser

class DocumentationGenerator(object):
    # Serializers defined in docstrings
    explicit_serializers = set()

    # Serializers defined in fields
    fields_serializers = set()

    # Response classes defined in docstrings
    explicit_response_types = dict()

    def generate(self, apis):
        """
        Returns documentation for a list of APIs
        """
        api_docs = []
        for api in apis:
            api_docs.append({
                'description': IntrospectorHelper.get_view_description(api['callback']),
                'path': api['path'],
                'operations': self.get_operations(api),
            })

        return api_docs

    def get_operations(self, api):
        """
        Returns docs for the allowed methods of an API endpoint
        """
        operations = []
        path = api['path']
        pattern = api['pattern']
        callback = api['callback']
        callback.request = HttpRequest()

        if issubclass(callback, viewsets.ViewSetMixin):
            introspector = ViewSetIntrospector(callback, path, pattern)
        else:
            introspector = APIViewIntrospector(callback, path, pattern)

        for method_introspector in introspector:
            if not isinstance(method_introspector, BaseMethodIntrospector) or \
                    method_introspector.get_http_method() == "OPTIONS":
                continue  # No one cares. I impose JSON.

            doc_parser = YAMLDocstringParser(
                docstring=method_introspector.get_docs())

            serializer = self._get_method_serializer(
                doc_parser, method_introspector)

            response_type = self._get_method_response_type(
                doc_parser, serializer, introspector, method_introspector)

            operation = {
                'method': method_introspector.get_http_method(),
                'summary': method_introspector.get_summary(),
                'nickname': method_introspector.get_nickname(),
                'notes': method_introspector.get_notes(),
                'type': response_type,
            }

            if doc_parser.yaml_error is not None:
                operation['notes'] += "
YAMLError:\n {err}
".format(
                    err=doc_parser.yaml_error)

            response_messages = doc_parser.get_response_messages()
            parameters = doc_parser.discover_parameters(
                inspector=method_introspector)

            if parameters:
                operation['parameters'] = parameters

            if response_messages:
                operation['responseMessages'] = response_messages

            operations.append(operation)

        return operations

    def get_models(self, apis):
        """
        Builds a list of Swagger 'models'. These represent
        DRF serializers and their fields
        """
        serializers = self._get_serializer_set(apis)
        serializers.update(self.explicit_serializers)
        serializers.update(
            self._find_field_serializers(serializers)
        )

        models = {}

        for serializer in serializers:
            data = self._get_serializer_fields(serializer)

            # Register 2 models with different subset of properties suitable
            # for data reading and writing.
            # i.e. rest framework does not output write_only fields in response
            # or require read_only fields in complex input.

            serializer_name = IntrospectorHelper.get_serializer_name(serializer)
            # Writing
            # no readonly fields
            w_name = "Write{serializer}".format(serializer=serializer_name)

            w_properties = dict((k, v) for k, v in data['fields'].items()
                                if k not in data['read_only'])

            models[w_name] = {
                'id': w_name,
                'required': [i for i in data['required'] if i in w_properties.keys()],
                'properties': w_properties,
            }

            # Reading
            # no write_only fields
            r_name = serializer_name

            r_properties = dict((k, v) for k, v in data['fields'].items()
                                if k not in data['write_only'])

            models[r_name] = {
                'id': r_name,
                'required': [i for i in r_properties.keys()],
                'properties': r_properties,
            }

            # Enable original model for testing purposes
            # models[serializer_name] = {
            #     'id': serializer_name,
            #     'required': data['required'],
            #     'properties': data['fields'],
            # }

        models.update(self.explicit_response_types)
        models.update(self.fields_serializers)
        return models

    def _get_method_serializer(self, doc_parser, method_inspector):
        """
        Returns serializer used in method.
        Registers custom serializer from docstring in scope.

        Serializer might be ignored if explicitly told in docstring
        """
        serializer = method_inspector.get_serializer_class()

        docstring_serializer = doc_parser.get_serializer_class(
            callback=method_inspector.callback
        )

        if doc_parser.get_response_type() is not None:
            # Custom response class detected
            return None

        if docstring_serializer is not None:
            self.explicit_serializers.add(docstring_serializer)
            serializer = docstring_serializer

        if doc_parser.should_omit_serializer():
            serializer = None

        return serializer

    def _get_method_response_type(self, doc_parser, serializer,
                                  view_inspector, method_inspector):
        """
        Returns response type for method.
        This might be custom `type` from docstring or discovered
        serializer class name.

        Once custom `type` found in docstring - it'd be
        registered in a scope
        """
        response_type = doc_parser.get_response_type()
        if response_type is not None:
            # Register class in scope
            view_name = view_inspector.callback.__name__
            view_name = view_name.replace('ViewSet', '')
            view_name = view_name.replace('APIView', '')
            view_name = view_name.replace('View', '')
            response_type_name = "{view}{method}Response".format(
                view=view_name,
                method=method_inspector.method.title().replace('_', '')
            )
            self.explicit_response_types.update({
                response_type_name: {
                    "id": response_type_name,
                    "properties": response_type
                }
            })
            return response_type_name
        else:
            serializer_name = IntrospectorHelper.get_serializer_name(serializer)
            if serializer_name is not None:
                return serializer_name

            return None

    def _get_serializer_set(self, apis):
        """
        Returns a set of serializer classes for a provided list
        of APIs
        """
        serializers = set()

        for api in apis:
            serializer = self._get_serializer_class(api['callback'])
            if serializer is not None:
                serializers.add(serializer)

        return serializers

    def _find_field_serializers(self, serializers):
        """
        Returns set of serializers discovered from fields
        """
        serializers_set = set()
        for serializer in serializers:
            fields = serializer().get_fields()
            for name, field in fields.items():
                if isinstance(field, BaseSerializer):
                    serializers_set.add(field)

        return serializers_set

    def _get_serializer_fields(self, serializer):
        """
        Returns serializer fields in the Swagger MODEL format
        """
        if serializer is None:
            return

        if hasattr(serializer, '__call__'):
            fields = serializer().get_fields()
        else:
            fields = serializer.get_fields()

        data = {
            'fields': {},
            'required': [],
            'write_only': [],
            'read_only': [],
        }
        for name, field in fields.items():
            if getattr(field, 'write_only', False):
                data['write_only'].append(name)

            if getattr(field, 'read_only', False):
                data['read_only'].append(name)

            if getattr(field, 'required', False):
                data['required'].append(name)

            data_type = field.type_label

            # guess format
            data_format = 'string'
            if data_type in BaseMethodIntrospector.PRIMITIVES:
                data_format = BaseMethodIntrospector.PRIMITIVES.get(data_type)[0]

            f = {
                'description': getattr(field, 'help_text', ''),
                'type': data_type,
                'format': data_format,
                'required': getattr(field, 'required', False),
                'defaultValue': get_resolved_value(field, 'default'),
                'readOnly': getattr(field, 'read_only', None),
            }

            # Min/Max values
            max_val = getattr(field, 'max_val', None)
            min_val = getattr(field, 'min_val', None)
            if max_val is not None and data_type == 'integer':
                f['minimum'] = min_val

            if max_val is not None and data_type == 'integer':
                f['maximum'] = max_val

            # ENUM options
            if field.type_label == 'multiple choice' \
                    and isinstance(field.choices, list):
                f['enum'] = [k for k, v in field.choices]

            # Support for complex types
            if isinstance(field, BaseSerializer):
                field_serializer = IntrospectorHelper.get_serializer_name(field)

                if getattr(field, 'write_only', False):
                    field_serializer = "Write{}".format(field_serializer)

                f['type'] = field_serializer
                if field.many:
                    f['type'] = 'array'
                    if data_type in BaseMethodIntrospector.PRIMITIVES:
                        f['items'] = {'type': data_type}
                    else:
                        f['items'] = {'$ref': field_serializer}

            # memorize discovered field
            data['fields'][name] = f

        return data

    def _get_serializer_class(self, callback):
        if hasattr(callback, 'get_serializer_class'):
            return callback().get_serializer_class()
<commit_msg>support nested serializers<commit_after>"""Generates API documentation by introspection."""
from django.http import HttpRequest

from rest_framework import viewsets
from rest_framework.serializers import BaseSerializer

from .introspectors import APIViewIntrospector, \
    ViewSetIntrospector, BaseMethodIntrospector, IntrospectorHelper, \
    get_resolved_value, YAMLDocstringParser

class DocumentationGenerator(object):
    # Serializers defined in docstrings
    explicit_serializers = set()

    # Serializers defined in fields
    fields_serializers = set()

    # Response classes defined in docstrings
    explicit_response_types = dict()

    def generate(self, apis):
        """
        Returns documentation for a list of APIs
        """
        api_docs = []
        for api in apis:
            api_docs.append({
                'description': IntrospectorHelper.get_view_description(api['callback']),
                'path': api['path'],
                'operations': self.get_operations(api),
            })

        return api_docs

    def get_operations(self, api):
        """
        Returns docs for the allowed methods of an API endpoint
        """
        operations = []
        path = api['path']
        pattern = api['pattern']
        callback = api['callback']
        callback.request = HttpRequest()

        if issubclass(callback, viewsets.ViewSetMixin):
            introspector = ViewSetIntrospector(callback, path, pattern)
        else:
            introspector = APIViewIntrospector(callback, path, pattern)

        for method_introspector in introspector:
            if not isinstance(method_introspector, BaseMethodIntrospector) or \
                    method_introspector.get_http_method() == "OPTIONS":
                continue  # No one cares. I impose JSON.

            doc_parser = YAMLDocstringParser(
                docstring=method_introspector.get_docs())

            serializer = self._get_method_serializer(
                doc_parser, method_introspector)

            response_type = self._get_method_response_type(
                doc_parser, serializer, introspector, method_introspector)

            operation = {
                'method': method_introspector.get_http_method(),
                'summary': method_introspector.get_summary(),
                'nickname': method_introspector.get_nickname(),
                'notes': method_introspector.get_notes(),
                'type': response_type,
            }

            if doc_parser.yaml_error is not None:
                operation['notes'] += "
YAMLError:\n {err}
".format(
                    err=doc_parser.yaml_error)

            response_messages = doc_parser.get_response_messages()
            parameters = doc_parser.discover_parameters(
                inspector=method_introspector)

            if parameters:
                operation['parameters'] = parameters

            if response_messages:
                operation['responseMessages'] = response_messages

            operations.append(operation)

        return operations

    def get_models(self, apis):
        """
        Builds a list of Swagger 'models'. These represent
        DRF serializers and their fields
        """
        serializers = self._get_serializer_set(apis)
        serializers.update(self.explicit_serializers)
        serializers.update(
            self._find_field_serializers(serializers)
        )

        models = {}

        for serializer in serializers:
            data = self._get_serializer_fields(serializer)

            # Register 2 models with different subset of properties suitable
            # for data reading and writing.
            # i.e. rest framework does not output write_only fields in response
            # or require read_only fields in complex input.

            serializer_name = IntrospectorHelper.get_serializer_name(serializer)
            # Writing
            # no readonly fields
            w_name = "Write{serializer}".format(serializer=serializer_name)

            w_properties = dict((k, v) for k, v in data['fields'].items()
                                if k not in data['read_only'])

            models[w_name] = {
                'id': w_name,
                'required': [i for i in data['required'] if i in w_properties.keys()],
                'properties': w_properties,
            }

            # Reading
            # no write_only fields
            r_name = serializer_name

            r_properties = dict((k, v) for k, v in data['fields'].items()
                                if k not in data['write_only'])

            models[r_name] = {
                'id': r_name,
                'required': [i for i in r_properties.keys()],
                'properties': r_properties,
            }

            # Enable original model for testing purposes
            # models[serializer_name] = {
            #     'id': serializer_name,
            #     'required': data['required'],
            #     'properties': data['fields'],
            # }

        models.update(self.explicit_response_types)
        models.update(self.fields_serializers)
        return models

    def _get_method_serializer(self, doc_parser, method_inspector):
        """
        Returns serializer used in method.
        Registers custom serializer from docstring in scope.

        Serializer might be ignored if explicitly told in docstring
        """
        serializer = method_inspector.get_serializer_class()

        docstring_serializer = doc_parser.get_serializer_class(
            callback=method_inspector.callback
        )

        if doc_parser.get_response_type() is not None:
            # Custom response class detected
            return None

        if docstring_serializer is not None:
            self.explicit_serializers.add(docstring_serializer)
            serializer = docstring_serializer

        if doc_parser.should_omit_serializer():
            serializer = None

        return serializer

    def _get_method_response_type(self, doc_parser, serializer,
                                  view_inspector, method_inspector):
        """
        Returns response type for method.
        This might be custom `type` from docstring or discovered
        serializer class name.

        Once custom `type` found in docstring - it'd be
        registered in a scope
        """
        response_type = doc_parser.get_response_type()
        if response_type is not None:
            # Register class in scope
            view_name = view_inspector.callback.__name__
            view_name = view_name.replace('ViewSet', '')
            view_name = view_name.replace('APIView', '')
            view_name = view_name.replace('View', '')
            response_type_name = "{view}{method}Response".format(
                view=view_name,
                method=method_inspector.method.title().replace('_', '')
            )
            self.explicit_response_types.update({
                response_type_name: {
                    "id": response_type_name,
                    "properties": response_type
                }
            })
            return response_type_name
        else:
            serializer_name = IntrospectorHelper.get_serializer_name(serializer)
            if serializer_name is not None:
                return serializer_name

            return None

    def _get_serializer_set(self, apis):
        """
        Returns a set of serializer classes for a provided list
        of APIs
        """
        serializers = set()

        for api in apis:
            serializer = self._get_serializer_class(api['callback'])
            if serializer is not None:
                serializers.add(serializer)

        return serializers

    def _find_field_serializers(self, serializers, found_serializers=set()):
        """
        Returns set of serializers discovered from fields
        """
        serializers_set = set()
        for serializer in serializers:
            fields = serializer().get_fields()
            for name, field in fields.items():
                if isinstance(field, BaseSerializer):
                    serializers_set.add(field)
                    if field not in found_serializers:
                        serializers_set.update(self._find_field_serializers((field.__class__, ), serializers_set))

        return serializers_set

    def _get_serializer_fields(self, serializer):
        """
        Returns serializer fields in the Swagger MODEL format
        """
        if serializer is None:
            return

        if hasattr(serializer, '__call__'):
            fields = serializer().get_fields()
        else:
            fields = serializer.get_fields()

        data = {
            'fields': {},
            'required': [],
            'write_only': [],
            'read_only': [],
        }
        for name, field in fields.items():
            if getattr(field, 'write_only', False):
                data['write_only'].append(name)

            if getattr(field, 'read_only', False):
                data['read_only'].append(name)

            if getattr(field, 'required', False):
                data['required'].append(name)

            data_type = field.type_label

            # guess format
            data_format = 'string'
            if data_type in BaseMethodIntrospector.PRIMITIVES:
                data_format = BaseMethodIntrospector.PRIMITIVES.get(data_type)[0]

            f = {
                'description': getattr(field, 'help_text', ''),
                'type': data_type,
                'format': data_format,
                'required': getattr(field, 'required', False),
                'defaultValue': get_resolved_value(field, 'default'),
                'readOnly': getattr(field, 'read_only', None),
            }

            # Min/Max values
            max_val = getattr(field, 'max_val', None)
            min_val = getattr(field, 'min_val', None)
            if max_val is not None and data_type == 'integer':
                f['minimum'] = min_val

            if max_val is not None and data_type == 'integer':
                f['maximum'] = max_val

            # ENUM options
            if field.type_label == 'multiple choice' \
                    and isinstance(field.choices, list):
                f['enum'] = [k for k, v in field.choices]

            # Support for complex types
            if isinstance(field, BaseSerializer):
                field_serializer = IntrospectorHelper.get_serializer_name(field)

                if getattr(field, 'write_only', False):
                    field_serializer = "Write{}".format(field_serializer)

                f['type'] = field_serializer
                if field.many:
                    f['type'] = 'array'
                    if data_type in BaseMethodIntrospector.PRIMITIVES:
                        f['items'] = {'type': data_type}
                    else:
                        f['items'] = {'$ref': field_serializer}

            # memorize discovered field
            data['fields'][name] = f

        return data

    def _get_serializer_class(self, callback):
        if hasattr(callback, 'get_serializer_class'):
            return callback().get_serializer_class()
<|endoftext|>


<commit_before>"""
These meta-datasources operate on :class:`revscoring.Datasource`'s that
return `list`'s and apply a specific function to each item.

.. autoclass:: revscoring.datasources.meta.mappers.map

.. autoclass:: revscoring.datasources.meta.mappers.lower_case

.. autoclass:: revscoring.datasources.meta.mappers.derepeat

.. autoclass:: revscoring.datasources.meta.mappers.abs
"""
from itertools import groupby

from ..datasource import Datasource

absolute_value = abs

class map(Datasource):
    """
    Returns a :class:`revscoring.Datasource` that applies a function over
    a set of items generated by another datasource.

    :Parameters:
        apply : `func`
            A function to apply to each item generated by `items_datasource`
        items_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of some item
        name : `str`
            A name for the datasource.
    """
    def __init__(self, apply, items_datasource, name=None):
        self.apply = apply
        name = self._format_name(name, [apply, items_datasource])
        super().__init__(name, self.process, depends_on=[items_datasource])

    def process(self, items):
        return [self.apply(item) for item in items]

class lower_case(map):
    """
    Returns a :class:`revscoring.Datasource` that lower cases a list of `str`
    returned by another datasource.

    :Parameters:
        strs_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of `str`
        name : `str`
            A name for the datasource.
    """
    def __init__(self, strs_datasource, name=None):
        name = self._format_name(name, [strs_datasource])
        super().__init__(self.lower, strs_datasource, name=name)

    def lower(self, s):
        return s.lower()

class derepeat(map):
    """
    Returns a :class:`revscoring.Datasource` that prevents a list of `str` from
    having repeated characters (e.g. "foo" --> "fo").

    :Parameters:
        strs_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of `str`
        name : `str`
            A name for the datasource.
    """
    def __init__(self, strs_datasource, name=None):
        name = self._format_name(name, [strs_datasource])
        super().__init__(self.no_repeat, strs_datasource, name=name)

    def no_repeat(self, s):
        return ''.join(char for char, group in groupby(s))

class de1337(map):
    """
    Returns a :class:`revscoring.Datasource` that converts numbers in the
    middle of words into the characters they are often used
    to represent (e.g. "he7d3r" --> "hetder").

    :Parameters:
        strs_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of `str`
        name : `str`
            A name for the datasource.
    """
    MAP = {'1': "l", '3': "e", '4': "a", '5': "s",
           '6': "g", '7': "t", '0': "o", "#": "h", "(": "c"}

    def __init__(self, strs_datasource, name=None):
        name = self._format_name(name, [strs_datasource])
        super().__init__(self.no_1337, strs_datasource, name=name)

    def no_1337(self, s):
        return ''.join(self.MAP.get(c, c) for c in s)

class abs(map):
    """
    Returns a :class:`revscoring.Datasource` that converts a list of numeric
    values into a list of absolute numeric values.

    :Parameters:
        numbers_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of numeric values
        name : `str`
            A name for the datasource.
    """
    def __init__(self, numbers_datasource, name=None):
        super().__init__(absolute_value, numbers_datasource, name=name)
<commit_msg>Fixes lower() to work for weird turkish chars<commit_after>"""
These meta-datasources operate on :class:`revscoring.Datasource`'s that
return `list`'s and apply a specific function to each item.

.. autoclass:: revscoring.datasources.meta.mappers.map

.. autoclass:: revscoring.datasources.meta.mappers.lower_case

.. autoclass:: revscoring.datasources.meta.mappers.derepeat

.. autoclass:: revscoring.datasources.meta.mappers.abs
"""
from itertools import groupby

from ..datasource import Datasource

absolute_value = abs

class map(Datasource):
    """
    Returns a :class:`revscoring.Datasource` that applies a function over
    a set of items generated by another datasource.

    :Parameters:
        apply : `func`
            A function to apply to each item generated by `items_datasource`
        items_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of some item
        name : `str`
            A name for the datasource.
    """
    def __init__(self, apply, items_datasource, name=None):
        self.apply = apply
        name = self._format_name(name, [apply, items_datasource])
        super().__init__(name, self.process, depends_on=[items_datasource])

    def process(self, items):
        return [self.apply(item) for item in items]

class lower_case(map):
    """
    Returns a :class:`revscoring.Datasource` that lower cases a list of `str`
    returned by another datasource.

    :Parameters:
        strs_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of `str`
        name : `str`
            A name for the datasource.
    """
    def __init__(self, strs_datasource, name=None):
        name = self._format_name(name, [strs_datasource])
        super().__init__(self.lower, strs_datasource, name=name)

    def lower(self, s):
        return s.replace(u'İ', 'i').lower()

class derepeat(map):
    """
    Returns a :class:`revscoring.Datasource` that prevents a list of `str` from
    having repeated characters (e.g. "foo" --> "fo").

    :Parameters:
        strs_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of `str`
        name : `str`
            A name for the datasource.
    """
    def __init__(self, strs_datasource, name=None):
        name = self._format_name(name, [strs_datasource])
        super().__init__(self.no_repeat, strs_datasource, name=name)

    def no_repeat(self, s):
        return ''.join(char for char, group in groupby(s))

class de1337(map):
    """
    Returns a :class:`revscoring.Datasource` that converts numbers in the
    middle of words into the characters they are often used
    to represent (e.g. "he7d3r" --> "hetder").

    :Parameters:
        strs_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of `str`
        name : `str`
            A name for the datasource.
    """
    MAP = {'1': "l", '3': "e", '4': "a", '5': "s",
           '6': "g", '7': "t", '0': "o", "#": "h", "(": "c"}

    def __init__(self, strs_datasource, name=None):
        name = self._format_name(name, [strs_datasource])
        super().__init__(self.no_1337, strs_datasource, name=name)

    def no_1337(self, s):
        return ''.join(self.MAP.get(c, c) for c in s)

class abs(map):
    """
    Returns a :class:`revscoring.Datasource` that converts a list of numeric
    values into a list of absolute numeric values.

    :Parameters:
        numbers_datasource : :class:`revscoring.Datasource`
            A datasource that generates a list of numeric values
        name : `str`
            A name for the datasource.
    """
    def __init__(self, numbers_datasource, name=None):
        super().__init__(absolute_value, numbers_datasource, name=name)
<|endoftext|>