<commit_before># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A rule for creating a Go container image.

The signature of this rule is compatible with go_binary.
"""

load(
    "//lang:image.bzl",
    "app_layer",
)
load(
    "//repositories:repositories.bzl",
    _repositories = "repositories",
)
load(
    "//container:container.bzl",
    "container_pull",
)

# It is expected that the Go rules have been properly
# initialized before loading this file to initialize
# go_image.
load("@io_bazel_rules_go//go:def.bzl", "go_binary")

# Load the resolved digests.
load(":go.bzl", "DIGESTS")

def repositories():
    # Call the core "repositories" function to reduce boilerplate.
    # This is idempotent if folks call it themselves.
    _repositories()

    excludes = native.existing_rules().keys()
    if "go_image_base" not in excludes:
        container_pull(
            name = "go_image_base",
            registry = "gcr.io",
            repository = "distroless/base",
            digest = DIGESTS["latest"],
        )
    if "go_debug_image_base" not in excludes:
        container_pull(
            name = "go_debug_image_base",
            registry = "gcr.io",
            repository = "distroless/base",
            digest = DIGESTS["debug"],
        )

DEFAULT_BASE = select({
    "@io_bazel_rules_docker//:fastbuild": "@go_image_base//image",
    "@io_bazel_rules_docker//:debug": "@go_debug_image_base//image",
    "@io_bazel_rules_docker//:optimized": "@go_image_base//image",
    "//conditions:default": "@go_image_base//image",
})

def go_image(name, base = None, deps = [], layers = [], binary = None, **kwargs):
    """Constructs a container image wrapping a go_binary target.

  Args:
    binary: An alternative binary target to use instead of generating one.
    layers: Augments "deps" with dependencies that should be put into their own layers.
    **kwargs: See go_binary.
  """
    if layers:
        print("go_image does not benefit from layers=[], got: %s" % layers)

    if not binary:
        binary = name + ".binary"
        go_binary(name = binary, deps = deps + layers, **kwargs)
    elif deps:
        fail("kwarg does nothing when binary is specified", "deps")

    base = base or DEFAULT_BASE
    for index, dep in enumerate(layers):
        base = app_layer(name = "%s.%d" % (name, index), base = base, dep = dep)
        base = app_layer(name = "%s.%d-symlinks" % (name, index), base = base, dep = dep, binary = binary)

    visibility = kwargs.get("visibility", None)
    tags = kwargs.get("tags", None)
    app_layer(
        name = name,
        base = base,
        binary = binary,
        visibility = visibility,
        tags = tags,
        args = kwargs.get("args"),
        data = kwargs.get("data"),
        testonly = kwargs.get("testonly"),
    )
<commit_msg>go_image() now respects restricted_to and compatible_with (#669)<commit_after># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A rule for creating a Go container image.

The signature of this rule is compatible with go_binary.
"""

load(
    "//lang:image.bzl",
    "app_layer",
)
load(
    "//repositories:repositories.bzl",
    _repositories = "repositories",
)
load(
    "//container:container.bzl",
    "container_pull",
)

# It is expected that the Go rules have been properly
# initialized before loading this file to initialize
# go_image.
load("@io_bazel_rules_go//go:def.bzl", "go_binary")

# Load the resolved digests.
load(":go.bzl", "DIGESTS")

def repositories():
    # Call the core "repositories" function to reduce boilerplate.
    # This is idempotent if folks call it themselves.
    _repositories()

    excludes = native.existing_rules().keys()
    if "go_image_base" not in excludes:
        container_pull(
            name = "go_image_base",
            registry = "gcr.io",
            repository = "distroless/base",
            digest = DIGESTS["latest"],
        )
    if "go_debug_image_base" not in excludes:
        container_pull(
            name = "go_debug_image_base",
            registry = "gcr.io",
            repository = "distroless/base",
            digest = DIGESTS["debug"],
        )

DEFAULT_BASE = select({
    "@io_bazel_rules_docker//:fastbuild": "@go_image_base//image",
    "@io_bazel_rules_docker//:debug": "@go_debug_image_base//image",
    "@io_bazel_rules_docker//:optimized": "@go_image_base//image",
    "//conditions:default": "@go_image_base//image",
})

def go_image(name, base = None, deps = [], layers = [], binary = None, **kwargs):
    """Constructs a container image wrapping a go_binary target.

  Args:
    binary: An alternative binary target to use instead of generating one.
    layers: Augments "deps" with dependencies that should be put into their own layers.
    **kwargs: See go_binary.
  """
    if layers:
        print("go_image does not benefit from layers=[], got: %s" % layers)

    if not binary:
        binary = name + ".binary"
        go_binary(name = binary, deps = deps + layers, **kwargs)
    elif deps:
        fail("kwarg does nothing when binary is specified", "deps")

    base = base or DEFAULT_BASE
    for index, dep in enumerate(layers):
        base = app_layer(name = "%s.%d" % (name, index), base = base, dep = dep)
        base = app_layer(name = "%s.%d-symlinks" % (name, index), base = base, dep = dep, binary = binary)

    visibility = kwargs.get("visibility", None)
    tags = kwargs.get("tags", None)
    restricted_to = kwargs.get("restricted_to", None)
    compatible_with = kwargs.get("compatible_with", None)
    app_layer(
        name = name,
        base = base,
        binary = binary,
        visibility = visibility,
        tags = tags,
        args = kwargs.get("args"),
        data = kwargs.get("data"),
        testonly = kwargs.get("testonly"),
        restricted_to = restricted_to,
        compatible_with = compatible_with,
    )
<|endoftext|>


<commit_before>import unittest
import os, os.path

from usecases.TestUtils import *

class testBasicPersistentDiskLifecycle(unittest.TestCase):

    diskSize = 1 # in gigabytes
    tag = '"tag\'with}very$dangerous\\"characters"'

    def setUp(self):
        self.uuid = stratusCreateVolume(self.diskSize, self.tag)

    def tearDown(self):
        stratusDeleteVolume(self.uuid)

    def test_usecase(self):
        stratusDescribeVolumes(self.uuid)

def suite():
    return unittest.TestLoader().loadTestsFromTestCase(testBasicPersistentDiskLifecycle)
<commit_msg>add backslash as a dangerous character<commit_after>import unittest
import os, os.path

from usecases.TestUtils import *

class testBasicPersistentDiskLifecycle(unittest.TestCase):

    diskSize = 1 # in gigabytes
    tag = '"tag\'with}very\\\\dangerous\\"characters"'

    def setUp(self):
        self.uuid = stratusCreateVolume(self.diskSize, self.tag)

    def tearDown(self):
        stratusDeleteVolume(self.uuid)

    def test_usecase(self):
        stratusDescribeVolumes(self.uuid)

def suite():
    return unittest.TestLoader().loadTestsFromTestCase(testBasicPersistentDiskLifecycle)
<|endoftext|>


<commit_before># Copyright 2011 Gilt Groupe, INC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
various dns operations
"""

# import some useful stuff
import re
import os
import shutil
import sys
import time
from mothership.mothership_models import *

def filter_domain_query(query, table, domain):
    sub = domain.split('.')
    if len(sub) >= 4:
        query = query.filter(table.realm==sub[-4:-3][0])
    if len(sub) >= 3:
        query = query.filter(table.site_id==sub[-3:-2][0])
    return query, '.'.join(sub[-2:])

def generate_dns_header(domainname, contact):
    """Prints out a DNS Zone header for domainname"""
    serial = int(time.time())
    if '@' in contact: contact = contact.replace('@','.')
    return """
$ORIGIN %s.
$TTL 86400
@                       IN      SOA     %s. %s. (
                                        %s   ; Serial
                                        21600        ; Refresh
                                        3600         ; Retry
                                        604800       ; Expire
                                        86400        ; TTL
                                        )

                        IN      NS      ns1.%s.
                        IN      NS      ns2.%s.

""" % (domainname, domainname, contact, serial, domainname, domainname)

def generate_dns_output(cfg, domain, outdir, usecobbler=False):
    """
    Turns the dns_addendum table into usable zonefiles
    Optionally, will insert the zone into cobbler
    """
    head = []
    da = DnsAddendum
    check,tld = filter_domain_query(cfg.dbsess.query(da.realm,da.site_id), da, domain)
    if not outdir:
        if check.distinct().count() > 1:
            print 'More than one domain matches %s in mothership:' % domain
            count = 0
            selection = []
            for pair in check.distinct().all():
                count += 1
                print str(count) + ') %s.%s.' % pair + tld
                selection.append('%s.%s.' % pair + tld)
            ans = raw_input('Please select the one you want to generate: ')
            if int(ans) < 1 or int(ans) > count:
                print 'Generate DNS aborted.'
                return
            else:
                domain = selection[int(ans)-1]
    data,tld = filter_domain_query(cfg.dbsess.query(da), da, domain)
    data = data.order_by(da.site_id, da.realm, da.record_type, da.host).all()
    for dns in data:
        zone = '%s/%s.%s.%s' % (outdir, dns.realm, dns.site_id, tld)
        if '%s.%s' % (dns.realm,dns.site_id) not in head:
            head.append('%s.%s' % (dns.realm,dns.site_id))
            if outdir:
                if usecobbler:
                    if os.path.exists(zone+'_header'):
                        print 'Using header: %s_header' % zone
                        shutil.copyfile(zone+'_header', zone)
                        print 'Updating %s/%s.%s.%s with dns_addendum table' \
                            % (outdir, dns.realm, dns.site_id, tld)
                        f = open(zone, 'a')
                    else:
                        print '\nFile does not exist: %s_header' % zone
                        print 'Using stdout instead:'
                        f = sys.stdout
                        f.write(generate_dns_header('%s.%s.%s' \
                            % (dns.realm, dns.site_id, tld), cfg.contact))
                else:
                    print 'Generating boilerplate header for %s/%s.%s.%s' \
                        % (outdir, dns.realm, dns.site_id, tld)
                    f = open(zone, 'w')
                    f.write(generate_dns_header('%s.%s.%s' \
                        % (dns.realm, dns.site_id, tld), cfg.contact))
                    f = open(zone, 'a')
            else:
                f = sys.stdout
                f.write(generate_dns_header('%s.%s.%s' \
                    % (dns.realm, dns.site_id, tld), cfg.contact))
        # if last octet of target is non-numerical and does not end with
        # period, add one
        target = dns.target
        if not re.search('\d+',dns.target.split('.')[-1]):
            if not target.endswith('.'):
                target += '.'
        f.write("%-20s\tIN\t%-8s%-16s\n" % (dns.host, dns.record_type, target))
        if usecobbler:
            f.write("\n$host_record\n")

def update_table_dnsaddendum(cfg, info, delete=False):
    """
    updates the dns_addendum table
    can be used to add or delete dns records
    """
    if not delete:
        data = cfg.dbsess.query(DnsAddendum).\
            filter(DnsAddendum.site_id==info['site_id']).\
            filter(DnsAddendum.realm==info['realm']).\
            filter(DnsAddendum.target==info['target']).\
            filter(DnsAddendum.record_type==info['record_type']).\
            filter(DnsAddendum.host==info['host']).all()
        if data:
            print 'The following already exist in the database'
            for d in data:
                print '%d) %s IN %s %s' % (d.id, d.host, d.record_type, d.target)
            print 'Please delete first before adding again'
            return
        else:
            if len(info['target'].split('.')) < 4 and (info['record_type'] == 'A' or info['record_type'] == 'CNAME'):
                print 'Target should end with ip or fqdn for A and CNAME types, aborting'
                return
            cfg.dbconn.execute(DnsAddendum.__table__.insert(), [ info ])
    else:
        idlist = []
        print 'For %s.%s, we found:' % (info['realm'], info['site_id'])
        for d in cfg.dbsess.query(DnsAddendum).\
            filter(DnsAddendum.site_id==info['site_id']).\
            filter(DnsAddendum.realm==info['realm']).\
            filter(DnsAddendum.host==info['host']).all():
            idlist.append(d.id)
            print '%d) %s IN %s %s' % (d.id, d.host, d.record_type, d.target)
        if len(idlist) == 0:
            print '    no DNS entries for %s.%s.%s' % (info['host'],
                info['realm'], info['site_id'])
        else:
            ans = raw_input('Which ID do you wish to delete? ')
            if int(ans) not in idlist:
                print 'Delete DNS entry aborted.'
                return
            else:
                print 'Deleting DNS entry %d' % int(ans)
                cfg.dbconn.execute(DnsAddendum.__table__.delete().\
                    where(DnsAddendum.id==int(ans)))
<commit_msg>3461317: move cobbler $host_record outside of recurring loop to avoid excess duplication during gen_dns/cobbler_sync<commit_after># Copyright 2011 Gilt Groupe, INC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
various dns operations
"""

# import some useful stuff
import re
import os
import shutil
import sys
import time
from mothership.mothership_models import *

def filter_domain_query(query, table, domain):
    sub = domain.split('.')
    if len(sub) >= 4:
        query = query.filter(table.realm==sub[-4:-3][0])
    if len(sub) >= 3:
        query = query.filter(table.site_id==sub[-3:-2][0])
    return query, '.'.join(sub[-2:])

def generate_dns_header(domainname, contact):
    """Prints out a DNS Zone header for domainname"""
    serial = int(time.time())
    if '@' in contact: contact = contact.replace('@','.')
    return """
$ORIGIN %s.
$TTL 86400
@                       IN      SOA     %s. %s. (
                                        %s   ; Serial
                                        21600        ; Refresh
                                        3600         ; Retry
                                        604800       ; Expire
                                        86400        ; TTL
                                        )

                        IN      NS      ns1.%s.
                        IN      NS      ns2.%s.

""" % (domainname, domainname, contact, serial, domainname, domainname)

def generate_dns_output(cfg, domain, outdir, usecobbler=False):
    """
    Turns the dns_addendum table into usable zonefiles
    Optionally, will insert the zone into cobbler
    """
    head = []
    da = DnsAddendum
    check,tld = filter_domain_query(cfg.dbsess.query(da.realm,da.site_id), da, domain)
    if not outdir:
        if check.distinct().count() > 1:
            print 'More than one domain matches %s in mothership:' % domain
            count = 0
            selection = []
            for pair in check.distinct().all():
                count += 1
                print str(count) + ') %s.%s.' % pair + tld
                selection.append('%s.%s.' % pair + tld)
            ans = raw_input('Please select the one you want to generate: ')
            if int(ans) < 1 or int(ans) > count:
                print 'Generate DNS aborted.'
                return
            else:
                domain = selection[int(ans)-1]
    data,tld = filter_domain_query(cfg.dbsess.query(da), da, domain)
    data = data.order_by(da.site_id, da.realm, da.record_type, da.host).all()
    for dns in data:
        zone = '%s/%s.%s.%s' % (outdir, dns.realm, dns.site_id, tld)
        if '%s.%s' % (dns.realm,dns.site_id) not in head:
            head.append('%s.%s' % (dns.realm,dns.site_id))
            if outdir:
                if usecobbler:
                    if os.path.exists(zone+'_header'):
                        print 'Using header: %s_header' % zone
                        shutil.copyfile(zone+'_header', zone)
                        print 'Updating %s/%s.%s.%s with dns_addendum table' \
                            % (outdir, dns.realm, dns.site_id, tld)
                        f = open(zone, 'a')
                    else:
                        print '\nFile does not exist: %s_header' % zone
                        print 'Using stdout instead:'
                        f = sys.stdout
                        f.write(generate_dns_header('%s.%s.%s' \
                            % (dns.realm, dns.site_id, tld), cfg.contact))
                else:
                    print 'Generating boilerplate header for %s/%s.%s.%s' \
                        % (outdir, dns.realm, dns.site_id, tld)
                    f = open(zone, 'w')
                    f.write(generate_dns_header('%s.%s.%s' \
                        % (dns.realm, dns.site_id, tld), cfg.contact))
                    f = open(zone, 'a')
            else:
                f = sys.stdout
                f.write(generate_dns_header('%s.%s.%s' \
                    % (dns.realm, dns.site_id, tld), cfg.contact))
        # if last octet of target is non-numerical and does not end with
        # period, add one
        target = dns.target
        if not re.search('\d+',dns.target.split('.')[-1]):
            if not target.endswith('.'):
                target += '.'
        f.write("%-20s\tIN\t%-8s%-16s\n" % (dns.host, dns.record_type, target))
    if usecobbler:
        f.write("\n$host_record\n")

def update_table_dnsaddendum(cfg, info, delete=False):
    """
    updates the dns_addendum table
    can be used to add or delete dns records
    """
    if not delete:
        data = cfg.dbsess.query(DnsAddendum).\
            filter(DnsAddendum.site_id==info['site_id']).\
            filter(DnsAddendum.realm==info['realm']).\
            filter(DnsAddendum.target==info['target']).\
            filter(DnsAddendum.record_type==info['record_type']).\
            filter(DnsAddendum.host==info['host']).all()
        if data:
            print 'The following already exist in the database'
            for d in data:
                print '%d) %s IN %s %s' % (d.id, d.host, d.record_type, d.target)
            print 'Please delete first before adding again'
            return
        else:
            if len(info['target'].split('.')) < 4 and (info['record_type'] == 'A' or info['record_type'] == 'CNAME'):
                print 'Target should end with ip or fqdn for A and CNAME types, aborting'
                return
            cfg.dbconn.execute(DnsAddendum.__table__.insert(), [ info ])
    else:
        idlist = []
        print 'For %s.%s, we found:' % (info['realm'], info['site_id'])
        for d in cfg.dbsess.query(DnsAddendum).\
            filter(DnsAddendum.site_id==info['site_id']).\
            filter(DnsAddendum.realm==info['realm']).\
            filter(DnsAddendum.host==info['host']).all():
            idlist.append(d.id)
            print '%d) %s IN %s %s' % (d.id, d.host, d.record_type, d.target)
        if len(idlist) == 0:
            print '    no DNS entries for %s.%s.%s' % (info['host'],
                info['realm'], info['site_id'])
        else:
            ans = raw_input('Which ID do you wish to delete? ')
            if int(ans) not in idlist:
                print 'Delete DNS entry aborted.'
                return
            else:
                print 'Deleting DNS entry %d' % int(ans)
                cfg.dbconn.execute(DnsAddendum.__table__.delete().\
                    where(DnsAddendum.id==int(ans)))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# name:             test_node.py
# author:           Harold Bradley III
# email:            harold@bradleystudio.net
# created on:       11/08/2015

# pylint:           disable=invalid-name,line-too-long

"""
A unit test for ext_pylib file module's Node class and methods.
"""

import pytest
from mock import patch

from ext_pylib.files.node import Node
from ext_pylib.user import get_current_username, get_current_groupname

CURRENT_USER = get_current_username()
CURRENT_GROUP = get_current_groupname()

DEFUALT_ATTS = {'path' : '/etc/path/file'}

INIT_ARGS = [
    ({'path' : None},
     {'path' : '<files.Node:stub>', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this/path/file'},
     {'path' : '/this/path/file', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this/path/'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this//path/'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this//path//'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this///path////'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file'},
     {'path' : '/etc/path/file', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file'},
     {'path' : '/etc/path/file', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file', 'perms' : 0o655},
     {'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'},
     {'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'}),
]
@pytest.mark.parametrize(("atts", "expected"), INIT_ARGS)
def test_node_initialize(atts, expected):
    """Test initialize Node."""
    node = Node(atts)
    assert str(node) == expected['path']
    assert node.perms == expected['perms']
    assert node.owner == expected['owner']
    assert node.group == expected['group']

REPR_ARGS = [
    ({'path' : None},
     "{'path' : None, 'perms' : None, 'owner' : None, 'group' : None}"),
    ({'path' : '/this/path/file'},
     "{'path' : '/this/path/file', 'perms' : None, 'owner' : None, 'group' : None}"),
    ({'path' : '/etc/path/file', 'perms' : 0o655},
     "{'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : None, 'group' : None}"),
    ({'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'},
     "{'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'}"),
]
@pytest.mark.parametrize(("atts", "expected"), REPR_ARGS)
def test_node_repr(atts, expected):
    """Test Node repr."""
    node = Node(atts)
    assert node.__repr__() == "Node(" + expected + ")"

@pytest.mark.parametrize(("atts", "expected"), REPR_ARGS)
def test_node_get_atts(atts, expected):
    """Test Node get_atts method."""
    node = Node(atts)
    if 'perms' not in atts:
        atts['perms'] = None
    if 'owner' not in atts:
        atts['owner'] = None
    if 'group' not in atts:
        atts['group'] = None
    assert node.get_atts(string=True) == expected
    assert node.get_atts(string=False) == atts

CONCAT_ARGS = [
    ({'path' : None}, '<files.Node:stub>'),
    ({'path' : '/this/path/'}, '/this/path/'),
    ({'path' : '/etc/path/file'}, '/etc/path/file'),
]
@pytest.mark.parametrize(("atts", "expected"), CONCAT_ARGS)
def test_node_concatenate(atts, expected):
    """Test concatenate Node objects."""
    node = Node(atts)
    assert node + 'string' == expected + 'string'
    assert 'string' + node == 'string' + expected

def test_node_create():
    """Test that Node throws an error when calling create()."""
    node = Node({'path' : '/the/path'})
    with pytest.raises(NotImplementedError):
        node.create()

def test_node_remove():
    """Test that Node throws an error when calling remove()."""
    node = Node({'path' : '/the/path'})
    with pytest.raises(NotImplementedError):
        node.remove()

VERIFY_ARGS = [
    ({'path' : None}),
    ({'path' : '/this/path/file'}),
    ({'path' : '/this/path/', 'perms' : 0o655}),
    ({'path' : '/this/path/', 'owner' : 'root'}),
    ({'path' : '/this/path/', 'group' : 'root'}),
    ({'path' : '/this/path/', 'perms' : 0o655, 'owner' : 'root'}),
    ({'path' : '/this/path/', 'perms' : 0o655, 'group' : 'root'}),
    ({'path' : '/this/path/', 'owner' : 'root', 'group' : 'root'}),
    ({'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'}),
]
@pytest.mark.parametrize(("atts"), VERIFY_ARGS)
@patch('ext_pylib.files.node.Node.actual_group')
@patch('ext_pylib.files.node.Node.actual_owner')
@patch('ext_pylib.files.node.Node.actual_perms')
@patch('ext_pylib.files.node.Node.exists')
def test_node_verify(mock_exists, mock_actual_perms, mock_actual_owner, mock_actual_group, atts):
    """Test Node's method verify."""
    # Test passing verification
    mock_exists.return_value = True
    node = Node(atts)
    node.actual_perms = None if 'perms' not in atts else atts['perms']
    node.actual_owner = None if 'owner' not in atts else atts['owner']
    node.actual_group = None if 'group' not in atts else atts['group']
    assert node.verify(False)

    # Test failing verification
    if atts['path']:
        # If there is no path, it's a stub and should always verify true.
        mock_exists.return_value = False
        mock_actual_perms.return_value = None
        mock_actual_owner.return_value = None
        mock_actual_group.return_value = None
        bad_node = Node(atts)
        assert not bad_node.verify(False)

@patch('ext_pylib.files.node.Node.verify')
def test_node_repair(mock_verify):
    """Test that repair() method calls verify(True)."""
    node = Node()
    node.repair()
    mock_verify.assert_called_once_with(True)

CHMOD_ARGS = [
    ({'path' : None, 'perms' : 0o600}, True),
    ({'path' : '/this/path/file', 'perms' : 0o700}, True),
    ({'path' : '/this/path/file'}, True),
]
@pytest.mark.parametrize(("atts", "expected"), CHMOD_ARGS)
@patch('ext_pylib.files.node.Node.exists')
@patch('os.chmod')
def test_node_chmod(mock_chmod, mock_path_exists, atts, expected):
    """Tests Node's chmod method."""
    mock_path_exists.return_value = True # Assume this is working for this test
    node = Node(atts)
    assert expected == node.chmod()
    if atts['path'] and node.perms:
        mock_chmod.assert_called_once_with(atts['path'], atts['perms'])
    else:
        assert not mock_chmod.called

@patch('ext_pylib.files.node.Node.exists')
def test_node_chmod_nonexisting(mock_path_exists):
    """Tests Node's chown method with a nonexisting node."""
    mock_path_exists.return_value = False
    node = Node(DEFUALT_ATTS)
    with pytest.raises(IOError):
        node.chmod()

CHOWN_ARGS = [
    ({'path' : None, 'owner' : 'www-data', 'group' : 'root'}, True),
    ({'path' : '/this/path/file', 'owner' : 'www-data', 'group' : 'root'}, True),
    ({'path' : '/this/path/file', 'owner' : None, 'group' : 'root'}, True),
    ({'path' : '/this/path/file', 'owner' : 'www-data', 'group' : None}, True),
    ({'path' : '/this/path/file', 'owner' : None, 'group' : None}, True),
    ({'path' : '/this/path/file'}, True),
    ({'path' : '/this/path/file', 'owner' : 'www-data'}, True),
    ({'path' : '/this/path/file', 'group' : 'www-data'}, True),
    ({'path' : '/this/path/file', 'owner' : None, 'group' : 'www-data'}, True),
]
@pytest.mark.parametrize(("atts", "expected"), CHOWN_ARGS)
@patch('ext_pylib.files.node.Node.exists')
@patch('pwd.getpwnam')
@patch('grp.getgrnam')
@patch('os.chown')
def test_node_chown(mock_chown, mock_getgrnam, mock_getpwnam, mock_path_exists, atts, expected):  # pylint: disable=too-many-arguments
    """Tests Node's chown method."""
    node = Node(atts)
    if 'owner' not in atts:
        atts['owner'] = CURRENT_USER
    if 'group' not in atts:
        atts['group'] = 'nogroup' if atts['owner'] == 'nobody' else CURRENT_GROUP
    mock_path_exists.return_value = True # Assume this is working for this test
    mock_getpwnam(atts['owner']).pw_uid = 123 # Just a number to use for mocking
    mock_getgrnam(atts['group']).gr_gid = 123
    assert expected == node.chown()
    if atts['path'] is not None:
        mock_getpwnam.assert_called_with(CURRENT_USER if not atts['owner'] else atts['owner'])
        mock_getgrnam.assert_called_with(CURRENT_GROUP if not atts['group'] else atts['group'])
        mock_chown.assert_called_once_with(atts['path'], 123, 123)

@patch('ext_pylib.files.node.Node.exists')
def test_node_chown_nonexisting(mock_path_exists):
    """Tests Node's chown method with a nonexisting node."""
    mock_path_exists.return_value = False
    node = Node(DEFUALT_ATTS)
    with pytest.raises(IOError):
        node.chown()

EXISTS_ARGS = [
    ({'path' : None}, False),
    ({'path' : '/this/path/file'}, False),
    ({'path' : '/this/path/file'}, True),
]
@pytest.mark.parametrize(("atts", "expected"), EXISTS_ARGS)
@patch('os.path.exists')
def test_node_exists(mock_path_exists, atts, expected):
    """Tests node's exist method."""
    mock_path_exists.return_value = expected
    node = Node(atts)
    assert expected == node.exists()
    if atts['path'] is not None:
        mock_path_exists.assert_called_once_with(atts['path'])

def test_node_set_path_empty():
    """Test that Node throws an error when setting path to empty string."""
    node = Node()
    with pytest.raises(ValueError):
        node.path = ''
    with pytest.raises(ValueError):
        node = Node({'path' : ''})

@pytest.mark.parametrize(('invalid_char'), [
    ('!'), ('@'), ('#'), ('$'), ('%'), ('^'), ('&'), ('*'), ('|'),
])
def test_node_set_path_invalid_char(invalid_char):
    """Test that Node throws an error when setting path to invalid character."""
    node = Node()
    with pytest.raises(ValueError):
        node.path = '/path/to' + invalid_char
    with pytest.raises(ValueError):
        node = Node({'path' : '/path/to' + invalid_char})

PARENT_DIRS_ARGS = [
    ({'path' : None}, None),
    ({'path' : '/this/path/'}, '/this/'),
    ({'path' : '/etc/path/file'}, '/etc/path/'),
    ({'path' : '//etc//path//file'}, '/etc/path/'),
]
@pytest.mark.parametrize(("atts", "expected"), PARENT_DIRS_ARGS)
def test_node_parent_node(atts, expected):
    """Test parent_node method. Should return a Node object."""
    node = Node(atts)
    if node.path:
        assert node.parent_node.path == expected
    else:
        assert node.parent_node == expected

def test_node_set_perms_invalid():
    """Tests setting node's perms as invalid values."""
    node = Node(DEFUALT_ATTS)
    with pytest.raises(ValueError):
        node.perms = 'a'
    with pytest.raises(ValueError):
        node.perms = 9999  # pylint: disable=redefined-variable-type
    with pytest.raises(ValueError):
        node.perms = -9999

def test_node_set_bad_owner():
    """Tests setting node's owner to an invalid user."""
    node = Node(DEFUALT_ATTS)
    with pytest.raises(KeyError):
        node.owner = 'HopefullyNot_a_RealUser'

def test_node_set_owner_root():
    """Tests setting node's owner to root."""
    node = Node(DEFUALT_ATTS)
    node.owner = 'root'
    assert node.owner == 'root'

def test_node_set_bad_group():
    """Tests setting node's group to an invalid user."""
    node = Node(DEFUALT_ATTS)
    with pytest.raises(KeyError):
        node.group = 'HopefullyNot_a_RealGroup'

def test_node_set_group_root():
    """Tests setting node's group to root."""
    node = Node(DEFUALT_ATTS)
    node.group = 'root'
    assert node.group == 'root'
<commit_msg>Clarifies comment.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# name:             test_node.py
# author:           Harold Bradley III
# email:            harold@bradleystudio.net
# created on:       11/08/2015

# pylint:           disable=invalid-name,line-too-long

"""
A unit test for ext_pylib file module's Node class and methods.
"""

import pytest
from mock import patch

from ext_pylib.files.node import Node
from ext_pylib.user import get_current_username, get_current_groupname

CURRENT_USER = get_current_username()
CURRENT_GROUP = get_current_groupname()

DEFUALT_ATTS = {'path' : '/etc/path/file'}

INIT_ARGS = [
    ({'path' : None},
     {'path' : '<files.Node:stub>', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this/path/file'},
     {'path' : '/this/path/file', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this/path/'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this//path/'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this//path//'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/this///path////'},
     {'path' : '/this/path/', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file'},
     {'path' : '/etc/path/file', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file'},
     {'path' : '/etc/path/file', 'perms' : None, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file', 'perms' : 0o655},
     {'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : None, 'group' : None}),
    ({'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'},
     {'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'}),
]
@pytest.mark.parametrize(("atts", "expected"), INIT_ARGS)
def test_node_initialize(atts, expected):
    """Test initialize Node."""
    node = Node(atts)
    assert str(node) == expected['path']
    assert node.perms == expected['perms']
    assert node.owner == expected['owner']
    assert node.group == expected['group']

REPR_ARGS = [
    ({'path' : None},
     "{'path' : None, 'perms' : None, 'owner' : None, 'group' : None}"),
    ({'path' : '/this/path/file'},
     "{'path' : '/this/path/file', 'perms' : None, 'owner' : None, 'group' : None}"),
    ({'path' : '/etc/path/file', 'perms' : 0o655},
     "{'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : None, 'group' : None}"),
    ({'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'},
     "{'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'}"),
]
@pytest.mark.parametrize(("atts", "expected"), REPR_ARGS)
def test_node_repr(atts, expected):
    """Test Node repr."""
    node = Node(atts)
    assert node.__repr__() == "Node(" + expected + ")"

@pytest.mark.parametrize(("atts", "expected"), REPR_ARGS)
def test_node_get_atts(atts, expected):
    """Test Node get_atts method."""
    node = Node(atts)
    if 'perms' not in atts:
        atts['perms'] = None
    if 'owner' not in atts:
        atts['owner'] = None
    if 'group' not in atts:
        atts['group'] = None
    assert node.get_atts(string=True) == expected
    assert node.get_atts(string=False) == atts

CONCAT_ARGS = [
    ({'path' : None}, '<files.Node:stub>'),
    ({'path' : '/this/path/'}, '/this/path/'),
    ({'path' : '/etc/path/file'}, '/etc/path/file'),
]
@pytest.mark.parametrize(("atts", "expected"), CONCAT_ARGS)
def test_node_concatenate(atts, expected):
    """Test concatenate Node objects."""
    node = Node(atts)
    assert node + 'string' == expected + 'string'
    assert 'string' + node == 'string' + expected

def test_node_create():
    """Test that Node throws an error when calling create()."""
    node = Node({'path' : '/the/path'})
    with pytest.raises(NotImplementedError):
        node.create()

def test_node_remove():
    """Test that Node throws an error when calling remove()."""
    node = Node({'path' : '/the/path'})
    with pytest.raises(NotImplementedError):
        node.remove()

VERIFY_ARGS = [
    ({'path' : None}),
    ({'path' : '/this/path/file'}),
    ({'path' : '/this/path/', 'perms' : 0o655}),
    ({'path' : '/this/path/', 'owner' : 'root'}),
    ({'path' : '/this/path/', 'group' : 'root'}),
    ({'path' : '/this/path/', 'perms' : 0o655, 'owner' : 'root'}),
    ({'path' : '/this/path/', 'perms' : 0o655, 'group' : 'root'}),
    ({'path' : '/this/path/', 'owner' : 'root', 'group' : 'root'}),
    ({'path' : '/etc/path/file', 'perms' : 0o655, 'owner' : 'root', 'group' : 'root'}),
]
@pytest.mark.parametrize(("atts"), VERIFY_ARGS)
@patch('ext_pylib.files.node.Node.actual_group')
@patch('ext_pylib.files.node.Node.actual_owner')
@patch('ext_pylib.files.node.Node.actual_perms')
@patch('ext_pylib.files.node.Node.exists')
def test_node_verify(mock_exists, mock_actual_perms, mock_actual_owner, mock_actual_group, atts):
    """Test Node's method verify."""
    # Test passing verification
    mock_exists.return_value = True
    node = Node(atts)
    node.actual_perms = None if 'perms' not in atts else atts['perms']
    node.actual_owner = None if 'owner' not in atts else atts['owner']
    node.actual_group = None if 'group' not in atts else atts['group']
    assert node.verify(False)

    # Test failing verification
    if atts['path']:
        # If there is no path, it's a stub and should always verify true.
        mock_exists.return_value = False
        mock_actual_perms.return_value = None
        mock_actual_owner.return_value = None
        mock_actual_group.return_value = None
        bad_node = Node(atts)
        assert not bad_node.verify(False)

@patch('ext_pylib.files.node.Node.verify')
def test_node_repair(mock_verify):
    """Test that repair() method calls verify(True)."""
    node = Node()
    node.repair()
    mock_verify.assert_called_once_with(True)

CHMOD_ARGS = [
    ({'path' : None, 'perms' : 0o600}, True),
    ({'path' : '/this/path/file', 'perms' : 0o700}, True),
    ({'path' : '/this/path/file'}, True),
]
@pytest.mark.parametrize(("atts", "expected"), CHMOD_ARGS)
@patch('ext_pylib.files.node.Node.exists')
@patch('os.chmod')
def test_node_chmod(mock_chmod, mock_path_exists, atts, expected):
    """Tests Node's chmod method."""
    mock_path_exists.return_value = True # Assume this is working for this test
    node = Node(atts)
    assert expected == node.chmod()
    if atts['path'] and node.perms:
        mock_chmod.assert_called_once_with(atts['path'], atts['perms'])
    else:
        assert not mock_chmod.called

@patch('ext_pylib.files.node.Node.exists')
def test_node_chmod_nonexisting(mock_path_exists):
    """Tests Node's chown method with a nonexisting node."""
    mock_path_exists.return_value = False
    node = Node(DEFUALT_ATTS)
    with pytest.raises(IOError):
        node.chmod()

CHOWN_ARGS = [
    ({'path' : None, 'owner' : 'www-data', 'group' : 'root'}, True),
    ({'path' : '/this/path/file', 'owner' : 'www-data', 'group' : 'root'}, True),
    ({'path' : '/this/path/file', 'owner' : None, 'group' : 'root'}, True),
    ({'path' : '/this/path/file', 'owner' : 'www-data', 'group' : None}, True),
    ({'path' : '/this/path/file', 'owner' : None, 'group' : None}, True),
    ({'path' : '/this/path/file'}, True),
    ({'path' : '/this/path/file', 'owner' : 'www-data'}, True),
    ({'path' : '/this/path/file', 'group' : 'www-data'}, True),
    ({'path' : '/this/path/file', 'owner' : None, 'group' : 'www-data'}, True),
]
@pytest.mark.parametrize(("atts", "expected"), CHOWN_ARGS)
@patch('ext_pylib.files.node.Node.exists')
@patch('pwd.getpwnam')
@patch('grp.getgrnam')
@patch('os.chown')
def test_node_chown(mock_chown, mock_getgrnam, mock_getpwnam, mock_path_exists, atts, expected):  # pylint: disable=too-many-arguments
    """Tests Node's chown method."""
    node = Node(atts)
    if 'owner' not in atts:
        atts['owner'] = CURRENT_USER
    if 'group' not in atts:
        atts['group'] = 'nogroup' if atts['owner'] == 'nobody' else CURRENT_GROUP
    mock_path_exists.return_value = True # Assume this is working for this test
    mock_getpwnam(atts['owner']).pw_uid = 123 # Just a number to use for mocking
    mock_getgrnam(atts['group']).gr_gid = 123
    assert expected == node.chown()
    if atts['path'] is not None:
        mock_getpwnam.assert_called_with(CURRENT_USER if not atts['owner'] else atts['owner'])
        mock_getgrnam.assert_called_with(CURRENT_GROUP if not atts['group'] else atts['group'])
        mock_chown.assert_called_once_with(atts['path'], 123, 123)

@patch('ext_pylib.files.node.Node.exists')
def test_node_chown_nonexisting(mock_path_exists):
    """Tests Node's chown method with a nonexisting node."""
    mock_path_exists.return_value = False
    node = Node(DEFUALT_ATTS)
    with pytest.raises(IOError):
        node.chown()

EXISTS_ARGS = [
    ({'path' : None}, False),
    ({'path' : '/this/path/file'}, False),
    ({'path' : '/this/path/file'}, True),
]
@pytest.mark.parametrize(("atts", "expected"), EXISTS_ARGS)
@patch('os.path.exists')
def test_node_exists(mock_path_exists, atts, expected):
    """Tests node's exist method."""
    mock_path_exists.return_value = expected
    node = Node(atts)
    assert expected == node.exists()
    if atts['path'] is not None:
        mock_path_exists.assert_called_once_with(atts['path'])

def test_node_set_path_empty():
    """Test that Node throws an error when setting path to empty string."""
    node = Node()
    with pytest.raises(ValueError):
        node.path = ''
    with pytest.raises(ValueError):
        node = Node({'path' : ''})

@pytest.mark.parametrize(('invalid_char'), [
    ('!'), ('@'), ('#'), ('$'), ('%'), ('^'), ('&'), ('*'), ('|'),
])
def test_node_set_path_invalid_char(invalid_char):
    """Test that Node throws an error when setting path to invalid character."""
    node = Node()
    with pytest.raises(ValueError):
        node.path = '/path/to' + invalid_char
    with pytest.raises(ValueError):
        node = Node({'path' : '/path/to' + invalid_char})

PARENT_DIRS_ARGS = [
    ({'path' : None}, None),
    ({'path' : '/this/path/'}, '/this/'),
    ({'path' : '/etc/path/file'}, '/etc/path/'),
    ({'path' : '//etc//path//file'}, '/etc/path/'),
]
@pytest.mark.parametrize(("atts", "expected"), PARENT_DIRS_ARGS)
def test_node_parent_node(atts, expected):
    """Test parent_node attribute. Should return a Node object."""
    node = Node(atts)
    if node.path:
        assert node.parent_node.path == expected
    else:
        assert node.parent_node == expected

def test_node_set_perms_invalid():
    """Tests setting node's perms as invalid values."""
    node = Node(DEFUALT_ATTS)
    with pytest.raises(ValueError):
        node.perms = 'a'
    with pytest.raises(ValueError):
        node.perms = 9999  # pylint: disable=redefined-variable-type
    with pytest.raises(ValueError):
        node.perms = -9999

def test_node_set_bad_owner():
    """Tests setting node's owner to an invalid user."""
    node = Node(DEFUALT_ATTS)
    with pytest.raises(KeyError):
        node.owner = 'HopefullyNot_a_RealUser'

def test_node_set_owner_root():
    """Tests setting node's owner to root."""
    node = Node(DEFUALT_ATTS)
    node.owner = 'root'
    assert node.owner == 'root'

def test_node_set_bad_group():
    """Tests setting node's group to an invalid user."""
    node = Node(DEFUALT_ATTS)
    with pytest.raises(KeyError):
        node.group = 'HopefullyNot_a_RealGroup'

def test_node_set_group_root():
    """Tests setting node's group to root."""
    node = Node(DEFUALT_ATTS)
    node.group = 'root'
    assert node.group == 'root'
<|endoftext|>


<commit_before>import datetime
import json
import unittest

import requests

URL = ("http://", "127.0.0.1", ":", "8000", "/")

HEADERS = {'Content-Type': 'application/json'}

class FilledUserTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        requests.get(''.join(URL + ('debug/clear/',)))
        params = {'name': 'gert', 'mailAddress': 'gertMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_1 = result['id']
        params = {'name': 'bar', 'mailAddress': 'barMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_2 = result['id']
        cls.names = ['gert', 'bar']
        cls.ids = [user_1, user_2]

    def test_list_2_users(self):
        r = requests.get(''.join(URL + ('user',)), headers=HEADERS)
        self.assertEqual(200, r.status_code)
        self.assertEqual('application/json', r.headers['Content-Type'])
        users = json.loads(r.text)
        self.assertEqual(2, users.get('overall_count'))
        self.assertEqual(100, users.get('limit'))
        self.assertEqual(0, users.get('offset'))
        entries = users['entries']
        self.assertIsInstance(entries, list)
        self.assertEqual(2, len(entries))
        for i in range(0, 1):
            with self.subTest(i=i):
                self.assertEqual(self.names[i], entries[i].get('name'))
                self.assertEqual(self.ids[i], entries[i]['id'])
                self.assertEqual(0, entries[i].get('balance'))
                self.assertIsNone(entries[i].get('last_transaction'))

    def test_list_users_empty_offset(self):
        r = requests.get(''.join(URL + ('user',)), headers=HEADERS, params={'offset': 2, 'limit': 1})
        self.assertEqual(200, r.status_code)
        self.assertEqual('application/json', r.headers['Content-Type'])
        users = json.loads(r.text)
        self.assertEqual(2, users.get('overall_count'))
        self.assertEqual(1, users.get('limit'))
        self.assertEqual(2, users.get('offset'))
        entries = users['entries']
        self.assertIsInstance(entries, list)
        self.assertEqual(0, len(entries))

    def test_load_user_by_id(self):
        for i, name in enumerate(self.names):
            r = requests.get(''.join(URL + ('user', '/', str(self.ids[i]))), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            user = json.loads(r.text)
            self.assertEqual(name, user.get('name'))
            self.assertEqual(self.ids[i], user.get('id'))
            self.assertEqual(0, user.get('balance'))
            self.assertIsNone(user.get('last_transaction'))

class FilledTransactionsTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        requests.get(''.join(URL + ('debug/clear/',)))
        params = {'name': 'gert', 'mailAddress': 'gertMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_1 = result['id']
        params = {'name': 'bar', 'mailAddress': 'barMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_2 = result['id']
        cls.names = ['gert', 'bar']
        cls.ids = [user_1, user_2]
        cls.transactions = {user_1: [1000, -500, -150],
                            user_2: [2000, -250, -50]}
        for user_id, value_list in cls.transactions.items():
            for value in value_list:
                FilledTransactionsTests.add_transaction_to_user(user_id, value=value)

    @staticmethod
    def add_transaction_to_user(user_id, value):
        params = {'value': value}
        r = requests.post(''.join((URL + ('user', '/', str(user_id), '/', 'transaction', '/'))),
                          headers=HEADERS,
                          data=json.dumps(params))
        assert 201 == r.status_code

    def test_load_user_by_id(self):
        for i, name in enumerate(self.names):
            user_id = self.ids[i]
            r = requests.get(''.join(URL + ('user', '/', str(user_id))), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            user = json.loads(r.text)
            self.assertEqual(name, user.get('name'))
            self.assertEqual(self.ids[i], user.get('id'))
            balance = sum(self.transactions.get(user_id, []))
            self.assertEqual(balance, user.get('balance'))
            self.assertIsNotNone(user.get('last_transaction'))
            now = datetime.datetime.utcnow()
            last_transaction = datetime.datetime.strptime(user['last_transaction'], '%Y-%m-%dT%H:%M:%S.%fZ')
            self.assertGreater(20, (now - last_transaction).total_seconds())

    def test_load_user_transactions(self):
        for i, name in enumerate(self.names):
            user_id = self.ids[i]
            r = requests.get(''.join(URL + ('user', '/', str(user_id), '/', 'transaction',)), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            transactions = json.loads(r.text)
            self.assertEqual(len(self.transactions.get(user_id)), transactions.get('overall_count'))
            self.assertEqual(100, transactions.get('limit'))
            self.assertEqual(0, transactions.get('offset'))
            self.assertIsInstance(transactions['entries'], list)
            entries = transactions['entries']
            self.assertEqual(len(self.transactions.get(user_id)), len(entries))
            for j, entry in enumerate(entries):
                self.assertEqual(self.transactions.get(user_id)[j], entry['value'])
                self.assertEqual(user_id, entry['user'])

    def test_load_user_transactions_offset(self):
        for i, name in enumerate(self.names):
            user_id = self.ids[i]
            r = requests.get(''.join(URL + ('user', '/', str(user_id), '/', 'transaction',)), headers=HEADERS,
                             params={'limit': 1, 'offset': 2})
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            transactions = json.loads(r.text)
            self.assertEqual(len(self.transactions.get(user_id)), transactions.get('overall_count'))
            self.assertEqual(1, transactions.get('limit'))
            self.assertEqual(2, transactions.get('offset'))
            self.assertIsInstance(transactions['entries'], list)
            entries = transactions['entries']
            self.assertEqual(1, len(entries))
            for j, entry in enumerate(entries):
                self.assertEqual(self.transactions.get(user_id)[j + 2], entry['value'])
                self.assertEqual(user_id, entry['user'])

    def test_load_user_single_transaction(self):
        for user_id in self.transactions.keys():
            r = requests.get(''.join(URL + ('user', '/', str(user_id), '/', 'transaction',)), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            transactions = json.loads(r.text)
            entries = transactions['entries']
            for j, entry in enumerate(entries):
                with self.subTest(transaction_id=entry['id']):
                    self.assertEqual(self.transactions.get(user_id)[j], entry['value'])
                    self.assertEqual(user_id, entry['user'])
                    self.assert_user_transaction(entry['id'], entry['value'], user_id)

    def assert_user_transaction(self, transaction_id, value, user_id):
        r = requests.get(
            ''.join(URL + ('user', '/', str(user_id), '/', 'transaction', '/', str(transaction_id))),
            headers=HEADERS)
        self.assertEqual(200, r.status_code, msg=r.text)
        self.assertEqual('application/json', r.headers['Content-Type'])
        transaction = r.json()
        self.assertEqual(transaction_id, transaction['id'])
        self.assertEqual(user_id, transaction['user'])
        self.assertEqual(value, transaction['value'])
        now = datetime.datetime.utcnow()
        last_transaction = datetime.datetime.strptime(transaction['create_date'], '%Y-%m-%dT%H:%M:%S.%fZ')
        self.assertGreater(20, (now - last_transaction).total_seconds())
<commit_msg>Test filled global transaction list<commit_after>import datetime
import json
import unittest

import requests

URL = ("http://", "127.0.0.1", ":", "8000", "/")

HEADERS = {'Content-Type': 'application/json'}

class FilledUserTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        requests.get(''.join(URL + ('debug/clear/',)))
        params = {'name': 'gert', 'mailAddress': 'gertMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_1 = result['id']
        params = {'name': 'bar', 'mailAddress': 'barMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_2 = result['id']
        cls.names = ['gert', 'bar']
        cls.ids = [user_1, user_2]

    def test_list_2_users(self):
        r = requests.get(''.join(URL + ('user',)), headers=HEADERS)
        self.assertEqual(200, r.status_code)
        self.assertEqual('application/json', r.headers['Content-Type'])
        users = json.loads(r.text)
        self.assertEqual(2, users.get('overall_count'))
        self.assertEqual(100, users.get('limit'))
        self.assertEqual(0, users.get('offset'))
        entries = users['entries']
        self.assertIsInstance(entries, list)
        self.assertEqual(2, len(entries))
        for i in range(0, 1):
            with self.subTest(i=i):
                self.assertEqual(self.names[i], entries[i].get('name'))
                self.assertEqual(self.ids[i], entries[i]['id'])
                self.assertEqual(0, entries[i].get('balance'))
                self.assertIsNone(entries[i].get('last_transaction'))

    def test_list_users_empty_offset(self):
        r = requests.get(''.join(URL + ('user',)), headers=HEADERS, params={'offset': 2, 'limit': 1})
        self.assertEqual(200, r.status_code)
        self.assertEqual('application/json', r.headers['Content-Type'])
        users = json.loads(r.text)
        self.assertEqual(2, users.get('overall_count'))
        self.assertEqual(1, users.get('limit'))
        self.assertEqual(2, users.get('offset'))
        entries = users['entries']
        self.assertIsInstance(entries, list)
        self.assertEqual(0, len(entries))

    def test_load_user_by_id(self):
        for i, name in enumerate(self.names):
            r = requests.get(''.join(URL + ('user', '/', str(self.ids[i]))), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            user = json.loads(r.text)
            self.assertEqual(name, user.get('name'))
            self.assertEqual(self.ids[i], user.get('id'))
            self.assertEqual(0, user.get('balance'))
            self.assertIsNone(user.get('last_transaction'))

class FilledTransactionsTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        requests.get(''.join(URL + ('debug/clear/',)))
        params = {'name': 'gert', 'mailAddress': 'gertMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_1 = result['id']
        params = {'name': 'bar', 'mailAddress': 'barMail'}
        r = requests.post(''.join(URL + ('user', '/')), headers=HEADERS, data=json.dumps(params))
        result = r.json()
        user_2 = result['id']
        cls.names = ['gert', 'bar']
        cls.ids = [user_1, user_2]
        cls.transactions = {user_1: [1000, -500, -150],
                            user_2: [2000, -250, -50]}
        for user_id, value_list in cls.transactions.items():
            for value in value_list:
                FilledTransactionsTests.add_transaction_to_user(user_id, value=value)

    @staticmethod
    def add_transaction_to_user(user_id, value):
        params = {'value': value}
        r = requests.post(''.join((URL + ('user', '/', str(user_id), '/', 'transaction', '/'))),
                          headers=HEADERS,
                          data=json.dumps(params))
        assert 201 == r.status_code

    def test_load_user_by_id(self):
        for i, name in enumerate(self.names):
            user_id = self.ids[i]
            r = requests.get(''.join(URL + ('user', '/', str(user_id))), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            user = json.loads(r.text)
            self.assertEqual(name, user.get('name'))
            self.assertEqual(self.ids[i], user.get('id'))
            balance = sum(self.transactions.get(user_id, []))
            self.assertEqual(balance, user.get('balance'))
            self.assertIsNotNone(user.get('last_transaction'))
            now = datetime.datetime.utcnow()
            last_transaction = datetime.datetime.strptime(user['last_transaction'], '%Y-%m-%dT%H:%M:%S.%fZ')
            self.assertGreater(20, (now - last_transaction).total_seconds())

    def test_load_user_transactions(self):
        for i, name in enumerate(self.names):
            user_id = self.ids[i]
            r = requests.get(''.join(URL + ('user', '/', str(user_id), '/', 'transaction',)), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            transactions = json.loads(r.text)
            self.assertEqual(len(self.transactions.get(user_id)), transactions.get('overall_count'))
            self.assertEqual(100, transactions.get('limit'))
            self.assertEqual(0, transactions.get('offset'))
            self.assertIsInstance(transactions['entries'], list)
            entries = transactions['entries']
            self.assertEqual(len(self.transactions.get(user_id)), len(entries))
            for j, entry in enumerate(entries):
                self.assertEqual(self.transactions.get(user_id)[j], entry['value'])
                self.assertEqual(user_id, entry['user'])

    def test_load_user_transactions_offset(self):
        for i, name in enumerate(self.names):
            user_id = self.ids[i]
            r = requests.get(''.join(URL + ('user', '/', str(user_id), '/', 'transaction',)), headers=HEADERS,
                             params={'limit': 1, 'offset': 2})
            self.assertEqual(200, r.status_code)
            self.assertEqual('application/json', r.headers['Content-Type'])
            transactions = json.loads(r.text)
            self.assertEqual(len(self.transactions.get(user_id)), transactions.get('overall_count'))
            self.assertEqual(1, transactions.get('limit'))
            self.assertEqual(2, transactions.get('offset'))
            self.assertIsInstance(transactions['entries'], list)
            entries = transactions['entries']
            self.assertEqual(1, len(entries))
            for j, entry in enumerate(entries):
                self.assertEqual(self.transactions.get(user_id)[j + 2], entry['value'])
                self.assertEqual(user_id, entry['user'])

    def test_load_user_single_transaction(self):
        for user_id in self.transactions.keys():
            r = requests.get(''.join(URL + ('user', '/', str(user_id), '/', 'transaction',)), headers=HEADERS)
            self.assertEqual(200, r.status_code)
            transactions = json.loads(r.text)
            entries = transactions['entries']
            for j, entry in enumerate(entries):
                with self.subTest(transaction_id=entry['id']):
                    self.assertEqual(self.transactions.get(user_id)[j], entry['value'])
                    self.assertEqual(user_id, entry['user'])
                    self.assert_user_transaction(entry['id'], entry['value'], user_id)

    def test_load_transactions_offset(self):
        r = requests.get(''.join(URL + ('transaction',)), headers=HEADERS,
                         params={'limit': 1, 'offset': 2})
        self.assertEqual(200, r.status_code)
        self.assertEqual('application/json', r.headers['Content-Type'])
        transactions = json.loads(r.text)
        total_count = sum([len(x) for x in self.transactions.values()])
        self.assertEqual(total_count, transactions.get('overall_count'))
        self.assertEqual(1, transactions.get('limit'))
        self.assertEqual(2, transactions.get('offset'))
        self.assertIsInstance(transactions['entries'], list)
        entries = transactions['entries']
        self.assertEqual(1, len(entries))
        for j, entry in enumerate(entries):
            user_id = entry['user']
            self.assertIn(entry['value'], self.transactions.get(user_id))

    def test_load_single_transaction(self):
        r = requests.get(''.join(URL + ('transaction',)), headers=HEADERS)
        self.assertEqual(200, r.status_code)
        transactions = json.loads(r.text)
        entries = transactions['entries']
        for j, entry in enumerate(entries):
            with self.subTest(transaction_id=entry['id']):
                user_id = entry['user']
                self.assertIn(entry['value'], self.transactions.get(user_id))
                self.assert_user_transaction(entry['id'], entry['value'], user_id)

    def assert_user_transaction(self, transaction_id, value, user_id):
        r = requests.get(
            ''.join(URL + ('user', '/', str(user_id), '/', 'transaction', '/', str(transaction_id))),
            headers=HEADERS)
        self.assertEqual(200, r.status_code, msg=r.text)
        self.assertEqual('application/json', r.headers['Content-Type'])
        transaction = r.json()
        self.assertEqual(transaction_id, transaction['id'])
        self.assertEqual(user_id, transaction['user'])
        self.assertEqual(value, transaction['value'])
        now = datetime.datetime.utcnow()
        last_transaction = datetime.datetime.strptime(transaction['create_date'], '%Y-%m-%dT%H:%M:%S.%fZ')
        self.assertGreater(20, (now - last_transaction).total_seconds())
<|endoftext|>


<commit_before>'''
c++ ../carbon-budget/emissions/cpp_util/calc_gross_emissions_generic.cpp -o ../carbon-budget/emissions/cpp_util//calc_gross_emissions_generic.exe -lgdal
python mp_run_full_model_short.py -t std -s all -r true -d 20200303 -l 00N_000E -ce loss -p biomass_soil -tcd 30
'''

import argparse
import os
import datetime
import constants_and_names as cn
import universal_util as uu
from gain.mp_forest_age_category_natrl_forest import mp_forest_age_category_natrl_forest
from gain.mp_gain_year_count_natrl_forest import mp_gain_year_count_natrl_forest
from gain.mp_annual_gain_rate_natrl_forest import mp_annual_gain_rate_natrl_forest
from gain.mp_cumulative_gain_natrl_forest import mp_cumulative_gain_natrl_forest
from gain.mp_merge_cumulative_annual_gain_all_forest_types import mp_merge_cumulative_annual_gain_all_forest_types
from carbon_pools.mp_create_carbon_pools import mp_create_carbon_pools
from emissions.mp_calculate_gross_emissions import mp_calculate_gross_emissions
from analyses.mp_net_flux import mp_net_flux
from analyses.mp_aggregate_results_to_10_km import mp_aggregate_results_to_10_km

def main ():

    model_stages = ['all', 'forest_age_category_natrl_forest', 'gain_year_count_natrl_forest',
                    'annual_gain_rate_natrl_forest', 'cumulative_gain_natrl_forest', 'removals_merged',
                    'carbon_pools', 'gross_emissions',
                    'net_flux', 'aggregate']

    # The argument for what kind of model run is being done: standard conditions or a sensitivity analysis run
    parser = argparse.ArgumentParser(description='Create tiles of the number of years of carbon gain for mangrove forests')
    parser.add_argument('--model-type', '-t', required=True, help='{}'.format(cn.model_type_arg_help))
    parser.add_argument('--stages', '-s', required=True,
                        help='Stages of creating Brazil legal Amazon-specific gross cumulative removals. Options are {}'.format(model_stages))
    parser.add_argument('--run_through', '-r', required=True,
                        help='Options: true or false. true: run named stage and following stages. false: run only named stage.')
    parser.add_argument('--run_date', '-d', required=True,
                        help='Date of run. Must be format YYYYMMDD.')
    parser.add_argument('--tile_id_list', '-l', required=True,
                        help='List of tile ids to use in the model. Should be of form 00N_110E or all.')
    parser.add_argument('--carbon-pool-extent', '-ce', required=False,
                        help='Extent over which carbon pools should be calculated: loss or 2000')
    parser.add_argument('--pools-to-use', '-p', required=False,
                        help='Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil.')
    parser.add_argument('--tcd-threshold', '-tcd', required=False,
                        help='Tree cover density threshold above which pixels will be included in the aggregation.')
    parser.add_argument('--std-net-flux-aggreg', '-sagg', required=False,
                        help='The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map')
    args = parser.parse_args()
    sensit_type = args.model_type
    stage_input = args.stages
    run_through = args.run_through
    run_date = args.run_date
    carbon_pool_extent = args.carbon_pool_extent
    pools = args.pools_to_use
    tile_id_list = args.tile_id_list
    thresh = args.tcd_threshold
    if thresh is not None:
        thresh = int(thresh)
    std_net_flux = args.std_net_flux_aggreg

    # Working directory
    working_dir = os.getcwd()

    # Checks the validity of the model stage arguments. If either one is invalid, the script ends.
    if (stage_input not in model_stages):
        raise Exception('Invalid stage selection. Please provide a stage from {}.'.format(model_stages))
    else:
        pass
    if (run_through not in ['true', 'false']):
        raise Exception('Invalid run through option. Please enter true or false.')
    else:
        pass

    # Generates the list of stages to run
    actual_stages = uu.analysis_stages(model_stages, stage_input, run_through)
    print "Analysis stages to run:", actual_stages

    # Checks whether the sensitivity analysis argument is valid
    uu.check_sensit_type(sensit_type)

    # Checks if the carbon pool type is specified if the stages to run includes carbon pool generation.
    # Does this up front so the user knows before the run begins that information is missing.
    if 'carbon_pools' in actual_stages and carbon_pool_extent not in ['loss', '2000']:
        raise Exception('Carbon pool year not specified for carbon pool creation step')

    # Checks if the correct c++ script has been compiled for the pool option selected.
    # Does this up front so that the user is prompted to compile the C++ before the script starts running, if necessary.
    if 'gross_emissions' in actual_stages:

        if pools == 'biomass_soil':
            # Some sensitivity analyses have specific gross emissions scripts.
            # The rest of the sensitivity analyses and the standard model can all use the same, generic gross emissions script.
            if sensit_type in ['no_shifting_ag', 'convert_to_grassland']:
                if os.path.exists('../carbon-budget/emissions/cpp_util/calc_gross_emissions_{}.exe'.format(sensit_type)):
                    print "C++ for {} already compiled.".format(sensit_type)
                else:
                    raise Exception('Must compile standard {} model C++...'.format(sensit_type))
            else:
                if os.path.exists('../carbon-budget/emissions/cpp_util/calc_gross_emissions_generic.exe'):
                    print "C++ for generic emissions already compiled."
                else:
                    raise Exception('Must compile generic emissions C++...')

        elif (pools == 'soil_only') & (sensit_type == 'std'):
            if os.path.exists('../carbon-budget/emissions/cpp_util/calc_gross_emissions_soil_only.exe'):
                print "C++ for soil_only already compiled."
            else:
                raise Exception('Must compile soil_only C++...')

        else:
            raise Exception('Pool and/or sensitivity analysis option not valid for gross emissions')

    # Checks whether the canopy cover argument is valid up front.
    if 'aggregate' in actual_stages:
        if thresh < 0 or thresh > 99:
            raise Exception('Invalid tcd. Please provide an integer between 0 and 99.')
        else:
            pass

    # If the tile_list argument is an s3 folder, the list of tiles in it is created
    if 's3://' in tile_id_list:
        tile_id_list = uu.tile_list_s3(tile_id_list, 'std')
        print tile_id_list
        print "There are {} tiles to process".format(str(len(tile_id_list))) + "\n"
    # Otherwise, check that the tile list argument is valid
    else:
        tile_id_list = uu.tile_id_list_check(tile_id_list)

    # List of output directories and output file name patterns
    raw_output_dir_list = [
                       cn.age_cat_natrl_forest_dir, cn.gain_year_count_natrl_forest_dir,
                       cn.annual_gain_AGB_natrl_forest_dir, cn.annual_gain_BGB_natrl_forest_dir,
                       cn.cumul_gain_AGCO2_natrl_forest_dir, cn.cumul_gain_BGCO2_natrl_forest_dir,
                       cn.annual_gain_AGB_BGB_all_types_dir, cn.cumul_gain_AGCO2_BGCO2_all_types_dir,
                       cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir,
                       cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir, cn.total_C_emis_year_dir,
                       cn.net_flux_dir
                       ]

    raw_output_pattern_list = [
                           cn.pattern_age_cat_natrl_forest, cn.pattern_gain_year_count_natrl_forest,
                           cn.pattern_annual_gain_AGB_natrl_forest, cn.pattern_annual_gain_BGB_natrl_forest,
                           cn.pattern_cumul_gain_AGCO2_natrl_forest, cn.pattern_cumul_gain_BGCO2_natrl_forest,
                           cn.pattern_annual_gain_AGB_BGB_all_types, cn.pattern_cumul_gain_AGCO2_BGCO2_all_types,
                           cn.pattern_AGC_emis_year, cn.pattern_BGC_emis_year, cn.pattern_deadwood_emis_year_2000,
                           cn.pattern_litter_emis_year_2000, cn.pattern_soil_C_emis_year_2000, cn.pattern_total_C_emis_year,
                           cn.pattern_net_flux
                           ]

    # Creates age category tiles for natural forests
    if 'forest_age_category_natrl_forest' in actual_stages:

        print ':::::Creating forest age category for natural forest tiles'
        start = datetime.datetime.now()

        mp_forest_age_category_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for forest_age_category_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of the number of years of removals for natural forests
    if 'gain_year_count_natrl_forest' in actual_stages:

        print ':::::Creating gain year count for natural forest tiles'
        start = datetime.datetime.now()

        mp_gain_year_count_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for gain_year_count_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of annual AGB and BGB gain rate for non-mangrove, non-planted forest using the standard model
    # removal function
    if 'annual_gain_rate_natrl_forest' in actual_stages:

        print ':::::Creating annual removals for natural forest tiles'
        start = datetime.datetime.now()

        mp_annual_gain_rate_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for annual_gain_rate_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of cumulative AGCO2 and BGCO2 gain rate for non-mangrove, non-planted forest using the standard model
    # removal function
    if 'cumulative_gain_natrl_forest' in actual_stages:

        print ':::::Creating cumulative removals for natural forest tiles'
        start = datetime.datetime.now()

        mp_cumulative_gain_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for cumulative_gain_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of annual gain rate and cumulative removals for all forest types (above + belowground)
    if 'removals_merged' in actual_stages:

        print ':::::Creating annual and cumulative removals for all forest types combined (above + belowground) tiles'
        start = datetime.datetime.now()

        mp_merge_cumulative_annual_gain_all_forest_types(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for removals_merged:", elapsed_time, "\n"

    # Creates carbon pools in loss year
    if 'carbon_pools' in actual_stages:

        print ':::::Creating emissions year carbon pools tiles'
        start = datetime.datetime.now()

        mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for carbon_pools:", elapsed_time, "\n"

    if 'gross_emissions' in actual_stages:

        print ':::::Creating gross emissions tiles'
        start = datetime.datetime.now()

        mp_calculate_gross_emissions(sensit_type, tile_id_list, pools, run_date = run_date, working_dir = working_dir)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for gross_emissions:", elapsed_time, "\n"

    if 'net_flux' in actual_stages:

        print ':::::Creating net flux tiles'
        start = datetime.datetime.now()

        mp_net_flux(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for net_flux:", elapsed_time, "\n"

    if 'aggregate' in actual_stages:

        print ':::::Creating 10km aggregate maps'
        start = datetime.datetime.now()

        mp_aggregate_results_to_10_km(sensit_type, thresh, std_net_flux, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for aggregate:", elapsed_time, "\n"

if __name__ == '__main__':
    main()<commit_msg>Trying to get gross emissions to work in feature_single_script.<commit_after>'''
c++ ../carbon-budget/emissions/cpp_util/calc_gross_emissions_generic.cpp -o ../carbon-budget/emissions/cpp_util/calc_gross_emissions_generic.exe -lgdal
python mp_run_full_model_short.py -t std -s all -r true -d 20200303 -l 00N_000E -ce loss -p biomass_soil -tcd 30
'''

import argparse
import os
import datetime
import constants_and_names as cn
import universal_util as uu
from gain.mp_forest_age_category_natrl_forest import mp_forest_age_category_natrl_forest
from gain.mp_gain_year_count_natrl_forest import mp_gain_year_count_natrl_forest
from gain.mp_annual_gain_rate_natrl_forest import mp_annual_gain_rate_natrl_forest
from gain.mp_cumulative_gain_natrl_forest import mp_cumulative_gain_natrl_forest
from gain.mp_merge_cumulative_annual_gain_all_forest_types import mp_merge_cumulative_annual_gain_all_forest_types
from carbon_pools.mp_create_carbon_pools import mp_create_carbon_pools
from emissions.mp_calculate_gross_emissions import mp_calculate_gross_emissions
from analyses.mp_net_flux import mp_net_flux
from analyses.mp_aggregate_results_to_10_km import mp_aggregate_results_to_10_km

def main ():

    model_stages = ['all', 'forest_age_category_natrl_forest', 'gain_year_count_natrl_forest',
                    'annual_gain_rate_natrl_forest', 'cumulative_gain_natrl_forest', 'removals_merged',
                    'carbon_pools', 'gross_emissions',
                    'net_flux', 'aggregate']

    # The argument for what kind of model run is being done: standard conditions or a sensitivity analysis run
    parser = argparse.ArgumentParser(description='Create tiles of the number of years of carbon gain for mangrove forests')
    parser.add_argument('--model-type', '-t', required=True, help='{}'.format(cn.model_type_arg_help))
    parser.add_argument('--stages', '-s', required=True,
                        help='Stages of creating Brazil legal Amazon-specific gross cumulative removals. Options are {}'.format(model_stages))
    parser.add_argument('--run_through', '-r', required=True,
                        help='Options: true or false. true: run named stage and following stages. false: run only named stage.')
    parser.add_argument('--run_date', '-d', required=True,
                        help='Date of run. Must be format YYYYMMDD.')
    parser.add_argument('--tile_id_list', '-l', required=True,
                        help='List of tile ids to use in the model. Should be of form 00N_110E or all.')
    parser.add_argument('--carbon-pool-extent', '-ce', required=False,
                        help='Extent over which carbon pools should be calculated: loss or 2000')
    parser.add_argument('--pools-to-use', '-p', required=False,
                        help='Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil.')
    parser.add_argument('--tcd-threshold', '-tcd', required=False,
                        help='Tree cover density threshold above which pixels will be included in the aggregation.')
    parser.add_argument('--std-net-flux-aggreg', '-sagg', required=False,
                        help='The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map')
    args = parser.parse_args()
    sensit_type = args.model_type
    stage_input = args.stages
    run_through = args.run_through
    run_date = args.run_date
    carbon_pool_extent = args.carbon_pool_extent
    pools = args.pools_to_use
    tile_id_list = args.tile_id_list
    thresh = args.tcd_threshold
    if thresh is not None:
        thresh = int(thresh)
    std_net_flux = args.std_net_flux_aggreg

    # Working directory
    working_dir = os.getcwd()

    # Checks the validity of the model stage arguments. If either one is invalid, the script ends.
    if (stage_input not in model_stages):
        raise Exception('Invalid stage selection. Please provide a stage from {}.'.format(model_stages))
    else:
        pass
    if (run_through not in ['true', 'false']):
        raise Exception('Invalid run through option. Please enter true or false.')
    else:
        pass

    # Generates the list of stages to run
    actual_stages = uu.analysis_stages(model_stages, stage_input, run_through)
    print "Analysis stages to run:", actual_stages

    # Checks whether the sensitivity analysis argument is valid
    uu.check_sensit_type(sensit_type)

    # Checks if the carbon pool type is specified if the stages to run includes carbon pool generation.
    # Does this up front so the user knows before the run begins that information is missing.
    if 'carbon_pools' in actual_stages and carbon_pool_extent not in ['loss', '2000']:
        raise Exception('Carbon pool year not specified for carbon pool creation step')

    # Checks if the correct c++ script has been compiled for the pool option selected.
    # Does this up front so that the user is prompted to compile the C++ before the script starts running, if necessary.
    if 'gross_emissions' in actual_stages:

        if pools == 'biomass_soil':
            # Some sensitivity analyses have specific gross emissions scripts.
            # The rest of the sensitivity analyses and the standard model can all use the same, generic gross emissions script.
            if sensit_type in ['no_shifting_ag', 'convert_to_grassland']:
                if os.path.exists('../carbon-budget/emissions/cpp_util/calc_gross_emissions_{}.exe'.format(sensit_type)):
                    print "C++ for {} already compiled.".format(sensit_type)
                else:
                    raise Exception('Must compile standard {} model C++...'.format(sensit_type))
            else:
                if os.path.exists('../carbon-budget/emissions/cpp_util/calc_gross_emissions_generic.exe'):
                    print "C++ for generic emissions already compiled."
                else:
                    raise Exception('Must compile generic emissions C++...')

        elif (pools == 'soil_only') & (sensit_type == 'std'):
            if os.path.exists('../carbon-budget/emissions/cpp_util/calc_gross_emissions_soil_only.exe'):
                print "C++ for soil_only already compiled."
            else:
                raise Exception('Must compile soil_only C++...')

        else:
            raise Exception('Pool and/or sensitivity analysis option not valid for gross emissions')

    # Checks whether the canopy cover argument is valid up front.
    if 'aggregate' in actual_stages:
        if thresh < 0 or thresh > 99:
            raise Exception('Invalid tcd. Please provide an integer between 0 and 99.')
        else:
            pass

    # If the tile_list argument is an s3 folder, the list of tiles in it is created
    if 's3://' in tile_id_list:
        tile_id_list = uu.tile_list_s3(tile_id_list, 'std')
        print tile_id_list
        print "There are {} tiles to process".format(str(len(tile_id_list))) + "\n"
    # Otherwise, check that the tile list argument is valid
    else:
        tile_id_list = uu.tile_id_list_check(tile_id_list)

    # List of output directories and output file name patterns
    raw_output_dir_list = [
                       cn.age_cat_natrl_forest_dir, cn.gain_year_count_natrl_forest_dir,
                       cn.annual_gain_AGB_natrl_forest_dir, cn.annual_gain_BGB_natrl_forest_dir,
                       cn.cumul_gain_AGCO2_natrl_forest_dir, cn.cumul_gain_BGCO2_natrl_forest_dir,
                       cn.annual_gain_AGB_BGB_all_types_dir, cn.cumul_gain_AGCO2_BGCO2_all_types_dir,
                       cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir,
                       cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir, cn.total_C_emis_year_dir,
                       cn.net_flux_dir
                       ]

    raw_output_pattern_list = [
                           cn.pattern_age_cat_natrl_forest, cn.pattern_gain_year_count_natrl_forest,
                           cn.pattern_annual_gain_AGB_natrl_forest, cn.pattern_annual_gain_BGB_natrl_forest,
                           cn.pattern_cumul_gain_AGCO2_natrl_forest, cn.pattern_cumul_gain_BGCO2_natrl_forest,
                           cn.pattern_annual_gain_AGB_BGB_all_types, cn.pattern_cumul_gain_AGCO2_BGCO2_all_types,
                           cn.pattern_AGC_emis_year, cn.pattern_BGC_emis_year, cn.pattern_deadwood_emis_year_2000,
                           cn.pattern_litter_emis_year_2000, cn.pattern_soil_C_emis_year_2000, cn.pattern_total_C_emis_year,
                           cn.pattern_net_flux
                           ]

    # Creates age category tiles for natural forests
    if 'forest_age_category_natrl_forest' in actual_stages:

        print ':::::Creating forest age category for natural forest tiles'
        start = datetime.datetime.now()

        mp_forest_age_category_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for forest_age_category_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of the number of years of removals for natural forests
    if 'gain_year_count_natrl_forest' in actual_stages:

        print ':::::Creating gain year count for natural forest tiles'
        start = datetime.datetime.now()

        mp_gain_year_count_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for gain_year_count_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of annual AGB and BGB gain rate for non-mangrove, non-planted forest using the standard model
    # removal function
    if 'annual_gain_rate_natrl_forest' in actual_stages:

        print ':::::Creating annual removals for natural forest tiles'
        start = datetime.datetime.now()

        mp_annual_gain_rate_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for annual_gain_rate_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of cumulative AGCO2 and BGCO2 gain rate for non-mangrove, non-planted forest using the standard model
    # removal function
    if 'cumulative_gain_natrl_forest' in actual_stages:

        print ':::::Creating cumulative removals for natural forest tiles'
        start = datetime.datetime.now()

        mp_cumulative_gain_natrl_forest(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for cumulative_gain_natrl_forest:", elapsed_time, "\n"

    # Creates tiles of annual gain rate and cumulative removals for all forest types (above + belowground)
    if 'removals_merged' in actual_stages:

        print ':::::Creating annual and cumulative removals for all forest types combined (above + belowground) tiles'
        start = datetime.datetime.now()

        mp_merge_cumulative_annual_gain_all_forest_types(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for removals_merged:", elapsed_time, "\n"

    # Creates carbon pools in loss year
    if 'carbon_pools' in actual_stages:

        print ':::::Creating emissions year carbon pools tiles'
        start = datetime.datetime.now()

        mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for carbon_pools:", elapsed_time, "\n"

    if 'gross_emissions' in actual_stages:

        print ':::::Creating gross emissions tiles'
        start = datetime.datetime.now()

        mp_calculate_gross_emissions(sensit_type, tile_id_list, pools, run_date = run_date, working_dir = working_dir)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for gross_emissions:", elapsed_time, "\n"

    if 'net_flux' in actual_stages:

        print ':::::Creating net flux tiles'
        start = datetime.datetime.now()

        mp_net_flux(sensit_type, tile_id_list, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for net_flux:", elapsed_time, "\n"

    if 'aggregate' in actual_stages:

        print ':::::Creating 10km aggregate maps'
        start = datetime.datetime.now()

        mp_aggregate_results_to_10_km(sensit_type, thresh, std_net_flux, run_date = run_date)

        end = datetime.datetime.now()
        elapsed_time = end - start
        print ":::::Processing time for aggregate:", elapsed_time, "\n"

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>"""This module provides the views for the portal."""

import os
from django.shortcuts import render_to_response, redirect
from django.template import RequestContext
from django.core.urlresolvers import reverse
from mpcontribs.rest.views import get_endpoint
from mpcontribs.rest.rester import MPContribsRester

def index(request):
    ctx = RequestContext(request)
    if request.user.is_authenticated():
        from webtzite.models import RegisteredUser
        user = RegisteredUser.objects.get(username=request.user.username)
        ctx['landing_pages'] = []
        rester = MPContribsRester(user.api_key, endpoint=get_endpoint(request))
        for doc in rester.get_landing_pages():
            explorer = 'mpcontribs_users_{}_explorer_index'.format(doc['project'])
            entry = {'project': doc['project'], 'url': reverse(explorer)}
            content = doc['content']
            entry['title'] = content.get('title', doc['project'])
            authors = content.get('authors', 'No authors available.').split(',', 1)
            provenance = '{}'.format(authors[0])
            if len(authors) > 1:
                provenance += '''<button class="btn btn-sm btn-link" data-html="true"
                data-toggle="tooltip" data-placement="bottom" data-container="body"
                title="{}" style="padding: 0px 0px 2px 5px;">et al.'''.format(
                    authors[1].strip().replace(', ', '
'))
                provenance += ''
                entry['provenance'] = provenance
            ctx['landing_pages'].append(entry) # consider everything in DB released
    else:
        #ctx.update({'alert': 'Please log in!'})
        return redirect('{}?next={}'.format(reverse('cas_ng_login'), reverse('mpcontribs_portal_index')))
    return render_to_response("mpcontribs_portal_index.html", ctx)

def groupadd(request, token):
    if request.user.is_authenticated():
        from webtzite.models import RegisteredUser
        from mpcontribs.rest.rester import MPContribsRester
        user = RegisteredUser.objects.get(username=request.user.username)
        r = MPContribsRester(user.api_key, endpoint=get_endpoint(request))
        r.groupadd(token)
        return redirect(reverse('mpcontribs_portal_index'))
    else:
        return redirect('{}?next={}'.format(
            reverse('cas_ng_login'),
            reverse('mpcontribs_portal_groupadd', kwargs={'token': token})
        ))
<commit_msg>portal: disable auto login<commit_after>"""This module provides the views for the portal."""

import os
from django.shortcuts import render_to_response, redirect
from django.template import RequestContext
from django.core.urlresolvers import reverse
from mpcontribs.rest.views import get_endpoint
from mpcontribs.rest.rester import MPContribsRester

def index(request):
    ctx = RequestContext(request)
    if request.user.is_authenticated():
        from webtzite.models import RegisteredUser
        user = RegisteredUser.objects.get(username=request.user.username)
        ctx['landing_pages'] = []
        rester = MPContribsRester(user.api_key, endpoint=get_endpoint(request))
        for doc in rester.get_landing_pages():
            explorer = 'mpcontribs_users_{}_explorer_index'.format(doc['project'])
            entry = {'project': doc['project'], 'url': reverse(explorer)}
            content = doc['content']
            entry['title'] = content.get('title', doc['project'])
            authors = content.get('authors', 'No authors available.').split(',', 1)
            provenance = '{}'.format(authors[0])
            if len(authors) > 1:
                provenance += '''<button class="btn btn-sm btn-link" data-html="true"
                data-toggle="tooltip" data-placement="bottom" data-container="body"
                title="{}" style="padding: 0px 0px 2px 5px;">et al.'''.format(
                    authors[1].strip().replace(', ', '
'))
                provenance += ''
                entry['provenance'] = provenance
            ctx['landing_pages'].append(entry) # consider everything in DB released
    else:
        ctx.update({'alert': 'Please log in!'})
        #return redirect('{}?next={}'.format(reverse('cas_ng_login'), reverse('mpcontribs_portal_index')))
    return render_to_response("mpcontribs_portal_index.html", ctx)

def groupadd(request, token):
    if request.user.is_authenticated():
        from webtzite.models import RegisteredUser
        from mpcontribs.rest.rester import MPContribsRester
        user = RegisteredUser.objects.get(username=request.user.username)
        r = MPContribsRester(user.api_key, endpoint=get_endpoint(request))
        r.groupadd(token)
        return redirect(reverse('mpcontribs_portal_index'))
    else:
        return redirect('{}?next={}'.format(
            reverse('cas_ng_login'),
            reverse('mpcontribs_portal_groupadd', kwargs={'token': token})
        ))
<|endoftext|>