<commit_before>import logging
from graph import *
import pprint

from parser import standard_deviation, variance, avg

env_types = {
    "BH_VE_CPU_JIT_FUSION": int,
    "OMP_NUM_THREADS": int
}

engine_ord  = ['numpy', 'fusion', 'sij']
sample_size = 3
data_points = 6

def brange(begin, end):
    c = i = begin
    while i <= end:
        yield i
        i = 2**c
        c += 1

def extract_parameters(data):

    # Extract script aliases
    script_aliases = list(set([result["script_alias"] for result in data]))
    script_aliases.sort()

    # Extract bridge aliases
    bridge_aliases = list(set([result["bridge_alias"] for result in data]))
    bridge_aliases.sort()

    # Extract engine aliases
    engine_aliases = list(set([result["engine_alias"] for result in data]))
    engine_aliases.sort()

    # Extract manager aliases
    manager_aliases = list(set([result["manager_alias"] for result in data]))
    manager_aliases.sort()

    # Extract environment variables
    env_vars = list(set([env for result in data for env in result["envs_overwrite"] ]))
    env_vars.sort()

    # Extract values of environment variables and convert their types
    env_values = {env: [] for env in env_vars}
    for result in data:
        for env in env_vars:
            if env in result["envs_overwrite"]:
                env_values[env].append(
                    env_types[env](result["envs_overwrite"][env])
                )

    # Sort the extracted environment variables
    for env in env_vars:
        env_values[env] = list(set(env_values[env]))
        env_values[env].sort()

    return {
        "env_vars": env_vars,
        "env_values": env_values,
        "script_aliases": script_aliases,
        "bridge_aliases": bridge_aliases,
        "engine_aliases": engine_aliases,
        "manager_aliases": manager_aliases
    }

def flatten(data):
    """
    Flatten the input-data into sorted rows with eight columns of:

    (script_alias, bridge_alias, engine_alias, BH_VE_CPU_JIT_FUSION,
     OMP_NUM_THREADS, average(elapsed), var(elapsed), std_dev(elapsed))

    :param mixed data: Result file as produced by press.py
    :returns: List of tuples organized as described above.
    :rvalue: Mixed
    """

    def has_holes(elapsed):

        for etime in elapsed:
            if etime == None:
                return True

        return False

    flattened = []
    for result in data:

        elapsed = result["elapsed"]
        if has_holes(elapsed):  # Skip result if it "has holes" in elapsed time
            print("Result discarded elapsed time missing.")
            continue

        var = variance(elapsed) # Extract some time information
        average = avg(elapsed)
        std_dev = standard_deviation(elapsed)

        fusion = 0              # Other properties
        nthreads = 0
        if 'envs_overwrite' in result and 'OMP_NUM_THREADS' in result['envs_overwrite']:
            nthreads = int(result["envs_overwrite"]["OMP_NUM_THREADS"])
        if 'envs_overwrite' in result and 'BH_VE_CPU_JIT_FUSION' in result['envs_overwrite']:
            fusion = int(result["envs_overwrite"]["BH_VE_CPU_JIT_FUSION"])

        flattened.append((
            result['script_alias'],
            result['bridge_alias'],
            result['engine_alias'],
            fusion,
            nthreads,
            average,
            var,
            std_dev
        ))
    flattened.sort()

    return flattened

def restructure(data_flattened):
    """Take the flattened input and structure it somehow."""

    labels = ["numpy", "sij", "fusion"]

    structured = {}
    for data in data_flattened:

        script, bridge, engine, fusion, nthreads, average, var, std_dev = data
        if script not in structured:
            structured[script] = {
                'avg': {'fusion': [], 'sij': [], 'numpy': []},
                'dev': {'fusion': [], 'sij': [], 'numpy': []},
                'var': {'fusion': [], 'sij': [], 'numpy': []},

                'rel_first': {
                    'numpy':    {'fusion': [], 'sij': [], 'numpy': []},
                    'sij':      {'fusion': [], 'sij': [], 'numpy': []},
                    'fusion':   {'fusion': [], 'sij': [], 'numpy': []}
                },
                'rel_all': {
                    'numpy':    {'fusion': [], 'sij': [], 'numpy': []},
                    'sij':      {'fusion': [], 'sij': [], 'numpy': []},
                    'fusion':   {'fusion': [], 'sij': [], 'numpy': []}
                },
                'max': {},
                'min': {}
            }

        # Sort the results
        if 'fusion' in engine:
            structured[script]['avg']['fusion'].append(average)
            structured[script]['var']['fusion'].append(var)
            structured[script]['dev']['fusion'].append(std_dev)
        elif 'omp' in engine:
            structured[script]['avg']['sij'].append(average)
            structured[script]['var']['sij'].append(var)
            structured[script]['dev']['sij'].append(std_dev)
        else:
            # Create pseudo-samples for NumPy
            for _ in xrange(data_points):
                structured[script]['avg']['numpy'].append(average)
                structured[script]['var']['numpy'].append(var)
                structured[script]['dev']['numpy'].append(std_dev)

    discarded_results = {}
    for script in structured:   # Verify that we have all absolute numbers
        for label in labels:
            if len(structured[script]["avg"][label]) != data_points:
                if script not in discarded_results:
                    discarded_results[script] = []

                discarded_results[script].append(label)

    for script in discarded_results:    # Discard missing script
        print(
            "Dropping '%s' data-points are missing, culprit=%s." % (
            script,
            ','.join(discarded_results[script])
        ))
        del structured[script]

    for script in structured:       # Compute relative numbers, min, max

        for bsl_label in labels:    # To first value
            baseline = [structured[script]["avg"][bsl_label][0]]*data_points

            structured[script]['rel_first'][bsl_label] = {}

            for other_label in labels:
                other = structured[script]["avg"][other_label]
                structured[script]["rel_first"][bsl_label][other_label] = [
                    bsl / oth for bsl, oth in zip(baseline, other)
                ]

        for bsl_label in labels:    # To each value
            baseline = structured[script]["avg"][bsl_label]

            structured[script]['rel_all'][bsl_label] = {
                bsl_label: [1.0 for _ in baseline]
            }

            for other_label in (label for label in labels if label != bsl_label):
                other = structured[script]["avg"][other_label]
                structured[script]["rel_all"][bsl_label][other_label] = [
                    bsl / oth for bsl, oth in zip(baseline, other)
                ]

        for bsl_label in labels:    # Min, max speedup
            structured[script]['min'][bsl_label] = {}
            structured[script]['max'][bsl_label] = {}
            for other_label in labels:
                numbers = structured[script]['rel_first'][bsl_label][other_label]
                structured[script]['max'][bsl_label][other_label] = max(numbers)
                structured[script]['min'][bsl_label][other_label] = min(numbers)

    return structured

class Cpu(Graph):
    """Create a graph that illustrates scalabiltity."""

    def render_rel(self, data, parameters, script, rel_type, rel_engine):
        self.graph_title = script
        self.prep()

        min_threads     = 1
        max_threads     = max(parameters["env_values"]["OMP_NUM_THREADS"])
        linear          = list(brange(min_threads, max_threads))
        plot_count      = len(linear)

        legends = {'plots': [], 'legends': []}
        for i, engine in enumerate(engine_ord):
            legend_txt = "%s: %.1fx" % (
                engine,
                data[script]['max'][rel_engine][engine]
            )
            elapsed = data[script][rel_type][rel_engine][engine]
            p, = plot(
                linear,
                elapsed,
                "-*",
                label=engine,
                color=colors[i]
            )
            legends['plots'].append(p)
            legends['legends'].append(legend_txt)

        plot(linear, linear, "--", color='gray')  # Linear speedup

        script_max = 0.0
        for engine in engine_ord:
            script_max = max(script_max, data[script]['max'][rel_engine][engine])

        #
        # Scale y-axis with a neat border
        ylabel("Speedup in relation to '%s'." % rel_engine)
        yscale("symlog")
        yt_range = list(brange(min_threads, max(script_max, max_threads)))
        yticks(yt_range, yt_range)
        ylim(
            ymin=min_threads*0.15,
            ymax=max(script_max, max_threads)*1.5
        )

        #
        # Scale x-axis with a neat border
        xlabel("Threads")
        xscale("symlog")
        xticks(linear, linear)
        xlim(xmin=min_threads*0.75, xmax=max_threads*1.5)

        #
        # Plot-legends and their positions
        lgd = legend(
            legends['plots'],
            legends['legends'],
            loc=3,
            ncol=3,
            bbox_to_anchor=(0.015, 0.95, 0.9, 0.102),
            borderaxespad=0.0,
        )

        t = title(script)
        t.set_y(1.05)

        tight_layout()                  # Spit them out to file
        return self.to_file("%s_%s_%s" % (script, rel_type, rel_engine))

    def render_absolute(self, data, parameters, script):

        self.graph_title = script
        self.prep()

        min_threads     = 1
        max_threads     = max(parameters["env_values"]["OMP_NUM_THREADS"])
        linear          = list(brange(min_threads, max_threads))
        plot_count      = len(linear)

        max_runtime = max(
            [max(data[script]['avg'][engine]) for engine in engine_ord]
        )

        elapsed = data[script]['avg']
        dev = data[script]['dev']
        var = data[script]['var']

        ylabel("Elapsed wall-clock in seconds")

        ind = range(data_points)                # Group start locations
        width = 0.3                             # Width of the bars
        group_width = width * (len(engine_ord)) # Width of the group
        group_center = group_width / 2.0        # Center of groups

        fig, ax = plt.subplots()

        rects = {}                          # Draw the bars
        for i, engine in enumerate(engine_ord):
            rects[engine] = ax.bar(
                [x+width*i for x in ind],
                elapsed[engine],
                width,
                color=colors[i],
                yerr=dev[engine]
            )

        # add some text for labels, title and axes ticks
        ax.set_ylabel('Elapsed wall-clock time in seconds')
        ax.set_title(script)
        ax.set_xticks([x+group_center for x in ind])
        ax.set_xticklabels([str(x) for x in linear])

        ax.legend(
            [rects[engine] for engine in engine_ord],
            [engine for engine in engine_ord],
            loc=3,
            ncol=3,
            bbox_to_anchor=(0.01, 0.95, 0.9, 0.102),
            borderaxespad=0.0
        )

        def autolabel(rectangles):
            # attach some text labels
            for rect in rectangles:
                height = rect.get_height()
                ax.text(
                    rect.get_x()+rect.get_width()/2.,
                    1.015 * height,
                    '%d'%int(height),
                    ha='center',
                    va='bottom'
                )

        for engine in engine_ord:
            autolabel(rects[engine])

        xlabel("Threads")
        t = title(script)
        t.set_y(1.05)

        ylim(ymin=0, ymax=max_runtime*1.13)

        tight_layout()                  # Spit them out to file
        return self.to_file("%s_%s" % (script, 'absolute'))

    def render_html(self, data):

        doc = """

        <title>Benchmark Suite CPU Results</title>

Benchmark Suite CPU

        __RESULTS__

        """

        results = ""
        scripts = [script for script in data]
        scripts.sort()
        for script in scripts:
            results += "
%s
" % script
            results += "
"
            results += """

            """ % (
                data[script][0].replace("graphs"+os.sep, ""),
                data[script][1].replace("graphs"+os.sep, ""),
                data[script][2].replace("graphs"+os.sep, ""),
                data[script][3].replace("graphs"+os.sep, "")
            )
            results += "
\n"

        html = doc.replace("__RESULTS__", results)
        with open('graphs/cpu.html', 'w') as fd:
            fd.write(html)

    def render(self, raw, data, processed=None, params=None):

        raw = raw["runs"]
        data_flattened = flatten(raw)
        parameters = extract_parameters(raw)
        data = restructure(data_flattened)

        graph_filenames = {}
        for script in data:
            graph_filenames[script] = []
            fns = self.render_absolute(data, parameters, script)    # Absolute
            graph_filenames[script].extend(fns)
            for engine in engine_ord:                               # Relative
                fns = self.render_rel(data, parameters, script, 'rel_first', engine)
                graph_filenames[script].extend(fns)

        self.render_html(graph_filenames)                           # HTML
<commit_msg>grapher-cpu: Font and axis tweaking.<commit_after>import logging
from graph import *
import pprint
from matplotlib import rcParams

rcParams['axes.labelsize'] = 14
rcParams['axes.titlesize'] = 16
rcParams['xtick.labelsize'] = 14
rcParams['ytick.labelsize'] = 14
rcParams['legend.fontsize'] = 14
rcParams['font.family'] = 'serif'
rcParams['font.serif'] = ['Computer Modern Roman']
rcParams['text.usetex'] = True

from parser import standard_deviation, variance, avg

env_types = {
    "BH_VE_CPU_JIT_FUSION": int,
    "OMP_NUM_THREADS": int
}

engine_ord  = ['numpy', 'fusion', 'sij']
sample_size = 3
data_points = 6

def brange(begin, end):
    c = i = begin
    while i <= end:
        yield i
        i = 2**c
        c += 1

def extract_parameters(data):

    # Extract script aliases
    script_aliases = list(set([result["script_alias"] for result in data]))
    script_aliases.sort()

    # Extract script size-args from cmd
    script_sizes = list(set([
        (result["script_alias"], cmd) for cmd in result["cmd"]
                                      for result in data
                                      if 'size' in cmd
    ]))
    if len(script_sizes) != len(script_aliases):
        print "NO GOOD! Erroneous benchmark results!"
    script_sizes = dict(script_sizes)

    # Extract bridge aliases
    bridge_aliases = list(set([result["bridge_alias"] for result in data]))
    bridge_aliases.sort()

    # Extract engine aliases
    engine_aliases = list(set([result["engine_alias"] for result in data]))
    engine_aliases.sort()

    # Extract manager aliases
    manager_aliases = list(set([result["manager_alias"] for result in data]))
    manager_aliases.sort()

    # Extract environment variables
    env_vars = list(set([env for result in data for env in result["envs_overwrite"] ]))
    env_vars.sort()

    # Extract values of environment variables and convert their types
    env_values = {env: [] for env in env_vars}
    for result in data:
        for env in env_vars:
            if env in result["envs_overwrite"]:
                env_values[env].append(
                    env_types[env](result["envs_overwrite"][env])
                )

    # Sort the extracted environment variables
    for env in env_vars:
        env_values[env] = list(set(env_values[env]))
        env_values[env].sort()

    return {
        "env_vars": env_vars,
        "env_values": env_values,
        "script_aliases": script_aliases,
        "script_sizes": script_sizes,
        "bridge_aliases": bridge_aliases,
        "engine_aliases": engine_aliases,
        "manager_aliases": manager_aliases
    }

def flatten(data):
    """
    Flatten the input-data into sorted rows with eight columns of:

    (script_alias, bridge_alias, engine_alias, BH_VE_CPU_JIT_FUSION,
     OMP_NUM_THREADS, average(elapsed), var(elapsed), std_dev(elapsed))

    :param mixed data: Result file as produced by press.py
    :returns: List of tuples organized as described above.
    :rvalue: Mixed
    """

    def has_holes(elapsed):

        for etime in elapsed:
            if etime == None:
                return True

        return False

    flattened = []
    for result in data:

        elapsed = result["elapsed"]
        if has_holes(elapsed):  # Skip result if it "has holes" in elapsed time
            print("Result discarded elapsed time missing.")
            continue

        var = variance(elapsed) # Extract some time information
        average = avg(elapsed)
        std_dev = standard_deviation(elapsed)

        fusion = 0              # Other properties
        nthreads = 0
        if 'envs_overwrite' in result and 'OMP_NUM_THREADS' in result['envs_overwrite']:
            nthreads = int(result["envs_overwrite"]["OMP_NUM_THREADS"])
        if 'envs_overwrite' in result and 'BH_VE_CPU_JIT_FUSION' in result['envs_overwrite']:
            fusion = int(result["envs_overwrite"]["BH_VE_CPU_JIT_FUSION"])

        flattened.append((
            result['script_alias'],
            result['bridge_alias'],
            result['engine_alias'],
            fusion,
            nthreads,
            average,
            var,
            std_dev
        ))
    flattened.sort()

    return flattened

def restructure(data_flattened):
    """Take the flattened input and structure it somehow."""

    labels = ["numpy", "sij", "fusion"]

    structured = {}
    for data in data_flattened:

        script, bridge, engine, fusion, nthreads, average, var, std_dev = data
        if script not in structured:
            structured[script] = {
                'avg': {'fusion': [], 'sij': [], 'numpy': []},
                'dev': {'fusion': [], 'sij': [], 'numpy': []},
                'var': {'fusion': [], 'sij': [], 'numpy': []},

                'rel_first': {
                    'numpy':    {'fusion': [], 'sij': [], 'numpy': []},
                    'sij':      {'fusion': [], 'sij': [], 'numpy': []},
                    'fusion':   {'fusion': [], 'sij': [], 'numpy': []}
                },
                'rel_all': {
                    'numpy':    {'fusion': [], 'sij': [], 'numpy': []},
                    'sij':      {'fusion': [], 'sij': [], 'numpy': []},
                    'fusion':   {'fusion': [], 'sij': [], 'numpy': []}
                },
                'max': {},
                'min': {}
            }

        # Sort the results
        if 'fusion' in engine:
            structured[script]['avg']['fusion'].append(average)
            structured[script]['var']['fusion'].append(var)
            structured[script]['dev']['fusion'].append(std_dev)
        elif 'omp' in engine:
            structured[script]['avg']['sij'].append(average)
            structured[script]['var']['sij'].append(var)
            structured[script]['dev']['sij'].append(std_dev)
        else:
            # Create pseudo-samples for NumPy
            for _ in xrange(data_points):
                structured[script]['avg']['numpy'].append(average)
                structured[script]['var']['numpy'].append(var)
                structured[script]['dev']['numpy'].append(std_dev)

    discarded_results = {}
    for script in structured:   # Verify that we have all absolute numbers
        for label in labels:
            if len(structured[script]["avg"][label]) != data_points:
                if script not in discarded_results:
                    discarded_results[script] = []

                discarded_results[script].append(label)

    for script in discarded_results:    # Discard missing script
        print(
            "Dropping '%s' data-points are missing, culprit=%s." % (
            script,
            ','.join(discarded_results[script])
        ))
        del structured[script]

    for script in structured:       # Compute relative numbers, min, max

        for bsl_label in labels:    # To first value
            baseline = [structured[script]["avg"][bsl_label][0]]*data_points

            structured[script]['rel_first'][bsl_label] = {}

            for other_label in labels:
                other = structured[script]["avg"][other_label]
                structured[script]["rel_first"][bsl_label][other_label] = [
                    bsl / oth for bsl, oth in zip(baseline, other)
                ]

        for bsl_label in labels:    # To each value
            baseline = structured[script]["avg"][bsl_label]

            structured[script]['rel_all'][bsl_label] = {
                bsl_label: [1.0 for _ in baseline]
            }

            for other_label in (label for label in labels if label != bsl_label):
                other = structured[script]["avg"][other_label]
                structured[script]["rel_all"][bsl_label][other_label] = [
                    bsl / oth for bsl, oth in zip(baseline, other)
                ]

        for bsl_label in labels:    # Min, max speedup
            structured[script]['min'][bsl_label] = {}
            structured[script]['max'][bsl_label] = {}
            for other_label in labels:
                numbers = structured[script]['rel_first'][bsl_label][other_label]
                structured[script]['max'][bsl_label][other_label] = max(numbers)
                structured[script]['min'][bsl_label][other_label] = min(numbers)

    return structured

class Cpu(Graph):
    """Create a graph that illustrates scalabiltity."""

    def render_rel(self, data, parameters, script, rel_type, rel_engine):
        self.graph_title = script
        self.prep()

        min_threads     = 1
        max_threads     = max(parameters["env_values"]["OMP_NUM_THREADS"])
        linear          = list(brange(min_threads, max_threads))
        plot_count      = len(linear)

        legends = {'plots': [], 'legends': []}
        for i, engine in enumerate(engine_ord):
            legend_txt = r"%s: \textbf{%.1f} - \textbf{%.1f}" % (
                engine,
                data[script]['min'][rel_engine][engine],
                data[script]['max'][rel_engine][engine]
            )
            elapsed = data[script][rel_type][rel_engine][engine]
            p, = plot(
                linear,
                elapsed,
                "-*",
                label=engine,
                color=colors[i]
            )
            legends['plots'].append(p)
            legends['legends'].append(legend_txt)

        plot(linear, linear, "--", color='gray')  # Linear speedup

        script_max = 0.0
        for engine in engine_ord:
            for r_eng in engine_ord:
                script_max = max(script_max, data[script]['max'][r_eng][engine])

        #
        # Scale y-axis with a neat border
        ylabel("Speedup in relation to '%s'." % rel_engine)
        yscale("symlog")
        yt_range = list(brange(min_threads, max(script_max, max_threads)))
        yticks(yt_range, yt_range)
        ylim(
            ymin=min_threads*0.15,
            ymax=max(script_max, max_threads)*1.5
        )

        #
        # Scale x-axis with a neat border
        xlabel("Threads")
        xscale("symlog")
        xticks(linear, linear)
        xlim(xmin=min_threads*0.75, xmax=max_threads*1.5)

        #
        # Plot-legends and their positions
        lgd = legend(
            legends['plots'],
            legends['legends'],
            loc=3,
            ncol=3,
            bbox_to_anchor=(0.015, 0.95, 0.9, 0.102),
            borderaxespad=0.0,
        )

        t = title(script)
        t.set_y(1.05)

        tight_layout()                  # Spit them out to file
        return self.to_file("%s_%s_%s" % (script, rel_type, rel_engine))

    def render_absolute(self, data, parameters, script):

        self.graph_title = r"\textbf{%s}" % script
        self.prep()

        min_threads     = 1
        max_threads     = max(parameters["env_values"]["OMP_NUM_THREADS"])
        linear          = list(brange(min_threads, max_threads))
        plot_count      = len(linear)

        max_runtime = max(
            [max(data[script]['avg'][engine]) for engine in engine_ord]
        )

        elapsed = data[script]['avg']
        dev = data[script]['dev']
        var = data[script]['var']

        ylabel("Elapsed wall-clock in seconds")

        ind = range(data_points)                # Group start locations
        width = 0.3                             # Width of the bars
        group_width = width * (len(engine_ord)) # Width of the group
        group_center = group_width / 2.0        # Center of groups

        fig, ax = plt.subplots()

        rects = {}                          # Draw the bars
        for i, engine in enumerate(engine_ord):
            rects[engine] = ax.bar(
                [x+width*i for x in ind],
                elapsed[engine],
                width,
                color=colors[i],
                yerr=dev[engine]
            )

        # add some text for labels, title and axes ticks
        ax.set_ylabel('Elapsed wall-clock time in seconds')
        ax.set_title(script)
        ax.set_xticks([x+group_center for x in ind])
        ax.set_xticklabels([str(x) for x in linear])

        ax.legend(
            [rects[engine] for engine in engine_ord],
            [engine for engine in engine_ord],
            loc=3,
            ncol=3,
            bbox_to_anchor=(0.01, 0.95, 0.9, 0.102),
            borderaxespad=0.0
        )

        def autolabel(rectangles):
            # attach some text labels
            for rect in rectangles:
                height = rect.get_height()
                ax.text(
                    rect.get_x()+rect.get_width()/2.,
                    1.015 * height,
                    '%d'%int(height),
                    ha='center',
                    va='bottom'
                )

        for engine in engine_ord:
            autolabel(rects[engine])

        xlabel("Threads")
        t = title(script)
        t.set_y(1.05)

        ylim(ymin=0, ymax=max_runtime*1.13)

        tight_layout()                  # Spit them out to file
        return self.to_file("%s_%s" % (script, 'absolute'))

    def render_html(self, filenames, meta, parameters):

        doc = """

        <title>Benchmark Suite CPU Results - REV: %s</title>

Benchmark Suite CPU

        __RESULTS__

        """ % meta["rev"]

        results = ""
        scripts = [script for script in filenames]
        scripts.sort()
        for script in scripts:
            results += "
%s %s
" % (
                script,
                parameters["script_sizes"][script]
            )
            results += "
"
            results += """

            """ % (
                filenames[script][0].replace(self.output+os.sep, ""),
                filenames[script][1].replace(self.output+os.sep, ""),
                filenames[script][2].replace(self.output+os.sep, ""),
                filenames[script][3].replace(self.output+os.sep, "")
            )
            results += "
\n"

        html = doc.replace("__RESULTS__", results)
        with open('%s%scpu.html' % (self.output, os.sep), 'w') as fd:
            fd.write(html)

    def render(self, raw, data, processed=None, params=None):

        meta = raw["meta"]
        raw = raw["runs"]

        data_flattened = flatten(raw)
        parameters = extract_parameters(raw)
        data = restructure(data_flattened)

        graph_filenames = {}
        for script in data:
            graph_filenames[script] = []
            fns = self.render_absolute(data, parameters, script)    # Absolute
            graph_filenames[script].extend(fns)
            for engine in engine_ord:                               # Relative
                fns = self.render_rel(
                    data,
                    parameters,
                    script,
                    'rel_first',
                    engine
                )
                graph_filenames[script].extend(fns)

        self.render_html(graph_filenames, meta, parameters)          # HTML
<|endoftext|>


<commit_before>from flask import Blueprint
from flask.ext.restful import Api

api = Blueprint("api", __name__, template_folder="templates")
rest = Api(api)

# Import the non-restful resources
from grum.api import mailgun

# Let's get restful!
from grum.api.inbox import Inbox
rest.add_resource(Inbox, '/inbox')

from grum.api.messages import Messages
rest.add_resource(Messages, '/messages/<string:message_id>')

from grum.api.accounts import Account, Accounts
rest.add_resource(Accounts, '/accounts')
rest.add_resource(Account, '/account/<string:address>')<commit_msg>Move where to post for sending emails<commit_after>from flask import Blueprint
from flask.ext.restful import Api

api = Blueprint("api", __name__, template_folder="templates")
rest = Api(api)

# Import the non-restful resources
from grum.api import mailgun

# Let's get restful!
from grum.api.inbox import Inbox
rest.add_resource(Inbox, '/inbox')

from grum.api.messages import Messages
rest.add_resource(Messages, '/messages/<string:message_id>')

from grum.api.accounts import Account, Accounts
rest.add_resource(Accounts, '/accounts')
rest.add_resource(Account, '/accounts/<string:address>')<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of mb command for creating cloud storage buckets."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import re
import textwrap

from gslib.cloud_api import AccessDeniedException, BadRequestException
from gslib.command import Command
from gslib.command_argument import CommandArgument
from gslib.commands.rpo import VALID_RPO_VALUES
from gslib.commands.rpo import VALID_RPO_VALUES_STRING
from gslib.cs_api_map import ApiSelector
from gslib.exception import CommandException
from gslib.exception import InvalidUrlError
from gslib.storage_url import StorageUrlFromString
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
from gslib.utils.constants import NO_MAX
from gslib.utils.retention_util import RetentionInSeconds
from gslib.utils.text_util import InsistAscii
from gslib.utils.text_util import InsistOnOrOff
from gslib.utils.text_util import NormalizeStorageClass
from gslib.utils.encryption_helper import ValidateCMEK

_SYNOPSIS = """
  gsutil mb [-b (on|off)] [-c <class>] [-l <location>] [-p <project>]
            [--autoclass] [--retention <time>] [--pap <setting>]
            [--placement <region1>,<region2>]
            [--rpo {}] gs://<bucket_name>...
""".format(VALID_RPO_VALUES_STRING)

_DETAILED_HELP_TEXT = ("""
SYNOPSIS
""" + _SYNOPSIS + """

DESCRIPTION
  Create one or more new buckets. Google Cloud Storage has a single namespace,
  so you are not allowed to create a bucket with a name already in use by
  another user. You can, however, carve out parts of the bucket name space
  corresponding to your company's domain name (see "gsutil help naming").

  If you don't specify a project ID or project number using the -p option, the
  buckets are created using the default project ID specified in your `gsutil
  configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_.

  The -l option specifies the location for the buckets. Once a bucket is created
  in a given location, it cannot be moved to a different location. Instead, you
  need to create a new bucket, move the data over, and then delete the original
  bucket.

BUCKET STORAGE CLASSES
  You can specify one of the `storage classes
  <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket
  with the -c option.

  Example:

    gsutil mb -c nearline gs://some-bucket

  See online documentation for
  `pricing <https://cloud.google.com/storage/pricing>`_ and
  `SLA <https://cloud.google.com/storage/sla>`_ details.

  If you don't specify a -c option, the bucket is created with the
  default storage class Standard Storage.

BUCKET LOCATIONS
  You can specify one of the `available locations
  <https://cloud.google.com/storage/docs/locations>`_ for a bucket
  with the -l option.

  Examples:

    gsutil mb -l asia gs://some-bucket

    gsutil mb -c standard -l us-east1 gs://some-bucket

  If you don't specify a -l option, the bucket is created in the default
  location (US).

Retention Policy
  You can specify retention period in one of the following formats:

  --retention <number>s
      Specifies retention period of <number> seconds for objects in this bucket.

  --retention <number>d
      Specifies retention period of <number> days for objects in this bucket.

  --retention <number>m
      Specifies retention period of <number> months for objects in this bucket.

  --retention <number>y
      Specifies retention period of <number> years for objects in this bucket.

  Examples:

    gsutil mb --retention 1y gs://some-bucket

    gsutil mb --retention 36m gs://some-bucket

  If you don't specify a --retention option, the bucket is created with no
  retention policy.

OPTIONS
  --autoclass            Enables the Autoclass feature that automatically
                         sets object storage classes.

  -b <on|off>            Specifies the uniform bucket-level access setting.
                         When "on", ACLs assigned to objects in the bucket are
                         not evaluated. Consequently, only IAM policies grant
                         access to objects in these buckets. Default is "off".

  -c class               Specifies the default storage class.
                         Default is "Standard".

  -k <key>               Set the default KMS key using the full path to the key,
                         which has the following form:
                         ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]``

  -l location            Can be any supported location. See
                         https://cloud.google.com/storage/docs/locations
                         for a discussion of this distinction. Default is US.
                         Locations are case insensitive.

  -p project             Specifies the project ID or project number to create
                         the bucket under.

  -s class               Same as -c.

  --retention time       Specifies the retention policy. Default is no retention
                         policy. This can only be set on gs:// buckets and
                         requires using the JSON API. For more details about
                         retention policy see "gsutil help retention"

  --pap setting          Specifies the public access prevention setting. Valid
                         values are "enforced" or "unspecified". When
                         "enforced", objects in this bucket cannot be made
                         publicly accessible. Default is "unspecified".

  --placement reg1,reg2  Two regions that form the cutom dual-region.
                         Only regions within the same continent are or will ever
                         be valid. Invalid location pairs (such as
                         mixed-continent, or with unsupported regions)
                         will return an error.

  --rpo setting          Specifies the `replication setting <https://cloud.google.com/storage/docs/turbo-replication>`_.
                         This flag is not valid for single-region buckets,
                         and multi-region buckets only accept a value of
                         DEFAULT. Valid values for dual region buckets
                         are {rpo_values}. If unspecified, DEFAULT is applied
                         for dual-region and multi-region buckets.

""".format(rpo_values=VALID_RPO_VALUES_STRING))

# Regex to disallow buckets violating charset or not [3..255] chars total.
BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$')
# Regex to disallow buckets with individual DNS labels longer than 63.
TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}')

IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue
BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue

class MbCommand(Command):
  """Implementation of gsutil mb command."""

  # Command specification. See base class for documentation.
  command_spec = Command.CreateCommandSpec(
      'mb',
      command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
      usage_synopsis=_SYNOPSIS,
      min_args=1,
      max_args=NO_MAX,
      supported_sub_args='b:c:l:p:s:k:',
      supported_private_args=[
          'autoclass', 'retention=', 'pap=', 'placement=', 'rpo='
      ],
      file_url_ok=False,
      provider_url_ok=False,
      urls_start_arg=0,
      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
      gs_default_api=ApiSelector.JSON,
      argparse_arguments=[
          CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
      ],
  )
  # Help specification. See help_provider.py for documentation.
  help_spec = Command.HelpSpec(
      help_name='mb',
      help_name_aliases=[
          'createbucket',
          'makebucket',
          'md',
          'mkdir',
          'location',
          'dra',
          'dras',
          'reduced_availability',
          'durable_reduced_availability',
          'rr',
          'reduced_redundancy',
          'standard',
          'storage class',
          'nearline',
          'nl',
      ],
      help_type='command_help',
      help_one_line_summary='Make buckets',
      help_text=_DETAILED_HELP_TEXT,
      subcommand_help_text={},
  )

  def RunCommand(self):
    """Command entry point for the mb command."""
    autoclass = False
    bucket_policy_only = None
    kms_key = None
    location = None
    storage_class = None
    seconds = None
    public_access_prevention = None
    rpo = None
    json_only_flags_in_command = []
    placements = None
    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '--autoclass':
          autoclass = True
          json_only_flags_in_command.append(o)
        elif o == '-k':
          kms_key = a
          ValidateCMEK(kms_key)
          json_only_flags_in_command.append(o)
        elif o == '-l':
          location = a
        elif o == '-p':
          # Project IDs are sent as header values when using gs and s3 XML APIs.
          InsistAscii(a, 'Invalid non-ASCII character found in project ID')
          self.project_id = a
        elif o == '-c' or o == '-s':
          storage_class = NormalizeStorageClass(a)
        elif o == '--retention':
          seconds = RetentionInSeconds(a)
        elif o == '--rpo':
          rpo = a.strip()
          if rpo not in VALID_RPO_VALUES:
            raise CommandException(
                'Invalid value for --rpo. Must be one of: {},'
                ' provided: {}'.format(VALID_RPO_VALUES_STRING, a))
          json_only_flags_in_command.append(o)
        elif o == '-b':
          InsistOnOrOff(a, 'Only on and off values allowed for -b option')
          bucket_policy_only = (a == 'on')
          json_only_flags_in_command.append(o)
        elif o == '--pap':
          public_access_prevention = a
          json_only_flags_in_command.append(o)
        elif o == '--placement':
          placements = a.split(',')
          if len(placements) != 2:
            raise CommandException(
                'Please specify two regions separated by comma without space.'
                ' Specified: {}'.format(a))
          json_only_flags_in_command.append(o)

    bucket_metadata = apitools_messages.Bucket(location=location,
                                               rpo=rpo,
                                               storageClass=storage_class)
    if autoclass:
      bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue(
          enabled=autoclass)
    if bucket_policy_only or public_access_prevention:
      bucket_metadata.iamConfiguration = IamConfigurationValue()
      iam_config = bucket_metadata.iamConfiguration
      if bucket_policy_only:
        iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
        iam_config.bucketPolicyOnly.enabled = bucket_policy_only
      if public_access_prevention:
        iam_config.publicAccessPrevention = public_access_prevention

    if kms_key:
      encryption = apitools_messages.Bucket.EncryptionValue()
      encryption.defaultKmsKeyName = kms_key
      bucket_metadata.encryption = encryption

    if placements:
      placement_config = apitools_messages.Bucket.CustomPlacementConfigValue()
      placement_config.dataLocations = placements
      bucket_metadata.customPlacementConfig = placement_config

    for bucket_url_str in self.args:
      bucket_url = StorageUrlFromString(bucket_url_str)
      if seconds is not None:
        if bucket_url.scheme != 'gs':
          raise CommandException('Retention policy can only be specified for '
                                 'GCS buckets.')
        retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
            retentionPeriod=seconds))
        bucket_metadata.retentionPolicy = retention_policy

      if json_only_flags_in_command and self.gsutil_api.GetApiSelector(
          bucket_url.scheme) != ApiSelector.JSON:
        raise CommandException('The {} option(s) can only be used for GCS'
                               ' Buckets with the JSON API'.format(
                                   ', '.join(json_only_flags_in_command)))

      if not bucket_url.IsBucket():
        raise CommandException('The mb command requires a URL that specifies a '
                               'bucket.\n"%s" is not valid.' % bucket_url)
      if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or
          TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
        raise InvalidUrlError('Invalid bucket name in URL "%s"' %
                              bucket_url.bucket_name)

      self.logger.info('Creating %s...', bucket_url)
      # Pass storage_class param only if this is a GCS bucket. (In S3 the
      # storage class is specified on the key object.)
      try:
        self.gsutil_api.CreateBucket(bucket_url.bucket_name,
                                     project_id=self.project_id,
                                     metadata=bucket_metadata,
                                     provider=bucket_url.scheme)
      except AccessDeniedException as e:
        message = e.reason
        if 'key' in message:
          # This will print the error reason and append the following as a
          # suggested next step:
          #
          # To authorize, run:
          #   gsutil kms authorize \
          #     -k <kms_key> \
          #     -p <project_id>
          message += ' To authorize, run:\n  gsutil kms authorize'
          message += ' \\\n    -k %s' % kms_key
          if (self.project_id):
            message += ' \\\n    -p %s' % self.project_id
          raise CommandException(message)
        else:
          raise

      except BadRequestException as e:
        if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and
            bucket_url.scheme == 'gs'):
          bucket_name = bucket_url.bucket_name
          final_comp = bucket_name[bucket_name.rfind('.') + 1:]
          raise CommandException('\n'.join(
              textwrap.wrap(
                  'Buckets with "." in the name must be valid DNS names. The bucket'
                  ' you are attempting to create (%s) is not a valid DNS name,'
                  ' because the final component (%s) is not currently a valid part'
                  ' of the top-level DNS tree.' % (bucket_name, final_comp))))
        else:
          raise

    return 0
<commit_msg>Adding -k option to mb synopsis (#1447)<commit_after># -*- coding: utf-8 -*-
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of mb command for creating cloud storage buckets."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import re
import textwrap

from gslib.cloud_api import AccessDeniedException, BadRequestException
from gslib.command import Command
from gslib.command_argument import CommandArgument
from gslib.commands.rpo import VALID_RPO_VALUES
from gslib.commands.rpo import VALID_RPO_VALUES_STRING
from gslib.cs_api_map import ApiSelector
from gslib.exception import CommandException
from gslib.exception import InvalidUrlError
from gslib.storage_url import StorageUrlFromString
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
from gslib.utils.constants import NO_MAX
from gslib.utils.retention_util import RetentionInSeconds
from gslib.utils.text_util import InsistAscii
from gslib.utils.text_util import InsistOnOrOff
from gslib.utils.text_util import NormalizeStorageClass
from gslib.utils.encryption_helper import ValidateCMEK

_SYNOPSIS = """
  gsutil mb [-b (on|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>]
            [--autoclass] [--retention <time>] [--pap <setting>]
            [--placement <region1>,<region2>]
            [--rpo {}] gs://<bucket_name>...
""".format(VALID_RPO_VALUES_STRING)

_DETAILED_HELP_TEXT = ("""
SYNOPSIS
""" + _SYNOPSIS + """

DESCRIPTION
  Create one or more new buckets. Google Cloud Storage has a single namespace,
  so you are not allowed to create a bucket with a name already in use by
  another user. You can, however, carve out parts of the bucket name space
  corresponding to your company's domain name (see "gsutil help naming").

  If you don't specify a project ID or project number using the -p option, the
  buckets are created using the default project ID specified in your `gsutil
  configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_.

  The -l option specifies the location for the buckets. Once a bucket is created
  in a given location, it cannot be moved to a different location. Instead, you
  need to create a new bucket, move the data over, and then delete the original
  bucket.

BUCKET STORAGE CLASSES
  You can specify one of the `storage classes
  <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket
  with the -c option.

  Example:

    gsutil mb -c nearline gs://some-bucket

  See online documentation for
  `pricing <https://cloud.google.com/storage/pricing>`_ and
  `SLA <https://cloud.google.com/storage/sla>`_ details.

  If you don't specify a -c option, the bucket is created with the
  default storage class Standard Storage.

BUCKET LOCATIONS
  You can specify one of the `available locations
  <https://cloud.google.com/storage/docs/locations>`_ for a bucket
  with the -l option.

  Examples:

    gsutil mb -l asia gs://some-bucket

    gsutil mb -c standard -l us-east1 gs://some-bucket

  If you don't specify a -l option, the bucket is created in the default
  location (US).

Retention Policy
  You can specify retention period in one of the following formats:

  --retention <number>s
      Specifies retention period of <number> seconds for objects in this bucket.

  --retention <number>d
      Specifies retention period of <number> days for objects in this bucket.

  --retention <number>m
      Specifies retention period of <number> months for objects in this bucket.

  --retention <number>y
      Specifies retention period of <number> years for objects in this bucket.

  Examples:

    gsutil mb --retention 1y gs://some-bucket

    gsutil mb --retention 36m gs://some-bucket

  If you don't specify a --retention option, the bucket is created with no
  retention policy.

OPTIONS
  --autoclass            Enables the Autoclass feature that automatically
                         sets object storage classes.

  -b <on|off>            Specifies the uniform bucket-level access setting.
                         When "on", ACLs assigned to objects in the bucket are
                         not evaluated. Consequently, only IAM policies grant
                         access to objects in these buckets. Default is "off".

  -c class               Specifies the default storage class.
                         Default is "Standard".

  -k <key>               Set the default KMS key using the full path to the key,
                         which has the following form:
                         ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]``

  -l location            Can be any supported location. See
                         https://cloud.google.com/storage/docs/locations
                         for a discussion of this distinction. Default is US.
                         Locations are case insensitive.

  -p project             Specifies the project ID or project number to create
                         the bucket under.

  -s class               Same as -c.

  --retention time       Specifies the retention policy. Default is no retention
                         policy. This can only be set on gs:// buckets and
                         requires using the JSON API. For more details about
                         retention policy see "gsutil help retention"

  --pap setting          Specifies the public access prevention setting. Valid
                         values are "enforced" or "unspecified". When
                         "enforced", objects in this bucket cannot be made
                         publicly accessible. Default is "unspecified".

  --placement reg1,reg2  Two regions that form the cutom dual-region.
                         Only regions within the same continent are or will ever
                         be valid. Invalid location pairs (such as
                         mixed-continent, or with unsupported regions)
                         will return an error.

  --rpo setting          Specifies the `replication setting <https://cloud.google.com/storage/docs/turbo-replication>`_.
                         This flag is not valid for single-region buckets,
                         and multi-region buckets only accept a value of
                         DEFAULT. Valid values for dual region buckets
                         are {rpo_values}. If unspecified, DEFAULT is applied
                         for dual-region and multi-region buckets.

""".format(rpo_values=VALID_RPO_VALUES_STRING))

# Regex to disallow buckets violating charset or not [3..255] chars total.
BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$')
# Regex to disallow buckets with individual DNS labels longer than 63.
TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}')

IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue
BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue

class MbCommand(Command):
  """Implementation of gsutil mb command."""

  # Command specification. See base class for documentation.
  command_spec = Command.CreateCommandSpec(
      'mb',
      command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
      usage_synopsis=_SYNOPSIS,
      min_args=1,
      max_args=NO_MAX,
      supported_sub_args='b:c:l:p:s:k:',
      supported_private_args=[
          'autoclass', 'retention=', 'pap=', 'placement=', 'rpo='
      ],
      file_url_ok=False,
      provider_url_ok=False,
      urls_start_arg=0,
      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
      gs_default_api=ApiSelector.JSON,
      argparse_arguments=[
          CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
      ],
  )
  # Help specification. See help_provider.py for documentation.
  help_spec = Command.HelpSpec(
      help_name='mb',
      help_name_aliases=[
          'createbucket',
          'makebucket',
          'md',
          'mkdir',
          'location',
          'dra',
          'dras',
          'reduced_availability',
          'durable_reduced_availability',
          'rr',
          'reduced_redundancy',
          'standard',
          'storage class',
          'nearline',
          'nl',
      ],
      help_type='command_help',
      help_one_line_summary='Make buckets',
      help_text=_DETAILED_HELP_TEXT,
      subcommand_help_text={},
  )

  def RunCommand(self):
    """Command entry point for the mb command."""
    autoclass = False
    bucket_policy_only = None
    kms_key = None
    location = None
    storage_class = None
    seconds = None
    public_access_prevention = None
    rpo = None
    json_only_flags_in_command = []
    placements = None
    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '--autoclass':
          autoclass = True
          json_only_flags_in_command.append(o)
        elif o == '-k':
          kms_key = a
          ValidateCMEK(kms_key)
          json_only_flags_in_command.append(o)
        elif o == '-l':
          location = a
        elif o == '-p':
          # Project IDs are sent as header values when using gs and s3 XML APIs.
          InsistAscii(a, 'Invalid non-ASCII character found in project ID')
          self.project_id = a
        elif o == '-c' or o == '-s':
          storage_class = NormalizeStorageClass(a)
        elif o == '--retention':
          seconds = RetentionInSeconds(a)
        elif o == '--rpo':
          rpo = a.strip()
          if rpo not in VALID_RPO_VALUES:
            raise CommandException(
                'Invalid value for --rpo. Must be one of: {},'
                ' provided: {}'.format(VALID_RPO_VALUES_STRING, a))
          json_only_flags_in_command.append(o)
        elif o == '-b':
          InsistOnOrOff(a, 'Only on and off values allowed for -b option')
          bucket_policy_only = (a == 'on')
          json_only_flags_in_command.append(o)
        elif o == '--pap':
          public_access_prevention = a
          json_only_flags_in_command.append(o)
        elif o == '--placement':
          placements = a.split(',')
          if len(placements) != 2:
            raise CommandException(
                'Please specify two regions separated by comma without space.'
                ' Specified: {}'.format(a))
          json_only_flags_in_command.append(o)

    bucket_metadata = apitools_messages.Bucket(location=location,
                                               rpo=rpo,
                                               storageClass=storage_class)
    if autoclass:
      bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue(
          enabled=autoclass)
    if bucket_policy_only or public_access_prevention:
      bucket_metadata.iamConfiguration = IamConfigurationValue()
      iam_config = bucket_metadata.iamConfiguration
      if bucket_policy_only:
        iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
        iam_config.bucketPolicyOnly.enabled = bucket_policy_only
      if public_access_prevention:
        iam_config.publicAccessPrevention = public_access_prevention

    if kms_key:
      encryption = apitools_messages.Bucket.EncryptionValue()
      encryption.defaultKmsKeyName = kms_key
      bucket_metadata.encryption = encryption

    if placements:
      placement_config = apitools_messages.Bucket.CustomPlacementConfigValue()
      placement_config.dataLocations = placements
      bucket_metadata.customPlacementConfig = placement_config

    for bucket_url_str in self.args:
      bucket_url = StorageUrlFromString(bucket_url_str)
      if seconds is not None:
        if bucket_url.scheme != 'gs':
          raise CommandException('Retention policy can only be specified for '
                                 'GCS buckets.')
        retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
            retentionPeriod=seconds))
        bucket_metadata.retentionPolicy = retention_policy

      if json_only_flags_in_command and self.gsutil_api.GetApiSelector(
          bucket_url.scheme) != ApiSelector.JSON:
        raise CommandException('The {} option(s) can only be used for GCS'
                               ' Buckets with the JSON API'.format(
                                   ', '.join(json_only_flags_in_command)))

      if not bucket_url.IsBucket():
        raise CommandException('The mb command requires a URL that specifies a '
                               'bucket.\n"%s" is not valid.' % bucket_url)
      if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or
          TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
        raise InvalidUrlError('Invalid bucket name in URL "%s"' %
                              bucket_url.bucket_name)

      self.logger.info('Creating %s...', bucket_url)
      # Pass storage_class param only if this is a GCS bucket. (In S3 the
      # storage class is specified on the key object.)
      try:
        self.gsutil_api.CreateBucket(bucket_url.bucket_name,
                                     project_id=self.project_id,
                                     metadata=bucket_metadata,
                                     provider=bucket_url.scheme)
      except AccessDeniedException as e:
        message = e.reason
        if 'key' in message:
          # This will print the error reason and append the following as a
          # suggested next step:
          #
          # To authorize, run:
          #   gsutil kms authorize \
          #     -k <kms_key> \
          #     -p <project_id>
          message += ' To authorize, run:\n  gsutil kms authorize'
          message += ' \\\n    -k %s' % kms_key
          if (self.project_id):
            message += ' \\\n    -p %s' % self.project_id
          raise CommandException(message)
        else:
          raise

      except BadRequestException as e:
        if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and
            bucket_url.scheme == 'gs'):
          bucket_name = bucket_url.bucket_name
          final_comp = bucket_name[bucket_name.rfind('.') + 1:]
          raise CommandException('\n'.join(
              textwrap.wrap(
                  'Buckets with "." in the name must be valid DNS names. The bucket'
                  ' you are attempting to create (%s) is not a valid DNS name,'
                  ' because the final component (%s) is not currently a valid part'
                  ' of the top-level DNS tree.' % (bucket_name, final_comp))))
        else:
          raise

    return 0
<|endoftext|>


<commit_before>from .data_classes import BaseGLTFDataClass
from .data_classes import TextureInfoData
from .data_classes import NormalTextureInfoData

class KHR_materials_transmission(BaseGLTFDataClass):
    """glTF extension that defines the optical transmission of a material.

    https://github.com/KhronosGroup/glTF/blob/master/extensions/2.0/Khronos/KHR_materials_transmission
    """

    key = "KHR_materials_transmission"

    def __init__(
        self,
        transmission_factor=None,
        transmission_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_transmission, self).__init__(extras, extensions)
        self.transmission_factor = transmission_factor
        self.transmission_texture = transmission_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.transmission_factor is not None:
            dct["transmissionFactor"] = self.transmission_factor
        if self.transmission_texture is not None:
            dct["transmissionTexture"] = self.transmission_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            transmission_factor=dct.get("transmissionFactor"),
            transmission_texture=TextureInfoData.from_data(
                dct.get("transmissionTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_specular(BaseGLTFDataClass):
    """glTF extension that defines the optical transmission of a material.

    https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_specular
    """

    key = "KHR_materials_specular"

    def __init__(
        self,
        specular_factor=None,
        specular_texture=None,
        specular_color_factor=None,
        specular_color_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_specular, self).__init__(extras, extensions)
        self.specular_factor = specular_factor
        self.specular_texture = specular_texture
        self.specular_color_factor = specular_color_factor
        self.specular_color_texture = specular_color_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.specular_factor is not None:
            dct["specularFactor"] = self.specular_factor
        if self.specular_texture is not None:
            dct["specularTexture"] = self.specular_texture.to_data(texture_index_by_key)
        if self.specular_color_factor is not None:
            dct["specularColorFactor"] = self.specular_color_factor
        if self.specular_color_texture is not None:
            dct["specularColorTexture"] = self.specular_color_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            specular_factor=dct.get("specularFactor"),
            specular_texture=TextureInfoData.from_data(dct.get("specularTexture")),
            specular_color_factor=dct.get("specularColorFactor"),
            specular_color_texture=TextureInfoData.from_data(
                dct.get("specularColorTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_clearcoat(BaseGLTFDataClass):
    """glTF extension that defines the clearcoat material layer.

    https://github.com/KhronosGroup/glTF/blob/master/extensions/2.0/Khronos/KHR_materials_clearcoat
    """

    key = "KHR_materials_clearcoat"

    def __init__(
        self,
        clearcoat_factor=None,
        clearcoat_texture=None,
        clearcoat_roughness_factor=None,
        clearcoat_roughness_texture=None,
        clearcoat_normal_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_clearcoat, self).__init__(extras, extensions)
        self.clearcoat_factor = clearcoat_factor
        self.clearcoat_texture = clearcoat_texture
        self.clearcoat_roughness_factor = clearcoat_roughness_factor
        self.clearcoat_roughness_texture = clearcoat_roughness_texture
        self.clearcoat_normal_texture = clearcoat_normal_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.clearcoat_factor is not None:
            dct["clearcoatFactor"] = self.clearcoat_factor
        if self.clearcoat_texture is not None:
            dct["clearcoatTexture"] = self.clearcoat_texture.to_data(
                texture_index_by_key
            )
        if self.clearcoat_roughness_factor is not None:
            dct["clearcoatRoughnessFactor"] = self.clearcoat_roughness_factor
        if self.clearcoat_roughness_texture is not None:
            dct["clearcoatRoughnessTexture"] = self.clearcoat_roughness_texture.to_data(
                texture_index_by_key
            )
        if self.clearcoat_normal_texture is not None:
            dct["clearcoatNormalTexture"] = self.clearcoat_normal_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            clearcoat_factor=dct.get("clearcoatFactor"),
            clearcoat_texture=TextureInfoData.from_data(dct.get("clearcoatTexture")),
            clearcoat_roughness_factor=dct.get("clearcoatRoughnessFactor"),
            clearcoat_roughness_texture=TextureInfoData.from_data(
                dct.get("clearcoatRoughnessTexture")
            ),
            clearcoat_normal_texture=NormalTextureInfoData.from_data(
                dct.get("clearcoatNormalTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_Texture_Transform(BaseGLTFDataClass):
    """glTF extension that enables shifting and scaling UV coordinates on a per-texture basis.

    https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_texture_transform
    """

    key = "KHR_texture_transform"

    def __init__(
        self,
        offset=None,
        rotation=None,
        scale=None,
        tex_coord=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_Texture_Transform, self).__init__(extras, extensions)
        self.offset = offset  # or [0.0, 0.0]
        self.rotation = rotation  # or 0.
        self.scale = scale  # or [1., 1.]
        self.tex_coord = tex_coord

    def to_data(self, **kwargs):
        dct = {}
        if self.offset is not None:
            dct["offset"] = self.offset
        if self.rotation is not None:
            dct["rotation"] = self.rotation
        if self.scale is not None:
            dct["scale"] = self.scale
        if self.tex_coord is not None:
            dct["texCoord"] = self.tex_coord
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            offset=dct.get("offset"),
            rotation=dct.get("rotation"),
            scale=dct.get("scale"),
            tex_coord=dct.get("texCoord"),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_pbrSpecularGlossiness(BaseGLTFDataClass):
    """glTF extension that defines the specular-glossiness material model from Physically-Based Rendering (PBR) methodology."""

    key = "KHR_materials_pbrSpecularGlossiness"

    def __init__(
        self,
        diffuse_factor=None,
        diffuse_texture=None,
        specular_factor=None,
        glossiness_factor=None,
        specular_glossiness_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_pbrSpecularGlossiness, self).__init__(extras, extensions)
        self.diffuse_factor = diffuse_factor or [1.0, 1.0, 1.0, 1.0]
        self.diffuse_texture = diffuse_texture
        self.specular_factor = specular_factor or [1.0, 1.0, 1.0]
        self.glossiness_factor = glossiness_factor or 1.0
        self.specular_glossiness_texture = specular_glossiness_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.diffuse_factor is not None:
            dct["diffuseFactor"] = self.diffuse_factor
        if self.diffuse_texture is not None:
            dct["diffuseTexture"] = self.diffuse_texture.to_data(texture_index_by_key)
        if self.specular_factor is not None:
            dct["specularFactor"] = self.specular_factor
        if self.glossiness_factor is not None:
            dct["glossinessFactor"] = self.glossiness_factor
        if self.specular_glossiness_texture is not None:
            dct["specularGlossinessTexture"] = self.specular_glossiness_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            diffuse_factor=dct.get("diffuseFactor"),
            diffuse_texture=TextureInfoData.from_data(dct.get("diffuseTexture")),
            specular_factor=dct.get("specularFactor"),
            glossiness_factor=dct.get("glossinessFactor"),
            specular_glossiness_texture=TextureInfoData.from_data(
                dct.get("specularGlossinessTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

SUPPORTED_EXTENSIONS = {
    KHR_materials_transmission.key: KHR_materials_transmission,
    KHR_materials_clearcoat.key: KHR_materials_clearcoat,
    KHR_Texture_Transform.key: KHR_Texture_Transform,
    KHR_materials_pbrSpecularGlossiness.key: KHR_materials_pbrSpecularGlossiness,
    KHR_materials_specular.key: KHR_materials_specular,
}
<commit_msg>add extension KHR_materials_ior<commit_after>from .data_classes import BaseGLTFDataClass
from .data_classes import TextureInfoData
from .data_classes import NormalTextureInfoData

class KHR_materials_transmission(BaseGLTFDataClass):
    """glTF extension that defines the optical transmission of a material.

    https://github.com/KhronosGroup/glTF/blob/master/extensions/2.0/Khronos/KHR_materials_transmission
    """

    key = "KHR_materials_transmission"

    def __init__(
        self,
        transmission_factor=None,
        transmission_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_transmission, self).__init__(extras, extensions)
        self.transmission_factor = transmission_factor
        self.transmission_texture = transmission_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.transmission_factor is not None:
            dct["transmissionFactor"] = self.transmission_factor
        if self.transmission_texture is not None:
            dct["transmissionTexture"] = self.transmission_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            transmission_factor=dct.get("transmissionFactor"),
            transmission_texture=TextureInfoData.from_data(
                dct.get("transmissionTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_specular(BaseGLTFDataClass):
    """glTF extension that defines the optical transmission of a material.

    https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_specular
    """

    key = "KHR_materials_specular"

    def __init__(
        self,
        specular_factor=None,
        specular_texture=None,
        specular_color_factor=None,
        specular_color_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_specular, self).__init__(extras, extensions)
        self.specular_factor = specular_factor
        self.specular_texture = specular_texture
        self.specular_color_factor = specular_color_factor
        self.specular_color_texture = specular_color_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.specular_factor is not None:
            dct["specularFactor"] = self.specular_factor
        if self.specular_texture is not None:
            dct["specularTexture"] = self.specular_texture.to_data(texture_index_by_key)
        if self.specular_color_factor is not None:
            dct["specularColorFactor"] = self.specular_color_factor
        if self.specular_color_texture is not None:
            dct["specularColorTexture"] = self.specular_color_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            specular_factor=dct.get("specularFactor"),
            specular_texture=TextureInfoData.from_data(dct.get("specularTexture")),
            specular_color_factor=dct.get("specularColorFactor"),
            specular_color_texture=TextureInfoData.from_data(
                dct.get("specularColorTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_ior(BaseGLTFDataClass):
    """glTF extension that defines the optical transmission of a material.

    https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_ior
    """

    key = "KHR_materials_ior"

    def __init__(
        self,
        ior=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_ior, self).__init__(extras, extensions)
        self.ior = ior

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.ior is not None:
            dct["ior"] = self.ior
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            ior=dct.get("ior"),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_clearcoat(BaseGLTFDataClass):
    """glTF extension that defines the clearcoat material layer.

    https://github.com/KhronosGroup/glTF/blob/master/extensions/2.0/Khronos/KHR_materials_clearcoat
    """

    key = "KHR_materials_clearcoat"

    def __init__(
        self,
        clearcoat_factor=None,
        clearcoat_texture=None,
        clearcoat_roughness_factor=None,
        clearcoat_roughness_texture=None,
        clearcoat_normal_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_clearcoat, self).__init__(extras, extensions)
        self.clearcoat_factor = clearcoat_factor
        self.clearcoat_texture = clearcoat_texture
        self.clearcoat_roughness_factor = clearcoat_roughness_factor
        self.clearcoat_roughness_texture = clearcoat_roughness_texture
        self.clearcoat_normal_texture = clearcoat_normal_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.clearcoat_factor is not None:
            dct["clearcoatFactor"] = self.clearcoat_factor
        if self.clearcoat_texture is not None:
            dct["clearcoatTexture"] = self.clearcoat_texture.to_data(
                texture_index_by_key
            )
        if self.clearcoat_roughness_factor is not None:
            dct["clearcoatRoughnessFactor"] = self.clearcoat_roughness_factor
        if self.clearcoat_roughness_texture is not None:
            dct["clearcoatRoughnessTexture"] = self.clearcoat_roughness_texture.to_data(
                texture_index_by_key
            )
        if self.clearcoat_normal_texture is not None:
            dct["clearcoatNormalTexture"] = self.clearcoat_normal_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            clearcoat_factor=dct.get("clearcoatFactor"),
            clearcoat_texture=TextureInfoData.from_data(dct.get("clearcoatTexture")),
            clearcoat_roughness_factor=dct.get("clearcoatRoughnessFactor"),
            clearcoat_roughness_texture=TextureInfoData.from_data(
                dct.get("clearcoatRoughnessTexture")
            ),
            clearcoat_normal_texture=NormalTextureInfoData.from_data(
                dct.get("clearcoatNormalTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_Texture_Transform(BaseGLTFDataClass):
    """glTF extension that enables shifting and scaling UV coordinates on a per-texture basis.

    https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_texture_transform
    """

    key = "KHR_texture_transform"

    def __init__(
        self,
        offset=None,
        rotation=None,
        scale=None,
        tex_coord=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_Texture_Transform, self).__init__(extras, extensions)
        self.offset = offset  # or [0.0, 0.0]
        self.rotation = rotation  # or 0.
        self.scale = scale  # or [1., 1.]
        self.tex_coord = tex_coord

    def to_data(self, **kwargs):
        dct = {}
        if self.offset is not None:
            dct["offset"] = self.offset
        if self.rotation is not None:
            dct["rotation"] = self.rotation
        if self.scale is not None:
            dct["scale"] = self.scale
        if self.tex_coord is not None:
            dct["texCoord"] = self.tex_coord
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            offset=dct.get("offset"),
            rotation=dct.get("rotation"),
            scale=dct.get("scale"),
            tex_coord=dct.get("texCoord"),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

class KHR_materials_pbrSpecularGlossiness(BaseGLTFDataClass):
    """glTF extension that defines the specular-glossiness material model from Physically-Based Rendering (PBR) methodology."""

    key = "KHR_materials_pbrSpecularGlossiness"

    def __init__(
        self,
        diffuse_factor=None,
        diffuse_texture=None,
        specular_factor=None,
        glossiness_factor=None,
        specular_glossiness_texture=None,
        extensions=None,
        extras=None,
    ):
        super(KHR_materials_pbrSpecularGlossiness, self).__init__(extras, extensions)
        self.diffuse_factor = diffuse_factor or [1.0, 1.0, 1.0, 1.0]
        self.diffuse_texture = diffuse_texture
        self.specular_factor = specular_factor or [1.0, 1.0, 1.0]
        self.glossiness_factor = glossiness_factor or 1.0
        self.specular_glossiness_texture = specular_glossiness_texture

    def to_data(self, texture_index_by_key, **kwargs):
        dct = {}
        if self.diffuse_factor is not None:
            dct["diffuseFactor"] = self.diffuse_factor
        if self.diffuse_texture is not None:
            dct["diffuseTexture"] = self.diffuse_texture.to_data(texture_index_by_key)
        if self.specular_factor is not None:
            dct["specularFactor"] = self.specular_factor
        if self.glossiness_factor is not None:
            dct["glossinessFactor"] = self.glossiness_factor
        if self.specular_glossiness_texture is not None:
            dct["specularGlossinessTexture"] = self.specular_glossiness_texture.to_data(
                texture_index_by_key
            )
        if self.extras is not None:
            dct["extras"] = self.extras
        if self.extensions is not None:
            dct["extensions"] = self.extensions_to_data()
        return dct

    @classmethod
    def from_data(cls, dct):
        if dct is None:
            return None
        return cls(
            diffuse_factor=dct.get("diffuseFactor"),
            diffuse_texture=TextureInfoData.from_data(dct.get("diffuseTexture")),
            specular_factor=dct.get("specularFactor"),
            glossiness_factor=dct.get("glossinessFactor"),
            specular_glossiness_texture=TextureInfoData.from_data(
                dct.get("specularGlossinessTexture")
            ),
            extensions=cls.extensions_from_data(dct.get("extensions")),
            extras=dct.get("extras"),
        )

SUPPORTED_EXTENSIONS = {
    KHR_materials_transmission.key: KHR_materials_transmission,
    KHR_materials_clearcoat.key: KHR_materials_clearcoat,
    KHR_Texture_Transform.key: KHR_Texture_Transform,
    KHR_materials_pbrSpecularGlossiness.key: KHR_materials_pbrSpecularGlossiness,
    KHR_materials_specular.key: KHR_materials_specular,
    KHR_materials_ior.key: KHR_materials_ior,
}
<|endoftext|>