<commit_before>"""
Neuroimaging tools for Python (NiPy).

The aim of NiPy is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.01a'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__    = "$LastChangedDate$"
__url__     = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

from neuroimaging import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms',
  'neuroimaging.algorithms.tests',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.modalities.fmri',
  'neuroimaging.modalities.fmri.tests',
  'neuroimaging.modalities.fmri.fmristat',
  'neuroimaging.modalities.fmri.fmristat.tests',
  'neuroimaging.utils',
  'neuroimaging.utils.tests'
  'neuroimaging.utils.tests.data')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr):
            return False
    return True

def haslength(obj):
    return hasattr(obj,"__len__")

def flatten(arr, dim=0):
    if len(arr.shape) < 2:
        return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

def reorder(seq, order):
    return [seq[i] for i in order]

def reverse(seq):
    return reorder(seq, range(len(seq)-1, -1, -1))

def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try:
        return getattr(module, objectname)
    except AttributeError:
        return None

<commit_msg>Fix missing comma<commit_after>"""
Neuroimaging tools for Python (NiPy).

The aim of NiPy is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.01a'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__    = "$LastChangedDate$"
__url__     = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

from neuroimaging import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms',
  'neuroimaging.algorithms.tests',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.modalities.fmri',
  'neuroimaging.modalities.fmri.tests',
  'neuroimaging.modalities.fmri.fmristat',
  'neuroimaging.modalities.fmri.fmristat.tests',
  'neuroimaging.utils',
  'neuroimaging.utils.tests',
  'neuroimaging.utils.tests.data')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr):
            return False
    return True

def haslength(obj):
    return hasattr(obj,"__len__")

def flatten(arr, dim=0):
    if len(arr.shape) < 2:
        return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

def reorder(seq, order):
    return [seq[i] for i in order]

def reverse(seq):
    return reorder(seq, range(len(seq)-1, -1, -1))

def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try:
        return getattr(module, objectname)
    except AttributeError:
        return None

<|endoftext|>


<commit_before>import json
import urllib2
import urllib

class API:

  def __init__(self, app_id, app_key):
    self.app_id = app_id
    self.app_key = app_key
    self.base_url = "https://beta.openphacts.org/1.3/"

  def getCompoundInfo(self, uri):
    uri_encoded = urllib.quote_plus(uri)
    #print uri_encoded
    url_str = self.base_url+ "compound?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_format=json"
    #print url_str
    results = json.load(urllib2.urlopen(url_str))
    return results

  def getTargetInfo(self, uri):
    uri_encoded = urllib.quote_plus(uri)
    url_str = self.base_url+ "target?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_format=json"
    #print url_str
    results = json.load(urllib2.urlopen(url_str))
    return results

  def getCompoundPharmacology(self, uri):
     uri_encoded = urllib.quote_plus(uri)
     page_num = "1"
     url_str = self.base_url+ "compound/pharmacology/pages?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_page="+ page_num +"&_format=json"
     for x in self.getCompoundPharmacologyIter(url_str):
       yield x

  def getCompoundPharmacologyIter(self, url_str):
     results = json.load(urllib2.urlopen(url_str))
     items = results['result']['items']

     for item in items:
          yield item

     if "next" in results['result']:
        url_str_next = results['result']['next']
        for x in self.getCompoundPharmacologyIter(url_str_next):
          yield x
<commit_msg>added functions for target pharmacology<commit_after>import json
import urllib2
import urllib
from compound import *
from target import *

class API:

  def __init__(self, app_id, app_key):
    self.app_id = app_id
    self.app_key = app_key
    self.base_url = "https://beta.openphacts.org/1.4/"

  def getCompoundInfo(self, uri):
    uri_encoded = urllib.quote_plus(uri)
    #print uri_encoded
    url_str = self.base_url+ "compound?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_format=json"
    #print url_str
    results = json.load(urllib2.urlopen(url_str))
    return results

  def getTargetInfo(self, uri):
    uri_encoded = urllib.quote_plus(uri)
    url_str = self.base_url+ "target?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_format=json"
    #print url_str
    results = json.load(urllib2.urlopen(url_str))
    return results

  def getCompoundPharmacology(self, uri):
     uri_encoded = urllib.quote_plus(uri)
     page_num = "1"
     url_str = self.base_url+ "compound/pharmacology/pages?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_page="+ page_num +"&_format=json"
     for x in self.getCompoundPharmacologyIter(url_str):
       yield x

  def getCompoundPharmacologyIter(self, url_str):
     results = json.load(urllib2.urlopen(url_str))
     items = results['result']['items']

     for item in items:
          yield item

     if "next" in results['result']:
        url_str_next = results['result']['next']
        for x in self.getCompoundPharmacologyIter(url_str_next):
          yield x

  def getTargetPharmacology(self, uri):
     uri_encoded = urllib.quote_plus(uri)
     page_num = "1"
     url_str = self.base_url+ "target/pharmacology/pages?app_id=" + self.app_id + "&app_key=" + self.app_key + "&uri="+ uri_encoded + "&_page="+ page_num +"&_format=json"
     for x in self.getCompoundPharmacologyIter(url_str):
       yield x

  def getTargetPharmacologyIter(self, url_str):
     results = json.load(urllib2.urlopen(url_str))
     items = results['result']['items']

     for item in items:
          yield item

     if "next" in results['result']:
        url_str_next = results['result']['next']
        for x in self.getTargetPharmacology(url_str_next):
          yield x
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""
Example Airflow DAG for Google Cloud Dataflow service
"""
from __future__ import annotations

import os
from datetime import datetime
from typing import Callable
from urllib.parse import urlparse

from airflow import models
from airflow.exceptions import AirflowException
from airflow.providers.apache.beam.operators.beam import (
    BeamRunJavaPipelineOperator,
    BeamRunPythonPipelineOperator,
)
from airflow.providers.google.cloud.hooks.dataflow import DataflowJobStatus
from airflow.providers.google.cloud.operators.dataflow import (
    CheckJobRunning,
    DataflowTemplatedJobStartOperator,
)
from airflow.providers.google.cloud.sensors.dataflow import (
    DataflowJobAutoScalingEventsSensor,
    DataflowJobMessagesSensor,
    DataflowJobMetricsSensor,
    DataflowJobStatusSensor,
)
from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator

START_DATE = datetime(2021, 1, 1)

GCS_TMP = os.environ.get("GCP_DATAFLOW_GCS_TMP", "gs://INVALID BUCKET NAME/temp/")
GCS_STAGING = os.environ.get("GCP_DATAFLOW_GCS_STAGING", "gs://INVALID BUCKET NAME/staging/")
GCS_OUTPUT = os.environ.get("GCP_DATAFLOW_GCS_OUTPUT", "gs://INVALID BUCKET NAME/output")
GCS_JAR = os.environ.get("GCP_DATAFLOW_JAR", "gs://INVALID BUCKET NAME/word-count-beam-bundled-0.1.jar")
GCS_PYTHON = os.environ.get("GCP_DATAFLOW_PYTHON", "gs://INVALID BUCKET NAME/wordcount_debugging.py")

GCS_JAR_PARTS = urlparse(GCS_JAR)
GCS_JAR_BUCKET_NAME = GCS_JAR_PARTS.netloc
GCS_JAR_OBJECT_NAME = GCS_JAR_PARTS.path[1:]

default_args = {
    "dataflow_default_options": {
        "tempLocation": GCS_TMP,
        "stagingLocation": GCS_STAGING,
    }
}

with models.DAG(
    "example_gcp_dataflow_native_java",
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_native_java:

    # [START howto_operator_start_java_job_jar_on_gcs]
    start_java_job = BeamRunJavaPipelineOperator(
        task_id="start-java-job",
        jar=GCS_JAR,
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        job_class="org.apache.beam.examples.WordCount",
        dataflow_config={
            "check_if_running": CheckJobRunning.IgnoreJob,
            "location": "europe-west3",
            "poll_sleep": 10,
        },
    )
    # [END howto_operator_start_java_job_jar_on_gcs]

    # [START howto_operator_start_java_job_local_jar]
    jar_to_local = GCSToLocalFilesystemOperator(
        task_id="jar-to-local",
        bucket=GCS_JAR_BUCKET_NAME,
        object_name=GCS_JAR_OBJECT_NAME,
        filename="/tmp/dataflow-{{ ds_nodash }}.jar",
    )

    start_java_job_local = BeamRunJavaPipelineOperator(
        task_id="start-java-job-local",
        jar="/tmp/dataflow-{{ ds_nodash }}.jar",
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        job_class="org.apache.beam.examples.WordCount",
        dataflow_config={
            "check_if_running": CheckJobRunning.WaitForRun,
            "location": "europe-west3",
            "poll_sleep": 10,
        },
    )
    jar_to_local >> start_java_job_local
    # [END howto_operator_start_java_job_local_jar]

with models.DAG(
    "example_gcp_dataflow_native_python",
    default_args=default_args,
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_native_python:

    # [START howto_operator_start_python_job]
    start_python_job = BeamRunPythonPipelineOperator(
        task_id="start-python-job",
        py_file=GCS_PYTHON,
        py_options=[],
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        py_requirements=["apache-beam[gcp]==2.21.0"],
        py_interpreter="python3",
        py_system_site_packages=False,
        dataflow_config={"location": "europe-west3"},
    )
    # [END howto_operator_start_python_job]

    start_python_job_local = BeamRunPythonPipelineOperator(
        task_id="start-python-job-local",
        py_file="apache_beam.examples.wordcount",
        py_options=["-m"],
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        py_requirements=["apache-beam[gcp]==2.14.0"],
        py_interpreter="python3",
        py_system_site_packages=False,
    )

with models.DAG(
    "example_gcp_dataflow_native_python_async",
    default_args=default_args,
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_native_python_async:
    # [START howto_operator_start_python_job_async]
    start_python_job_async = BeamRunPythonPipelineOperator(
        task_id="start-python-job-async",
        runner="DataflowRunner",
        py_file=GCS_PYTHON,
        py_options=[],
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        py_requirements=["apache-beam[gcp]==2.25.0"],
        py_interpreter="python3",
        py_system_site_packages=False,
        dataflow_config={
            "job_name": "start-python-job-async",
            "location": "europe-west3",
            "wait_until_finished": False,
        },
    )
    # [END howto_operator_start_python_job_async]

    # [START howto_sensor_wait_for_job_status]
    wait_for_python_job_async_done = DataflowJobStatusSensor(
        task_id="wait-for-python-job-async-done",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['dataflow_job_id']}}",
        expected_statuses={DataflowJobStatus.JOB_STATE_DONE},
        location="europe-west3",
    )
    # [END howto_sensor_wait_for_job_status]

    # [START howto_sensor_wait_for_job_metric]
    def check_metric_scalar_gte(metric_name: str, value: int) -> Callable:
        """Check is metric greater than equals to given value."""

        def callback(metrics: list[dict]) -> bool:
            dag_native_python_async.log.info("Looking for '%s' >= %d", metric_name, value)
            for metric in metrics:
                context = metric.get("name", {}).get("context", {})
                original_name = context.get("original_name", "")
                tentative = context.get("tentative", "")
                if original_name == "Service-cpu_num_seconds" and not tentative:
                    return metric["scalar"] >= value
            raise AirflowException(f"Metric '{metric_name}' not found in metrics")

        return callback

    wait_for_python_job_async_metric = DataflowJobMetricsSensor(
        task_id="wait-for-python-job-async-metric",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['dataflow_job_id']}}",
        location="europe-west3",
        callback=check_metric_scalar_gte(metric_name="Service-cpu_num_seconds", value=100),
        fail_on_terminal_state=False,
    )
    # [END howto_sensor_wait_for_job_metric]

    # [START howto_sensor_wait_for_job_message]
    def check_message(messages: list[dict]) -> bool:
        """Check message"""
        for message in messages:
            if "Adding workflow start and stop steps." in message.get("messageText", ""):
                return True
        return False

    wait_for_python_job_async_message = DataflowJobMessagesSensor(
        task_id="wait-for-python-job-async-message",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['dataflow_job_id']}}",
        location="europe-west3",
        callback=check_message,
        fail_on_terminal_state=False,
    )
    # [END howto_sensor_wait_for_job_message]

    # [START howto_sensor_wait_for_job_autoscaling_event]
    def check_autoscaling_event(autoscaling_events: list[dict]) -> bool:
        """Check autoscaling event"""
        for autoscaling_event in autoscaling_events:
            if "Worker pool started." in autoscaling_event.get("description", {}).get("messageText", ""):
                return True
        return False

    wait_for_python_job_async_autoscaling_event = DataflowJobAutoScalingEventsSensor(
        task_id="wait-for-python-job-async-autoscaling-event",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['dataflow_job_id']}}",
        location="europe-west3",
        callback=check_autoscaling_event,
        fail_on_terminal_state=False,
    )
    # [END howto_sensor_wait_for_job_autoscaling_event]

    start_python_job_async >> wait_for_python_job_async_done
    start_python_job_async >> wait_for_python_job_async_metric
    start_python_job_async >> wait_for_python_job_async_message
    start_python_job_async >> wait_for_python_job_async_autoscaling_event

with models.DAG(
    "example_gcp_dataflow_template",
    default_args=default_args,
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_template:
    # [START howto_operator_start_template_job]
    start_template_job = DataflowTemplatedJobStartOperator(
        task_id="start-template-job",
        template="gs://dataflow-templates/latest/Word_Count",
        parameters={"inputFile": "gs://dataflow-samples/shakespeare/kinglear.txt", "output": GCS_OUTPUT},
        location="europe-west3",
    )
    # [END howto_operator_start_template_job]
<commit_msg>use the proper key to retrieve the dataflow job_id (#27336)<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""
Example Airflow DAG for Google Cloud Dataflow service
"""
from __future__ import annotations

import os
from datetime import datetime
from typing import Callable
from urllib.parse import urlparse

from airflow import models
from airflow.exceptions import AirflowException
from airflow.providers.apache.beam.operators.beam import (
    BeamRunJavaPipelineOperator,
    BeamRunPythonPipelineOperator,
)
from airflow.providers.google.cloud.hooks.dataflow import DataflowJobStatus
from airflow.providers.google.cloud.operators.dataflow import (
    CheckJobRunning,
    DataflowTemplatedJobStartOperator,
)
from airflow.providers.google.cloud.sensors.dataflow import (
    DataflowJobAutoScalingEventsSensor,
    DataflowJobMessagesSensor,
    DataflowJobMetricsSensor,
    DataflowJobStatusSensor,
)
from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator

START_DATE = datetime(2021, 1, 1)

GCS_TMP = os.environ.get("GCP_DATAFLOW_GCS_TMP", "gs://INVALID BUCKET NAME/temp/")
GCS_STAGING = os.environ.get("GCP_DATAFLOW_GCS_STAGING", "gs://INVALID BUCKET NAME/staging/")
GCS_OUTPUT = os.environ.get("GCP_DATAFLOW_GCS_OUTPUT", "gs://INVALID BUCKET NAME/output")
GCS_JAR = os.environ.get("GCP_DATAFLOW_JAR", "gs://INVALID BUCKET NAME/word-count-beam-bundled-0.1.jar")
GCS_PYTHON = os.environ.get("GCP_DATAFLOW_PYTHON", "gs://INVALID BUCKET NAME/wordcount_debugging.py")

GCS_JAR_PARTS = urlparse(GCS_JAR)
GCS_JAR_BUCKET_NAME = GCS_JAR_PARTS.netloc
GCS_JAR_OBJECT_NAME = GCS_JAR_PARTS.path[1:]

default_args = {
    "dataflow_default_options": {
        "tempLocation": GCS_TMP,
        "stagingLocation": GCS_STAGING,
    }
}

with models.DAG(
    "example_gcp_dataflow_native_java",
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_native_java:

    # [START howto_operator_start_java_job_jar_on_gcs]
    start_java_job = BeamRunJavaPipelineOperator(
        task_id="start-java-job",
        jar=GCS_JAR,
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        job_class="org.apache.beam.examples.WordCount",
        dataflow_config={
            "check_if_running": CheckJobRunning.IgnoreJob,
            "location": "europe-west3",
            "poll_sleep": 10,
        },
    )
    # [END howto_operator_start_java_job_jar_on_gcs]

    # [START howto_operator_start_java_job_local_jar]
    jar_to_local = GCSToLocalFilesystemOperator(
        task_id="jar-to-local",
        bucket=GCS_JAR_BUCKET_NAME,
        object_name=GCS_JAR_OBJECT_NAME,
        filename="/tmp/dataflow-{{ ds_nodash }}.jar",
    )

    start_java_job_local = BeamRunJavaPipelineOperator(
        task_id="start-java-job-local",
        jar="/tmp/dataflow-{{ ds_nodash }}.jar",
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        job_class="org.apache.beam.examples.WordCount",
        dataflow_config={
            "check_if_running": CheckJobRunning.WaitForRun,
            "location": "europe-west3",
            "poll_sleep": 10,
        },
    )
    jar_to_local >> start_java_job_local
    # [END howto_operator_start_java_job_local_jar]

with models.DAG(
    "example_gcp_dataflow_native_python",
    default_args=default_args,
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_native_python:

    # [START howto_operator_start_python_job]
    start_python_job = BeamRunPythonPipelineOperator(
        task_id="start-python-job",
        py_file=GCS_PYTHON,
        py_options=[],
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        py_requirements=["apache-beam[gcp]==2.21.0"],
        py_interpreter="python3",
        py_system_site_packages=False,
        dataflow_config={"location": "europe-west3"},
    )
    # [END howto_operator_start_python_job]

    start_python_job_local = BeamRunPythonPipelineOperator(
        task_id="start-python-job-local",
        py_file="apache_beam.examples.wordcount",
        py_options=["-m"],
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        py_requirements=["apache-beam[gcp]==2.14.0"],
        py_interpreter="python3",
        py_system_site_packages=False,
    )

with models.DAG(
    "example_gcp_dataflow_native_python_async",
    default_args=default_args,
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_native_python_async:
    # [START howto_operator_start_python_job_async]
    start_python_job_async = BeamRunPythonPipelineOperator(
        task_id="start-python-job-async",
        runner="DataflowRunner",
        py_file=GCS_PYTHON,
        py_options=[],
        pipeline_options={
            "output": GCS_OUTPUT,
        },
        py_requirements=["apache-beam[gcp]==2.25.0"],
        py_interpreter="python3",
        py_system_site_packages=False,
        dataflow_config={
            "job_name": "start-python-job-async",
            "location": "europe-west3",
            "wait_until_finished": False,
        },
    )
    # [END howto_operator_start_python_job_async]

    # [START howto_sensor_wait_for_job_status]
    wait_for_python_job_async_done = DataflowJobStatusSensor(
        task_id="wait-for-python-job-async-done",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['id']}}",
        expected_statuses={DataflowJobStatus.JOB_STATE_DONE},
        location="europe-west3",
    )
    # [END howto_sensor_wait_for_job_status]

    # [START howto_sensor_wait_for_job_metric]
    def check_metric_scalar_gte(metric_name: str, value: int) -> Callable:
        """Check is metric greater than equals to given value."""

        def callback(metrics: list[dict]) -> bool:
            dag_native_python_async.log.info("Looking for '%s' >= %d", metric_name, value)
            for metric in metrics:
                context = metric.get("name", {}).get("context", {})
                original_name = context.get("original_name", "")
                tentative = context.get("tentative", "")
                if original_name == "Service-cpu_num_seconds" and not tentative:
                    return metric["scalar"] >= value
            raise AirflowException(f"Metric '{metric_name}' not found in metrics")

        return callback

    wait_for_python_job_async_metric = DataflowJobMetricsSensor(
        task_id="wait-for-python-job-async-metric",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['id']}}",
        location="europe-west3",
        callback=check_metric_scalar_gte(metric_name="Service-cpu_num_seconds", value=100),
        fail_on_terminal_state=False,
    )
    # [END howto_sensor_wait_for_job_metric]

    # [START howto_sensor_wait_for_job_message]
    def check_message(messages: list[dict]) -> bool:
        """Check message"""
        for message in messages:
            if "Adding workflow start and stop steps." in message.get("messageText", ""):
                return True
        return False

    wait_for_python_job_async_message = DataflowJobMessagesSensor(
        task_id="wait-for-python-job-async-message",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['id']}}",
        location="europe-west3",
        callback=check_message,
        fail_on_terminal_state=False,
    )
    # [END howto_sensor_wait_for_job_message]

    # [START howto_sensor_wait_for_job_autoscaling_event]
    def check_autoscaling_event(autoscaling_events: list[dict]) -> bool:
        """Check autoscaling event"""
        for autoscaling_event in autoscaling_events:
            if "Worker pool started." in autoscaling_event.get("description", {}).get("messageText", ""):
                return True
        return False

    wait_for_python_job_async_autoscaling_event = DataflowJobAutoScalingEventsSensor(
        task_id="wait-for-python-job-async-autoscaling-event",
        job_id="{{task_instance.xcom_pull('start-python-job-async')['id']}}",
        location="europe-west3",
        callback=check_autoscaling_event,
        fail_on_terminal_state=False,
    )
    # [END howto_sensor_wait_for_job_autoscaling_event]

    start_python_job_async >> wait_for_python_job_async_done
    start_python_job_async >> wait_for_python_job_async_metric
    start_python_job_async >> wait_for_python_job_async_message
    start_python_job_async >> wait_for_python_job_async_autoscaling_event

with models.DAG(
    "example_gcp_dataflow_template",
    default_args=default_args,
    start_date=START_DATE,
    catchup=False,
    tags=["example"],
) as dag_template:
    # [START howto_operator_start_template_job]
    start_template_job = DataflowTemplatedJobStartOperator(
        task_id="start-template-job",
        template="gs://dataflow-templates/latest/Word_Count",
        parameters={"inputFile": "gs://dataflow-samples/shakespeare/kinglear.txt", "output": GCS_OUTPUT},
        location="europe-west3",
    )
    # [END howto_operator_start_template_job]
<|endoftext|>


<commit_before>#! /usr/bin/env python

import pybindgen

def add_VectorInterface(module, name='Dune::Pymor::LA::VectorInterface'):
    assert(isinstance(module, pybindgen.module.Module))
    assert(name is not None)
    name = str(name)
    assert(len(name) > 0)
    namespace = module
    namespaces = [nspace.strip() for nspace in name.split('::')[:-1]]
    name = name.split('::')[-1].strip()
    for nspace in namespaces:
        namespace = namespace.add_cpp_namespace(nspace)
    Vector = namespace.add_class(name)
    Vector.add_constructor([pybindgen.param('int', 'ss')])
    Vector.add_method('type', pybindgen.retval('std::string'), [], is_const=True)
    Vector.add_method('dim', pybindgen.retval('int'), [], is_const=True)
    return module
<commit_msg>[dune.pymor.la.container] finished add_Vector()<commit_after>#! /usr/bin/env python

import pybindgen

def add_VectorInterface(module, name='Dune::Pymor::LA::VectorInterface'):
    assert(isinstance(module, pybindgen.module.Module))
    assert(name is not None)
    name = str(name)
    assert(len(name) > 0)
    namespace = module
    namespaces = [nspace.strip() for nspace in name.split('::')[:-1]]
    name = name.split('::')[-1].strip()
    for nspace in namespaces:
        namespace = namespace.add_cpp_namespace(nspace)
    Vector = namespace.add_class(name)
    Vector.add_constructor([pybindgen.param('int', 'ss')])
    Vector.add_method('type',
                      pybindgen.retval('std::string'),
                      [],
                      is_const=True)
    Vector.add_method('dim',
                      pybindgen.retval('int'),
                      [],
                      is_const=True)
    Vector.add_method('almost_equal',
                      pybindgen.retval('bool'),
                      [pybindgen.param(name + ' *', 'other', transfer_ownership=False),
                       pybindgen.param('double', 'epsilon')],
                      is_const=True)
    Vector.add_method('scal',
                      None,
                      [pybindgen.param('double', 'alpha')])
    Vector.add_method('axpy',
                      None,
                      [pybindgen.param('double', 'alpha'),
                       pybindgen.param(name + ' *', 'x', transfer_ownership=False)])
    Vector.add_method('dot',
                      pybindgen.retval('double'),
                      [pybindgen.param(name + ' *', 'other', transfer_ownership=False)],
                      is_const=True)
    Vector.add_method('l1_norm',
                      pybindgen.retval('double'),
                      [],
                      is_const=True)
    Vector.add_method('l2_norm',
                      pybindgen.retval('double'),
                      [],
                      is_const=True)
    Vector.add_method('sup_norm',
                      pybindgen.retval('double'),
                      [],
                      is_const=True)
    Vector.add_method('components',
                      pybindgen.retval('std::vector< double >'),
                      [pybindgen.param('std::vector< int >', 'component_indices')],
                      is_const=True)
    Vector.add_method('amax',
                      pybindgen.retval('std::vector< double >'),
                      [],
                      is_const=True)
    Vector.add_method('add',
                      pybindgen.retval(name + ' *', caller_owns_return=True),
                      [pybindgen.param(name + ' *', 'other', transfer_ownership=False)],
                      is_const=True)
    Vector.add_method('iadd',
                      None,
                      [pybindgen.param(name + ' *', 'other', transfer_ownership=False)])
    Vector.add_method('sub',
                      pybindgen.retval(name + ' *', caller_owns_return=True),
                      [pybindgen.param(name + ' *', 'other', transfer_ownership=False)],
                      is_const=True)
    Vector.add_method('isub',
                      None,
                      [pybindgen.param(name + ' *', 'other', transfer_ownership=False)])
    return module, Vector
<|endoftext|>


<commit_before>from __future__ import absolute_import
import os
import six
import json
import django
import boto
from boto.s3.key import Key
from moto import mock_s3
from .. import views, feeds
from django.db import models
from .. import models as bmodels
from django.conf import settings
from django.test import TestCase
from django.http import HttpResponse
from django.core.management import call_command
from django.core.management.base import CommandError
from django.contrib.contenttypes.models import ContentType

try:
    from mock import patch
except ImportError:
    from unittest.mock import patch

class MockObject(bmodels.BuildableModel):
    detail_views = ['bakery.tests.MockDetailView']
    name = models.CharField(max_length=500)

    def get_absolute_url(self):
        super(MockObject, self).get_absolute_url()  # Just for test coverage
        return '/%s/' % self.id

class AutoMockObject(bmodels.AutoPublishingBuildableModel):
    detail_views = ['bakery.tests.MockDetailView']
    name = models.CharField(max_length=500)
    is_published = models.BooleanField(default=False)

    def get_absolute_url(self):
        return '/%s/' % self.id

class MockDetailView(views.BuildableDetailView):
    model = MockObject
    template_name = 'detailview.html'

class MockRedirectView(views.BuildableRedirectView):
    build_path = "detail/badurl.html"
    url = "/detail/"

class MockRSSFeed(feeds.BuildableFeed):
    link = '/latest.xml'

    def items(self):
        return MockObject.objects.all()

class JSONResponseMixin(object):

    def render_to_response(self, context, **response_kwargs):
        return HttpResponse(
            self.convert_context_to_json(context),
            content_type='application/json',
            **response_kwargs
        )

    def convert_context_to_json(self, context):
        return json.dumps(context)

class MockJSONView(JSONResponseMixin, views.BuildableTemplateView):
    build_path = 'jsonview.json'

    def get_content(self):
        return self.get(self.request).content

    def get_context_data(self, **kwargs):
        return {'hello': 'tests'}

class BakeryTest(TestCase):

    def setUp(self):
        for m in [MockObject, AutoMockObject]:
            m.objects.create(name=1)
            m.objects.create(name=2)
            m.objects.create(name=3)

    def test_models(self):
        for m in [MockObject, AutoMockObject]:
            obj = m.objects.all()[0]
            obj.build()
            obj.unbuild()
            obj.get_absolute_url()

    def test_template_view(self):
        v = views.BuildableTemplateView(
            template_name='templateview.html',
            build_path='foo.html',
        )
        v.build_method
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, 'foo.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)
        v = views.BuildableTemplateView(
            template_name='templateview.html',
            build_path='foo/bar.html',
        )
        v.build_method
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, 'foo', 'bar.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_list_view(self):
        v = views.BuildableListView(
            queryset=[1, 2, 3],
            template_name='listview.html',
            build_path='foo.html',
        )
        v.build_method
        v.build_queryset()
        build_path = os.path.join(settings.BUILD_DIR, 'foo.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)
        v = views.BuildableListView(
            queryset=[1, 2, 3],
            template_name='listview.html',
            build_path='foo/bar.html',
        )
        v.build_method
        v.build_queryset()
        build_path = os.path.join(settings.BUILD_DIR, 'foo', 'bar.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_detail_view(self):
        v = views.BuildableDetailView(
            queryset=MockObject.objects.all(),
            template_name='detailview.html',
        )
        v.build_method
        v.build_queryset()
        for o in MockObject.objects.all():
            build_path = os.path.join(
                settings.BUILD_DIR,
                o.get_absolute_url()[1:],
                'index.html',
            )
            self.assertTrue(os.path.exists(build_path))
            v.unbuild_object(o)

    def test_redirect_view(self):
        v = views.BuildableRedirectView(
            build_path="detail/badurl.html",
            url="/detail/"
        )
        v.build_method
        v.build()
        MockRedirectView().build()
        build_path = os.path.join(
            settings.BUILD_DIR,
            "detail/badurl.html"
        )
        self.assertTrue(os.path.exists(build_path))

    def test_404_view(self):
        v = views.Buildable404View()
        v.build_method
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, '404.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_json_view(self):
        v = MockJSONView()
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, 'jsonview.json')
        self.assertTrue(os.path.exists(build_path))
        self.assertEqual(
            json.loads(open(build_path, 'rb').read().decode()),
            {"hello": "tests"}
        )
        os.remove(build_path)

    def test_rss_feed(self):
        f = MockRSSFeed()
        f.build_method
        f.build_queryset()
        build_path = os.path.join(settings.BUILD_DIR, 'feed.xml')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_build_cmd(self):
        call_command("build", **{'skip_media': True, 'verbosity': 3})
        call_command("build", **{'skip_static': True, 'verbosity': 3})
        call_command("build", **{'skip_static': True, 'skip_media': True})
        call_command("build", **{
            'skip_static': True,
            'skip_media': True,
            'verbosity': 3,
        })
        call_command("build", **{
            'skip_static': True,
            'skip_media': True,
            'build_dir': settings.BUILD_DIR,
        })
        call_command("build", 'bakery.tests.MockDetailView')
        foobar_path = os.path.join(
            settings.BUILD_DIR,
            'static',
            'foo.bar'
        )
        self.assertTrue(os.path.exists(foobar_path))
        self.assertEqual(
            open(foobar_path, 'rb').read().strip(),
            six.b('Hello tests')
        )
        robots_path = os.path.join(settings.BUILD_DIR, 'robots.txt')
        self.assertTrue(os.path.exists(robots_path))
        favicon_path = os.path.join(settings.BUILD_DIR, 'favicon.ico')
        self.assertTrue(os.path.exists(favicon_path))
        if django.VERSION >= (1, 5):
            self.assertRaises(
                CommandError,
                call_command,
                'build',
                'FooView',
            )

    def test_unbuild_cmd(self):
        call_command("unbuild")

    def test_gzipped(self):
        if django.VERSION >= (1, 4):
            with self.settings(BAKERY_GZIP=True):
                six.print_("testing gzipped files")
                self.test_models()
                self.test_template_view()
                self.test_list_view()
                self.test_detail_view()
                self.test_404_view()
                self.test_build_cmd()

    def test_buildserver_cmd(self):
        pass

    def test_publish_cmd(self):
        pass

    @mock_s3
    def test_publish_cmd_keep_files(self):
        """
        Test that running the publish management command with the
        --keep-files option does not delete remote files that
        don't exist in the build directory
        """
        aws_bucket_name = 'fake.s3.example.com'

        # Since we're mocking S3, we need to create the mock bucket
        conn = boto.connect_s3()
        bucket = conn.create_bucket(aws_bucket_name)

        # Create an item in S3 that isn't in our build directory
        contents = 'This is a file not in the Django Bakery build'
        key = 'not_in_build/test.html'
        k = Key(bucket)
        k.key = key
        k.set_contents_from_string(contents)

        # Build our mock site via the build management command
        call_command("build", 'bakery.tests.MockDetailView')

        # HACK: Running the publish command causes this exception to be raised:
        # S3DataError: BotoClientError: ETag from S3 did not match computed MD5.
        # I suspect, but haven't confirmed that this is due to the combination
        # of mocking S3 and threading.  We don't really care about that part
        # of the code, so just mock the offending method
        with patch.object(Key, 'should_retry', return_value=True):
            # Publish the built site
            cmd_kwargs = {
                'keep_files': True,
                'aws_bucket_name': aws_bucket_name,
            }
            call_command('publish', **cmd_kwargs)

            # The key we created should still be in the bucket after publishing
            k = bucket.get_key(key)
            self.assertEqual(k.get_contents_as_string(), contents)

    def test_unpublish_cmd(self):
        pass

    def test_tasks(self):
        from bakery import tasks
        obj = AutoMockObject.objects.all()[0]
        ct = ContentType.objects.get_for_model(obj)
        tasks.publish_object(ct.id, obj.id)
        tasks.unpublish_object(ct.id, obj.id)
        # Some save overrides tests
        obj = AutoMockObject.objects.all()[0]
        obj.save(publish=False)
        # obj.is_published = True
        # obj.save()
        obj.delete(unpublish=False)
<commit_msg>Fix PEP8<commit_after>from __future__ import absolute_import
import os
import six
import json
import django
import boto
from boto.s3.key import Key
from moto import mock_s3
from .. import views, feeds
from django.db import models
from .. import models as bmodels
from django.conf import settings
from django.test import TestCase
from django.http import HttpResponse
from django.core.management import call_command
from django.core.management.base import CommandError
from django.contrib.contenttypes.models import ContentType

try:
    from mock import patch
except ImportError:
    from unittest.mock import patch

class MockObject(bmodels.BuildableModel):
    detail_views = ['bakery.tests.MockDetailView']
    name = models.CharField(max_length=500)

    def get_absolute_url(self):
        super(MockObject, self).get_absolute_url()  # Just for test coverage
        return '/%s/' % self.id

class AutoMockObject(bmodels.AutoPublishingBuildableModel):
    detail_views = ['bakery.tests.MockDetailView']
    name = models.CharField(max_length=500)
    is_published = models.BooleanField(default=False)

    def get_absolute_url(self):
        return '/%s/' % self.id

class MockDetailView(views.BuildableDetailView):
    model = MockObject
    template_name = 'detailview.html'

class MockRedirectView(views.BuildableRedirectView):
    build_path = "detail/badurl.html"
    url = "/detail/"

class MockRSSFeed(feeds.BuildableFeed):
    link = '/latest.xml'

    def items(self):
        return MockObject.objects.all()

class JSONResponseMixin(object):

    def render_to_response(self, context, **response_kwargs):
        return HttpResponse(
            self.convert_context_to_json(context),
            content_type='application/json',
            **response_kwargs
        )

    def convert_context_to_json(self, context):
        return json.dumps(context)

class MockJSONView(JSONResponseMixin, views.BuildableTemplateView):
    build_path = 'jsonview.json'

    def get_content(self):
        return self.get(self.request).content

    def get_context_data(self, **kwargs):
        return {'hello': 'tests'}

class BakeryTest(TestCase):

    def setUp(self):
        for m in [MockObject, AutoMockObject]:
            m.objects.create(name=1)
            m.objects.create(name=2)
            m.objects.create(name=3)

    def test_models(self):
        for m in [MockObject, AutoMockObject]:
            obj = m.objects.all()[0]
            obj.build()
            obj.unbuild()
            obj.get_absolute_url()

    def test_template_view(self):
        v = views.BuildableTemplateView(
            template_name='templateview.html',
            build_path='foo.html',
        )
        v.build_method
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, 'foo.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)
        v = views.BuildableTemplateView(
            template_name='templateview.html',
            build_path='foo/bar.html',
        )
        v.build_method
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, 'foo', 'bar.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_list_view(self):
        v = views.BuildableListView(
            queryset=[1, 2, 3],
            template_name='listview.html',
            build_path='foo.html',
        )
        v.build_method
        v.build_queryset()
        build_path = os.path.join(settings.BUILD_DIR, 'foo.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)
        v = views.BuildableListView(
            queryset=[1, 2, 3],
            template_name='listview.html',
            build_path='foo/bar.html',
        )
        v.build_method
        v.build_queryset()
        build_path = os.path.join(settings.BUILD_DIR, 'foo', 'bar.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_detail_view(self):
        v = views.BuildableDetailView(
            queryset=MockObject.objects.all(),
            template_name='detailview.html',
        )
        v.build_method
        v.build_queryset()
        for o in MockObject.objects.all():
            build_path = os.path.join(
                settings.BUILD_DIR,
                o.get_absolute_url()[1:],
                'index.html',
            )
            self.assertTrue(os.path.exists(build_path))
            v.unbuild_object(o)

    def test_redirect_view(self):
        v = views.BuildableRedirectView(
            build_path="detail/badurl.html",
            url="/detail/"
        )
        v.build_method
        v.build()
        MockRedirectView().build()
        build_path = os.path.join(
            settings.BUILD_DIR,
            "detail/badurl.html"
        )
        self.assertTrue(os.path.exists(build_path))

    def test_404_view(self):
        v = views.Buildable404View()
        v.build_method
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, '404.html')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_json_view(self):
        v = MockJSONView()
        v.build()
        build_path = os.path.join(settings.BUILD_DIR, 'jsonview.json')
        self.assertTrue(os.path.exists(build_path))
        self.assertEqual(
            json.loads(open(build_path, 'rb').read().decode()),
            {"hello": "tests"}
        )
        os.remove(build_path)

    def test_rss_feed(self):
        f = MockRSSFeed()
        f.build_method
        f.build_queryset()
        build_path = os.path.join(settings.BUILD_DIR, 'feed.xml')
        self.assertTrue(os.path.exists(build_path))
        os.remove(build_path)

    def test_build_cmd(self):
        call_command("build", **{'skip_media': True, 'verbosity': 3})
        call_command("build", **{'skip_static': True, 'verbosity': 3})
        call_command("build", **{'skip_static': True, 'skip_media': True})
        call_command("build", **{
            'skip_static': True,
            'skip_media': True,
            'verbosity': 3,
        })
        call_command("build", **{
            'skip_static': True,
            'skip_media': True,
            'build_dir': settings.BUILD_DIR,
        })
        call_command("build", 'bakery.tests.MockDetailView')
        foobar_path = os.path.join(
            settings.BUILD_DIR,
            'static',
            'foo.bar'
        )
        self.assertTrue(os.path.exists(foobar_path))
        self.assertEqual(
            open(foobar_path, 'rb').read().strip(),
            six.b('Hello tests')
        )
        robots_path = os.path.join(settings.BUILD_DIR, 'robots.txt')
        self.assertTrue(os.path.exists(robots_path))
        favicon_path = os.path.join(settings.BUILD_DIR, 'favicon.ico')
        self.assertTrue(os.path.exists(favicon_path))
        if django.VERSION >= (1, 5):
            self.assertRaises(
                CommandError,
                call_command,
                'build',
                'FooView',
            )

    def test_unbuild_cmd(self):
        call_command("unbuild")

    def test_gzipped(self):
        if django.VERSION >= (1, 4):
            with self.settings(BAKERY_GZIP=True):
                six.print_("testing gzipped files")
                self.test_models()
                self.test_template_view()
                self.test_list_view()
                self.test_detail_view()
                self.test_404_view()
                self.test_build_cmd()

    def test_buildserver_cmd(self):
        pass

    def test_publish_cmd(self):
        pass

    @mock_s3
    def test_publish_cmd_keep_files(self):
        """
        Test that running the publish management command with the
        --keep-files option does not delete remote files that
        don't exist in the build directory
        """
        aws_bucket_name = 'fake.s3.example.com'

        # Since we're mocking S3, we need to create the mock bucket
        conn = boto.connect_s3()
        bucket = conn.create_bucket(aws_bucket_name)

        # Create an item in S3 that isn't in our build directory
        contents = 'This is a file not in the Django Bakery build'
        key = 'not_in_build/test.html'
        k = Key(bucket)
        k.key = key
        k.set_contents_from_string(contents)

        # Build our mock site via the build management command
        call_command("build", 'bakery.tests.MockDetailView')

        # HACK: Running the publish command causes this exception to be raised:
        # S3DataError: BotoClientError: ETag from S3 did not match computed
        # MD5.
        # I suspect, but haven't confirmed that this is due to the combination
        # of mocking S3 and threading.  We don't really care about that part
        # of the code, so just mock the offending method
        with patch.object(Key, 'should_retry', return_value=True):
            # Publish the built site
            cmd_kwargs = {
                'keep_files': True,
                'aws_bucket_name': aws_bucket_name,
            }
            call_command('publish', **cmd_kwargs)

            # The key we created should still be in the bucket after publishing
            k = bucket.get_key(key)
            self.assertEqual(k.get_contents_as_string(), contents)

    def test_unpublish_cmd(self):
        pass

    def test_tasks(self):
        from bakery import tasks
        obj = AutoMockObject.objects.all()[0]
        ct = ContentType.objects.get_for_model(obj)
        tasks.publish_object(ct.id, obj.id)
        tasks.unpublish_object(ct.id, obj.id)
        # Some save overrides tests
        obj = AutoMockObject.objects.all()[0]
        obj.save(publish=False)
        # obj.is_published = True
        # obj.save()
        obj.delete(unpublish=False)
<|endoftext|>


<commit_before>#!/usr/bin/env python2
#
# Copyright (C) Microsoft Corporation, All rights reserved.

import ConfigParser
import os
import sys

import serializerfactory

json = serializerfactory.get_serializer(sys.version_info)

CONFIG_ENV_KEY = "WORKERCONF"
WORKER_REQUIRED_CONFIG_SECTION = "worker-required"
WORKER_OPTIONAL_CONFIG_SECTION = "worker-optional"

# manually set configuration values
SOURCE_DIRECTORY_PATH = "source_directory_path"
COMPONENT = "component"

# required configuration keys
CERT_PATH = "jrds_cert_path"
KEY_PATH = "jrds_key_path"
BASE_URI = "jrds_base_uri"
ACCOUNT_ID = "account_id"
MACHINE_ID = "machine_id"
HYBRID_WORKER_GROUP_NAME = "hybrid_worker_group_name"
WORKER_VERSION = "worker_version"
WORKING_DIRECTORY_PATH = "working_directory_path"

# optional configuration keys
DEBUG_TRACES = "debug_traces"
BYPASS_CERTIFICATE_VERIFICATION = "bypass_certificate_verification"
ENFORCE_RUNBOOK_SIGNATURE_VALIDATION = "enforce_runbook_signature_validation"
GPG_PUBLIC_KEYRING_PATH = "gpg_public_keyring_path"
STATE_DIRECTORY_PATH = "state_directory_path"
JRDS_POLLING_FREQUENCY = "jrds_polling_frequency"
PROXY_CONFIGURATION_PATH = "proxy_configuration_path"

# optional configuration default values
DEFAULT_EMPTY = ""
DEFAULT_DEBUG_TRACES = "false"
DEFAUTL_BYPASS_CERTIFICATE_VERIFICATION = "false"
DEFAULT_ENFORCE_RUNBOOK_SIGNATURE_VALIDATION = "true"
DEFAULT_GPG_PUBLIC_KEYRING_PATH = DEFAULT_EMPTY
DEFAULT_STATE_DIRECTORY_PATH = DEFAULT_EMPTY
DEFAULT_PROXY_CONFIGURATION_PATH = DEFAULT_EMPTY
DEFAULT_COMPONENT = "Unknown"
DEFAULT_WORKER_VERSION = "1.0.0.3"
DEFAULT_JRDS_POLLING_FREQUENCY = "30"

# state configuration keys
STATE_PID = "pid"
STATE_RESOURCE_VERSION = "resource_version"
STATE_WORKSPACE_ID = "workspace_id"
STATE_WORKER_VERSION = "worker_version"

def read_and_set_configuration(config_path):
    """Reads the worker configuration from the file at config_path and sets the read configuration to
    the env variable.

    The configuration is read of the path, put into a dictionary which will be serialized and set in the env
    variable.

    Notes:
        The WORKER_VERSION has to be set manually for now.
        The COMPONENT has to be set manually at the entry point of each component (worker/sandbox).

    Args:
        config_path: string, the configuration file path.
    """
    clear_config()

    # init and set default values for optional configuration keys
    config = ConfigParser.SafeConfigParser({DEBUG_TRACES: DEFAULT_DEBUG_TRACES,
                                            BYPASS_CERTIFICATE_VERIFICATION: DEFAUTL_BYPASS_CERTIFICATE_VERIFICATION,
                                            ENFORCE_RUNBOOK_SIGNATURE_VALIDATION: DEFAULT_ENFORCE_RUNBOOK_SIGNATURE_VALIDATION,
                                            GPG_PUBLIC_KEYRING_PATH: DEFAULT_GPG_PUBLIC_KEYRING_PATH,
                                            STATE_DIRECTORY_PATH: DEFAULT_STATE_DIRECTORY_PATH,
                                            JRDS_POLLING_FREQUENCY: DEFAULT_JRDS_POLLING_FREQUENCY,
                                            PROXY_CONFIGURATION_PATH : DEFAULT_PROXY_CONFIGURATION_PATH})

    # load the worker configuration file
    config.read(config_path)

    # create the configuration dictionary
    # read required configuration values
    configuration = {CERT_PATH: os.path.abspath(config.get(WORKER_REQUIRED_CONFIG_SECTION, CERT_PATH)),
                     KEY_PATH: os.path.abspath(config.get(WORKER_REQUIRED_CONFIG_SECTION, KEY_PATH)),
                     BASE_URI: config.get(WORKER_REQUIRED_CONFIG_SECTION, BASE_URI),
                     ACCOUNT_ID: config.get(WORKER_REQUIRED_CONFIG_SECTION, ACCOUNT_ID),
                     MACHINE_ID: config.get(WORKER_REQUIRED_CONFIG_SECTION, MACHINE_ID),
                     HYBRID_WORKER_GROUP_NAME: config.get(WORKER_REQUIRED_CONFIG_SECTION, HYBRID_WORKER_GROUP_NAME),
                     WORKING_DIRECTORY_PATH: os.path.abspath(
                         config.get(WORKER_REQUIRED_CONFIG_SECTION, WORKING_DIRECTORY_PATH)),
                     SOURCE_DIRECTORY_PATH: os.path.dirname(os.path.realpath(__file__)),
                     WORKER_VERSION: DEFAULT_WORKER_VERSION,
                     COMPONENT: DEFAULT_COMPONENT}

    # read optional configuration section
    configuration.update({DEBUG_TRACES: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION, DEBUG_TRACES),
                          BYPASS_CERTIFICATE_VERIFICATION: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION,
                                                                             BYPASS_CERTIFICATE_VERIFICATION),
                          ENFORCE_RUNBOOK_SIGNATURE_VALIDATION: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION,
                                                                                ENFORCE_RUNBOOK_SIGNATURE_VALIDATION),
                          GPG_PUBLIC_KEYRING_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, GPG_PUBLIC_KEYRING_PATH),
                          STATE_DIRECTORY_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, STATE_DIRECTORY_PATH),
                          JRDS_POLLING_FREQUENCY: config.getint(WORKER_OPTIONAL_CONFIG_SECTION, JRDS_POLLING_FREQUENCY),
                          PROXY_CONFIGURATION_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, PROXY_CONFIGURATION_PATH)})

    # set the worker conf to env var
    set_config(configuration)

def set_config(configuration):
    """Sets the worker configuration to the env variable.

    This method will merge the provided dictionary to any existent value in the environment variable.

    Args:
        configuration: dictionary(string), the configuration key value pairs.
    """
    try:
        env_config = os.environ[CONFIG_ENV_KEY]
        config = json.loads(env_config)
        config.update(configuration)
        configuration = config
    except KeyError:
        pass

    os.environ[CONFIG_ENV_KEY] = json.dumps(configuration)

def clear_config():
    try:
        del os.environ[CONFIG_ENV_KEY]
    except Exception:
        pass

def get_value(key):
    """Gets a specific value from the configuration value in the environment variable.

    This method will merge the provided dictionary to any existent value in the environment variable.

    Args:
        key: string, the configuration key value.

    Returns:
        The configuration value.
    """
    try:
        return json.loads(os.environ[CONFIG_ENV_KEY])[key]
    except KeyError:
        raise KeyError("Configuration environment variable not found. [key=" + key + "].")

def get_jrds_get_sandbox_actions_polling_freq():
    return get_value(JRDS_POLLING_FREQUENCY)

def get_jrds_get_job_actions_polling_freq():
    return get_value(JRDS_POLLING_FREQUENCY)

def get_component():
    return get_value(COMPONENT)

def get_jrds_cert_path():
    return get_value(CERT_PATH)

def get_jrds_key_path():
    return get_value(KEY_PATH)

def get_jrds_base_uri():
    return get_value(BASE_URI)

def get_account_id():
    return get_value(ACCOUNT_ID)

def get_machine_id():
    return get_value(MACHINE_ID)

def get_hybrid_worker_name():
    return get_value(HYBRID_WORKER_GROUP_NAME)

def get_worker_version():
    return get_value(WORKER_VERSION)

def get_working_directory_path():
    return get_value(WORKING_DIRECTORY_PATH)

def get_debug_traces():
    return get_value(DEBUG_TRACES)

def get_verify_certificates():
    return get_value(BYPASS_CERTIFICATE_VERIFICATION)

def get_source_directory_path():
    return get_value(SOURCE_DIRECTORY_PATH)

def get_enforce_runbook_signature_validation():
    return get_value(ENFORCE_RUNBOOK_SIGNATURE_VALIDATION)

def get_gpg_public_keyring_path():
    return get_value(GPG_PUBLIC_KEYRING_PATH)

def get_state_directory_path():
    return get_value(STATE_DIRECTORY_PATH)

def get_temporary_request_directory_path():
    return os.path.join(get_working_directory_path(), "req_tmp")

def get_proxy_configuration_path():
    return get_value(PROXY_CONFIGURATION_PATH)
<commit_msg>Bumped the version number of the Linux Hybrid Worker contained in nxOMSAutomationWorker<commit_after>#!/usr/bin/env python2
#
# Copyright (C) Microsoft Corporation, All rights reserved.

import ConfigParser
import os
import sys

import serializerfactory

json = serializerfactory.get_serializer(sys.version_info)

CONFIG_ENV_KEY = "WORKERCONF"
WORKER_REQUIRED_CONFIG_SECTION = "worker-required"
WORKER_OPTIONAL_CONFIG_SECTION = "worker-optional"

# manually set configuration values
SOURCE_DIRECTORY_PATH = "source_directory_path"
COMPONENT = "component"

# required configuration keys
CERT_PATH = "jrds_cert_path"
KEY_PATH = "jrds_key_path"
BASE_URI = "jrds_base_uri"
ACCOUNT_ID = "account_id"
MACHINE_ID = "machine_id"
HYBRID_WORKER_GROUP_NAME = "hybrid_worker_group_name"
WORKER_VERSION = "worker_version"
WORKING_DIRECTORY_PATH = "working_directory_path"

# optional configuration keys
DEBUG_TRACES = "debug_traces"
BYPASS_CERTIFICATE_VERIFICATION = "bypass_certificate_verification"
ENFORCE_RUNBOOK_SIGNATURE_VALIDATION = "enforce_runbook_signature_validation"
GPG_PUBLIC_KEYRING_PATH = "gpg_public_keyring_path"
STATE_DIRECTORY_PATH = "state_directory_path"
JRDS_POLLING_FREQUENCY = "jrds_polling_frequency"
PROXY_CONFIGURATION_PATH = "proxy_configuration_path"

# optional configuration default values
DEFAULT_EMPTY = ""
DEFAULT_DEBUG_TRACES = "false"
DEFAUTL_BYPASS_CERTIFICATE_VERIFICATION = "false"
DEFAULT_ENFORCE_RUNBOOK_SIGNATURE_VALIDATION = "true"
DEFAULT_GPG_PUBLIC_KEYRING_PATH = DEFAULT_EMPTY
DEFAULT_STATE_DIRECTORY_PATH = DEFAULT_EMPTY
DEFAULT_PROXY_CONFIGURATION_PATH = DEFAULT_EMPTY
DEFAULT_COMPONENT = "Unknown"
DEFAULT_WORKER_VERSION = "1.1.0.0"
DEFAULT_JRDS_POLLING_FREQUENCY = "30"

# state configuration keys
STATE_PID = "pid"
STATE_RESOURCE_VERSION = "resource_version"
STATE_WORKSPACE_ID = "workspace_id"
STATE_WORKER_VERSION = "worker_version"

def read_and_set_configuration(config_path):
    """Reads the worker configuration from the file at config_path and sets the read configuration to
    the env variable.

    The configuration is read of the path, put into a dictionary which will be serialized and set in the env
    variable.

    Notes:
        The WORKER_VERSION has to be set manually for now.
        The COMPONENT has to be set manually at the entry point of each component (worker/sandbox).

    Args:
        config_path: string, the configuration file path.
    """
    clear_config()

    # init and set default values for optional configuration keys
    config = ConfigParser.SafeConfigParser({DEBUG_TRACES: DEFAULT_DEBUG_TRACES,
                                            BYPASS_CERTIFICATE_VERIFICATION: DEFAUTL_BYPASS_CERTIFICATE_VERIFICATION,
                                            ENFORCE_RUNBOOK_SIGNATURE_VALIDATION: DEFAULT_ENFORCE_RUNBOOK_SIGNATURE_VALIDATION,
                                            GPG_PUBLIC_KEYRING_PATH: DEFAULT_GPG_PUBLIC_KEYRING_PATH,
                                            STATE_DIRECTORY_PATH: DEFAULT_STATE_DIRECTORY_PATH,
                                            JRDS_POLLING_FREQUENCY: DEFAULT_JRDS_POLLING_FREQUENCY,
                                            PROXY_CONFIGURATION_PATH : DEFAULT_PROXY_CONFIGURATION_PATH})

    # load the worker configuration file
    config.read(config_path)

    # create the configuration dictionary
    # read required configuration values
    configuration = {CERT_PATH: os.path.abspath(config.get(WORKER_REQUIRED_CONFIG_SECTION, CERT_PATH)),
                     KEY_PATH: os.path.abspath(config.get(WORKER_REQUIRED_CONFIG_SECTION, KEY_PATH)),
                     BASE_URI: config.get(WORKER_REQUIRED_CONFIG_SECTION, BASE_URI),
                     ACCOUNT_ID: config.get(WORKER_REQUIRED_CONFIG_SECTION, ACCOUNT_ID),
                     MACHINE_ID: config.get(WORKER_REQUIRED_CONFIG_SECTION, MACHINE_ID),
                     HYBRID_WORKER_GROUP_NAME: config.get(WORKER_REQUIRED_CONFIG_SECTION, HYBRID_WORKER_GROUP_NAME),
                     WORKING_DIRECTORY_PATH: os.path.abspath(
                         config.get(WORKER_REQUIRED_CONFIG_SECTION, WORKING_DIRECTORY_PATH)),
                     SOURCE_DIRECTORY_PATH: os.path.dirname(os.path.realpath(__file__)),
                     WORKER_VERSION: DEFAULT_WORKER_VERSION,
                     COMPONENT: DEFAULT_COMPONENT}

    # read optional configuration section
    configuration.update({DEBUG_TRACES: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION, DEBUG_TRACES),
                          BYPASS_CERTIFICATE_VERIFICATION: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION,
                                                                             BYPASS_CERTIFICATE_VERIFICATION),
                          ENFORCE_RUNBOOK_SIGNATURE_VALIDATION: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION,
                                                                                ENFORCE_RUNBOOK_SIGNATURE_VALIDATION),
                          GPG_PUBLIC_KEYRING_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, GPG_PUBLIC_KEYRING_PATH),
                          STATE_DIRECTORY_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, STATE_DIRECTORY_PATH),
                          JRDS_POLLING_FREQUENCY: config.getint(WORKER_OPTIONAL_CONFIG_SECTION, JRDS_POLLING_FREQUENCY),
                          PROXY_CONFIGURATION_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, PROXY_CONFIGURATION_PATH)})

    # set the worker conf to env var
    set_config(configuration)

def set_config(configuration):
    """Sets the worker configuration to the env variable.

    This method will merge the provided dictionary to any existent value in the environment variable.

    Args:
        configuration: dictionary(string), the configuration key value pairs.
    """
    try:
        env_config = os.environ[CONFIG_ENV_KEY]
        config = json.loads(env_config)
        config.update(configuration)
        configuration = config
    except KeyError:
        pass

    os.environ[CONFIG_ENV_KEY] = json.dumps(configuration)

def clear_config():
    try:
        del os.environ[CONFIG_ENV_KEY]
    except Exception:
        pass

def get_value(key):
    """Gets a specific value from the configuration value in the environment variable.

    This method will merge the provided dictionary to any existent value in the environment variable.

    Args:
        key: string, the configuration key value.

    Returns:
        The configuration value.
    """
    try:
        return json.loads(os.environ[CONFIG_ENV_KEY])[key]
    except KeyError:
        raise KeyError("Configuration environment variable not found. [key=" + key + "].")

def get_jrds_get_sandbox_actions_polling_freq():
    return get_value(JRDS_POLLING_FREQUENCY)

def get_jrds_get_job_actions_polling_freq():
    return get_value(JRDS_POLLING_FREQUENCY)

def get_component():
    return get_value(COMPONENT)

def get_jrds_cert_path():
    return get_value(CERT_PATH)

def get_jrds_key_path():
    return get_value(KEY_PATH)

def get_jrds_base_uri():
    return get_value(BASE_URI)

def get_account_id():
    return get_value(ACCOUNT_ID)

def get_machine_id():
    return get_value(MACHINE_ID)

def get_hybrid_worker_name():
    return get_value(HYBRID_WORKER_GROUP_NAME)

def get_worker_version():
    return get_value(WORKER_VERSION)

def get_working_directory_path():
    return get_value(WORKING_DIRECTORY_PATH)

def get_debug_traces():
    return get_value(DEBUG_TRACES)

def get_verify_certificates():
    return get_value(BYPASS_CERTIFICATE_VERIFICATION)

def get_source_directory_path():
    return get_value(SOURCE_DIRECTORY_PATH)

def get_enforce_runbook_signature_validation():
    return get_value(ENFORCE_RUNBOOK_SIGNATURE_VALIDATION)

def get_gpg_public_keyring_path():
    return get_value(GPG_PUBLIC_KEYRING_PATH)

def get_state_directory_path():
    return get_value(STATE_DIRECTORY_PATH)

def get_temporary_request_directory_path():
    return os.path.join(get_working_directory_path(), "req_tmp")

def get_proxy_configuration_path():
    return get_value(PROXY_CONFIGURATION_PATH)
<|endoftext|>


<commit_before>from flask import Response, abort, flash, redirect, render_template, request

from baseframe import _
from baseframe.filters import date_filter
from baseframe.forms import render_form, render_redirect
from coaster.auth import current_auth
from coaster.views import (
    ModelView,
    UrlChangeCheck,
    UrlForView,
    get_next_url,
    render_with,
    requestargs,
    requires_roles,
    route,
)

from .. import app, funnelapp
from ..forms import ProfileBannerForm, ProfileForm, ProfileLogoForm, SavedProjectForm
from ..models import Profile, Project, Proposal, db
from .decorators import legacy_redirect
from .login_session import requires_login
from .mixins import ProfileViewMixin

@Profile.features('new_project')
def feature_profile_new_project(obj):
    return (
        obj.is_organization_profile
        and obj.current_roles.admin
        and bool(obj.state.PUBLIC)
    )

@Profile.features('new_user_project')
def feature_profile_new_user_project(obj):
    return obj.is_user_profile and obj.current_roles.admin and bool(obj.state.PUBLIC)

@Profile.features('make_public')
def feature_profile_make_public(obj):
    return obj.current_roles.admin and not bool(obj.state.PUBLIC)

@Profile.features('make_private')
def feature_profile_make_private(obj):
    return obj.current_roles.admin and bool(obj.state.PUBLIC)

def template_switcher(templateargs):
    template = templateargs.pop('template')
    return Response(render_template(template, **templateargs), mimetype='text/html')

@Profile.views('main')
@route('/<profile>')
class ProfileView(ProfileViewMixin, UrlChangeCheck, UrlForView, ModelView):
    __decorators__ = [legacy_redirect]

    @route('')
    @render_with({'*/*': template_switcher}, json=True)
    @requires_roles({'reader', 'admin'})
    def view(self):
        template_name = None
        ctx = {}

        if self.obj.is_user_profile:
            template_name = 'user_profile.html.jinja2'

            submitted_proposals = self.obj.user.speaker_at.filter(
                ~(Proposal.state.DRAFT), ~(Proposal.state.DELETED)
            ).all()

            tagged_sessions = [
                proposal.session
                for proposal in submitted_proposals
                if proposal.session is not None
            ]

            ctx = {
                'template': template_name,
                'profile': self.obj.current_access(datasets=('primary', 'related')),
                'tagged_sessions': [
                    session.current_access() for session in tagged_sessions
                ],
            }

        elif self.obj.is_organization_profile:
            template_name = 'profile.html.jinja2'

            # `order_by(None)` clears any existing order defined in relationship.
            # We're using it because we want to define our own order here.
            projects = self.obj.listed_projects.order_by(None)
            all_projects = (
                projects.filter(
                    Project.state.PUBLISHED,
                    db.or_(
                        Project.schedule_state.LIVE, Project.schedule_state.UPCOMING
                    ),
                )
                .order_by(Project.schedule_start_at.asc())
                .all()
            )
            upcoming_projects = all_projects[:3]
            all_projects = all_projects[3:]
            featured_project = (
                projects.filter(
                    Project.state.PUBLISHED,
                    db.or_(
                        Project.schedule_state.LIVE, Project.schedule_state.UPCOMING
                    ),
                    Project.featured.is_(True),
                )
                .order_by(Project.schedule_start_at.asc())
                .limit(1)
                .first()
            )
            if featured_project in upcoming_projects:
                upcoming_projects.remove(featured_project)
            open_cfp_projects = (
                projects.filter(Project.state.PUBLISHED, Project.cfp_state.OPEN)
                .order_by(Project.schedule_start_at.asc())
                .all()
            )

            # If the user is an admin of this profile, show all draft projects.
            # Else, only show the drafts they have a crew role in
            if self.obj.current_roles.admin:
                draft_projects = self.obj.draft_projects
                unscheduled_projects = self.obj.projects.filter(
                    Project.schedule_state.PUBLISHED_WITHOUT_SESSIONS
                ).all()
            else:
                draft_projects = self.obj.draft_projects_for(current_auth.user)
                unscheduled_projects = self.obj.unscheduled_projects_for(
                    current_auth.user
                )

            ctx = {
                'template': template_name,
                'profile': self.obj.current_access(datasets=('primary', 'related')),
                'all_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in all_projects
                ],
                'unscheduled_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in unscheduled_projects
                ],
                'upcoming_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in upcoming_projects
                ],
                'open_cfp_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in open_cfp_projects
                ],
                'draft_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in draft_projects
                ],
                'featured_project': (
                    featured_project.current_access(
                        datasets=('without_parent', 'related')
                    )
                    if featured_project
                    else None
                ),
                'project_save_form': SavedProjectForm(),
            }
        else:
            abort(404)  # Reserved profile

        return ctx

    @route('in/projects')
    @render_with('user_profile_projects.html.jinja2', json=True)
    def user_participated_projects(self):
        if self.obj.is_organization_profile:
            abort(404)

        submitted_proposals = self.obj.user.speaker_at.filter(
            ~(Proposal.state.DRAFT), ~(Proposal.state.DELETED)
        ).all()

        participated_project_ids = [
            proposal.project_id for proposal in submitted_proposals
        ] + [project.id for project in self.obj.user.projects_as_crew]

        participated_projects = Project.query.filter(
            Project.state.PUBLISHED,
            Project.schedule_start_at.isnot(None),
            Project.id.in_(set(participated_project_ids)),
        ).order_by(Project.schedule_start_at.desc())

        return {
            'profile': self.obj.current_access(datasets=('primary', 'related')),
            'participated_projects': [
                project.current_access(datasets=('without_parent', 'related'))
                for project in participated_projects
            ],
        }

    @route('in/proposals')
    @render_with('user_profile_proposals.html.jinja2', json=True)
    def user_proposals(self):
        if self.obj.is_organization_profile:
            abort(404)

        submitted_proposals = self.obj.user.speaker_at.filter(
            ~(Proposal.state.DRAFT), ~(Proposal.state.DELETED)
        ).all()

        return {
            'profile': self.obj.current_access(datasets=('primary', 'related')),
            'submitted_proposals': [
                proposal.current_access(datasets=('without_parent', 'related'))
                for proposal in submitted_proposals
            ],
        }

    @route('past.json')
    @requestargs(('page', int), ('per_page', int))
    def past_projects_json(self, page=1, per_page=10):
        projects = self.obj.listed_projects.order_by(None)
        past_projects = projects.filter(
            Project.state.PUBLISHED, Project.schedule_state.PAST
        ).order_by(Project.schedule_start_at.desc())
        pagination = past_projects.paginate(page=page, per_page=per_page)
        return {
            'status': 'ok',
            'title': 'Past projects',
            'headings': ['Date', 'Project', 'Location'],
            'next_page': (
                pagination.page + 1 if pagination.page < pagination.pages else ''
            ),
            'total_pages': pagination.pages,
            'past_projects': [
                {
                    'title': p.title,
                    'datetime': date_filter(
                        p.schedule_start_at_localized, format='dd MMM yyyy'
                    ),
                    'venue': p.primary_venue.city if p.primary_venue else p.location,
                    'url': p.url_for(),
                }
                for p in pagination.items
            ],
        }

    @route('edit', methods=['GET', 'POST'])
    @requires_roles({'admin'})
    def edit(self):
        form = ProfileForm(obj=self.obj, model=Profile)
        if self.obj.user:
            form.make_for_user()
        if form.validate_on_submit():
            form.populate_obj(self.obj)
            db.session.commit()
            flash(_("Your changes have been saved"), 'info')
            return redirect(self.obj.url_for(), code=303)
        return render_form(
            form=form,
            title=_("Edit profile details"),
            submit=_("Save changes"),
            cancel_url=self.obj.url_for(),
            ajax=False,
        )

    @route('edit_logo', methods=['GET', 'POST'])
    @requires_roles({'admin'})
    def edit_logo_url(self):
        form = ProfileLogoForm(obj=self.obj)
        if request.method == 'POST':
            if form.validate_on_submit():
                form.populate_obj(self.obj)
                db.session.commit()
                flash(_("Your changes have been saved"), 'info')
                return render_redirect(self.obj.url_for(), code=303)
            else:
                return render_form(
                    form=form, title=_(""), submit=_("Save logo"), ajax=True,
                )
        return render_form(
            form=form,
            title=_(""),
            submit=_("Save logo"),
            ajax=True,
            template='img_upload_formlayout.html.jinja2',
        )

    @route('edit_banner', methods=['GET', 'POST'])
    @requires_roles({'admin'})
    def edit_banner_image_url(self):
        form = ProfileBannerForm(obj=self.obj)
        if request.method == 'POST':
            if form.validate_on_submit():
                form.populate_obj(self.obj)
                db.session.commit()
                flash(_("Your changes have been saved"), 'info')
                return render_redirect(self.obj.url_for(), code=303)
            else:
                return render_form(
                    form=form, title=_(""), submit=_("Save banner"), ajax=True,
                )
        return render_form(
            form=form,
            title=_(""),
            submit=_("Save banner"),
            ajax=True,
            template='img_upload_formlayout.html.jinja2',
        )

    @route('transition', methods=['POST'])
    @requires_login
    @requires_roles({'owner'})
    def transition(self):
        form = self.obj.forms.transition(obj=self.obj)
        if form.validate_on_submit():
            transition_name = form.transition.data
            getattr(self.obj, transition_name)()
            db.session.commit()
            flash(_("Your changes have been saved"), 'info')
        else:
            flash(
                _("There was a problem saving your changes. Please try again"), 'error'
            )
        return redirect(get_next_url(referrer=True), code=303)

@route('/', subdomain='<profile>')
class FunnelProfileView(ProfileView):
    @route('')
    @render_with('funnelindex.html.jinja2')
    @requires_roles({'reader'})
    def view(self):
        return {'profile': self.obj, 'projects': self.obj.listed_projects}

ProfileView.init_app(app)
FunnelProfileView.init_app(funnelapp)
<commit_msg>fixed role for user profile views (#812)<commit_after>from flask import Response, abort, flash, redirect, render_template, request

from baseframe import _
from baseframe.filters import date_filter
from baseframe.forms import render_form, render_redirect
from coaster.auth import current_auth
from coaster.views import (
    ModelView,
    UrlChangeCheck,
    UrlForView,
    get_next_url,
    render_with,
    requestargs,
    requires_roles,
    route,
)

from .. import app, funnelapp
from ..forms import ProfileBannerForm, ProfileForm, ProfileLogoForm, SavedProjectForm
from ..models import Profile, Project, Proposal, db
from .decorators import legacy_redirect
from .login_session import requires_login
from .mixins import ProfileViewMixin

@Profile.features('new_project')
def feature_profile_new_project(obj):
    return (
        obj.is_organization_profile
        and obj.current_roles.admin
        and bool(obj.state.PUBLIC)
    )

@Profile.features('new_user_project')
def feature_profile_new_user_project(obj):
    return obj.is_user_profile and obj.current_roles.admin and bool(obj.state.PUBLIC)

@Profile.features('make_public')
def feature_profile_make_public(obj):
    return obj.current_roles.admin and not bool(obj.state.PUBLIC)

@Profile.features('make_private')
def feature_profile_make_private(obj):
    return obj.current_roles.admin and bool(obj.state.PUBLIC)

def template_switcher(templateargs):
    template = templateargs.pop('template')
    return Response(render_template(template, **templateargs), mimetype='text/html')

@Profile.views('main')
@route('/<profile>')
class ProfileView(ProfileViewMixin, UrlChangeCheck, UrlForView, ModelView):
    __decorators__ = [legacy_redirect]

    @route('')
    @render_with({'*/*': template_switcher}, json=True)
    @requires_roles({'reader', 'admin'})
    def view(self):
        template_name = None
        ctx = {}

        if self.obj.is_user_profile:
            template_name = 'user_profile.html.jinja2'

            submitted_proposals = self.obj.user.speaker_at.filter(
                ~(Proposal.state.DRAFT), ~(Proposal.state.DELETED)
            ).all()

            tagged_sessions = [
                proposal.session
                for proposal in submitted_proposals
                if proposal.session is not None
            ]

            ctx = {
                'template': template_name,
                'profile': self.obj.current_access(datasets=('primary', 'related')),
                'tagged_sessions': [
                    session.current_access() for session in tagged_sessions
                ],
            }

        elif self.obj.is_organization_profile:
            template_name = 'profile.html.jinja2'

            # `order_by(None)` clears any existing order defined in relationship.
            # We're using it because we want to define our own order here.
            projects = self.obj.listed_projects.order_by(None)
            all_projects = (
                projects.filter(
                    Project.state.PUBLISHED,
                    db.or_(
                        Project.schedule_state.LIVE, Project.schedule_state.UPCOMING
                    ),
                )
                .order_by(Project.schedule_start_at.asc())
                .all()
            )
            upcoming_projects = all_projects[:3]
            all_projects = all_projects[3:]
            featured_project = (
                projects.filter(
                    Project.state.PUBLISHED,
                    db.or_(
                        Project.schedule_state.LIVE, Project.schedule_state.UPCOMING
                    ),
                    Project.featured.is_(True),
                )
                .order_by(Project.schedule_start_at.asc())
                .limit(1)
                .first()
            )
            if featured_project in upcoming_projects:
                upcoming_projects.remove(featured_project)
            open_cfp_projects = (
                projects.filter(Project.state.PUBLISHED, Project.cfp_state.OPEN)
                .order_by(Project.schedule_start_at.asc())
                .all()
            )

            # If the user is an admin of this profile, show all draft projects.
            # Else, only show the drafts they have a crew role in
            if self.obj.current_roles.admin:
                draft_projects = self.obj.draft_projects
                unscheduled_projects = self.obj.projects.filter(
                    Project.schedule_state.PUBLISHED_WITHOUT_SESSIONS
                ).all()
            else:
                draft_projects = self.obj.draft_projects_for(current_auth.user)
                unscheduled_projects = self.obj.unscheduled_projects_for(
                    current_auth.user
                )

            ctx = {
                'template': template_name,
                'profile': self.obj.current_access(datasets=('primary', 'related')),
                'all_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in all_projects
                ],
                'unscheduled_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in unscheduled_projects
                ],
                'upcoming_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in upcoming_projects
                ],
                'open_cfp_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in open_cfp_projects
                ],
                'draft_projects': [
                    p.current_access(datasets=('without_parent', 'related'))
                    for p in draft_projects
                ],
                'featured_project': (
                    featured_project.current_access(
                        datasets=('without_parent', 'related')
                    )
                    if featured_project
                    else None
                ),
                'project_save_form': SavedProjectForm(),
            }
        else:
            abort(404)  # Reserved profile

        return ctx

    @route('in/projects')
    @render_with('user_profile_projects.html.jinja2', json=True)
    @requires_roles({'reader', 'admin'})
    def user_participated_projects(self):
        if self.obj.is_organization_profile:
            abort(404)

        submitted_proposals = self.obj.user.speaker_at.filter(
            ~(Proposal.state.DRAFT), ~(Proposal.state.DELETED)
        ).all()

        participated_project_ids = [
            proposal.project_id for proposal in submitted_proposals
        ] + [project.id for project in self.obj.user.projects_as_crew]

        participated_projects = Project.query.filter(
            Project.state.PUBLISHED,
            Project.schedule_start_at.isnot(None),
            Project.id.in_(set(participated_project_ids)),
        ).order_by(Project.schedule_start_at.desc())

        return {
            'profile': self.obj.current_access(datasets=('primary', 'related')),
            'participated_projects': [
                project.current_access(datasets=('without_parent', 'related'))
                for project in participated_projects
            ],
        }

    @route('in/proposals')
    @render_with('user_profile_proposals.html.jinja2', json=True)
    @requires_roles({'reader', 'admin'})
    def user_proposals(self):
        if self.obj.is_organization_profile:
            abort(404)

        submitted_proposals = self.obj.user.speaker_at.filter(
            ~(Proposal.state.DRAFT), ~(Proposal.state.DELETED)
        ).all()

        return {
            'profile': self.obj.current_access(datasets=('primary', 'related')),
            'submitted_proposals': [
                proposal.current_access(datasets=('without_parent', 'related'))
                for proposal in submitted_proposals
            ],
        }

    @route('past.json')
    @requestargs(('page', int), ('per_page', int))
    def past_projects_json(self, page=1, per_page=10):
        projects = self.obj.listed_projects.order_by(None)
        past_projects = projects.filter(
            Project.state.PUBLISHED, Project.schedule_state.PAST
        ).order_by(Project.schedule_start_at.desc())
        pagination = past_projects.paginate(page=page, per_page=per_page)
        return {
            'status': 'ok',
            'title': 'Past projects',
            'headings': ['Date', 'Project', 'Location'],
            'next_page': (
                pagination.page + 1 if pagination.page < pagination.pages else ''
            ),
            'total_pages': pagination.pages,
            'past_projects': [
                {
                    'title': p.title,
                    'datetime': date_filter(
                        p.schedule_start_at_localized, format='dd MMM yyyy'
                    ),
                    'venue': p.primary_venue.city if p.primary_venue else p.location,
                    'url': p.url_for(),
                }
                for p in pagination.items
            ],
        }

    @route('edit', methods=['GET', 'POST'])
    @requires_roles({'admin'})
    def edit(self):
        form = ProfileForm(obj=self.obj, model=Profile)
        if self.obj.user:
            form.make_for_user()
        if form.validate_on_submit():
            form.populate_obj(self.obj)
            db.session.commit()
            flash(_("Your changes have been saved"), 'info')
            return redirect(self.obj.url_for(), code=303)
        return render_form(
            form=form,
            title=_("Edit profile details"),
            submit=_("Save changes"),
            cancel_url=self.obj.url_for(),
            ajax=False,
        )

    @route('edit_logo', methods=['GET', 'POST'])
    @requires_roles({'admin'})
    def edit_logo_url(self):
        form = ProfileLogoForm(obj=self.obj)
        if request.method == 'POST':
            if form.validate_on_submit():
                form.populate_obj(self.obj)
                db.session.commit()
                flash(_("Your changes have been saved"), 'info')
                return render_redirect(self.obj.url_for(), code=303)
            else:
                return render_form(
                    form=form, title=_(""), submit=_("Save logo"), ajax=True,
                )
        return render_form(
            form=form,
            title=_(""),
            submit=_("Save logo"),
            ajax=True,
            template='img_upload_formlayout.html.jinja2',
        )

    @route('edit_banner', methods=['GET', 'POST'])
    @requires_roles({'admin'})
    def edit_banner_image_url(self):
        form = ProfileBannerForm(obj=self.obj)
        if request.method == 'POST':
            if form.validate_on_submit():
                form.populate_obj(self.obj)
                db.session.commit()
                flash(_("Your changes have been saved"), 'info')
                return render_redirect(self.obj.url_for(), code=303)
            else:
                return render_form(
                    form=form, title=_(""), submit=_("Save banner"), ajax=True,
                )
        return render_form(
            form=form,
            title=_(""),
            submit=_("Save banner"),
            ajax=True,
            template='img_upload_formlayout.html.jinja2',
        )

    @route('transition', methods=['POST'])
    @requires_login
    @requires_roles({'owner'})
    def transition(self):
        form = self.obj.forms.transition(obj=self.obj)
        if form.validate_on_submit():
            transition_name = form.transition.data
            getattr(self.obj, transition_name)()
            db.session.commit()
            flash(_("Your changes have been saved"), 'info')
        else:
            flash(
                _("There was a problem saving your changes. Please try again"), 'error'
            )
        return redirect(get_next_url(referrer=True), code=303)

@route('/', subdomain='<profile>')
class FunnelProfileView(ProfileView):
    @route('')
    @render_with('funnelindex.html.jinja2')
    @requires_roles({'reader'})
    def view(self):
        return {'profile': self.obj, 'projects': self.obj.listed_projects}

ProfileView.init_app(app)
FunnelProfileView.init_app(funnelapp)
<|endoftext|>