<commit_before># -*- coding: utf-8 -*-

#	SAGA Modules plugin for Quantum GIS
#
#	plugin.py (C) Camilo Polymeris
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#   MA 02110-1301, USA.

from PyQt4.QtCore import *
from PyQt4.QtGui import *
from qgis.core import *

import os
import processingplugin.processing as processing
import processingplugin.processing.parameters as parameters
import otbApplication as otb

class OTBPlugin():
    def __init__(self, iface):
        self._modules = None
        self._iface = iface

    def initGui(self):
        processing.framework.registerModuleProvider(self)

    def unload(self):
        pass # TODO : unload the modules

    def modules(self):
        if self._modules is None:
            apps = otb.Registry.GetAvailableApplications()
            print "available apps : " + str(apps)
            self._modules = set()
            for app in apps:
                self._modules.add(OTBModule(app, self._iface))
        return self._modules

class OTBModule(processing.Module):
    def __init__(self, modulename, iface):
        print "OTBModule __init__ " + modulename

        self._app = otb.Registry.CreateApplication(modulename)
        self._iface = iface
        processing.Module.__init__(self,
            self._app.GetName(),
            self._app.GetDescription())

        self._parameters = []
        self._instance = OTBModuleInstance(self)

        self.layerRegistry = QgsMapLayerRegistry.instance()

        for p in self._app.GetParametersKeys():
            #print p
            self.addParameter(p)

    def instance(self):
        return self._instance

    def addParameter(self, otbParamKey):
        print "addParameter " + otbParamKey
        otbToQGisParam = {
            otb.ParameterType_Empty:               parameters.BooleanParameter,
            otb.ParameterType_Int:                 parameters.NumericParameter,
            otb.ParameterType_Float:               parameters.NumericParameter,
            otb.ParameterType_String:              parameters.StringParameter,
            otb.ParameterType_Filename:            parameters.PathParameter,
            otb.ParameterType_Directory:           parameters.PathParameter,
            otb.ParameterType_Choice:              parameters.ChoiceParameter,
            otb.ParameterType_InputImage:          parameters.RasterLayerParameter,
            otb.ParameterType_InputComplexImage:   parameters.RasterLayerParameter,
            otb.ParameterType_InputVectorData:     parameters.VectorLayerParameter,
            otb.ParameterType_OutputImage:         parameters.PathParameter,
            otb.ParameterType_OutputVectorData:    parameters.PathParameter,
            otb.ParameterType_Radius:              parameters.NumericParameter
        }

        name = otbParamKey
        descr = self._app.GetParameterName(otbParamKey)
        typ = self._app.GetParameterType(otbParamKey)
        role = parameters.Parameter.Role.input # TODO
        mandatory = self._app.IsMandatory(otbParamKey)

        try:
            qgisParamClass = otbToQGisParam[typ]

            if not qgisParamClass: # We are ignoring certain types of parameters
                print "Ignoring type " + str(typ)
                return

            qgisParam = qgisParamClass(name, descr, role=role)

            qgisParam._app = self._app
            qgisParam._key = otbParamKey

            if qgisParamClass == parameters.NumericParameter:
                # TODO (handle no value case) qgisParam.setDefaultValue(self._app.GetParameterFloat(otbParamKey))
                qgisParam.setDefaultValue(0)

            elif qgisParamClass == parameters.ChoiceParameter:
                qgisParam.setChoices(self._app.GetChoiceNames(otbParamKey))
                qgisParam.setDefaultValue(self._app.GetParameterInt(otbParamKey))

            elif (qgisParamClass == parameters.VectorLayerParameter or
                qgisParamClass == parameters.RasterLayerParameter):
                if role == parameters.Parameter.Role.output:
                    self._instance.outLayer.append(qgisParam)
                else:
                    self._instance.inLayer.append(qgisParam)

                # force update of layer
                self.onParameterChanged(qgisParam, None)

            # TODO validator

            qgisParam.setMandatory(mandatory)
            self._parameters.append(qgisParam)

            # register callback to instance for parameter
            print "connecting signal " + str(self._instance.valueChangedSignal(qgisParam))
            QObject.connect(self._instance,
                            self._instance.valueChangedSignal(qgisParam),
                            lambda x: self.onParameterChanged(qgisParam, x))

        except KeyError:
            print otbParamKey + " is of unhandled parameter type."
            pass

    def parameters(self):
        return self._parameters

    def tags(self):
        return processing.Module.tags(self) | set([processing.Tag('otb')])

    def onParameterChanged(self, qgisParam, value):
        pc = qgisParam.__class__
        app = qgisParam._app
        key = qgisParam._key
        ptype = app.GetParameterType(key)
        print "onParameterChanged " + key + " " + str(value)

        if ptype == otb.ParameterType_Empty:
            if value :
              print "Enable " + key
              app.EnableParameter(key)
            else :
              print "Disable " + key
              app.DisableParameter(key)

        elif ptype == otb.ParameterType_Int or ptype == otb.ParameterType_Radius:
            print "SetParameterInt " + key
            app.SetParameterInt(key, int(value))

        elif ptype == otb.ParameterType_Float:
            print "SetParameterFloat " + key
            app.SetParameterFloat(key, float(value))

        elif ptype == otb.ParameterType_String \
             or ptype == otb.ParameterType_Filename \
             or ptype == otb.ParameterType_Directory \
             or ptype == otb.ParameterType_String:
            print "SetParameterString " + key
            app.SetParameterString(key, value)

        elif ptype == otb.ParameterType_InputImage \
             or ptype == otb.ParameterType_InputComplexImage:
            if value is None: return
            qgisParam.layer = value
            dpUri = str(qgisParam.layer.dataProvider().dataSourceUri())
            dpDescription = qgisParam.layer.dataProvider().description()
            isLocal = dpDescription.startsWith('GDAL provider')
            if isLocal:
              print "Setting " + dpUri + " to param RasterLayerParameter " + key
              app.SetParameterString(key, dpUri)

        elif ptype == otb.ParameterType_InputVectorData:
            if value is None: return
            qgisParam.layer = value
            dpUri = str(qgisParam.layer.dataProvider().dataSourceUri())
            dpDescription = qgisParam.layer.dataProvider().description()
            isLocal = dpDescription.startsWith('OGR data provider')
            if isLocal:
              print "Setting " + dpUri + " to param VectorLayerParameter " + key
              app.SetParameterString(key, dpUri)

        elif ptype == otb.ParameterType_OutputImage:
            print "SetParameterString " + key
            app.SetParameterString(key, str(value))

        elif ptype == otb.ParameterType_OutputVectorData:
            print "SetParameterString " + key
            app.SetParameterString(key, str(value))

class OTBModuleInstance(processing.ModuleInstance):
    def __init__(self, module):
        print "OTBModuleInstance  " + module.name()
        processing.ModuleInstance.__init__(self, module)
        QObject.connect(
            self, self.valueChangedSignal(self.stateParameter),
            self.stateParameterValueChanged)
        self.inLayer = list()
        self.outLayer = list()

    def stateParameterValueChanged(self, state):
        """ Only reacts to start running state, ignore others.
        """
        print "stateParameterValueChanged " + str(state)
        if state != parameters.StateParameter.State.running:
            return
        modName = self.module().name()

        print "Executing " + modName
        app = self.module()._app

        #try:
        app.ExecuteAndWriteOutput()

        # umm- what if there is no iface?
        iface = self.module()._iface

        # now import output layers
        for key in app.GetParametersKeys():
            if app.GetParameterType(key) == otb.ParameterType_OutputImage:
                iface.addRasterLayer(app.GetParameterString(key))
            elif app.GetParameterType(key) == otb.ParameterType_OutputVectorData:
                basename = os.path.splitext( os.path.basename(app.GetParameterString(key)) )[0]
                iface.addVectorLayer(app.GetParameterString(key), basename, "ogr")

        self.setFeedback("OTB Module execution suceeded.")
        #except:
        #  self.setFeedback("OTB Module execution failed.")

        self.setState(parameters.StateParameter.State.stopped)

<commit_msg>ENH: several improvements to QGis plugin - add progress reporting - initialize parameters to their defaultValue - update all parameters widget each time one parameter is changed<commit_after># -*- coding: utf-8 -*-

#	SAGA Modules plugin for Quantum GIS
#
#	plugin.py (C) Camilo Polymeris
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#   MA 02110-1301, USA.

from PyQt4.QtCore import *
from PyQt4.QtGui import *
from qgis.core import *

import os
import processing
import processing.parameters as parameters
import otbApplication as otb

class OTBPlugin():
    def __init__(self, iface):
        self._modules = None
        self._iface = iface

    def initGui(self):
        processing.framework.registerModuleProvider(self)

    def unload(self):
        pass # TODO : unload the modules

    def modules(self):
        if self._modules is None:
            apps = otb.Registry.GetAvailableApplications()
            self._modules = set()
            for app in apps:
                self._modules.add(OTBModule(app, self._iface))
        return self._modules

class OTBModule(processing.Module):
    def __init__(self, modulename, iface):
        self._app = otb.Registry.CreateApplication(modulename)
        self._iface = iface
        processing.Module.__init__(self,
            self._app.GetName(),
            self._app.GetDescription())

        self._blockSignals = False
        self._parameters = []
        self._instance = OTBModuleInstance(self)

        self.layerRegistry = QgsMapLayerRegistry.instance()

        for p in self._app.GetParametersKeys():
            self.addParameter(p)

    def instance(self):
        return self._instance

    def addParameter(self, otbParamKey):
        otbToQGisParam = {
            otb.ParameterType_Empty:               parameters.BooleanParameter,
            otb.ParameterType_Int:                 parameters.NumericParameter,
            otb.ParameterType_Float:               parameters.NumericParameter,
            otb.ParameterType_String:              parameters.StringParameter,
            otb.ParameterType_Filename:            parameters.PathParameter,
            otb.ParameterType_Directory:           parameters.PathParameter,
            otb.ParameterType_Choice:              parameters.ChoiceParameter,
            otb.ParameterType_InputImage:          parameters.RasterLayerParameter,
            otb.ParameterType_InputComplexImage:   parameters.RasterLayerParameter,
            otb.ParameterType_InputVectorData:     parameters.VectorLayerParameter,
            otb.ParameterType_OutputImage:         parameters.PathParameter,
            otb.ParameterType_OutputVectorData:    parameters.PathParameter,
            otb.ParameterType_Radius:              parameters.NumericParameter
        }

        name = otbParamKey
        descr = self._app.GetParameterName(otbParamKey)
        typ = self._app.GetParameterType(otbParamKey)
        role = parameters.Parameter.Role.input # TODO
        mandatory = self._app.IsMandatory(otbParamKey)

        try:
            qgisParamClass = otbToQGisParam[typ]

            if not qgisParamClass: # We are ignoring certain types of parameters
                print "Ignoring type " + str(typ)
                return

            qgisParam = qgisParamClass(name, descr, role=role)

            qgisParam._app = self._app
            qgisParam._key = otbParamKey

            try:
              if typ == otb.ParameterType_Empty:
                qgisParam.setDefaultValue(self._app.IsParameterEnabled(otbParamKey))
              elif typ == otb.ParameterType_Int:
                qgisParam.setDefaultValue(self._app.GetParameterInt(otbParamKey))
              elif typ == otb.ParameterType_Float:
                qgisParam.setDefaultValue(self._app.GetParameterFloat(otbParamKey))
              elif typ == otb.ParameterType_String:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_Filename:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_Directory:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_Choice:
                qgisParam.setChoices(self._app.GetChoiceNames(otbParamKey))
                qgisParam.setDefaultValue(self._app.GetParameterInt(otbParamKey))
              elif typ == otb.ParameterType_InputImage:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_InputComplexImage:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_InputVectorData:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_OutputImage:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_OutputVectorData:
                qgisParam.setDefaultValue(self._app.GetParameterString(otbParamKey))
              elif typ == otb.ParameterType_Radius:
                qgisParam.setDefaultValue(self._app.GetParameterInt(otbParamKey))
            except:
              pass
            # TODO validator

            qgisParam.setMandatory(mandatory)
            self._parameters.append(qgisParam)

            # register callback to instance for parameter
            QObject.connect(self._instance,
                            self._instance.valueChangedSignal(qgisParam),
                            lambda x: self.onParameterChanged(qgisParam, x))

        except KeyError:
            print otbParamKey + " is of unhandled parameter type."
            pass

    def parameters(self):
        return self._parameters

    def tags(self):
        return processing.Module.tags(self) | set([processing.Tag('otb')])

    def onParameterChanged(self, qgisParam, value):
        if self._blockSignals == True:
            print "signals blocked"
            return

        self._blockSignals = True

        pc = qgisParam.__class__
        app = qgisParam._app
        key = qgisParam._key
        ptype = app.GetParameterType(key)
        print "onParameterChanged " + key + " " + str(value)

        if ptype == otb.ParameterType_Empty:
            if value :
              print "Enable " + key
              app.EnableParameter(key)
            else :
              print "Disable " + key
              app.DisableParameter(key)

        elif ptype == otb.ParameterType_Int or ptype == otb.ParameterType_Radius:
            print "SetParameterInt " + key
            app.SetParameterInt(key, int(value))

        elif ptype == otb.ParameterType_Float:
            print "SetParameterFloat " + key
            app.SetParameterFloat(key, float(value))

        elif ptype == otb.ParameterType_Choice:
            print "SetParameterInt " + key
            app.SetParameterInt(key, int(value))

        elif ptype == otb.ParameterType_String \
             or ptype == otb.ParameterType_Filename \
             or ptype == otb.ParameterType_Directory:
            print "SetParameterString " + key
            app.SetParameterString(key, str(value))

        elif ptype == otb.ParameterType_InputImage \
             or ptype == otb.ParameterType_InputComplexImage:
            if value is None: return
            qgisParam.layer = value
            dpUri = str(qgisParam.layer.dataProvider().dataSourceUri())
            dpDescription = qgisParam.layer.dataProvider().description()
            isLocal = dpDescription.startsWith('GDAL provider')
            if isLocal:
              print "Setting " + dpUri + " to param RasterLayerParameter " + key
              app.SetParameterString(key, dpUri)

        elif ptype == otb.ParameterType_InputVectorData:
            if value is None: return
            qgisParam.layer = value
            dpUri = str(qgisParam.layer.dataProvider().dataSourceUri())
            dpDescription = qgisParam.layer.dataProvider().description()
            isLocal = dpDescription.startsWith('OGR data provider')
            if isLocal:
              print "Setting " + dpUri + " to param VectorLayerParameter " + key
              app.SetParameterString(key, dpUri)

        elif ptype == otb.ParameterType_OutputImage:
            print "SetParameterString " + key
            app.SetParameterString(key, str(value))

        elif ptype == otb.ParameterType_OutputVectorData:
            print "SetParameterString " + key
            app.SetParameterString(key, str(value))

        elif ptype == otb.ParameterType_StringList:
            print "StringList not supported yet"

        elif ptype == otb.ParameterType_InputImageList:
            print "InputImageList not supported yet"

        elif ptype == otb.ParameterType_ListView:
            print "ListView not supported yet"

        # trigger global update
        app.UpdateParameters()
        self.notifyInternalUpdate()
        self._blockSignals = False

    def notifyInternalUpdate(self):
        """ To update the GUI from application instance content
        """
        for param in self._parameters:
            key = param.name()
            typ = self._app.GetParameterType(key)
            value = None

            try:
              if typ == otb.ParameterType_Empty:
                value = self._app.IsParameterEnabled(key)
              elif typ == otb.ParameterType_Int:
                value = float(self._app.GetParameterInt(key))
              elif typ == otb.ParameterType_Float:
                value = float(self._app.GetParameterFloat(key))
              elif typ == otb.ParameterType_String:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_Filename:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_Directory:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_Choice:
                value = self._app.GetParameterInt(key)
              elif typ == otb.ParameterType_InputImage:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_InputComplexImage:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_InputVectorData:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_OutputImage:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_OutputVectorData:
                value = self._app.GetParameterString(key)
              elif typ == otb.ParameterType_Radius:
                value = self._app.GetParameterInt(key)
            except:
              pass

            signal = self.instance().valueChangedSignal(param)

            if typ != otb.ParameterType_InputImage \
               and  typ != otb.ParameterType_InputComplexImage \
               and typ != otb.ParameterType_InputVectorData \
               and value is not None:
                QObject.emit( self.instance(), signal, value )

class OTBModuleInstance(processing.ModuleInstance):
    def __init__(self, module):
        print "OTBModuleInstance  " + module.name()
        processing.ModuleInstance.__init__(self, module)
        QObject.connect(
            self, self.valueChangedSignal(self.stateParameter),
            self.stateParameterValueChanged)

    def stateParameterValueChanged(self, state):
        """ Only reacts to start running state, ignore others.
        """
        print "stateParameterValueChanged " + str(state)
        if state != parameters.StateParameter.State.running:
            return
        modName = self.module().name()

        print "Executing " + modName
        app = self.module()._app

        if not app.IsApplicationReady():
          self.setFeedback("Missing mandatory parameters")
          self.setState(parameters.StateParameter.State.stopped)
          return

        command = otb.itkPyCommand.New()
        command.SetCommandCallable(self.newProgressSource)
        app.AddObserver( otb.AddProcessToWatchEvent(), command.GetPointer() )
        #try:
        app.ExecuteAndWriteOutput()

        # umm- what if there is no iface?
        iface = self.module()._iface

        # now import output layers
        for key in app.GetParametersKeys():
            if app.GetParameterType(key) == otb.ParameterType_OutputImage:
                iface.addRasterLayer(app.GetParameterString(key))
            elif app.GetParameterType(key) == otb.ParameterType_OutputVectorData:
                basename = os.path.splitext( os.path.basename(app.GetParameterString(key)) )[0]
                iface.addVectorLayer(app.GetParameterString(key), basename, "ogr")

        self.setFeedback("OTB Module execution suceeded.")
        #except:
        #  self.setFeedback("OTB Module execution failed.")

        self.setState(parameters.StateParameter.State.stopped)

    def newProgressSource(self):
        # Setup the feedback with a null progress
        self.setFeedback(self.module()._app.GetProgressDescription(), 0)
        command = otb.itkPyCommand.New()
        command.SetCommandCallable(self.newProgressCallback)
        self.module()._app.GetProgressSource().AddObserver( otb.itkProgressEvent(), command.GetPointer() )

    def newProgressCallback(self):
        # Setup the feedback and the associated progress
        self.setFeedback(self.module()._app.GetProgressDescription(), False, int(100 * self.module()._app.GetProgressSource().GetProgress()))

<|endoftext|>


<commit_before>"""
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Example Genomics Map Reduce
"""

import cloudstorage as gcs
import logging
import os

from common import Common
from genomicsapi import GenomicsAPI
from mapreduce import base_handler
from mapreduce import mapreduce_pipeline

Common.initialize()

def generate_coverage_map(data):
  """Generate coverage map function."""
  (content, sequenceStart, sequenceEnd) = data
  reads = content['reads'] if 'reads' in content else []
  coverage = GenomicsAPI.compute_coverage(reads, sequenceStart,
                                          sequenceEnd)
  for key, value in coverage.iteritems():
    yield (key, value)

def generate_coverage_reduce(key, values):
  """Generate coverage reduce function."""
  yield "%d: %d\n" % (int(key), sum(int(value) for value in values))

def consolidate_output_map(file):
  """Consolidate output map function."""
  for line in file:
    data = line.split(":")
    if len(data) == 2:
      # So that we get the data properly sorted numerically in the end.
      key = str.format("%09d" % int(data[0]))
      yield (key, data[1])

def consolidate_output_reduce(key, values):
  """Consolidate output reduce function."""
  yield "%d: %d\n" % (int(key), sum(int(value) for value in values))

class PipelineGenerateCoverage(base_handler.PipelineBase):
  """A pipeline to generate coverage data

  Args:
    readsetId: the Id of the readset
  """

  def run(self, readsetId, sequenceName, sequenceStart, sequenceEnd,
          useMockData):
    #logging.debug("Running Pipeline for readsetId: %s", readsetId)
    bucket = os.environ['BUCKET']
    shards = os.environ['MAPREDUCE_SHARDS']

    # In the first pipeline, generate the raw coverage data.
    raw_coverage_data = yield mapreduce_pipeline.MapreducePipeline(
      "generate_coverage",
      "pipeline.generate_coverage_map",
      "pipeline.generate_coverage_reduce",
      "input_reader.GenomicsAPIInputReader",
      "mapreduce.output_writers._GoogleCloudStorageOutputWriter",
      mapper_params={
        "input_reader": {
          "readsetId": readsetId,
          "sequenceName": sequenceName,
          "sequenceStart": sequenceStart,
          "sequenceEnd": sequenceEnd,
          "useMockData": useMockData,
        },
      },
      reducer_params={
        "output_writer": {
          "bucket_name": bucket,
          "content_type": "text/plain",
        },
      },
      shards=shards)

    # Pass the results on to the output consolidator.
    yield PipelineConsolidateOutput(readsetId, sequenceName, sequenceStart,
                                    sequenceEnd, raw_coverage_data)

class PipelineConsolidateOutput(base_handler.PipelineBase):
  """A pipeline to proecss the result of the MapReduce job.

  Args:
    raw_coverage_data: the raw coverage data that is to be consolidated.
  """

  def run(self, readsetId, sequenceName, sequenceStart, sequenceEnd,
          raw_coverage_data):
    bucket = os.environ['BUCKET']
    #logging.debug("Got %d raw coverage data output files to consolidate.",
    #              len(raw_coverage_data))

    # Remove bucket from filenames. (Would be nice if you didn't have to do
    # this.
    paths = []
    for file in raw_coverage_data:
      paths.append(str.replace(str(file), "/" + bucket + "/", ""))

    # Create another pipeline to combine the raw coverage data into a single
    # file.
    output = yield mapreduce_pipeline.MapreducePipeline(
      "consolidate_output",
      "pipeline.consolidate_output_map",
      "pipeline.consolidate_output_reduce",
      "mapreduce.input_readers._GoogleCloudStorageInputReader",
      "mapreduce.output_writers._GoogleCloudStorageOutputWriter",
      mapper_params={
        "input_reader": {
           "bucket_name": bucket,
           "objects": paths,
        },
      },
      reducer_params={
        "output_writer": {
          "bucket_name": bucket,
          "content_type": "text/plain",
        },
      },
      shards=1)

    # Return back the final output results.
    yield PipelineReturnResults(readsetId, sequenceName, sequenceStart,
                                sequenceEnd, output)

class PipelineReturnResults(base_handler.PipelineBase):
  """A pipeline to proecss the result of the MapReduce job.

  Args:
    output: the blobstore location where the output of the job is stored
  """

  def run(self, readsetId, sequenceName, sequenceStart, sequenceEnd, output):
    #logging.debug('Number of output files: %d', len(output))
    file = output[0]

    # If you have a setting to copy it over, do so
    dir = os.environ['OUTPUT_DIRECTORY']
    if dir is not None:
      bucket = os.environ['BUCKET']
      src = file
      dst = str.format("/%s/%s/%s_%s_%s-%s.txt" % (
        bucket, dir, str(readsetId), str(sequenceName), sequenceStart,
        sequenceEnd))
      gcs._copy2(src, dst)
      logging.info("Copied output file to: %s" % dst)
      file = dst

    if os.environ['SERVER_SOFTWARE'].startswith('Development'):
      url = "http://localhost:8080/_ah/gcs" + file
    else:
      url = "https://storage.cloud.google.com" + file
    logging.info("Genomics pipeline completed. Results: %s", url)

<commit_msg>Comment typo<commit_after>"""
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Example Genomics Map Reduce
"""

import cloudstorage as gcs
import logging
import os

from common import Common
from genomicsapi import GenomicsAPI
from mapreduce import base_handler
from mapreduce import mapreduce_pipeline

Common.initialize()

def generate_coverage_map(data):
  """Generate coverage map function."""
  (content, sequenceStart, sequenceEnd) = data
  reads = content['reads'] if 'reads' in content else []
  coverage = GenomicsAPI.compute_coverage(reads, sequenceStart,
                                          sequenceEnd)
  for key, value in coverage.iteritems():
    yield (key, value)

def generate_coverage_reduce(key, values):
  """Generate coverage reduce function."""
  yield "%d: %d\n" % (int(key), sum(int(value) for value in values))

def consolidate_output_map(file):
  """Consolidate output map function."""
  for line in file:
    data = line.split(":")
    if len(data) == 2:
      # So that we get the data properly sorted numerically in the end.
      key = str.format("%09d" % int(data[0]))
      yield (key, data[1])

def consolidate_output_reduce(key, values):
  """Consolidate output reduce function."""
  yield "%d: %d\n" % (int(key), sum(int(value) for value in values))

class PipelineGenerateCoverage(base_handler.PipelineBase):
  """A pipeline to generate coverage data

  Args:
    readsetId: the Id of the readset
  """

  def run(self, readsetId, sequenceName, sequenceStart, sequenceEnd,
          useMockData):
    #logging.debug("Running Pipeline for readsetId: %s", readsetId)
    bucket = os.environ['BUCKET']
    shards = os.environ['MAPREDUCE_SHARDS']

    # In the first pipeline, generate the raw coverage data.
    raw_coverage_data = yield mapreduce_pipeline.MapreducePipeline(
      "generate_coverage",
      "pipeline.generate_coverage_map",
      "pipeline.generate_coverage_reduce",
      "input_reader.GenomicsAPIInputReader",
      "mapreduce.output_writers._GoogleCloudStorageOutputWriter",
      mapper_params={
        "input_reader": {
          "readsetId": readsetId,
          "sequenceName": sequenceName,
          "sequenceStart": sequenceStart,
          "sequenceEnd": sequenceEnd,
          "useMockData": useMockData,
        },
      },
      reducer_params={
        "output_writer": {
          "bucket_name": bucket,
          "content_type": "text/plain",
        },
      },
      shards=shards)

    # Pass the results on to the output consolidator.
    yield PipelineConsolidateOutput(readsetId, sequenceName, sequenceStart,
                                    sequenceEnd, raw_coverage_data)

class PipelineConsolidateOutput(base_handler.PipelineBase):
  """A pipeline to process the result of the MapReduce job.

  Args:
    raw_coverage_data: the raw coverage data that is to be consolidated.
  """

  def run(self, readsetId, sequenceName, sequenceStart, sequenceEnd,
          raw_coverage_data):
    bucket = os.environ['BUCKET']
    #logging.debug("Got %d raw coverage data output files to consolidate.",
    #              len(raw_coverage_data))

    # Remove bucket from filenames. (Would be nice if you didn't have to do
    # this.
    paths = []
    for file in raw_coverage_data:
      paths.append(str.replace(str(file), "/" + bucket + "/", ""))

    # Create another pipeline to combine the raw coverage data into a single
    # file.
    output = yield mapreduce_pipeline.MapreducePipeline(
      "consolidate_output",
      "pipeline.consolidate_output_map",
      "pipeline.consolidate_output_reduce",
      "mapreduce.input_readers._GoogleCloudStorageInputReader",
      "mapreduce.output_writers._GoogleCloudStorageOutputWriter",
      mapper_params={
        "input_reader": {
           "bucket_name": bucket,
           "objects": paths,
        },
      },
      reducer_params={
        "output_writer": {
          "bucket_name": bucket,
          "content_type": "text/plain",
        },
      },
      shards=1)

    # Return back the final output results.
    yield PipelineReturnResults(readsetId, sequenceName, sequenceStart,
                                sequenceEnd, output)

class PipelineReturnResults(base_handler.PipelineBase):
  """A pipeline to proecss the result of the MapReduce job.

  Args:
    output: the blobstore location where the output of the job is stored
  """

  def run(self, readsetId, sequenceName, sequenceStart, sequenceEnd, output):
    #logging.debug('Number of output files: %d', len(output))
    file = output[0]

    # If you have a setting to copy it over, do so
    dir = os.environ['OUTPUT_DIRECTORY']
    if dir is not None:
      bucket = os.environ['BUCKET']
      src = file
      dst = str.format("/%s/%s/%s_%s_%s-%s.txt" % (
        bucket, dir, str(readsetId), str(sequenceName), sequenceStart,
        sequenceEnd))
      gcs._copy2(src, dst)
      logging.info("Copied output file to: %s" % dst)
      file = dst

    if os.environ['SERVER_SOFTWARE'].startswith('Development'):
      url = "http://localhost:8080/_ah/gcs" + file
    else:
      url = "https://storage.cloud.google.com" + file
    logging.info("Genomics pipeline completed. Results: %s", url)

<|endoftext|>


<commit_before># Copyright 2013 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import marconi
from marconi.tests import util as testing

class TestSimple(testing.TestBase):

    def test_simple(self):
        """Doesn't really test much."""
        conf_file = self.conf_path('wsgi_reference.conf')
        boot = marconi.Bootstrap(conf_file)
        transport = boot.transport
        wsgi_app = transport.app  # NOQA
        self.assertTrue(True)
<commit_msg>chore: Remove unused test<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2013-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Generate seeds.txt from Pieter's DNS seeder
#

import collections
import dns.resolver
import re
import sys

NSEEDS = 512

MAX_SEEDS_PER_ASN = 2

MIN_BLOCKS = 540000

# These are hosts that have been observed to be behaving strangely (e.g.
# aggressively connecting to every node).
SUSPICIOUS_HOSTS = {
    "23.92.36.9", "72.36.89.11", "130.211.129.106", "178.63.107.226",
    "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
    "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
    "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
    "54.94.195.96", "54.94.200.247"
}

PATTERN_IPV4 = re.compile(
    r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
PATTERN_ONION = re.compile(
    r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")

# Used to only select nodes with a user agent string compatible with the
# BCH/UAHF specification.
PATTERN_AGENT = re.compile(
    r"^(/Bitcoin ABC:0.21.(\d+)\(.+\)/|/bchd:0.15.(\d+)\(.+\)/|/BUCash:1.7.(\d+)\(.+\)/)")

def parseline(line):
    sline = line.split()
    if len(sline) < 11:
        return None

    # The user agent is at the end of the line. It may contain space, so we
    # concatenate.
    for i in range(12, len(sline)):
        sline[11] += ' ' + sline[i]

    # Remove leftovers
    del sline[12:]

    m = PATTERN_IPV4.match(sline[0])
    sortkey = None
    ip = None
    if m is None:
        m = PATTERN_IPV6.match(sline[0])
        if m is None:
            m = PATTERN_ONION.match(sline[0])
            if m is None:
                return None
            else:
                net = 'onion'
                ipstr = sortkey = m.group(1)
                port = int(m.group(2))
        else:
            net = 'ipv6'
            # Not interested in localhost
            if m.group(1) in ['::']:
                return None
            ipstr = m.group(1)
            # XXX parse IPv6 into number, could use name_to_ipv6 from
            # generate-seeds
            sortkey = ipstr
            port = int(m.group(2))
    else:
        # Do IPv4 sanity check
        ip = 0
        for i in range(0, 4):
            if int(m.group(i + 2)) < 0 or int(m.group(i + 2)) > 255:
                return None
            ip = ip + (int(m.group(i + 2)) << (8 * (3 - i)))
        if ip == 0:
            return None
        net = 'ipv4'
        sortkey = ip
        ipstr = m.group(1)
        port = int(m.group(6))
    # Skip bad results.
    if sline[1] == 0:
        return None
    # Extract uptime %.
    uptime30 = float(sline[7][:-1])
    # Extract Unix timestamp of last success.
    lastsuccess = int(sline[2])
    # Extract protocol version.
    version = int(sline[10])
    # Extract user agent.
    agent = sline[11][1:-1]
    # Extract service flags.
    service = int(sline[9], 16)
    # Extract blocks.
    blocks = int(sline[8])
    # Construct result.
    return {
        'net': net,
        'ip': ipstr,
        'port': port,
        'ipnum': ip,
        'uptime': uptime30,
        'lastsuccess': lastsuccess,
        'version': version,
        'agent': agent,
        'service': service,
        'blocks': blocks,
        'sortkey': sortkey,
    }

def filtermultiport(ips):
    '''Filter out hosts with more nodes per IP'''
    hist = collections.defaultdict(list)
    for ip in ips:
        hist[ip['sortkey']].append(ip)
    return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]

# Based on Greg Maxwell's seed_filter.py

def filterbyasn(ips, max_per_asn, max_total):
    # Sift out ips by type
    ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
    ips_onion = [ip for ip in ips if ip['net'] == 'onion']

    # Filter IPv46 by ASN
    result = []
    asn_count = {}
    for ip in ips_ipv46:
        if len(result) == max_total:
            break
        try:
            if ip['net'] == 'ipv4':
                ipaddr = ip['ip']
                prefix = '.origin'
            else:
                # http://www.team-cymru.com/IP-ASN-mapping.html
                # 2001:4860:b002:23::68
                res = str()
                # pick the first 4 nibbles
                for nb in ip['ip'].split(':')[:4]:
                    # right padded with '0'
                    for c in nb.zfill(4):
                        # 2001 4860 b002 0023
                        res += c + '.'
                # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
                ipaddr = res.rstrip('.')
                prefix = '.origin6'

            asn = int([x.to_text() for x in dns.resolver.query('.'.join(
                       reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
                'TXT').response.answer][0].split('\"')[1].split(' ')[0])
            if asn not in asn_count:
                asn_count[asn] = 0
            if asn_count[asn] == max_per_asn:
                continue
            asn_count[asn] += 1
            result.append(ip)
        except Exception:
            sys.stderr.write(
                'ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')

    # Add back Onions
    result.extend(ips_onion)
    return result

def main():
    lines = sys.stdin.readlines()
    ips = [parseline(line) for line in lines]

    # Skip entries with valid address.
    ips = [ip for ip in ips if ip is not None]
    # Skip entries from suspicious hosts.
    ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
    # Enforce minimal number of blocks.
    ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
    # Require service bit 1.
    ips = [ip for ip in ips if (ip['service'] & 1) == 1]
    # Require at least 50% 30-day uptime.
    ips = [ip for ip in ips if ip['uptime'] > 50]
    # Require a known and recent user agent.
    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]

    # Sort by availability (and use last success as tie breaker)
    ips.sort(key=lambda x:
             (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
    # Filter out hosts with multiple bitcoin ports, these are likely abusive
    ips = filtermultiport(ips)
    # Look up ASNs and limit results, both per ASN and globally.
    ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
    # Sort the results by IP address (for deterministic output).
    ips.sort(key=lambda x: (x['net'], x['sortkey']))

    for ip in ips:
        if ip['net'] == 'ipv6':
            print('[{}]:{}'.format(ip['ip'], ip['port']))
        else:
            print('{}:{}'.format(ip['ip'], ip['port']))

if __name__ == '__main__':
    main()
<commit_msg>Update makeseeds<commit_after>#!/usr/bin/env python3
# Copyright (c) 2013-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Generate seeds.txt from Pieter's DNS seeder
#

import collections
import dns.resolver
import re
import sys

NSEEDS = 512

MAX_SEEDS_PER_ASN = 2

MIN_BLOCKS = 540000

# These are hosts that have been observed to be behaving strangely (e.g.
# aggressively connecting to every node).
SUSPICIOUS_HOSTS = {
    "23.92.36.9", "72.36.89.11", "130.211.129.106", "178.63.107.226",
    "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
    "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
    "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
    "54.94.195.96", "54.94.200.247"
}

PATTERN_IPV4 = re.compile(
    r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
PATTERN_ONION = re.compile(
    r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")

# Used to only select nodes with a user agent string compatible with the
# BCH/UAHF specification.
PATTERN_AGENT = re.compile(r"^(/Bitcoin ABC:0.(21|22).(\d+)\(.+\)/)")

def parseline(line):
    sline = line.split()
    if len(sline) < 11:
        return None

    # The user agent is at the end of the line. It may contain space, so we
    # concatenate.
    for i in range(12, len(sline)):
        sline[11] += ' ' + sline[i]

    # Remove leftovers
    del sline[12:]

    m = PATTERN_IPV4.match(sline[0])
    sortkey = None
    ip = None
    if m is None:
        m = PATTERN_IPV6.match(sline[0])
        if m is None:
            m = PATTERN_ONION.match(sline[0])
            if m is None:
                return None
            else:
                net = 'onion'
                ipstr = sortkey = m.group(1)
                port = int(m.group(2))
        else:
            net = 'ipv6'
            # Not interested in localhost
            if m.group(1) in ['::']:
                return None
            ipstr = m.group(1)
            # XXX parse IPv6 into number, could use name_to_ipv6 from
            # generate-seeds
            sortkey = ipstr
            port = int(m.group(2))
    else:
        # Do IPv4 sanity check
        ip = 0
        for i in range(0, 4):
            if int(m.group(i + 2)) < 0 or int(m.group(i + 2)) > 255:
                return None
            ip = ip + (int(m.group(i + 2)) << (8 * (3 - i)))
        if ip == 0:
            return None
        net = 'ipv4'
        sortkey = ip
        ipstr = m.group(1)
        port = int(m.group(6))
    # Skip bad results.
    if sline[1] == 0:
        return None
    # Extract uptime %.
    uptime30 = float(sline[7][:-1])
    # Extract Unix timestamp of last success.
    lastsuccess = int(sline[2])
    # Extract protocol version.
    version = int(sline[10])
    # Extract user agent.
    agent = sline[11][1:-1]
    # Extract service flags.
    service = int(sline[9], 16)
    # Extract blocks.
    blocks = int(sline[8])
    # Construct result.
    return {
        'net': net,
        'ip': ipstr,
        'port': port,
        'ipnum': ip,
        'uptime': uptime30,
        'lastsuccess': lastsuccess,
        'version': version,
        'agent': agent,
        'service': service,
        'blocks': blocks,
        'sortkey': sortkey,
    }

def filtermultiport(ips):
    '''Filter out hosts with more nodes per IP'''
    hist = collections.defaultdict(list)
    for ip in ips:
        hist[ip['sortkey']].append(ip)
    return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]

# Based on Greg Maxwell's seed_filter.py

def filterbyasn(ips, max_per_asn, max_total):
    # Sift out ips by type
    ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
    ips_onion = [ip for ip in ips if ip['net'] == 'onion']

    # Filter IPv46 by ASN
    result = []
    asn_count = {}
    for ip in ips_ipv46:
        if len(result) == max_total:
            break
        try:
            if ip['net'] == 'ipv4':
                ipaddr = ip['ip']
                prefix = '.origin'
            else:
                # http://www.team-cymru.com/IP-ASN-mapping.html
                # 2001:4860:b002:23::68
                res = str()
                # pick the first 4 nibbles
                for nb in ip['ip'].split(':')[:4]:
                    # right padded with '0'
                    for c in nb.zfill(4):
                        # 2001 4860 b002 0023
                        res += c + '.'
                # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
                ipaddr = res.rstrip('.')
                prefix = '.origin6'

            asn = int([x.to_text() for x in dns.resolver.query('.'.join(
                       reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
                'TXT').response.answer][0].split('\"')[1].split(' ')[0])
            if asn not in asn_count:
                asn_count[asn] = 0
            if asn_count[asn] == max_per_asn:
                continue
            asn_count[asn] += 1
            result.append(ip)
        except Exception:
            sys.stderr.write(
                'ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')

    # Add back Onions
    result.extend(ips_onion)
    return result

def main():
    lines = sys.stdin.readlines()
    ips = [parseline(line) for line in lines]

    # Skip entries with valid address.
    ips = [ip for ip in ips if ip is not None]
    # Skip entries from suspicious hosts.
    ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
    # Enforce minimal number of blocks.
    ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
    # Require service bit 1.
    ips = [ip for ip in ips if (ip['service'] & 1) == 1]
    # Require at least 50% 30-day uptime.
    ips = [ip for ip in ips if ip['uptime'] > 50]
    # Require a known and recent user agent.
    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]

    # Sort by availability (and use last success as tie breaker)
    ips.sort(key=lambda x:
             (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
    # Filter out hosts with multiple bitcoin ports, these are likely abusive
    ips = filtermultiport(ips)
    # Look up ASNs and limit results, both per ASN and globally.
    ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
    # Sort the results by IP address (for deterministic output).
    ips.sort(key=lambda x: (x['net'], x['sortkey']))

    for ip in ips:
        if ip['net'] == 'ipv6':
            print('[{}]:{}'.format(ip['ip'], ip['port']))
        else:
            print('{}:{}'.format(ip['ip'], ip['port']))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
This script executes the task of estimating the building type, based solely on the geometry for that building.
The data for this script can be generated by running the prep/get-data.sh and prep/preprocess-buildings.py scripts,
which will take about an hour or two.
"""
import os
from datetime import datetime

import numpy as np
from keras import Input
from keras.callbacks import TensorBoard, EarlyStopping
from keras.engine import Model
from keras.layers import LSTM, TimeDistributed, Dense, Flatten
from keras.optimizers import Adam

from topoml_util.geom_scaler import localized_mean, localized_normal
from topoml_util.slack_send import notify

SCRIPT_VERSION = '0.1.21'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
SIGNATURE = SCRIPT_NAME + ' ' + TIMESTAMP
DATA_FOLDER = '../files/buildings/'
FILENAME_PREFIX = 'buildings-train'

# Hyperparameters
BATCH_SIZE = int(os.getenv('BATCH_SIZE', 512))
TRAIN_VALIDATE_SPLIT = float(os.getenv('TRAIN_VALIDATE_SPLIT', 0.1))
REPEAT_DEEP_ARCH = int(os.getenv('REPEAT_DEEP_ARCH', 1))
LSTM_SIZE = int(os.getenv('LSTM_SIZE', 256))
DENSE_SIZE = int(os.getenv('DENSE_SIZE', 64))
EPOCHS = int(os.getenv('EPOCHS', 100))
LEARNING_RATE = float(os.getenv('LEARNING_RATE', 3e-4))
GEOM_SCALE = int(os.getenv('GEOM_SCALE', 0))  # Default 0, overridden when data is known
OPTIMIZER = Adam(lr=LEARNING_RATE)
PATIENCE = 40
RECURRENT_DROPOUT = 0.05

message = 'running {0} with ' \
          'batch size: {1} ' \
          'train/validate split: {2} ' \
          'repeat deep: {3} ' \
          'lstm size: {4} ' \
          'dense size: {5} ' \
          'epochs: {6} ' \
          'learning rate: {7}' \
    .format(
        SIGNATURE,
        BATCH_SIZE,
        TRAIN_VALIDATE_SPLIT,
        REPEAT_DEEP_ARCH,
        LSTM_SIZE,
        DENSE_SIZE,
        EPOCHS,
        LEARNING_RATE)
print(message)

# Load training data
train_geoms = []
train_building_type = []

for file in os.listdir(DATA_FOLDER):
    if file.startswith(FILENAME_PREFIX) and file.endswith('.npz'):
        train_loaded = np.load(DATA_FOLDER + file)
        if len(train_geoms):
            train_geoms = np.append(train_geoms, train_loaded['geoms'], axis=0)
            train_building_type = np.append(train_building_type, train_loaded['building_type'], axis=0)
        else:
            train_geoms = train_loaded['geoms']
            train_building_type = train_loaded['building_type']

# Normalize
means = localized_mean(train_geoms)
GEOM_SCALE = GEOM_SCALE or np.var(train_geoms[..., 0:2])
train_geoms = localized_normal(train_geoms, means, GEOM_SCALE)

# Map building types to one-hot vectors
train_targets = np.zeros((len(train_building_type), train_building_type.max() + 1))
for index, building_type in enumerate(train_building_type):
    train_targets[index, building_type] = 1

# Shape determination
geom_max_points, geom_vector_len = train_geoms.shape[1:]
output_seq_length = train_targets.shape[-1]

# Build model
inputs = Input(shape=(geom_max_points, geom_vector_len))
model = LSTM(LSTM_SIZE, activation='relu', return_sequences=True, recurrent_dropout=RECURRENT_DROPOUT)(inputs)

for layer in range(REPEAT_DEEP_ARCH):
    model = LSTM(LSTM_SIZE, activation='relu', return_sequences=True, recurrent_dropout=RECURRENT_DROPOUT)(model)
    # model = TimeDistributed(Dense(DENSE_SIZE, activation='relu'))(model)

model = Dense(DENSE_SIZE, activation='relu')(model)
model = Flatten()(model)
model = Dense(output_seq_length, activation='softmax')(model)

model = Model(inputs=inputs, outputs=model)
model.compile(
    loss='categorical_crossentropy',
    metrics=['accuracy'],
    optimizer=OPTIMIZER),
model.summary()

# Callbacks
callbacks = [
    TensorBoard(log_dir='./tensorboard_log/' + SIGNATURE, write_graph=False),
    EarlyStopping(patience=PATIENCE, min_delta=0.001)
]

history = model.fit(
    x=train_geoms,
    y=train_targets,
    epochs=EPOCHS,
    batch_size=BATCH_SIZE,
    validation_split=TRAIN_VALIDATE_SPLIT,
    callbacks=callbacks).history

# Run on unseen test data
TEST_DATA_FILE = '../files/buildings/buildings-test.npz'
test_loaded = np.load(TEST_DATA_FILE)
test_geoms = test_loaded['geoms']
test_building_types = test_loaded['building_type']

# Normalize
means = localized_mean(test_geoms)
test_geoms = localized_normal(test_geoms, means, GEOM_SCALE)  # re-use variance from training
test_pred = model.predict(test_geoms)

# Map test targets to one-hot vectors
test_targets = np.zeros((len(test_building_types), test_building_types.max() + 1))
for index, building_type in enumerate(test_building_types):
    test_targets[index, building_type] = 1

correct = 0
for prediction, expected in zip(test_pred, test_targets):
    if np.argmax(prediction) == np.argmax(expected):
        correct += 1

accuracy = correct / len(test_pred)
message = 'test accuracy of {0} with ' \
          'batch size: {1} ' \
          'train/validate split: {2} ' \
          'repeat deep: {3} ' \
          'lstm size: {4} ' \
          'dense size: {5} ' \
          'epochs: {6} ' \
          'learning rate: {7}' \
    .format(
        str(accuracy),
        BATCH_SIZE,
        TRAIN_VALIDATE_SPLIT,
        REPEAT_DEEP_ARCH,
        LSTM_SIZE,
        DENSE_SIZE,
        len(history['val_loss']),
        LEARNING_RATE)

notify(SIGNATURE, message)
print(SCRIPT_NAME, 'finished successfully')
<commit_msg>tweak batch size<commit_after>"""
This script executes the task of estimating the building type, based solely on the geometry for that building.
The data for this script can be generated by running the prep/get-data.sh and prep/preprocess-buildings.py scripts,
which will take about an hour or two.
"""
import os
from datetime import datetime

import numpy as np
from keras import Input
from keras.callbacks import TensorBoard, EarlyStopping
from keras.engine import Model
from keras.layers import LSTM, Dense, Flatten
from keras.optimizers import Adam

from topoml_util.geom_scaler import localized_mean, localized_normal
from topoml_util.slack_send import notify

SCRIPT_VERSION = '0.1.21'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
SIGNATURE = SCRIPT_NAME + ' ' + TIMESTAMP
DATA_FOLDER = '../files/buildings/'
FILENAME_PREFIX = 'buildings-train'

# Hyperparameters
BATCH_SIZE = int(os.getenv('BATCH_SIZE', 384))
TRAIN_VALIDATE_SPLIT = float(os.getenv('TRAIN_VALIDATE_SPLIT', 0.1))
REPEAT_DEEP_ARCH = int(os.getenv('REPEAT_DEEP_ARCH', 1))
LSTM_SIZE = int(os.getenv('LSTM_SIZE', 256))
DENSE_SIZE = int(os.getenv('DENSE_SIZE', 64))
EPOCHS = int(os.getenv('EPOCHS', 100))
LEARNING_RATE = float(os.getenv('LEARNING_RATE', 3e-4))
GEOM_SCALE = int(os.getenv('GEOM_SCALE', 0))  # Default 0, overridden when data is known
OPTIMIZER = Adam(lr=LEARNING_RATE)
PATIENCE = 40
RECURRENT_DROPOUT = 0.05

message = 'running {0} with ' \
          'batch size: {1} ' \
          'train/validate split: {2} ' \
          'repeat deep: {3} ' \
          'lstm size: {4} ' \
          'dense size: {5} ' \
          'epochs: {6} ' \
          'learning rate: {7}' \
    .format(
        SIGNATURE,
        BATCH_SIZE,
        TRAIN_VALIDATE_SPLIT,
        REPEAT_DEEP_ARCH,
        LSTM_SIZE,
        DENSE_SIZE,
        EPOCHS,
        LEARNING_RATE)
print(message)

# Load training data
train_geoms = []
train_building_type = []

for file in os.listdir(DATA_FOLDER):
    if file.startswith(FILENAME_PREFIX) and file.endswith('.npz'):
        train_loaded = np.load(DATA_FOLDER + file)
        if len(train_geoms):
            train_geoms = np.append(train_geoms, train_loaded['geoms'], axis=0)
            train_building_type = np.append(train_building_type, train_loaded['building_type'], axis=0)
        else:
            train_geoms = train_loaded['geoms']
            train_building_type = train_loaded['building_type']

# Normalize
means = localized_mean(train_geoms)
GEOM_SCALE = GEOM_SCALE or np.var(train_geoms[..., 0:2])
train_geoms = localized_normal(train_geoms, means, GEOM_SCALE)

# Map building types to one-hot vectors
train_targets = np.zeros((len(train_building_type), train_building_type.max() + 1))
for index, building_type in enumerate(train_building_type):
    train_targets[index, building_type] = 1

# Shape determination
geom_max_points, geom_vector_len = train_geoms.shape[1:]
output_seq_length = train_targets.shape[-1]

# Build model
inputs = Input(shape=(geom_max_points, geom_vector_len))
model = LSTM(LSTM_SIZE, activation='relu', return_sequences=True, recurrent_dropout=RECURRENT_DROPOUT)(inputs)

for layer in range(REPEAT_DEEP_ARCH):
    model = LSTM(LSTM_SIZE, activation='relu', return_sequences=True, recurrent_dropout=RECURRENT_DROPOUT)(model)
    # model = TimeDistributed(Dense(DENSE_SIZE, activation='relu'))(model)

model = Dense(DENSE_SIZE, activation='relu')(model)
model = Flatten()(model)
model = Dense(output_seq_length, activation='softmax')(model)

model = Model(inputs=inputs, outputs=model)
model.compile(
    loss='categorical_crossentropy',
    metrics=['accuracy'],
    optimizer=OPTIMIZER),
model.summary()

# Callbacks
callbacks = [
    TensorBoard(log_dir='./tensorboard_log/' + SIGNATURE, write_graph=False),
    EarlyStopping(patience=PATIENCE, min_delta=0.001)
]

history = model.fit(
    x=train_geoms,
    y=train_targets,
    epochs=EPOCHS,
    batch_size=BATCH_SIZE,
    validation_split=TRAIN_VALIDATE_SPLIT,
    callbacks=callbacks).history

# Run on unseen test data
TEST_DATA_FILE = '../files/buildings/buildings-test.npz'
test_loaded = np.load(TEST_DATA_FILE)
test_geoms = test_loaded['geoms']
test_building_types = test_loaded['building_type']

# Normalize
means = localized_mean(test_geoms)
test_geoms = localized_normal(test_geoms, means, GEOM_SCALE)  # re-use variance from training
test_pred = model.predict(test_geoms)

# Map test targets to one-hot vectors
test_targets = np.zeros((len(test_building_types), test_building_types.max() + 1))
for index, building_type in enumerate(test_building_types):
    test_targets[index, building_type] = 1

correct = 0
for prediction, expected in zip(test_pred, test_targets):
    if np.argmax(prediction) == np.argmax(expected):
        correct += 1

accuracy = correct / len(test_pred)
message = 'test accuracy of {0} with ' \
          'batch size: {1} ' \
          'train/validate split: {2} ' \
          'repeat deep: {3} ' \
          'lstm size: {4} ' \
          'dense size: {5} ' \
          'epochs: {6} ' \
          'learning rate: {7}' \
    .format(
        str(accuracy),
        BATCH_SIZE,
        TRAIN_VALIDATE_SPLIT,
        REPEAT_DEEP_ARCH,
        LSTM_SIZE,
        DENSE_SIZE,
        len(history['val_loss']),
        LEARNING_RATE)

notify(SIGNATURE, message)
print(SCRIPT_NAME, 'finished successfully')
<|endoftext|>


<commit_before>import sys
from core import osgi

__all__ = []

oh1_actions = osgi.find_services("org.openhab.core.scriptengine.action.ActionService", None) or []
oh2_actions = osgi.find_services("org.eclipse.smarthome.model.script.engine.action.ActionService", None) or []

_module = sys.modules[__name__]

for s in oh1_actions + oh2_actions:
    action_class = s.actionClass
    name = action_class.simpleName
    setattr(_module, name, action_class)
    __all__.append(name)

try:
    from org.openhab.core.model.script.actions import Exec
    from org.openhab.core.model.script.actions import HTTP
    from org.openhab.core.model.script.actions import LogAction
    from org.openhab.core.model.script.actions import Ping
except:
    from org.eclipse.smarthome.model.script.actions import Exec
    from org.eclipse.smarthome.model.script.actions import HTTP
    from org.eclipse.smarthome.model.script.actions import LogAction
    from org.eclipse.smarthome.model.script.actions import Ping

static_imports = [Exec, HTTP, LogAction, Ping]

for s in static_imports:
    name = s.simpleName
    setattr(_module, name, s)
    __all__.append(name)
<commit_msg>Add ScriptExecution Action (#118)<commit_after>import sys
from core import osgi

__all__ = []

oh1_actions = osgi.find_services("org.openhab.core.scriptengine.action.ActionService", None) or []
oh2_actions = osgi.find_services("org.eclipse.smarthome.model.script.engine.action.ActionService", None) or []

_module = sys.modules[__name__]

for s in oh1_actions + oh2_actions:
    action_class = s.actionClass
    name = action_class.simpleName
    setattr(_module, name, action_class)
    __all__.append(name)

try:
    from org.openhab.core.model.script.actions import Exec
    from org.openhab.core.model.script.actions import HTTP
    from org.openhab.core.model.script.actions import LogAction
    from org.openhab.core.model.script.actions import Ping
    from org.openhab.core.model.script.actions import ScriptExecution
except:
    from org.eclipse.smarthome.model.script.actions import Exec
    from org.eclipse.smarthome.model.script.actions import HTTP
    from org.eclipse.smarthome.model.script.actions import LogAction
    from org.eclipse.smarthome.model.script.actions import Ping
    from org.eclipse.smarthome.model.script.actions import ScriptExecution

static_imports = [Exec, HTTP, LogAction, Ping]

for s in static_imports:
    name = s.simpleName
    setattr(_module, name, s)
    __all__.append(name)
<|endoftext|>


<commit_before>#!/usr/bin/env python3.6
"""
Make a csv file that can be imported into Google Sheets, and then manipulated
with the Javascript code at the end of this file.  The result is a calendar
with bars indicating support windows of various pieces of software.

It's kind of silly how it works now, with a csv file output.  It would make
more sense to just output a list of bars to draw, and then use the Javascript
to draw them.

"""

import csv
import datetime
import itertools
import sys

EXTEND = '-'

class BaseCalendar:
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self.width = 12 * (end - start + 1)

    def column(self, year, month):
        return (year - self.start) * 12 + month - 1

    def bar(self, name, start, end=None, length=None, color=None, text_color="black"):
        istart = self.column(*start)
        if length is None:
            iend = self.column(*end)
        else:
            iend = istart + length - 1
        if istart >= self.width:
            return  # bar is entirely in the future.
        if iend < 0:
            return  # bar is entirely in the past.
        istart = max(0, istart)
        iend = min(self.width - 1, iend)
        self.rawbar(istart, iend, name, color, text_color)

class CsvCalendar(BaseCalendar):
    def __init__(self, start, end):
        super().__init__(start, end)
        self.rows = []

    def years_months(self):
        years = []
        months = []
        for year in range(self.start, self.end+1):
            years.append(year)
            years.extend([EXTEND]*11)
            months.extend("JFMAMJJASOND")
        self.rows.append(years)
        self.rows.append(months)

    def rawbar(self, istart, iend, name, color, text_color):
        row = [''] * self.width
        if color is not None:
            name += "|" + color
        row[istart] = name
        for ii in range(istart+1, iend+1):
            row[ii] = EXTEND
        self.rows.append(row)

    def write(self):
        writer = csv.writer(sys.stdout)
        for row in self.rows:
            writer.writerow(row)

class NewCalendar(BaseCalendar):
    def __init__(self, start, end):
        super().__init__(start, end)
        self.currow = 1
        self.cycling = None
        self.prologue()

    def prologue(self):
        print(f"""\
            function makeBarCalendar() {{
            var sheet = SpreadsheetApp.getActiveSheet();
            sheet.getDataRange().deleteCells(SpreadsheetApp.Dimension.ROWS);
            """)

    def epilog(self):
        print(f"""\
            range = sheet.getDataRange();
            sheet.setColumnWidths(range.getColumn(), range.getWidth(), 12);
            sheet.setRowHeights(range.getRow(), range.getHeight(), 18);
            range.setFontSize(9);

            var keepRows = 10;
            var tooMany = sheet.getMaxRows() - range.getLastRow() - keepRows;
            if (tooMany > 0) {{
                sheet.deleteRows(range.getLastRow() + keepRows + 1, tooMany);
            }}
            var keepCols = 1;
            var tooMany = sheet.getMaxColumns() - range.getLastColumn() - keepCols;
            if (tooMany > 0) {{
                sheet.deleteColumns(range.getLastColumn() + keepCols + 1, tooMany);
            }}

            for (var c = 12; c <= range.getLastColumn(); c += 12) {{
                sheet.getRange(1, c, sheet.getMaxRows(), 1)
                    .setBorder(null, null, null, true, null, null, "black", null);
            }}

            }}
            """)

    def years_months(self):
        yearrow = self.currow
        monthrow = self.currow + 1
        self.currow += 2

        print(f"""\
            sheet.insertColumns(1, {(self.end - self.start + 1) * 12});
            """)

        for year in range(self.start, self.end+1):
            iyear = self.column(year, 1) + 1
            print(f"""\
                sheet.getRange({yearrow}, {iyear}, 1, 12)
                    .merge()
                    .setBorder(null, null, null, true, null, null, "black", null)
                    .setValue("{year}");
                for (m = 0; m < 12; m++) {{
                    sheet.getRange({monthrow}, {iyear}+m).setValue("JFMAMJJASOND"[m]);
                }}
                """);
        print(f"""\
            sheet.getRange({yearrow}, 1, 1, {self.width})
                .setFontWeight("bold")
                .setHorizontalAlignment("center");
            sheet.getRange({monthrow}, 1, 1, {self.width})
                .setHorizontalAlignment("center");
            """);
        print(f"""\
            var rules = sheet.getConditionalFormatRules();
            rules.push(
            SpreadsheetApp.newConditionalFormatRule()
                .whenFormulaSatisfied("=(year(now())-$A$1)*12+1 = column()")
                .setBackground("#E9CECE")
                .setRanges([sheet.getRange({yearrow}, 1, 1, {self.width})])
                .build()
            );
            rules.push(
            SpreadsheetApp.newConditionalFormatRule()
                .whenFormulaSatisfied("=(year(now())-$A$1)*12 + (month(now())) = column()")
                .setBackground("#E9CECE")
                .setRanges([sheet.getRange({monthrow}, 1, 1, {self.width})])
                .build()
            );
            sheet.setConditionalFormatRules(rules);
            """)

    def rawbar(self, istart, iend, name, color, text_color):
        print(f"""\
            sheet.getRange({self.currow}, {istart + 1}, 1, {iend - istart + 1})
                .merge()
                .setBackground({color!r})
                .setFontColor({text_color!r})
                .setValue({name!r});
            """)
        self.next_bar()

    def set_cycling(self, cycling):
        if cycling:
            self.top_cycling_row = self.currow
        else:
            self.currow = self.top_cycling_row + self.cycling
        self.cycling = cycling

    def next_bar(self):
        self.currow += 1
        if self.cycling:
            if self.currow >= self.top_cycling_row + self.cycling:
                self.currow = self.top_cycling_row

    def text_line(self, text=""):
        print(f"""\
            sheet.getRange({self.currow}, 1).setValue({text!r})
            """)
        self.currow += 1

    def freeze_here(self):
        print(f"""\
            sheet.setFrozenRows({self.currow - 1});
            """)

    def write(self):
        self.epilog()

# Options
START_YEAR = 2016
END_YEAR = 2024
LTS_ONLY = True

if sys.argv[1] == "new":
    cal = NewCalendar(START_YEAR, END_YEAR)
else:
    cal = CsvCalendar(START_YEAR, END_YEAR)
cal.years_months()

# Open edX releases
cal.set_cycling(3)
names = [
    ('Aspen', 2014, 10),
    ('Birch', 2015, 2),
    ('Cypress', 2015, 8),
    ('Dogwood', 2016, 2),
    ('Eucalyptus', 2016, 8),
    ('Ficus', 2017, 2),
    ('Ginkgo', 2017, 8),
    ('Hawthorn', 2018, 8),
    ('Ironwood', 2019, 3),
    ('Juniper', 2019, 10),
    ]
future = ['Koa', 'Lilac', 'Maple'] + list('NOPQRST')
releases = list(itertools.chain(names, [(name, None, None) for name in future]))
last = (None, None)
for (name, year, month), (_, nextyear, nextmonth) in zip(releases, releases[1:]):
    if year is None:
        year, month = last
        month += 6
        yearplus, month = divmod(month, 12)
        year += yearplus
    if nextyear is None:
        length = 6
    else:
        length = (nextyear * 12 + nextmonth) - (year * 12 + month)
    cal.bar(name, start=(year, month), length=length, color="#fce5cd")
    last = (year, month)

cal.set_cycling(None)
cal.freeze_here()
cal.text_line("(this calendar is part of OEP-10, please don't change it without considering the impact there.)")

# Django releases
django_releases = [
    ('1.8', 2015, 4, True),
    ('1.9', 2016, 1, False),
    ('1.10', 2016, 8, False),
    ('1.11', 2017, 4, True),
    ('2.0', 2018, 1, False),
    ('2.1', 2018, 8, False),
    ('2.2', 2019, 4, True),
    ('3.0', 2020, 1, False),
    ('3.1', 2020, 8, False),
    ('3.2', 2021, 4, True),
    ('4.0', 2022, 1, False),
]
for name, year, month, lts in django_releases:
    if LTS_ONLY and not lts:
        continue
    length = 3*12 if lts else 16
    color = "#44b78b" if lts else "#c9f0df"
    cal.bar(f"Django {name}", start=(year, month), length=length, color=color)

# Python releases
python_releases = [
    ('2.7', 2010, 7, 2019, 12),
    ('3.5', 2015, 9, 2020, 9),          # https://www.python.org/dev/peps/pep-0478/
    ('3.6', 2016, 12, 2021, 12),        # https://www.python.org/dev/peps/pep-0494/
    ('3.7', 2018, 6, 2023, 6),          # https://www.python.org/dev/peps/pep-0537/
    ('3.8', 2019, 10, 2024, 10),        # https://www.python.org/dev/peps/pep-0569/
    ('3.9', 2020, 10, 2025, 10),        # https://www.python.org/dev/peps/pep-0596/
]
for name, syear, smonth, eyear, emonth in python_releases:
    cal.bar(f"Python {name}", start=(syear, smonth), end=(eyear, emonth), color="#ffd545")

# Ubuntu releases
for year, month in itertools.product(range(16, 23), [4, 10]):
    name = "Ubuntu {:d}.{:02d}".format(year, month)
    lts = (year % 2 == 0) and (month == 4)
    if LTS_ONLY and not lts:
        continue
    length = 5*12 if lts else 9
    color = "#E95420" if lts else "#F4AA90"     # http://design.ubuntu.com/brand/colour-palette
    cal.bar(name, (2000+year, month), length=length, color=color, text_color="white")

# Node releases: https://github.com/nodejs/Release
node_releases = [
    ('6.x', 2016, 4, 2019, 4),
    ('8.x', 2017, 5, 2019, 12),
    ('10.x', 2018, 4, 2021, 4),
    ('12.x', 2019, 4, 2022, 4),
    ('14.x', 2020, 4, 2023, 4),
]
for name, syear, smonth, eyear, emonth in node_releases:
    cal.bar(f"Node {name}", start=(syear, smonth), end=(eyear, emonth), color="#2f6c1b", text_color="white")

cal.write()

# The code for Google Sheets to turn this output into something nice.
"""
function mergeRangeBars(range) {
  var sheet = range.getSheet();
  for (var r = range.getRow(); r <= range.getLastRow(); r++) {
    for (var c = range.getColumn(); c <= range.getLastColumn(); c++) {
      var firstValue = sheet.getRange(r, c).getValue();
      if (firstValue !== "") {
        // Start of a range, look for dashes
        cend = c+1;
        while (sheet.getRange(r, cend).getValue() === "-") {
          cend++;
        }
        cend--;
        var fullRange = sheet.getRange(r, c, 1, cend-c+1);
        if (cend != c) {
          fullRange.merge();
        }
        // Apply colors
        if (typeof firstValue === 'string') {
          var parts = firstValue.split('|');
          fullRange.setValue(parts[0]);
          if (parts.length > 1) {
            fullRange.setBackground(parts[1]);
          }
        }
      }
    }
  }
}

function makeBarCalendar() {
  var sheet = SpreadsheetApp.getActiveSheet();
  var range = sheet.getDataRange();
  sheet.setColumnWidths(range.getColumn(), range.getWidth(), 12);
  sheet.setRowHeights(range.getRow(), range.getHeight(), 18);
  range.setFontSize(9);
  mergeRangeBars(range);
  sheet.setFrozenRows(2);
}
"""
# Also in the sheet:
# Open Script Editor. Select makeBarCalendar. Click the Run button. It's slow, be patient.
# Turn off gridlines
# Row 1: center and bold.
# Conditional formatting for row 2:
#   "Custom formula is:"
#   =(year(now())-$A$1)*12 + (month(now())) = column()
#   color red

# for row 1:
#   =(year(now())-$A$1)*12+1 = column()
#
# Put a dark outline around the current boxes by hand.
<commit_msg>Remove the old csv implementation<commit_after>#!/usr/bin/env python3.6
"""
Write JavaScript code to be pasted into a Google Sheet to draw a calendar.
"""

import csv
import datetime
import itertools
import sys

class BaseCalendar:
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self.width = 12 * (end - start + 1)

    def column(self, year, month):
        return (year - self.start) * 12 + month - 1

    def bar(self, name, start, end=None, length=None, color=None, text_color="black"):
        istart = self.column(*start)
        if length is None:
            iend = self.column(*end)
        else:
            iend = istart + length - 1
        if istart >= self.width:
            return  # bar is entirely in the future.
        if iend < 0:
            return  # bar is entirely in the past.
        istart = max(0, istart)
        iend = min(self.width - 1, iend)
        self.rawbar(istart, iend, name, color, text_color)

class GsheetCalendar(BaseCalendar):
    def __init__(self, start, end):
        super().__init__(start, end)
        self.currow = 1
        self.cycling = None
        self.prologue()

    def prologue(self):
        print(f"""\
            function makeBarCalendar() {{
            var sheet = SpreadsheetApp.getActiveSheet();
            sheet.getDataRange().deleteCells(SpreadsheetApp.Dimension.ROWS);
            """)

    def epilog(self):
        print(f"""\
            range = sheet.getDataRange();
            sheet.setColumnWidths(range.getColumn(), range.getWidth(), 12);
            sheet.setRowHeights(range.getRow(), range.getHeight(), 18);
            range.setFontSize(9);

            var keepRows = 10;
            var tooMany = sheet.getMaxRows() - range.getLastRow() - keepRows;
            if (tooMany > 0) {{
                sheet.deleteRows(range.getLastRow() + keepRows + 1, tooMany);
            }}
            var keepCols = 1;
            var tooMany = sheet.getMaxColumns() - range.getLastColumn() - keepCols;
            if (tooMany > 0) {{
                sheet.deleteColumns(range.getLastColumn() + keepCols + 1, tooMany);
            }}

            for (var c = 12; c <= range.getLastColumn(); c += 12) {{
                sheet.getRange(1, c, sheet.getMaxRows(), 1)
                    .setBorder(null, null, null, true, null, null, "black", null);
            }}

            }}
            """)

    def years_months(self):
        yearrow = self.currow
        monthrow = self.currow + 1
        self.currow += 2

        print(f"""\
            sheet.insertColumns(1, {(self.end - self.start + 1) * 12});
            """)

        for year in range(self.start, self.end+1):
            iyear = self.column(year, 1) + 1
            print(f"""\
                sheet.getRange({yearrow}, {iyear}, 1, 12)
                    .merge()
                    .setBorder(null, null, null, true, null, null, "black", null)
                    .setValue("{year}");
                for (m = 0; m < 12; m++) {{
                    sheet.getRange({monthrow}, {iyear}+m).setValue("JFMAMJJASOND"[m]);
                }}
                """);
        print(f"""\
            sheet.getRange({yearrow}, 1, 1, {self.width})
                .setFontWeight("bold")
                .setHorizontalAlignment("center");
            sheet.getRange({monthrow}, 1, 1, {self.width})
                .setHorizontalAlignment("center");
            """);
        print(f"""\
            var rules = sheet.getConditionalFormatRules();
            rules.push(
            SpreadsheetApp.newConditionalFormatRule()
                .whenFormulaSatisfied("=(year(now())-$A$1)*12+1 = column()")
                .setBackground("#E9CECE")
                .setRanges([sheet.getRange({yearrow}, 1, 1, {self.width})])
                .build()
            );
            rules.push(
            SpreadsheetApp.newConditionalFormatRule()
                .whenFormulaSatisfied("=(year(now())-$A$1)*12 + (month(now())) = column()")
                .setBackground("#E9CECE")
                .setRanges([sheet.getRange({monthrow}, 1, 1, {self.width})])
                .build()
            );
            sheet.setConditionalFormatRules(rules);
            """)

    def rawbar(self, istart, iend, name, color, text_color):
        print(f"""\
            sheet.getRange({self.currow}, {istart + 1}, 1, {iend - istart + 1})
                .merge()
                .setBackground({color!r})
                .setFontColor({text_color!r})
                .setValue({name!r});
            """)
        self.next_bar()

    def set_cycling(self, cycling):
        if cycling:
            self.top_cycling_row = self.currow
        else:
            self.currow = self.top_cycling_row + self.cycling
        self.cycling = cycling

    def next_bar(self):
        self.currow += 1
        if self.cycling:
            if self.currow >= self.top_cycling_row + self.cycling:
                self.currow = self.top_cycling_row

    def text_line(self, text=""):
        print(f"""\
            sheet.getRange({self.currow}, 1).setValue({text!r})
            """)
        self.currow += 1

    def freeze_here(self):
        print(f"""\
            sheet.setFrozenRows({self.currow - 1});
            """)

    def write(self):
        self.epilog()

# Options
START_YEAR = 2016
END_YEAR = 2024
LTS_ONLY = True

cal = GsheetCalendar(START_YEAR, END_YEAR)
cal.years_months()

# Open edX releases
cal.set_cycling(3)
names = [
    ('Aspen', 2014, 10),
    ('Birch', 2015, 2),
    ('Cypress', 2015, 8),
    ('Dogwood', 2016, 2),
    ('Eucalyptus', 2016, 8),
    ('Ficus', 2017, 2),
    ('Ginkgo', 2017, 8),
    ('Hawthorn', 2018, 8),
    ('Ironwood', 2019, 3),
    ('Juniper', 2019, 10),
    ]
future = ['Koa', 'Lilac', 'Maple'] + list('NOPQRST')
releases = list(itertools.chain(names, [(name, None, None) for name in future]))
last = (None, None)
for (name, year, month), (_, nextyear, nextmonth) in zip(releases, releases[1:]):
    if year is None:
        year, month = last
        month += 6
        yearplus, month = divmod(month, 12)
        year += yearplus
    if nextyear is None:
        length = 6
    else:
        length = (nextyear * 12 + nextmonth) - (year * 12 + month)
    cal.bar(name, start=(year, month), length=length, color="#fce5cd")
    last = (year, month)

cal.set_cycling(None)
cal.freeze_here()
cal.text_line("(this calendar is part of OEP-10, please don't change it without considering the impact there.)")

# Django releases
django_releases = [
    ('1.8', 2015, 4, True),
    ('1.9', 2016, 1, False),
    ('1.10', 2016, 8, False),
    ('1.11', 2017, 4, True),
    ('2.0', 2018, 1, False),
    ('2.1', 2018, 8, False),
    ('2.2', 2019, 4, True),
    ('3.0', 2020, 1, False),
    ('3.1', 2020, 8, False),
    ('3.2', 2021, 4, True),
    ('4.0', 2022, 1, False),
]
for name, year, month, lts in django_releases:
    if LTS_ONLY and not lts:
        continue
    length = 3*12 if lts else 16
    color = "#44b78b" if lts else "#c9f0df"
    cal.bar(f"Django {name}", start=(year, month), length=length, color=color)

# Python releases
python_releases = [
    ('2.7', 2010, 7, 2019, 12),
    ('3.5', 2015, 9, 2020, 9),          # https://www.python.org/dev/peps/pep-0478/
    ('3.6', 2016, 12, 2021, 12),        # https://www.python.org/dev/peps/pep-0494/
    ('3.7', 2018, 6, 2023, 6),          # https://www.python.org/dev/peps/pep-0537/
    ('3.8', 2019, 10, 2024, 10),        # https://www.python.org/dev/peps/pep-0569/
    ('3.9', 2020, 10, 2025, 10),        # https://www.python.org/dev/peps/pep-0596/
]
for name, syear, smonth, eyear, emonth in python_releases:
    cal.bar(f"Python {name}", start=(syear, smonth), end=(eyear, emonth), color="#ffd545")

# Ubuntu releases
for year, month in itertools.product(range(16, 23), [4, 10]):
    name = "Ubuntu {:d}.{:02d}".format(year, month)
    lts = (year % 2 == 0) and (month == 4)
    if LTS_ONLY and not lts:
        continue
    length = 5*12 if lts else 9
    color = "#E95420" if lts else "#F4AA90"     # http://design.ubuntu.com/brand/colour-palette
    cal.bar(name, (2000+year, month), length=length, color=color, text_color="white")

# Node releases: https://github.com/nodejs/Release
node_releases = [
    ('6.x', 2016, 4, 2019, 4),
    ('8.x', 2017, 5, 2019, 12),
    ('10.x', 2018, 4, 2021, 4),
    ('12.x', 2019, 4, 2022, 4),
    ('14.x', 2020, 4, 2023, 4),
]
for name, syear, smonth, eyear, emonth in node_releases:
    cal.bar(f"Node {name}", start=(syear, smonth), end=(eyear, emonth), color="#2f6c1b", text_color="white")

cal.write()
<|endoftext|>


<commit_before>import time
import requests
import json
import string
import os
from os import path
import time
from senpy.plugins import SentimentPlugin, SenpyPlugin
from senpy.models import Results, Entry, Sentiment,Error

class DaedalusPlugin(SentimentPlugin):

    def activate(self, *args, **kwargs):
        pass

    def deactivate(self, *args, **kwargs):
        self.close()

    def analyse(self, **params):

        txt = params["input"]
        model = params["model"] # general_es / general_es / general_fr
        api = 'http://api.meaningcloud.com/sentiment-2.1'
        lang = params.get("language")
        key = params["apiKey"]
        parameters = {'key': key,'model': model,'lang': lang,'of': 'json','txt': txt,'src': 'its-not-a-real-python-sdk'}
        r = requests.post(api, params=parameters)
        print(r.text)

        value = r.json().get('score_tag', None)
        if not value:
            raise Error(r.json())

        #Senpy Response
        response = Results()

        polarity = 'marl:Neutral'
        if 'N' in value:
            polarity = 'marl:Negative'
        elif 'P' in value:
            polarity = 'marl:Positive'
        entry = Entry(id="Entry0",nif_isString=txt)
        opinion = Sentiment(id="Opinion0",marl__hasPolarity=polarity)
        opinion["prov:wasGeneratedBy"] = self.id
        entry.sentiments = []
        entry.sentiments.append(opinion)
        response.entries.append(entry)
        return response
<commit_msg>Added polarityValue to meaningCloud plugin<commit_after>import time
import requests
import json
import string
import os
from os import path
import time
from senpy.plugins import SentimentPlugin, SenpyPlugin
from senpy.models import Results, Entry, Sentiment,Error

class DaedalusPlugin(SentimentPlugin):

    def activate(self, *args, **kwargs):
        pass

    def deactivate(self, *args, **kwargs):
        self.close()

    def analyse(self, **params):

        txt = params["input"]
        model = params["model"] # general_es / general_es / general_fr
        api = 'http://api.meaningcloud.com/sentiment-2.1'
        lang = params.get("language")
        key = params["apiKey"]
        parameters = {'key': key,'model': model,'lang': lang,'of': 'json','txt': txt,'src': 'its-not-a-real-python-sdk'}
        r = requests.post(api, params=parameters)
        print(r.text)

        value = r.json().get('score_tag', None)
        if not value:
            raise Error(r.json())

        #Senpy Response
        response = Results()
        polarityValue = 0
        polarity = 'marl:Neutral'
        if 'N' in value:
            polarity = 'marl:Negative'
            polarityValue = -1
        elif 'P' in value:
            polarity = 'marl:Positive'
            polarityValue = 1
        entry = Entry(id="Entry0",nif_isString=txt)
        opinion = Sentiment(id="Opinion0",marl__hasPolarity=polarity,marl__polarityValue = polarityValue)
        opinion["prov:wasGeneratedBy"] = self.id
        entry.sentiments = []
        entry.sentiments.append(opinion)
        response.entries.append(entry)
        return response
<|endoftext|>


<commit_before>from app import settings
from app.queue_publisher import QueuePublisher
from app.settings import session
from json import dumps
from requests.packages.urllib3.exceptions import MaxRetryError

def queue_receipt(logger, payload):
    publisher = QueuePublisher(logger, settings.RABBIT_URLS, settings.RABBIT_RRM_RECEIPT_QUEUE)
    return publisher.publish_message(payload)

class ResponseProcessor:
    def __init__(self, logger):
        self.logger = logger
        self.tx_id = ""

    def process(self, encrypted_survey):
        # decrypt
        decrypt_ok, decrypted_json = self.decrypt_survey(encrypted_survey)
        if not decrypt_ok:
            return False

        metadata = decrypted_json['metadata']
        self.logger = self.logger.bind(user_id=metadata['user_id'], ru_ref=metadata['ru_ref'])

        if 'tx_id' in decrypted_json:
            self.tx_id = decrypted_json['tx_id']
            self.logger = self.logger.bind(tx_id=self.tx_id)

        # validate
        validate_ok = self.validate_survey(decrypted_json)
        if not validate_ok:
            return False

        # store
        store_ok = self.store_survey(decrypted_json)
        if not store_ok:
            return False

        receipt_json = {
            'tx_id': decrypted_json['tx_id'],
            'collection': {'exercise_sid': decrypted_json['collection']['exercise_sid']},
            'metadata': {
                'ru_ref': decrypted_json['metadata']['ru_ref'],
                'user_id': decrypted_json['metadata']['user_id']}
        }

        queue_ok = queue_receipt(self.logger, dumps(receipt_json))

        if not queue_ok:
            return False

    def decrypt_survey(self, encrypted_survey):
        response = self.remote_call(settings.SDX_DECRYPT_URL, data=encrypted_survey)
        decrypt_ok = self.response_ok(response)
        if decrypt_ok:
            return (True, response.json())
        else:
            return (False, None)

    def validate_survey(self, decrypted_json):
        response = self.remote_call(settings.SDX_VALIDATE_URL, json=decrypted_json)
        return self.response_ok(response)

    def store_survey(self, decrypted_json):
        response = self.remote_call(settings.SDX_STORE_URL, json=decrypted_json)
        return self.response_ok(response)

    def remote_call(self, request_url, json=None, data=None, headers=None, verify=True, auth=None):
        try:
            self.logger.info("Calling service", request_url=request_url)
            r = None

            if json:
                r = session.post(request_url, json=json, headers=headers, verify=verify, auth=auth)
            elif data:
                r = session.post(request_url, data=data, headers=headers, verify=verify, auth=auth)
            else:
                r = session.get(request_url, headers=headers, verify=verify, auth=auth)

            return r

        except MaxRetryError:
            self.logger.error("Max retries exceeded (5)", request_url=request_url)

    def response_ok(self, res):
        if res.status_code == 200 or res.status_code == 201:
            self.logger.info("Returned from service", request_url=res.url, status_code=res.status_code)
            return True

        else:
            self.logger.error("Returned from service", request_url=res.url, status_code=res.status_code)
            return False
<commit_msg>Return True after successful publishing to queue<commit_after>from app import settings
from app.queue_publisher import QueuePublisher
from app.settings import session
from json import dumps
from requests.packages.urllib3.exceptions import MaxRetryError

def queue_receipt(logger, payload):
    publisher = QueuePublisher(logger, settings.RABBIT_URLS, settings.RABBIT_RRM_RECEIPT_QUEUE)
    return publisher.publish_message(payload)

class ResponseProcessor:
    def __init__(self, logger):
        self.logger = logger
        self.tx_id = ""

    def process(self, encrypted_survey):
        # decrypt
        decrypt_ok, decrypted_json = self.decrypt_survey(encrypted_survey)
        if not decrypt_ok:
            return False

        metadata = decrypted_json['metadata']
        self.logger = self.logger.bind(user_id=metadata['user_id'], ru_ref=metadata['ru_ref'])

        if 'tx_id' in decrypted_json:
            self.tx_id = decrypted_json['tx_id']
            self.logger = self.logger.bind(tx_id=self.tx_id)

        # validate
        validate_ok = self.validate_survey(decrypted_json)
        if not validate_ok:
            return False

        # store
        store_ok = self.store_survey(decrypted_json)
        if not store_ok:
            return False

        receipt_json = {
            'tx_id': decrypted_json['tx_id'],
            'collection': {'exercise_sid': decrypted_json['collection']['exercise_sid']},
            'metadata': {
                'ru_ref': decrypted_json['metadata']['ru_ref'],
                'user_id': decrypted_json['metadata']['user_id']}
        }

        queue_ok = queue_receipt(self.logger, dumps(receipt_json))

        if not queue_ok:
            return False
        else:
            return True

    def decrypt_survey(self, encrypted_survey):
        response = self.remote_call(settings.SDX_DECRYPT_URL, data=encrypted_survey)
        decrypt_ok = self.response_ok(response)
        if decrypt_ok:
            return (True, response.json())
        else:
            return (False, None)

    def validate_survey(self, decrypted_json):
        response = self.remote_call(settings.SDX_VALIDATE_URL, json=decrypted_json)
        return self.response_ok(response)

    def store_survey(self, decrypted_json):
        response = self.remote_call(settings.SDX_STORE_URL, json=decrypted_json)
        return self.response_ok(response)

    def remote_call(self, request_url, json=None, data=None, headers=None, verify=True, auth=None):
        try:
            self.logger.info("Calling service", request_url=request_url)
            r = None

            if json:
                r = session.post(request_url, json=json, headers=headers, verify=verify, auth=auth)
            elif data:
                r = session.post(request_url, data=data, headers=headers, verify=verify, auth=auth)
            else:
                r = session.get(request_url, headers=headers, verify=verify, auth=auth)

            return r

        except MaxRetryError:
            self.logger.error("Max retries exceeded (5)", request_url=request_url)

    def response_ok(self, res):
        if res.status_code == 200 or res.status_code == 201:
            self.logger.info("Returned from service", request_url=res.url, status_code=res.status_code)
            return True

        else:
            self.logger.error("Returned from service", request_url=res.url, status_code=res.status_code)
            return False
<|endoftext|>