<commit_before>import os
import json
import numpy as np
import tensorflow as tf

from wrn_cifar10 import data  # pylint: disable=g-bad-import-order
from wrn_cifar10 import model # pylint: disable=g-bad-import-order
from wrn_cifar10 import controller # pylint: disable=g-bad-import-order

app = tf.app
logging = tf.logging
flags = tf.flags
gfile = tf.gfile

IMAGE_SIZE = 32

flags.DEFINE_string('data_dir', './cifar-10-batches-py/',
                    """Path to the CIFAR-10 python data.""")
flags.DEFINE_string('model_dir', './train', """Directory where model checkpoint are stored.""")
flags.DEFINE_integer('epoch_id', 28125, """Epoch number""")
flags.DEFINE_string('save_dir', './output', """Directory where metrics will be stored.""")
flags.DEFINE_float('gpu_fraction', 0.95,
                   """The fraction of GPU memory to be allocated""")
flags.DEFINE_boolean('log_device_placement', False,
                     """Whether to log device placement.""")

FLAGS = tf.app.flags.FLAGS

def evaluate():

    hparams_dict = json.load(open(os.path.join(FLAGS.model_dir, "hparams.json")))

    hp = tf.contrib.training.HParams(
        # Optimization.
        optimizer=model.Optimizer.MOMENTUM,
        initial_lr=hparams_dict["initial_lr"],
        lr_decay=model.LRDecaySchedule[hparams_dict["lr_decay"]],
        decay_steps=hparams_dict["decay_steps"],
        weight_decay=hparams_dict["weight_decay"],
        momentum=hparams_dict["momentum"],
        use_nesterov=hparams_dict["use_nesterov"],
        # Architecture.
        n_filters_1=hparams_dict["n_filters_1"],
        n_filters_2=hparams_dict["n_filters_2"],
        n_filters_3=hparams_dict["n_filters_3"],
        stride_1=hparams_dict["stride_1"],
        stride_2=hparams_dict["stride_2"],
        stride_3=hparams_dict["stride_3"],
        depthwise=hparams_dict["depthwise"],
        num_residual_units_1=hparams_dict["num_residual_units_1"],
        num_residual_units_2=hparams_dict["num_residual_units_3"],
        num_residual_units_3=hparams_dict["num_residual_units_2"],
        k=hparams_dict["k"],
        activation_1=model.Activation[hparams_dict["activation_1"]],
        activation_2=model.Activation[hparams_dict["activation_2"]],
        activation_3=model.Activation[hparams_dict["activation_3"]],
        n_conv_layers_1=hparams_dict["n_conv_layers_1"],
        n_conv_layers_2=hparams_dict["n_conv_layers_2"],
        n_conv_layers_3=hparams_dict["n_conv_layers_3"],
        dropout_1=hparams_dict["dropout_1"],
        dropout_2=hparams_dict["dropout_2"],
        dropout_3=hparams_dict["dropout_3"],
        # Misc.
        batch_size=hparams_dict["batch_size"],
        num_epochs=hparams_dict["num_epochs"],
        replicate=hparams_dict["replicate"],
    )

    with tf.Graph().as_default() as g:

        run_meta = tf.RunMetadata()
        # Load the data
        # pylint: disable=invalid-name
        X_train, y_train, X_valid, y_valid, X_test, y_test = data.load_data(
            FLAGS.data_dir)
        # pylint: enable=invalid-name

        # Build a Graph that computes the predictions from the inference model.
        images = tf.placeholder(tf.float32,
                                [hp.batch_size, IMAGE_SIZE, IMAGE_SIZE, 3])
        labels = tf.placeholder(tf.int32, [hp.batch_size])

        # Build model
        network = model.ResNet(
            hps=hp,
            tpu_only_ops=False,
            use_tpu=False,
            is_training=None,
            images=images,
            labels=labels)
        network.build_graph()

        epoch = FLAGS.epoch_id
        # Start running operations on the Graph.
        sess = tf.Session(
            config=tf.ConfigProto(
                gpu_options=tf.GPUOptions(
                    per_process_gpu_memory_fraction=FLAGS.gpu_fraction),
                log_device_placement=FLAGS.log_device_placement))

        saver = tf.train.Saver(tf.all_variables(), max_to_keep=10000)

        saver.restore(sess, os.path.join(FLAGS.model_dir, "model.ckpt-%d" % epoch))

        # Validate
        duration_valid, valid_loss, valid_acc = 0.0, 0.0, 0.0

        updates_per_epoch = 0

        for batch in data.iterate_minibatches(sess, X_valid, y_valid, hp.batch_size, shuffle=False, augment=False):
            valid_images, valid_labels = batch
            loss_value, acc_value = sess.run([network.loss, network.acc],
                                             feed_dict={images: valid_images,
                                                        labels: valid_labels,
                                                        network.is_training: False})

            valid_loss += loss_value
            valid_acc += acc_value
            updates_per_epoch += 1

        valid_loss /= updates_per_epoch
        valid_acc /= updates_per_epoch

        # Test
        duration_test, test_loss, test_acc = 0.0, 0.0, 0.0

        updates_per_epoch = 0
        for batch in data.iterate_minibatches(sess, X_test, y_test, hp.batch_size, shuffle=False, augment=False):
            test_images, test_labels = batch
            loss_value, acc_value = sess.run([network.loss, network.acc],
                                             feed_dict={images: test_images,
                                                        labels: test_labels,
                                                        network.is_training: False})

            test_loss += loss_value
            test_acc += acc_value
            updates_per_epoch += 1

        test_loss /= updates_per_epoch
        test_acc /= updates_per_epoch

        # Save results
        results = dict()
        results["valid_loss"] = float(valid_loss)
        results["valid_acc"] = float(valid_acc)
        results["test_loss"] = float(test_loss)
        results["test_acc"] = float(test_acc)

        n_params = np.sum([np.prod(v.shape) for v in tf.trainable_variables()])
        results["number_of_parameters"] = int(n_params.value)
        results["configuration"] = controller.hparams_to_json(hp)
        # results["runtime_train_epochs"] = runtime_train_epochs
        # results["runtime_test_epochs"] = runtime_test_epochs
        # results["runtime_valid_epochs"] = runtime_valid_epochs

        results["epoch"] = epoch

        opts = tf.profiler.ProfileOptionBuilder.float_operation()
        flops = tf.profiler.profile(g, run_meta=run_meta, cmd='op', options=opts)

        results["flops"] = flops.total_float_ops

        fh = open(os.path.join(FLAGS.save_dir, "results.json"), "w")
        json.dump(results, fh)
        fh.close()

def main(argv=None):
    logging.set_verbosity('INFO')
    evaluate()

if __name__ == '__main__':
    tf.app.run()
<commit_msg>save results correctly<commit_after>import os
import json
import numpy as np
import tensorflow as tf

from wrn_cifar10 import data  # pylint: disable=g-bad-import-order
from wrn_cifar10 import model # pylint: disable=g-bad-import-order
from wrn_cifar10 import controller # pylint: disable=g-bad-import-order

app = tf.app
logging = tf.logging
flags = tf.flags
gfile = tf.gfile

IMAGE_SIZE = 32

flags.DEFINE_string('data_dir', './cifar-10-batches-py/',
                    """Path to the CIFAR-10 python data.""")
flags.DEFINE_string('model_dir', './train', """Directory where model checkpoint are stored.""")
flags.DEFINE_integer('epoch_id', 28125, """Epoch number""")
flags.DEFINE_string('save_dir', './output', """Directory where metrics will be stored.""")
flags.DEFINE_float('gpu_fraction', 0.95,
                   """The fraction of GPU memory to be allocated""")
flags.DEFINE_boolean('log_device_placement', False,
                     """Whether to log device placement.""")

FLAGS = tf.app.flags.FLAGS

def evaluate():

    hparams_dict = json.load(open(os.path.join(FLAGS.model_dir, "hparams.json")))

    hp = tf.contrib.training.HParams(
        # Optimization.
        optimizer=model.Optimizer.MOMENTUM,
        initial_lr=hparams_dict["initial_lr"],
        lr_decay=model.LRDecaySchedule[hparams_dict["lr_decay"]],
        decay_steps=hparams_dict["decay_steps"],
        weight_decay=hparams_dict["weight_decay"],
        momentum=hparams_dict["momentum"],
        use_nesterov=hparams_dict["use_nesterov"],
        # Architecture.
        n_filters_1=hparams_dict["n_filters_1"],
        n_filters_2=hparams_dict["n_filters_2"],
        n_filters_3=hparams_dict["n_filters_3"],
        stride_1=hparams_dict["stride_1"],
        stride_2=hparams_dict["stride_2"],
        stride_3=hparams_dict["stride_3"],
        depthwise=hparams_dict["depthwise"],
        num_residual_units_1=hparams_dict["num_residual_units_1"],
        num_residual_units_2=hparams_dict["num_residual_units_3"],
        num_residual_units_3=hparams_dict["num_residual_units_2"],
        k=hparams_dict["k"],
        activation_1=model.Activation[hparams_dict["activation_1"]],
        activation_2=model.Activation[hparams_dict["activation_2"]],
        activation_3=model.Activation[hparams_dict["activation_3"]],
        n_conv_layers_1=hparams_dict["n_conv_layers_1"],
        n_conv_layers_2=hparams_dict["n_conv_layers_2"],
        n_conv_layers_3=hparams_dict["n_conv_layers_3"],
        dropout_1=hparams_dict["dropout_1"],
        dropout_2=hparams_dict["dropout_2"],
        dropout_3=hparams_dict["dropout_3"],
        # Misc.
        batch_size=hparams_dict["batch_size"],
        num_epochs=hparams_dict["num_epochs"],
        replicate=hparams_dict["replicate"],
    )

    with tf.Graph().as_default() as g:

        run_meta = tf.RunMetadata()
        # Load the data
        # pylint: disable=invalid-name
        X_train, y_train, X_valid, y_valid, X_test, y_test = data.load_data(
            FLAGS.data_dir)
        # pylint: enable=invalid-name

        # Build a Graph that computes the predictions from the inference model.
        images = tf.placeholder(tf.float32,
                                [hp.batch_size, IMAGE_SIZE, IMAGE_SIZE, 3])
        labels = tf.placeholder(tf.int32, [hp.batch_size])

        # Build model
        network = model.ResNet(
            hps=hp,
            tpu_only_ops=False,
            use_tpu=False,
            is_training=None,
            images=images,
            labels=labels)
        network.build_graph()

        epoch = FLAGS.epoch_id
        # Start running operations on the Graph.
        sess = tf.Session(
            config=tf.ConfigProto(
                gpu_options=tf.GPUOptions(
                    per_process_gpu_memory_fraction=FLAGS.gpu_fraction),
                log_device_placement=FLAGS.log_device_placement))

        saver = tf.train.Saver(tf.all_variables(), max_to_keep=10000)

        saver.restore(sess, os.path.join(FLAGS.model_dir, "model.ckpt-%d" % epoch))

        # Validate
        duration_valid, valid_loss, valid_acc = 0.0, 0.0, 0.0

        updates_per_epoch = 0

        for batch in data.iterate_minibatches(sess, X_valid, y_valid, hp.batch_size, shuffle=False, augment=False):
            valid_images, valid_labels = batch
            loss_value, acc_value = sess.run([network.loss, network.acc],
                                             feed_dict={images: valid_images,
                                                        labels: valid_labels,
                                                        network.is_training: False})

            valid_loss += loss_value
            valid_acc += acc_value
            updates_per_epoch += 1

        valid_loss /= updates_per_epoch
        valid_acc /= updates_per_epoch

        # Test
        duration_test, test_loss, test_acc = 0.0, 0.0, 0.0

        updates_per_epoch = 0
        for batch in data.iterate_minibatches(sess, X_test, y_test, hp.batch_size, shuffle=False, augment=False):
            test_images, test_labels = batch
            loss_value, acc_value = sess.run([network.loss, network.acc],
                                             feed_dict={images: test_images,
                                                        labels: test_labels,
                                                        network.is_training: False})

            test_loss += loss_value
            test_acc += acc_value
            updates_per_epoch += 1

        test_loss /= updates_per_epoch
        test_acc /= updates_per_epoch

        # Save results
        results = dict()
        results["valid_loss"] = float(valid_loss)
        results["valid_acc"] = float(valid_acc)
        results["test_loss"] = float(test_loss)
        results["test_acc"] = float(test_acc)

        n_params = np.sum([np.prod(v.shape) for v in tf.trainable_variables()])
        results["number_of_parameters"] = int(n_params.value)
        results["configuration"] = controller.hparams_to_json(hp)
        # results["runtime_train_epochs"] = runtime_train_epochs
        # results["runtime_test_epochs"] = runtime_test_epochs
        # results["runtime_valid_epochs"] = runtime_valid_epochs

        results["epoch"] = epoch

        opts = tf.profiler.ProfileOptionBuilder.float_operation()
        flops = tf.profiler.profile(g, run_meta=run_meta, cmd='op', options=opts)

        results["flops"] = flops.total_float_ops

        os.makedirs(os.path.join(FLAGS.save_dir), exist_ok=True)
        fh = open(os.path.join(FLAGS.save_dir, "results_%d.json" % epoch), "w")
        json.dump(results, fh)
        fh.close()

def main(argv=None):
    logging.set_verbosity('INFO')
    evaluate()

if __name__ == '__main__':
    tf.app.run()
<|endoftext|>


<commit_before># coding: utf-8

# maposmatic, the web front-end of the MapOSMatic city map generation system
# Copyright (C) 2009  David Decotigny
# Copyright (C) 2009  Frédéric Lehobey
# Copyright (C) 2009  Pierre Mauduit
# Copyright (C) 2009  David Mentré
# Copyright (C) 2009  Maxime Petazzoni
# Copyright (C) 2009  Thomas Petazzoni
# Copyright (C) 2009  Gaël Utard
# Copyright (C) 2019  Hartmut Holzgraefe

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Forms for MapOSMatic

from django import forms
from django.utils.safestring import mark_safe
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _
from django.forms.utils import ErrorList
import time

import ocitysmap
from www.maposmatic import models, widgets
import www.settings

from multiupload.fields import MultiFileField

class MapSearchForm(forms.Form):
    """
    The map search form, allowing search through the rendered maps.
    """

    query = forms.CharField(min_length=1, required=True,
                widget=forms.TextInput(attrs=
                    {'placeholder': _('Search for a map'),
                     'class': 'span2'}))

class MapRenderingJobForm(forms.ModelForm):
    """
    The main map rendering form, displayed on the 'Create Map' page. It's a
    ModelForm based on the MapRenderingJob model.
    """
    class Meta:
        model = models.MapRenderingJob
        fields = ('layout', 'stylesheet', 'overlay',
                  'maptitle', 'administrative_city',
                  'lat_upper_left', 'lon_upper_left',
                  'lat_bottom_right', 'lon_bottom_right',
                  'submittermail')

    mode = forms.CharField(initial='bbox', widget=forms.HiddenInput)
    layout = forms.ChoiceField(choices=(), widget=forms.RadioSelect(attrs= { 'onchange' : 'clearPaperSize(this.value); $("#layout-preview").attr("src","/media/img/layout/"+this.value+".png");'}))
    stylesheet = forms.ChoiceField(choices=(), widget=forms.Select(attrs= { 'onchange' : '$("#style-preview").attr("src","/media/img/style/"+this.value+".jpg");'}))
    overlay = forms.MultipleChoiceField(choices=(), widget=forms.SelectMultiple(attrs= { 'class': 'multipleSelect' }), required=False)
    paper_width_mm = forms.IntegerField(widget=forms.NumberInput(attrs= {'onchange' : 'change_papersize();', 'style': 'width: 5em;'}), min_value=100)
    paper_height_mm = forms.IntegerField(widget=forms.NumberInput(attrs= {'onchange' : 'change_papersize();', 'style': 'width: 5em;'}), min_value=100)
    maptitle = forms.CharField(max_length=256, required=False)
    bbox = widgets.AreaField(label=_("Area"),
                             fields=(forms.FloatField(), forms.FloatField(),
                                     forms.FloatField(), forms.FloatField()))
    uploadfile = MultiFileField(required=False)

    map_lang_flag_list = []
    for lang_key, lang_name in www.settings.MAP_LANGUAGES_LIST:
        if lang_key == 'C':
            map_lang_flag_list.append((lang_key, lang_name))
        else:
            country_code = lang_key[3:5].lower()
            lang_html = mark_safe("  %s"
                                       % (country_code, lang_name))
            map_lang_flag_list.append((lang_key, lang_html))

    map_language = forms.ChoiceField(choices=map_lang_flag_list,
                                     widget=forms.Select(
                                        attrs={'style': 'min-width: 200px'}))

    administrative_osmid = forms.IntegerField(widget=forms.HiddenInput,
                                              required=False)

    def __init__(self, *args, **kwargs):
        super(MapRenderingJobForm, self).__init__(*args, **kwargs)

        self._ocitysmap = ocitysmap.OCitySMap(www.settings.OCITYSMAP_CFG_PATH)

        layout_renderers = self._ocitysmap.get_all_renderers()
        stylesheets = self._ocitysmap.get_all_style_configurations()
        overlays = self._ocitysmap.get_all_overlay_configurations()

        self.fields['layout'].choices = []
        # TODO move descriptions to ocitysmap side
        for r in layout_renderers:
            if r.name == 'plain':
                description = _(u"Full-page layout without street index")
            elif r.name == 'single_page_index_side':
                description = _(u"Full-page layout with the street index on the side")
            elif r.name == 'single_page_index_bottom':
                description = _(u"Full-page layout with the street index at the bottom")
            elif r.name == 'single_page_index_extra_page':
                description = _(u"Full-page layout with the street index on extra page (PDF only)")
            elif r.name == 'multi_page':
                description = _(u"Multi-page layout")
            else:
                continue
                # description = mark_safe(_(u"The %(layout_name)s layout") % {'layout_name':r.name})
            self.fields['layout'].choices.append((r.name, description))

        if not self.fields['layout'].initial:
            self.fields['layout'].initial = layout_renderers[0].name

        style_choices = {"": []}
        for s in stylesheets:
            if s.description is not None:
                description = mark_safe(escape(s.description))
            else:
                description = mark_safe(_("The %(stylesheet_name)s stylesheet") % {'stylesheet_name':s.name})

            if s.url:
                description = mark_safe("%s " % (description, s.url, _("more info")))

            if s.group not in style_choices:
                style_choices[s.group] = []
            style_choices[s.group].append((s.name, description))

        grouped_choices = []
        for name, members in style_choices.items():
            grouped_choices.append([name, members])
        self.fields['stylesheet'].choices = grouped_choices;

        if not self.fields['stylesheet'].initial:
            self.fields['stylesheet'].initial = stylesheets[0].name

        overlay_choices = {}
        for s in overlays:
            if s.description is not None:
                description = mark_safe(escape(s.description))
            else:
                description = mark_safe(_("The %(stylesheet_name)s overlay") % {'stylesheet_name':s.name})

            if s.url:
                description = mark_safe("%s " % (description, s.url, _("more info")))

            if s.group not in overlay_choices:
                overlay_choices[s.group] = []
            overlay_choices[s.group].append((s.name, description))

        grouped_choices = []
        for name, members in overlay_choices.items():
            grouped_choices.append([name, members])
        self.fields['overlay'].choices = grouped_choices;

        if not self.fields['overlay'].initial:
            self.fields['overlay'].initial = ''

    def clean(self):
        """Cleanup function for the map query form. Different checks are
        required depending on the selected mode (by admininstrative city, or by
        bounding box).

        Returns the cleaned_data array.
        """

        cleaned_data = self.cleaned_data

        mode = cleaned_data.get("mode")
        city = cleaned_data.get("administrative_city")
        title = cleaned_data.get("maptitle")
        layout = cleaned_data.get("layout")
        stylesheet = cleaned_data.get("stylesheet")
        overlay_array = []
        try:
            for overlay in cleaned_data.get("overlay"):
                overlay_array.append(overlay)
        except:
            pass
        overlay = ",".join(overlay_array)

        if layout == '':
            msg = _(u"Layout required")
            self._errors["layout"] = ErrorList([msg])
            del cleaned_data["layout"]

        if stylesheet == '':
            msg = _(u"Stylesheet required")
            self._errors["stylesheet"] = ErrorList([msg])
            del cleaned_data["stylesheet"]

        if mode == 'admin':
            if city == "":
                msg = _(u"Administrative city required")
                self._errors["administrative_city"] = ErrorList([msg])
                del cleaned_data["administrative_city"]

            # Make sure that bbox and admin modes are exclusive
            # TODO: we should maybe merge these two instead? (See also OcitysMap Github Issue #24)
            cleaned_data["lat_upper_left"] = None
            cleaned_data["lon_upper_left"] = None
            cleaned_data["lat_bottom_right"] = None
            cleaned_data["lon_bottom_right"] = None

            try:
                self._check_osm_id(cleaned_data.get("administrative_osmid"))
            except Exception as ex:
                msg = _(u"Error with osm city: %s" % ex)
                self._errors['administrative_osmid'] \
                    = ErrorList([msg])

        elif mode == 'bbox':
            # Check bounding box corners are provided
            for f in [ "lat_upper_left", "lon_upper_left",
                       "lat_bottom_right", "lon_bottom_right" ]:
                val = cleaned_data.get(f)
                if val is None:
                    msg = _(u"Required")
                    self._errors['bbox'] = ErrorList([msg])
                    if f in cleaned_data:
                        del cleaned_data[f]

            # Check latitude and longitude are different
            if (cleaned_data.get("lat_upper_left")
                == cleaned_data.get("lat_bottom_right")):
                msg = _(u"Same latitude")
                self._errors['bbox'] = ErrorList([msg])
                del cleaned_data["lat_upper_left"]
                del cleaned_data["lat_bottom_right"]

            if (cleaned_data.get("lon_upper_left")
                == cleaned_data.get("lon_bottom_right")):
                msg = _(u"Same longitude")
                self._errors['bbox'] = ErrorList([msg])
                del cleaned_data["lon_upper_left"]
                del cleaned_data["lon_bottom_right"]

            # Make sure that bbox and admin modes are exclusive
            cleaned_data["administrative_city"] = ''
            cleaned_data["administrative_osmid"] = None

            # Don't try to instanciate a bounding box with empty coordinates
            if self._errors:
                return cleaned_data

            lat_upper_left = cleaned_data.get("lat_upper_left")
            lon_upper_left = cleaned_data.get("lon_upper_left")
            lat_bottom_right = cleaned_data.get("lat_bottom_right")
            lon_bottom_right = cleaned_data.get("lon_bottom_right")

            boundingbox = ocitysmap.coords.BoundingBox(
                lat_upper_left, lon_upper_left,
                lat_bottom_right, lon_bottom_right)
            (metric_size_lat, metric_size_long) = boundingbox.spheric_sizes()
            if (metric_size_lat > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS
                or metric_size_long > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS):
                msg = _(u"Bounding Box too large")
                self._errors['bbox'] = ErrorList([msg])

        return cleaned_data

    def _check_osm_id(self, osm_id):
        """Make sure that the supplied OSM Id is valid and can be accepted for
        rendering (bounding box not too large, etc.). Raise an exception in
        case of error."""
        bbox_wkt, area_wkt = self._ocitysmap.get_geographic_info(osm_id)
        bbox = ocitysmap.coords.BoundingBox.parse_wkt(bbox_wkt)
        (metric_size_lat, metric_size_long) = bbox.spheric_sizes()
        if metric_size_lat > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS or \
                metric_size_long > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS:
            raise ValueError("Area too large")

class MapPaperSizeForm(forms.Form):
    """
    The map paper size form, which is only used to analyze the
    arguments of the POST request to /apis/papersize/
    """
    osmid            = forms.IntegerField(required=False)
    layout           = forms.CharField(max_length=256)
    stylesheet       = forms.CharField(max_length=256)
    lat_upper_left   = forms.FloatField(required=False, min_value=-90.0, max_value=90.0)
    lon_upper_left   = forms.FloatField(required=False, min_value=-180.0, max_value=180.0)
    lat_bottom_right = forms.FloatField(required=False, min_value=-90.0, max_value=90.0)
    lon_bottom_right = forms.FloatField(required=False, min_value=-180.0, max_value=180.0)

class MapRecreateForm(forms.Form):
    """
    The map recreate form, to reschedule an already processed job on the queue.
    """

    id = forms.IntegerField(widget=forms.HiddenInput, required=True)

    def clean(self):
        cleaned_data = self.cleaned_data

        try:
            cleaned_data["id"] = int(cleaned_data.get("id", 0))
        except ValueError:
            cleaned_data["id"] = 0

        return cleaned_data

class MapCancelForm(forms.Form):
    """
    The map cancel form, to cancel a job (when the user has the matching
    nonce).
    """

    id = forms.IntegerField(widget=forms.HiddenInput, required=True)
    nonce = forms.CharField(widget=forms.HiddenInput, required=True)

    def clean(self):
        cleaned_data = self.cleaned_data

        try:
            cleaned_data["id"] = int(cleaned_data.get("id", 0))
        except ValueError:
            cleaned_data["id"] = 0

        return cleaned_data
<commit_msg>make sure that default styles without explicit group show first<commit_after># coding: utf-8

# maposmatic, the web front-end of the MapOSMatic city map generation system
# Copyright (C) 2009  David Decotigny
# Copyright (C) 2009  Frédéric Lehobey
# Copyright (C) 2009  Pierre Mauduit
# Copyright (C) 2009  David Mentré
# Copyright (C) 2009  Maxime Petazzoni
# Copyright (C) 2009  Thomas Petazzoni
# Copyright (C) 2009  Gaël Utard
# Copyright (C) 2019  Hartmut Holzgraefe

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Forms for MapOSMatic

from django import forms
from django.utils.safestring import mark_safe
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _
from django.forms.utils import ErrorList
import time

import ocitysmap
from www.maposmatic import models, widgets
import www.settings

from multiupload.fields import MultiFileField

class MapSearchForm(forms.Form):
    """
    The map search form, allowing search through the rendered maps.
    """

    query = forms.CharField(min_length=1, required=True,
                widget=forms.TextInput(attrs=
                    {'placeholder': _('Search for a map'),
                     'class': 'span2'}))

class MapRenderingJobForm(forms.ModelForm):
    """
    The main map rendering form, displayed on the 'Create Map' page. It's a
    ModelForm based on the MapRenderingJob model.
    """
    class Meta:
        model = models.MapRenderingJob
        fields = ('layout', 'stylesheet', 'overlay',
                  'maptitle', 'administrative_city',
                  'lat_upper_left', 'lon_upper_left',
                  'lat_bottom_right', 'lon_bottom_right',
                  'submittermail')

    mode = forms.CharField(initial='bbox', widget=forms.HiddenInput)
    layout = forms.ChoiceField(choices=(), widget=forms.RadioSelect(attrs= { 'onchange' : 'clearPaperSize(this.value); $("#layout-preview").attr("src","/media/img/layout/"+this.value+".png");'}))
    stylesheet = forms.ChoiceField(choices=(), widget=forms.Select(attrs= { 'onchange' : '$("#style-preview").attr("src","/media/img/style/"+this.value+".jpg");'}))
    overlay = forms.MultipleChoiceField(choices=(), widget=forms.SelectMultiple(attrs= { 'class': 'multipleSelect' }), required=False)
    paper_width_mm = forms.IntegerField(widget=forms.NumberInput(attrs= {'onchange' : 'change_papersize();', 'style': 'width: 5em;'}), min_value=100)
    paper_height_mm = forms.IntegerField(widget=forms.NumberInput(attrs= {'onchange' : 'change_papersize();', 'style': 'width: 5em;'}), min_value=100)
    maptitle = forms.CharField(max_length=256, required=False)
    bbox = widgets.AreaField(label=_("Area"),
                             fields=(forms.FloatField(), forms.FloatField(),
                                     forms.FloatField(), forms.FloatField()))
    uploadfile = MultiFileField(required=False)

    map_lang_flag_list = []
    for lang_key, lang_name in www.settings.MAP_LANGUAGES_LIST:
        if lang_key == 'C':
            map_lang_flag_list.append((lang_key, lang_name))
        else:
            country_code = lang_key[3:5].lower()
            lang_html = mark_safe("  %s"
                                       % (country_code, lang_name))
            map_lang_flag_list.append((lang_key, lang_html))

    map_language = forms.ChoiceField(choices=map_lang_flag_list,
                                     widget=forms.Select(
                                        attrs={'style': 'min-width: 200px'}))

    administrative_osmid = forms.IntegerField(widget=forms.HiddenInput,
                                              required=False)

    def __init__(self, *args, **kwargs):
        super(MapRenderingJobForm, self).__init__(*args, **kwargs)

        self._ocitysmap = ocitysmap.OCitySMap(www.settings.OCITYSMAP_CFG_PATH)

        layout_renderers = self._ocitysmap.get_all_renderers()
        stylesheets = self._ocitysmap.get_all_style_configurations()
        overlays = self._ocitysmap.get_all_overlay_configurations()

        self.fields['layout'].choices = []
        # TODO move descriptions to ocitysmap side
        for r in layout_renderers:
            if r.name == 'plain':
                description = _(u"Full-page layout without street index")
            elif r.name == 'single_page_index_side':
                description = _(u"Full-page layout with the street index on the side")
            elif r.name == 'single_page_index_bottom':
                description = _(u"Full-page layout with the street index at the bottom")
            elif r.name == 'single_page_index_extra_page':
                description = _(u"Full-page layout with the street index on extra page (PDF only)")
            elif r.name == 'multi_page':
                description = _(u"Multi-page layout")
            else:
                continue
                # description = mark_safe(_(u"The %(layout_name)s layout") % {'layout_name':r.name})
            self.fields['layout'].choices.append((r.name, description))

        if not self.fields['layout'].initial:
            self.fields['layout'].initial = layout_renderers[0].name

        style_choices = {"": []}
        for s in stylesheets:
            if s.description is not None:
                description = mark_safe(escape(s.description))
            else:
                description = mark_safe(_("The %(stylesheet_name)s stylesheet") % {'stylesheet_name':s.name})

            if s.url:
                description = mark_safe("%s " % (description, s.url, _("more info")))

            if s.group:
                group = s.group
            else:
                group = ""

            if group not in style_choices:
                style_choices[group] = []
            style_choices[s.group].append((s.name, description))

        grouped_choices = []
        for name, members in style_choices.items():
            grouped_choices.append([name, members])
        self.fields['stylesheet'].choices = grouped_choices;

        if not self.fields['stylesheet'].initial:
            self.fields['stylesheet'].initial = stylesheets[0].name

        overlay_choices = {}
        for s in overlays:
            if s.description is not None:
                description = mark_safe(escape(s.description))
            else:
                description = mark_safe(_("The %(stylesheet_name)s overlay") % {'stylesheet_name':s.name})

            if s.url:
                description = mark_safe("%s " % (description, s.url, _("more info")))

            if s.group not in overlay_choices:
                overlay_choices[s.group] = []
            overlay_choices[s.group].append((s.name, description))

        grouped_choices = []
        for name, members in overlay_choices.items():
            grouped_choices.append([name, members])
        self.fields['overlay'].choices = grouped_choices;

        if not self.fields['overlay'].initial:
            self.fields['overlay'].initial = ''

    def clean(self):
        """Cleanup function for the map query form. Different checks are
        required depending on the selected mode (by admininstrative city, or by
        bounding box).

        Returns the cleaned_data array.
        """

        cleaned_data = self.cleaned_data

        mode = cleaned_data.get("mode")
        city = cleaned_data.get("administrative_city")
        title = cleaned_data.get("maptitle")
        layout = cleaned_data.get("layout")
        stylesheet = cleaned_data.get("stylesheet")
        overlay_array = []
        try:
            for overlay in cleaned_data.get("overlay"):
                overlay_array.append(overlay)
        except:
            pass
        overlay = ",".join(overlay_array)

        if layout == '':
            msg = _(u"Layout required")
            self._errors["layout"] = ErrorList([msg])
            del cleaned_data["layout"]

        if stylesheet == '':
            msg = _(u"Stylesheet required")
            self._errors["stylesheet"] = ErrorList([msg])
            del cleaned_data["stylesheet"]

        if mode == 'admin':
            if city == "":
                msg = _(u"Administrative city required")
                self._errors["administrative_city"] = ErrorList([msg])
                del cleaned_data["administrative_city"]

            # Make sure that bbox and admin modes are exclusive
            # TODO: we should maybe merge these two instead? (See also OcitysMap Github Issue #24)
            cleaned_data["lat_upper_left"] = None
            cleaned_data["lon_upper_left"] = None
            cleaned_data["lat_bottom_right"] = None
            cleaned_data["lon_bottom_right"] = None

            try:
                self._check_osm_id(cleaned_data.get("administrative_osmid"))
            except Exception as ex:
                msg = _(u"Error with osm city: %s" % ex)
                self._errors['administrative_osmid'] \
                    = ErrorList([msg])

        elif mode == 'bbox':
            # Check bounding box corners are provided
            for f in [ "lat_upper_left", "lon_upper_left",
                       "lat_bottom_right", "lon_bottom_right" ]:
                val = cleaned_data.get(f)
                if val is None:
                    msg = _(u"Required")
                    self._errors['bbox'] = ErrorList([msg])
                    if f in cleaned_data:
                        del cleaned_data[f]

            # Check latitude and longitude are different
            if (cleaned_data.get("lat_upper_left")
                == cleaned_data.get("lat_bottom_right")):
                msg = _(u"Same latitude")
                self._errors['bbox'] = ErrorList([msg])
                del cleaned_data["lat_upper_left"]
                del cleaned_data["lat_bottom_right"]

            if (cleaned_data.get("lon_upper_left")
                == cleaned_data.get("lon_bottom_right")):
                msg = _(u"Same longitude")
                self._errors['bbox'] = ErrorList([msg])
                del cleaned_data["lon_upper_left"]
                del cleaned_data["lon_bottom_right"]

            # Make sure that bbox and admin modes are exclusive
            cleaned_data["administrative_city"] = ''
            cleaned_data["administrative_osmid"] = None

            # Don't try to instanciate a bounding box with empty coordinates
            if self._errors:
                return cleaned_data

            lat_upper_left = cleaned_data.get("lat_upper_left")
            lon_upper_left = cleaned_data.get("lon_upper_left")
            lat_bottom_right = cleaned_data.get("lat_bottom_right")
            lon_bottom_right = cleaned_data.get("lon_bottom_right")

            boundingbox = ocitysmap.coords.BoundingBox(
                lat_upper_left, lon_upper_left,
                lat_bottom_right, lon_bottom_right)
            (metric_size_lat, metric_size_long) = boundingbox.spheric_sizes()
            if (metric_size_lat > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS
                or metric_size_long > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS):
                msg = _(u"Bounding Box too large")
                self._errors['bbox'] = ErrorList([msg])

        return cleaned_data

    def _check_osm_id(self, osm_id):
        """Make sure that the supplied OSM Id is valid and can be accepted for
        rendering (bounding box not too large, etc.). Raise an exception in
        case of error."""
        bbox_wkt, area_wkt = self._ocitysmap.get_geographic_info(osm_id)
        bbox = ocitysmap.coords.BoundingBox.parse_wkt(bbox_wkt)
        (metric_size_lat, metric_size_long) = bbox.spheric_sizes()
        if metric_size_lat > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS or \
                metric_size_long > www.settings.BBOX_MAXIMUM_LENGTH_IN_METERS:
            raise ValueError("Area too large")

class MapPaperSizeForm(forms.Form):
    """
    The map paper size form, which is only used to analyze the
    arguments of the POST request to /apis/papersize/
    """
    osmid            = forms.IntegerField(required=False)
    layout           = forms.CharField(max_length=256)
    stylesheet       = forms.CharField(max_length=256)
    lat_upper_left   = forms.FloatField(required=False, min_value=-90.0, max_value=90.0)
    lon_upper_left   = forms.FloatField(required=False, min_value=-180.0, max_value=180.0)
    lat_bottom_right = forms.FloatField(required=False, min_value=-90.0, max_value=90.0)
    lon_bottom_right = forms.FloatField(required=False, min_value=-180.0, max_value=180.0)

class MapRecreateForm(forms.Form):
    """
    The map recreate form, to reschedule an already processed job on the queue.
    """

    id = forms.IntegerField(widget=forms.HiddenInput, required=True)

    def clean(self):
        cleaned_data = self.cleaned_data

        try:
            cleaned_data["id"] = int(cleaned_data.get("id", 0))
        except ValueError:
            cleaned_data["id"] = 0

        return cleaned_data

class MapCancelForm(forms.Form):
    """
    The map cancel form, to cancel a job (when the user has the matching
    nonce).
    """

    id = forms.IntegerField(widget=forms.HiddenInput, required=True)
    nonce = forms.CharField(widget=forms.HiddenInput, required=True)

    def clean(self):
        cleaned_data = self.cleaned_data

        try:
            cleaned_data["id"] = int(cleaned_data.get("id", 0))
        except ValueError:
            cleaned_data["id"] = 0

        return cleaned_data
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
# Andre Anjos <andre.anjos@idiap.ch>
# Thu Mar 20 12:38:14 CET 2014

"""Helps looking for Boost on stock file-system locations"""

import os
import re
import sys
import glob
from distutils.version import LooseVersion

from .utils import uniq

def boost_version(boost_dir):

  fname = os.path.join(boost_dir, "version.hpp")
  if not os.path.exists(fname):
    fname = os.path.join(boost_dir, "boost/version.hpp")
  if not os.path.exists(fname):
    return None

  try:
    m = re.search("#\s*define\s+BOOST_VERSION\s+(\d+)", open(fname).read())
    if m:
      version_int = int(m.group(1))
      version_tuple = (version_int // 100000, (version_int // 100) % 1000,
                       version_int % 100)
      return '.'.join([str(k) for k in version_tuple])
  except:
    pass

  return None

class boost:
  """A class for capturing configuration information from boost

  Example usage:

  .. doctest::
     :options: +NORMALIZE_WHITESPACE +ELLIPSIS

     >>> from xbob.extension import boost
     >>> boost = boost('>= 1.35')
     >>> boost.include_directory
     '...'
     >>> boost.version
     '...'

  You can also use this class to retrieve information about installed Boost
  libraries and link information:

  .. doctest::
     :options: +NORMALIZE_WHITESPACE +ELLIPSIS

     >>> from xbob.extension import boost
     >>> boost = boost('>= 1.35')
     >>> boost.libconfig(['python', 'system'])
     (...)

  """

  def __init__ (self, requirement=''):
    """
    Searches for the Boost library in stock locations. Allows user to override.

    If the user sets the environment variable XBOB_PREFIX_PATH, that prefixes
    the standard path locations.
    """

    default_roots = [
        "/usr",
        "/usr/local",
        "/opt/local",
        ]

    if 'XBOB_PREFIX_PATH' in os.environ:
      roots = os.environ['XBOB_PREFIX_PATH'].split(os.pathsep) + default_roots
    else:
      roots = default_roots

    globs = []
    for k in roots:
      globs += [
          os.path.join(k, 'include', 'boost') + os.sep,
          os.path.join(k, 'include', 'boost?*') + os.sep,
          ]
    valid_globs = [glob.glob(k) for k in globs]
    candidates = [d for k in valid_globs for d in k] #flatten

    if not candidates:
      raise RuntimeError("could not find any version of boost on the file system (looked at: %s)" % (', '.join(candidates)))

    if not requirement:
      self.include_directory = candidates[0]
      self.version = boost_version(self.include_directory)

    else:

      # requirement is 'operator' 'version'
      operator, required = [k.strip() for k in requirement.split(' ', 1)]

      # now check for user requirements
      for path in candidates:
        version = boost_version(path)
        available = LooseVersion(version)
        if (operator == '<' and available < required) or \
           (operator == '<=' and available <= required) or \
           (operator == '>' and available > required) or \
           (operator == '>=' and available >= required) or \
           (operator == '==' and available == required):
          self.include_directory = path
          self.version = version
        else:
          raise RuntimeError("could not find the required (%s) version of boost on the file system (looked at: %s)" % (requirement, ', '.join(candidates)))

    # normalize
    self.include_directory = os.path.normpath(self.include_directory)

  def libconfig(self, modules, only_static=False,
      templates=['boost_%(name)s-mt-%(py)s',
        'boost_%(name)s-mt', 'boost_%(name)s']):
    """Returns a tuple containing the library configuration for requested
    modules.

    This function respects the path location where the include files for Boost
    are installed.

    Parameters:

    modules (list of strings)
      A list of string specifying the requested libraries to search for. For
      example, to search for `libboost_mpi.so`, pass only ``mpi``.

    static (bool)
      A boolean, indicating if we should try only to search for static versions
      of the libraries. If not set, any would do.

    templates (list of template strings)
      A list that defines in which order to search for libraries on the default
      search path, defined by ``self.include_directory``. Tune this list if you
      have compiled specific versions of Boost with support to multi-threading
      (``-mt``), debug (``-g``), STLPORT (``-p``) or required to insert
      compiler, the underlying thread API used or your own namespace.

      Here are the keywords you can use:

      %(name)s
        resolves to the module name you are searching for

      %(version)s
        resolves to the current boost version string (e.g. ``'1.50.0'``)

      %(py)s
        resolves to the string ``'pyXY'`` where ``XY`` represent the major and
        minor versions of the current python interpreter.

      Example templates:

      * ``'boost_%(name)s-mt'``
      * ``'boost_%(name)s'``
      * ``'boost_%(name)s-gcc43-%(version)s'``

    Returns:

    directories (list of strings)
      A list of directories indicating where the libraries are installed

    libs (list of strings)
      A list of strings indicating the names of the libraries you can use
    """

    if only_static:
      extensions = ['.a']
    else:
      if sys.platform == 'darwin':
        extensions = ['.dylib', '.a']
      elif sys.platform == 'win32':
        extensions = ['.dll', '.a']
      else: # linux like
        extensions = ['.so', '.a']

    prefix = os.path.dirname(os.path.dirname(self.include_directory))

    libpaths = [
        os.path.join(prefix, 'lib'),
        ]

    if __import__('platform').architecture()[0] == '32bit':
      libpaths += [
          os.path.join(prefix, 'lib', 'i386-linux-gnu'),
          os.path.join(prefix, 'lib32'),
          ]
    else:
      libpaths += [
          os.path.join(prefix, 'lib', 'x86_64-linux-gnu'),
          os.path.join(prefix, 'lib64'),
          ]

    py = 'py%d%d' % sys.version_info[:2]

    def paths(module):
      """Yields all possible paths for a module in good order"""

      for extension in extensions:
        for template in templates:
          for libpath in libpaths:
            modname = template % dict(name=module, version=self.version, py=py)
            yield modname, os.path.join(libpath, 'lib' + modname + extension)

    items = {}
    for module in modules:
      for modname, path in paths(module):
        if os.path.exists(path):
          items[module] = (os.path.dirname(path), modname)
          break

    # checks all modules were found, reports
    for module in modules:
      if module not in items:
        raise RuntimeError("cannot find required boost module `%s', searched as `%s'" % (module, ', '.join([k[1] for k in paths(module)])))

    libpaths, libraries = zip(*items.values())

    return uniq(libpaths), uniq(libraries)

  def macros(self):
    """Returns package availability and version number macros

    This method returns a python list with 2 macros indicating package
    availability and a version number, using standard GNU compatible names. For
    example, if the package is named ``foo`` and its version is ``1.4``, this
    command would return:

    .. doctest::
       :options: +NORMALIZE_WHITESPACE +ELLIPSIS

       >>> from xbob.extension import boost
       >>> pkg = boost('>= 1.34')
       >>> pkg.macros()
       [('HAVE_BOOST', '1'), ('BOOST_VERSION', '"..."')]

    """
    return [('HAVE_BOOST', '1'), ('BOOST_VERSION', '"%s"' % self.version)]

<commit_msg>Fix version requirement logic<commit_after>#!/usr/bin/env python
# encoding: utf-8
# Andre Anjos <andre.anjos@idiap.ch>
# Thu Mar 20 12:38:14 CET 2014

"""Helps looking for Boost on stock file-system locations"""

import os
import re
import sys
import glob
from distutils.version import LooseVersion

from .utils import uniq

def boost_version(boost_dir):

  fname = os.path.join(boost_dir, "version.hpp")
  if not os.path.exists(fname):
    fname = os.path.join(boost_dir, "boost/version.hpp")
  if not os.path.exists(fname):
    return None

  try:
    m = re.search("#\s*define\s+BOOST_VERSION\s+(\d+)", open(fname).read())
    if m:
      version_int = int(m.group(1))
      version_tuple = (version_int // 100000, (version_int // 100) % 1000,
                       version_int % 100)
      return '.'.join([str(k) for k in version_tuple])
  except:
    pass

  return None

class boost:
  """A class for capturing configuration information from boost

  Example usage:

  .. doctest::
     :options: +NORMALIZE_WHITESPACE +ELLIPSIS

     >>> from xbob.extension import boost
     >>> boost = boost('>= 1.35')
     >>> boost.include_directory
     '...'
     >>> boost.version
     '...'

  You can also use this class to retrieve information about installed Boost
  libraries and link information:

  .. doctest::
     :options: +NORMALIZE_WHITESPACE +ELLIPSIS

     >>> from xbob.extension import boost
     >>> boost = boost('>= 1.35')
     >>> boost.libconfig(['python', 'system'])
     (...)

  """

  def __init__ (self, requirement=''):
    """
    Searches for the Boost library in stock locations. Allows user to override.

    If the user sets the environment variable XBOB_PREFIX_PATH, that prefixes
    the standard path locations.
    """

    default_roots = [
        "/usr",
        "/usr/local",
        "/opt/local",
        ]

    if 'XBOB_PREFIX_PATH' in os.environ:
      roots = os.environ['XBOB_PREFIX_PATH'].split(os.pathsep) + default_roots
    else:
      roots = default_roots

    globs = []
    for k in roots:
      globs += [
          os.path.join(k, 'include', 'boost') + os.sep,
          os.path.join(k, 'include', 'boost?*') + os.sep,
          ]
    valid_globs = [glob.glob(k) for k in globs]
    candidates = [d for k in valid_globs for d in k] #flatten

    if not candidates:
      raise RuntimeError("could not find any version of boost on the file system (looked at: %s)" % (', '.join(candidates)))

    found = False

    if not requirement:
      self.include_directory = candidates[0]
      self.version = boost_version(self.include_directory)
      found = True

    else:

      # requirement is 'operator' 'version'
      operator, required = [k.strip() for k in requirement.split(' ', 1)]

      # now check for user requirements
      for path in candidates:
        version = boost_version(path)
        available = LooseVersion(version)
        if (operator == '<' and available < required) or \
           (operator == '<=' and available <= required) or \
           (operator == '>' and available > required) or \
           (operator == '>=' and available >= required) or \
           (operator == '==' and available == required):
          self.include_directory = path
          self.version = version
          found = True
          break

    if not found:
      raise RuntimeError("could not find the required (%s) version of boost on the file system (looked at: %s)" % (requirement, ', '.join(candidates)))

    # normalize
    self.include_directory = os.path.normpath(self.include_directory)

  def libconfig(self, modules, only_static=False,
      templates=['boost_%(name)s-mt-%(py)s',
        'boost_%(name)s-mt', 'boost_%(name)s']):
    """Returns a tuple containing the library configuration for requested
    modules.

    This function respects the path location where the include files for Boost
    are installed.

    Parameters:

    modules (list of strings)
      A list of string specifying the requested libraries to search for. For
      example, to search for `libboost_mpi.so`, pass only ``mpi``.

    static (bool)
      A boolean, indicating if we should try only to search for static versions
      of the libraries. If not set, any would do.

    templates (list of template strings)
      A list that defines in which order to search for libraries on the default
      search path, defined by ``self.include_directory``. Tune this list if you
      have compiled specific versions of Boost with support to multi-threading
      (``-mt``), debug (``-g``), STLPORT (``-p``) or required to insert
      compiler, the underlying thread API used or your own namespace.

      Here are the keywords you can use:

      %(name)s
        resolves to the module name you are searching for

      %(version)s
        resolves to the current boost version string (e.g. ``'1.50.0'``)

      %(py)s
        resolves to the string ``'pyXY'`` where ``XY`` represent the major and
        minor versions of the current python interpreter.

      Example templates:

      * ``'boost_%(name)s-mt'``
      * ``'boost_%(name)s'``
      * ``'boost_%(name)s-gcc43-%(version)s'``

    Returns:

    directories (list of strings)
      A list of directories indicating where the libraries are installed

    libs (list of strings)
      A list of strings indicating the names of the libraries you can use
    """

    if only_static:
      extensions = ['.a']
    else:
      if sys.platform == 'darwin':
        extensions = ['.dylib', '.a']
      elif sys.platform == 'win32':
        extensions = ['.dll', '.a']
      else: # linux like
        extensions = ['.so', '.a']

    prefix = os.path.dirname(os.path.dirname(self.include_directory))

    libpaths = [
        os.path.join(prefix, 'lib'),
        ]

    if __import__('platform').architecture()[0] == '32bit':
      libpaths += [
          os.path.join(prefix, 'lib', 'i386-linux-gnu'),
          os.path.join(prefix, 'lib32'),
          ]
    else:
      libpaths += [
          os.path.join(prefix, 'lib', 'x86_64-linux-gnu'),
          os.path.join(prefix, 'lib64'),
          ]

    py = 'py%d%d' % sys.version_info[:2]

    def paths(module):
      """Yields all possible paths for a module in good order"""

      for extension in extensions:
        for template in templates:
          for libpath in libpaths:
            modname = template % dict(name=module, version=self.version, py=py)
            yield modname, os.path.join(libpath, 'lib' + modname + extension)

    items = {}
    for module in modules:
      for modname, path in paths(module):
        if os.path.exists(path):
          items[module] = (os.path.dirname(path), modname)
          break

    # checks all modules were found, reports
    for module in modules:
      if module not in items:
        raise RuntimeError("cannot find required boost module `%s', searched as `%s'" % (module, ', '.join([k[1] for k in paths(module)])))

    libpaths, libraries = zip(*items.values())

    return uniq(libpaths), uniq(libraries)

  def macros(self):
    """Returns package availability and version number macros

    This method returns a python list with 2 macros indicating package
    availability and a version number, using standard GNU compatible names. For
    example, if the package is named ``foo`` and its version is ``1.4``, this
    command would return:

    .. doctest::
       :options: +NORMALIZE_WHITESPACE +ELLIPSIS

       >>> from xbob.extension import boost
       >>> pkg = boost('>= 1.34')
       >>> pkg.macros()
       [('HAVE_BOOST', '1'), ('BOOST_VERSION', '"..."')]

    """
    return [('HAVE_BOOST', '1'), ('BOOST_VERSION', '"%s"' % self.version)]

<|endoftext|>


<commit_before>"""
(c) RIKEN 2015. All rights reserved.
Author: Keitaro Yamashita

This software is released under the new BSD License; see LICENSE.
"""
import os
import sys
import re
import shutil
import subprocess
import commands
import glob
import tempfile
from libtbx.utils import null_out
import libtbx.load_env

def call(cmd, arg="",
         stdin=None, stdout=subprocess.PIPE,
         wdir=None,
         expects_in=[], expects_out=[]):

    ##
    # call the external program using subprocess.
    #
    # @param expects_in expected files before running
    # @param expects_out expected files after running
    #
    # expected_in/out must be written as relative path from wdir or absolute path.
    #

    def check_exist(files):
        is_exist = [os.path.isfile(f) for f in files]

        if sum(is_exist) != len(is_exist):
            not_founds = [ f for f, e in zip(files, is_exist) if not e ]
            raise Exception("Expected file(s) not found: " + " ".join(not_founds))

    # check_exist()

    if wdir is None:
        wdir = os.getcwd()

    # Go to working directory
    cwd = os.getcwd()
    os.chdir(wdir)

    # check before run
    check_exist(expects_in)

    # call the program
    p = subprocess.Popen("%s %s" % (cmd, arg),
                         shell=True,
                         stdin=subprocess.PIPE,
                         stdout=stdout,
                         stderr=stdout
                         )

    if stdin is not None:
        p.stdin.write(stdin)

    if stdout == subprocess.PIPE:
        out, err = p.communicate()
    else:
        out, err = None, None
        p.stdin.close()
        p.wait()

    if p.returncode < 0:
        print >>sys.stderr, cmd, ": returncode is", p.returncode

    # check after run
    check_exist(expects_out)

    # go back to the previous working directory
    os.chdir(cwd)

    return p.returncode, out, err
# call()

def rotate_file(filename, copy=False):
    """
    Rotate file like logrotate.
    If given filename already exists, rename it to "filename".n, n=1...
    Filename with larger n is older one.
    """

    # If not exist,
    if not os.path.isfile(filename):
        return

    # make list [ [filename, number], ... ]
    old_list = []
    dot_files = glob.glob(filename + ".*")
    for f in dot_files:
        suffix = f.replace(filename+".", "")
        try:
            i = int(suffix)
            if str(i) == suffix: # ignore if suffix was such as 003...
                old_list.append([f, i])
        except ValueError, e:
            continue

    old_list.sort(lambda x,y: x[1]-y[1])

    # rotate files
    for f, i in reversed(old_list):
        os.rename(f, "%s.%d" % (f[:f.rfind(".")], i+1))

    if copy:
        shutil.copyfile(filename, filename + ".1")
    else:
        os.rename(filename, filename + ".1")

    return filename + ".1"
# rotate_file()

def safe_copy(src, dst, move=False):
    """
    Don't reveal file before copy completed.
    """

    src_name = os.path.basename(src)
    if os.path.isdir(dst): dst = os.path.join(dst, src_name)

    tmpfd, tmp = tempfile.mkstemp(prefix="."+src_name, dir=os.path.dirname(dst))
    os.close(tmpfd)

    shutil.copy2(src, tmp)
    os.rename(tmp, dst)

    if move and os.path.isfile(dst) and not os.path.islink(dst) and os.path.getsize(src)==os.path.getsize(dst):
        os.remove(src)

# safe_copy()

def commonalize(Is):
    new_Is = []
    Is0 = Is[0]
    for I in Is[1:]:
        Is0, I = Is0.common_sets(I, assert_is_similar_symmetry=False)
        new_Is.append(I)

    Is = []

    for I in new_Is:
        I = I.common_set(Is0, assert_is_similar_symmetry=False)
        assert len(Is0.data()) == len(I.data())
        Is.append(I)

    return [Is0,] + Is
# commonalize()

def get_number_of_processors(default=4):
    nproc = default

    if os.path.isfile("/proc/cpuinfo"):
        nproc = len(filter(lambda x:x.startswith("processor"), open("/proc/cpuinfo")))
    else:
        try:
            nproc = int(commands.getoutput("sysctl -n hw.ncpu"))
        except:
            pass

    return nproc
# get_number_of_processors()

def safe_float(v):
    try:
        return float(v)
    except ValueError:
        return float("nan")
# safe_float()

def num_th_str(v):
    s = str(v)
    if s[-1] == "1": return s+"st"
    if s[-1] == "2": return s+"nd"
    if s[-1] == "3": return s+"rd"
    return s+"th"
# num_th_str()

def directory_included(path, topdir=None, include_dir=[], exclude_dir=[]):
    if topdir is None:
        for d in include_dir:
            if directory_included(path, d): return True
        return False

    l1 = filter(lambda x: x, path.split(os.sep))
    l2 = filter(lambda x: x, topdir.split(os.sep))

    lc = os.path.commonprefix([l1,l2])

    if len(lc) != len(l2): return False

    if include_dir == exclude_dir == []:
        return True

    if include_dir != []:
        for d in include_dir:
            if directory_included(path, d): return True
        return False

    if exclude_dir != []:
        for d in exclude_dir:
            if directory_included(path, d): return False
        return True
# directory_included()

def read_path_list(lstin, comment_strs=["#"], only_exists=False, as_abspath=False, err_out=null_out()):
    ret = []

    for l in open(lstin):
        for c in comment_strs:
            if c in l: l = l[:l.index(c)]

        l = l.strip()
        if not l: continue
        if only_exists and not os.path.exists(l):
            err_out.write("Error: file not found: %s\n"%l)
            continue
        ret.append(os.path.abspath(l) if as_abspath else l)

    return ret
# read_path_list()

def return_first_found_file(files, wd=None):
    for f in files:
        if wd is not None: f = os.path.join(wd, f)
        if os.path.isfile(f): return f
# return_first_found_file()

def expand_wildcard_in_list(fdlst, err_out=null_out()):
    ret = []
    for d in fdlst:
        gd = glob.glob(d)

        if len(gd) == 0:
            print >>err_out, "Error: No match!!: %s" % d
            continue
        ret.extend(gd)
    return ret
# expand_wildcard_in_list()

def check_disk_free_bytes(d):
    try:
        x = os.statvfs(d)
        return x.f_frsize * x.f_bavail
    except:
        return -1
# check_disk_free_bytes()

def get_temp_local_dir(prefix, min_bytes=None, min_kb=None, min_mb=None, min_gb=None, additional_tmpd=None):
    assert (min_bytes, min_kb, min_mb, min_gb).count(None) >= 2

    min_free_bytes = 0

    if min_bytes is not None: min_free_bytes = min_bytes
    if min_kb is not None: min_free_bytes = min_kb * 1024
    if min_mb is not None: min_free_bytes = min_mb * 1024**2
    if min_gb is not None: min_free_bytes = min_gb * 1024**3

    ramdisk = "/dev/shm"

    if os.path.isdir(ramdisk): tmpdirs = [ramdisk, tempfile.gettempdir()]
    else: tmpdirs = [tempfile.gettempdir()]

    if type(additional_tmpd) is str:
        tmpdirs.append(additional_tmpd)
    elif type(additional_tmpd) in (list, tuple):
        tmpdirs.extend(additional_tmpd)

    for tmpdir in tmpdirs:
        if check_disk_free_bytes(tmpdir) >= min_free_bytes:
            return tempfile.mkdtemp(prefix=prefix, dir=tmpdir)

    return None
# get_temp_local_dir()

def get_temp_filename(prefix="tmp", suffix="", wdir=None):
    tmpfd, tmp = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=wdir)
    os.close(tmpfd)
    return tmp
# get_temp_filename()

def replace_forbidden_chars(filename, repl="-"):
    return re.sub(r"[/><\*\\\?%:]", repl, filename)
# replace_forbidden_chars()

def human_readable_bytes(bytes):
    if bytes < 1024:
        return bytes, "B"
    elif bytes < 1024**2:
        return bytes/1024., "KB"
    elif bytes < 1024**3:
        return bytes/1024.**2, "MB"
    elif bytes < 1024**4:
        return bytes/1024.**3, "GB"
    elif bytes < 1024**5:
        return bytes/1024.**4, "TB"
    else:# if bytes < 1024**6:
        return bytes/1024.**5, "PB"

# human_readable_bytes()

def yamtbx_module_root():
    """
    Possible locations: modules/yamtbx or modules/yamtbx/yamtbx
    """
    tmp = libtbx.env.find_in_repositories("yamtbx/yamtbx")
    if tmp: return tmp
    tmp = libtbx.env.find_in_repositories("yamtbx")
    if tmp: return tmp
# yamtbx_module_root()
<commit_msg>forgot to include in Commit a6b9679c<commit_after>"""
(c) RIKEN 2015. All rights reserved.
Author: Keitaro Yamashita

This software is released under the new BSD License; see LICENSE.
"""
import os
import sys
import re
import time
import shutil
import subprocess
import commands
import glob
import tempfile
import traceback
from libtbx.utils import null_out
import libtbx.load_env

def call(cmd, arg="",
         stdin=None, stdout=subprocess.PIPE,
         wdir=None,
         expects_in=[], expects_out=[]):

    ##
    # call the external program using subprocess.
    #
    # @param expects_in expected files before running
    # @param expects_out expected files after running
    #
    # expected_in/out must be written as relative path from wdir or absolute path.
    #

    def check_exist(files):
        is_exist = [os.path.isfile(f) for f in files]

        if sum(is_exist) != len(is_exist):
            not_founds = [ f for f, e in zip(files, is_exist) if not e ]
            raise Exception("Expected file(s) not found: " + " ".join(not_founds))

    # check_exist()

    if wdir is None:
        wdir = os.getcwd()

    # Go to working directory
    cwd = os.getcwd()
    os.chdir(wdir)

    # check before run
    check_exist(expects_in)

    # call the program
    p = subprocess.Popen("%s %s" % (cmd, arg),
                         shell=True,
                         stdin=subprocess.PIPE,
                         stdout=stdout,
                         stderr=stdout
                         )

    if stdin is not None:
        p.stdin.write(stdin)

    if stdout == subprocess.PIPE:
        out, err = p.communicate()
    else:
        out, err = None, None
        p.stdin.close()
        p.wait()

    if p.returncode < 0:
        print >>sys.stderr, cmd, ": returncode is", p.returncode

    # check after run
    check_exist(expects_out)

    # go back to the previous working directory
    os.chdir(cwd)

    return p.returncode, out, err
# call()

def rotate_file(filename, copy=False):
    """
    Rotate file like logrotate.
    If given filename already exists, rename it to "filename".n, n=1...
    Filename with larger n is older one.
    """

    # If not exist,
    if not os.path.isfile(filename):
        return

    # make list [ [filename, number], ... ]
    old_list = []
    dot_files = glob.glob(filename + ".*")
    for f in dot_files:
        suffix = f.replace(filename+".", "")
        try:
            i = int(suffix)
            if str(i) == suffix: # ignore if suffix was such as 003...
                old_list.append([f, i])
        except ValueError, e:
            continue

    old_list.sort(lambda x,y: x[1]-y[1])

    # rotate files
    for f, i in reversed(old_list):
        os.rename(f, "%s.%d" % (f[:f.rfind(".")], i+1))

    if copy:
        shutil.copyfile(filename, filename + ".1")
    else:
        os.rename(filename, filename + ".1")

    return filename + ".1"
# rotate_file()

def safe_copy(src, dst, move=False):
    """
    Don't reveal file before copy completed.
    """

    src_name = os.path.basename(src)
    if os.path.isdir(dst): dst = os.path.join(dst, src_name)

    tmpfd, tmp = tempfile.mkstemp(prefix="."+src_name, dir=os.path.dirname(dst))
    os.close(tmpfd)

    shutil.copy2(src, tmp)
    os.rename(tmp, dst)

    if move and os.path.isfile(dst) and not os.path.islink(dst) and os.path.getsize(src)==os.path.getsize(dst):
        os.remove(src)

# safe_copy()

def commonalize(Is):
    new_Is = []
    Is0 = Is[0]
    for I in Is[1:]:
        Is0, I = Is0.common_sets(I, assert_is_similar_symmetry=False)
        new_Is.append(I)

    Is = []

    for I in new_Is:
        I = I.common_set(Is0, assert_is_similar_symmetry=False)
        assert len(Is0.data()) == len(I.data())
        Is.append(I)

    return [Is0,] + Is
# commonalize()

def get_number_of_processors(default=4):
    nproc = default

    if os.path.isfile("/proc/cpuinfo"):
        nproc = len(filter(lambda x:x.startswith("processor"), open("/proc/cpuinfo")))
    else:
        try:
            nproc = int(commands.getoutput("sysctl -n hw.ncpu"))
        except:
            pass

    return nproc
# get_number_of_processors()

def safe_float(v):
    try:
        return float(v)
    except ValueError:
        return float("nan")
# safe_float()

def num_th_str(v):
    s = str(v)
    if s[-1] == "1": return s+"st"
    if s[-1] == "2": return s+"nd"
    if s[-1] == "3": return s+"rd"
    return s+"th"
# num_th_str()

def directory_included(path, topdir=None, include_dir=[], exclude_dir=[]):
    if topdir is None:
        for d in include_dir:
            if directory_included(path, d): return True
        return False

    l1 = filter(lambda x: x, path.split(os.sep))
    l2 = filter(lambda x: x, topdir.split(os.sep))

    lc = os.path.commonprefix([l1,l2])

    if len(lc) != len(l2): return False

    if include_dir == exclude_dir == []:
        return True

    if include_dir != []:
        for d in include_dir:
            if directory_included(path, d): return True
        return False

    if exclude_dir != []:
        for d in exclude_dir:
            if directory_included(path, d): return False
        return True
# directory_included()

def read_path_list(lstin, comment_strs=["#"], only_exists=False, as_abspath=False, err_out=null_out()):
    ret = []

    for l in open(lstin):
        for c in comment_strs:
            if c in l: l = l[:l.index(c)]

        l = l.strip()
        if not l: continue
        if only_exists and not os.path.exists(l):
            err_out.write("Error: file not found: %s\n"%l)
            continue
        ret.append(os.path.abspath(l) if as_abspath else l)

    return ret
# read_path_list()

def return_first_found_file(files, wd=None):
    for f in files:
        if wd is not None: f = os.path.join(wd, f)
        if os.path.isfile(f): return f
# return_first_found_file()

def expand_wildcard_in_list(fdlst, err_out=null_out()):
    ret = []
    for d in fdlst:
        gd = glob.glob(d)

        if len(gd) == 0:
            print >>err_out, "Error: No match!!: %s" % d
            continue
        ret.extend(gd)
    return ret
# expand_wildcard_in_list()

def check_disk_free_bytes(d):
    try:
        x = os.statvfs(d)
        return x.f_frsize * x.f_bavail
    except:
        return -1
# check_disk_free_bytes()

def get_temp_local_dir(prefix, min_bytes=None, min_kb=None, min_mb=None, min_gb=None, additional_tmpd=None):
    assert (min_bytes, min_kb, min_mb, min_gb).count(None) >= 2

    min_free_bytes = 0

    if min_bytes is not None: min_free_bytes = min_bytes
    if min_kb is not None: min_free_bytes = min_kb * 1024
    if min_mb is not None: min_free_bytes = min_mb * 1024**2
    if min_gb is not None: min_free_bytes = min_gb * 1024**3

    ramdisk = "/dev/shm"

    if os.path.isdir(ramdisk): tmpdirs = [ramdisk, tempfile.gettempdir()]
    else: tmpdirs = [tempfile.gettempdir()]

    if type(additional_tmpd) is str:
        tmpdirs.append(additional_tmpd)
    elif type(additional_tmpd) in (list, tuple):
        tmpdirs.extend(additional_tmpd)

    for tmpdir in tmpdirs:
        if check_disk_free_bytes(tmpdir) >= min_free_bytes:
            return tempfile.mkdtemp(prefix=prefix, dir=tmpdir)

    return None
# get_temp_local_dir()

def get_temp_filename(prefix="tmp", suffix="", wdir=None):
    tmpfd, tmp = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=wdir)
    os.close(tmpfd)
    return tmp
# get_temp_filename()

def replace_forbidden_chars(filename, repl="-"):
    return re.sub(r"[/><\*\\\?%:]", repl, filename)
# replace_forbidden_chars()

def human_readable_bytes(bytes):
    if bytes < 1024:
        return bytes, "B"
    elif bytes < 1024**2:
        return bytes/1024., "KB"
    elif bytes < 1024**3:
        return bytes/1024.**2, "MB"
    elif bytes < 1024**4:
        return bytes/1024.**3, "GB"
    elif bytes < 1024**5:
        return bytes/1024.**4, "TB"
    else:# if bytes < 1024**6:
        return bytes/1024.**5, "PB"

# human_readable_bytes()

def yamtbx_module_root():
    """
    Possible locations: modules/yamtbx or modules/yamtbx/yamtbx
    """
    tmp = libtbx.env.find_in_repositories("yamtbx/yamtbx")
    if tmp: return tmp
    tmp = libtbx.env.find_in_repositories("yamtbx")
    if tmp: return tmp
# yamtbx_module_root()

def retry_until_noexc(f, args=(), kwargs={}, sleep=1, ntry=10, outf=lambda x:x):
    for i in xrange(ntry):
        try:
            return f(*args, **kwargs)
        except Exception, e:
            outf("%dth trial failed with an exception"%(i+1))
            outf(traceback.format_exc())
            if i == ntry-1: raise e

        time.sleep(sleep)
# retry_until_noexc()

def None_in(*args):
    for a in args:
        if a is None: return True
    return False
# None_in()

def touch_file(path):
    if not os.path.exists(path):
        with open(path, "w") as f:
            pass
    else:
        os.utime(path, None)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import glob
import copy
import sys

from distutils.core import Extension

from astropy_helpers import setup_helpers

pjoin = os.path.join
HERE = os.path.dirname(__file__)

# ASAN_PATH = '/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/8.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib'
# if os.path.exists(ASAN_PATH):
#     os.environ.setdefault('SHADY_USE_ASAN', 'yes')
#     os.environ.setdefault('DYLD_INSERT_LIBRARIES', ASAN_PATH)

def get_zmq_include_path():
    """Get the ZMQ include path in an import-safe manner."""
    try:
        import zmq
        includes = zmq.get_includes()
    except ImportError as e:
        includes = []
    if os.path.exists(pjoin(HERE, 'includes')):
        return includes + [ pjoin(HERE, 'includes') ]
    return includes

def get_zmq_library_path():
    """Get the ZMQ include path in an import-safe manner."""
    try:
        import zmq
    except ImportError as e:
        pass
    else:
        if len(glob.glob(pjoin(os.path.dirname(zmq.__file__), 'libzmq.*'))):
            return [os.path.dirname(zmq.__file__)]
    return []

def get_zmq_extension_args():
    """Get the ZMQ Distutils Extension Args"""
    cfg = setup_helpers.DistutilsExtensionArgs()
    cfg['include_dirs'] = get_zmq_include_path()
    cfg['library_dirs'] = get_zmq_library_path()
    cfg['libraries'] = ['zmq']
    if not sys.platform.startswith(('darwin', 'freebsd')):
        cfg['libraries'].append("rt")

    return cfg

def get_utils_extension_args():
    """Get utility module extension arguments"""
    directory = os.path.dirname(__file__)
    name = __name__.split(".")[:-1]
    cfg = setup_helpers.DistutilsExtensionArgs()
    cfg['include_dirs'] = [os.path.normpath(os.path.join(directory, "utils"))]
    return cfg

def _generate_cython_extensions(extension_args, directory, package_name):
    """Generate cython extensions"""

    if sys.platform == 'darwin' and os.environ.get("SHADY_USE_ASAN","") == 'yes':
        extension_args['extra_compile_args'].extend(['-fsanitize=address', '-fno-omit-frame-pointer'])
        extension_args['extra_link_args'].extend(['-fsanitize=address'])

    for component in glob.iglob(os.path.join(directory, "*.pyx")):
        # Component name and full module name.
        cfg = setup_helpers.DistutilsExtensionArgs(copy.deepcopy(dict(**extension_args)))
        cname = os.path.splitext(os.path.basename(component))[0]
        cfg['sources'].append(component)

        component_pxd = os.path.splitext(component)[0] + ".pxd"

        if cname.startswith("_"):
            cname = cname[1:]
            name = ".".join(package_name + ["_{0:s}".format(cname)])
        else:
            name = ".".join(package_name + [cname])
        # Extension object.
        yield Extension(name, **cfg)
    <commit_msg>Fix a bug in build_helpers for zmqheaders<commit_after># -*- coding: utf-8 -*-

import os
import glob
import copy
import sys

from distutils.core import Extension

from astropy_helpers import setup_helpers

pjoin = os.path.join
HERE = os.path.dirname(__file__)

# ASAN_PATH = '/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/8.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib'
# if os.path.exists(ASAN_PATH):
#     os.environ.setdefault('SHADY_USE_ASAN', 'yes')
#     os.environ.setdefault('DYLD_INSERT_LIBRARIES', ASAN_PATH)

def get_zmq_include_path():
    """Get the ZMQ include path in an import-safe manner."""
    try:
        import zmq
        includes = zmq.get_includes()
    except ImportError as e:
        includes = []
    if os.path.exists(pjoin(HERE, 'include')):
        return includes + [ pjoin(HERE, 'include') ]
    return includes

def get_zmq_library_path():
    """Get the ZMQ include path in an import-safe manner."""
    try:
        import zmq
    except ImportError as e:
        pass
    else:
        if len(glob.glob(pjoin(os.path.dirname(zmq.__file__), 'libzmq.*'))):
            return [os.path.dirname(zmq.__file__)]
    return []

def get_zmq_extension_args():
    """Get the ZMQ Distutils Extension Args"""
    cfg = setup_helpers.DistutilsExtensionArgs()
    cfg['include_dirs'] = get_zmq_include_path()
    cfg['library_dirs'] = get_zmq_library_path()
    cfg['libraries'] = ['zmq']
    if not (sys.platform.startswith('darwin') or sys.platform.startswith('freebsd')):
        cfg['libraries'].append("rt")

    return cfg

def get_utils_extension_args():
    """Get utility module extension arguments"""
    directory = os.path.dirname(__file__)
    name = __name__.split(".")[:-1]
    cfg = setup_helpers.DistutilsExtensionArgs()
    cfg['include_dirs'] = [os.path.normpath(os.path.join(directory, "utils"))]
    return cfg

def _generate_cython_extensions(extension_args, directory, package_name):
    """Generate cython extensions"""

    if sys.platform == 'darwin' and os.environ.get("SHADY_USE_ASAN","") == 'yes':
        extension_args['extra_compile_args'].extend(['-fsanitize=address', '-fno-omit-frame-pointer'])
        extension_args['extra_link_args'].extend(['-fsanitize=address'])

    for component in glob.iglob(os.path.join(directory, "*.pyx")):
        # Component name and full module name.
        cfg = setup_helpers.DistutilsExtensionArgs(copy.deepcopy(dict(**extension_args)))
        cname = os.path.splitext(os.path.basename(component))[0]
        cfg['sources'].append(component)

        component_pxd = os.path.splitext(component)[0] + ".pxd"

        if cname.startswith("_"):
            cname = cname[1:]
            name = ".".join(package_name + ["_{0:s}".format(cname)])
        else:
            name = ".".join(package_name + [cname])
        # Extension object.
        yield Extension(name, **cfg)
    <|endoftext|>


<commit_before>#!/usr/bin/python
import mechanize
import urllib
import sys
import logging
import zephyr
import traceback
import simplejson
import re
import time
import subprocess
import optparse

parser = optparse.OptionParser()
parser.add_option('--forward-class-messages',
                  dest='forward_class_messages',
                  default=False,
                  action='store_true')
parser.add_option('--resend-log',
                  dest='resend_log',
                  default=False,
                  action='store_true')
parser.add_option('--no-forward-personals',
                  dest='forward_personals',
                  default=True,
                  action='store_false')
parser.add_option('--forward-from-humbug',
                  dest='forward_to_humbug',
                  default=True,
                  action='store_false')
(options, args) = parser.parse_args()

browser = None
csrf_token = None

def browser_login():
    logger = logging.getLogger("mechanize")
    logger.addHandler(logging.StreamHandler(sys.stdout))
    logger.setLevel(logging.INFO)

    global browser
    browser = mechanize.Browser()
    browser.set_handle_robots(False)
    ## debugging code to consider
    # browser.set_debug_http(True)
    # browser.set_debug_responses(True)
    # browser.set_debug_redirects(True)
    # browser.set_handle_refresh(False)

    browser.add_password("https://app.humbughq.com/", "tabbott", "xxxxxxxxxxxxxxxxx", "wiki")
    browser.open("https://app.humbughq.com/")
    browser.follow_link(text_regex="\s*Log in\s*")
    browser.select_form(nr=0)
    browser["username"] = "starnine@mit.edu"
    browser["password"] = "xxxxxxxx"

    global csrf_token
    csrf_token = browser["csrfmiddlewaretoken"]

    browser.submit()

def send_zephyr(zeph):
    zeph['fullname']  = username_to_fullname(zeph['sender'])
    zeph['shortname'] = zeph['sender'].split('@')[0]

    browser.addheaders.append(('X-CSRFToken', csrf_token))
    zephyr_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in zeph.items()])
    browser.open("https://app.humbughq.com/forge_zephyr/", zephyr_data)

def fetch_fullname(username):
    try:
        match_user = re.match(r'([a-zA-Z0-9_]+)@mit\.edu', username)
        if match_user:
            proc = subprocess.Popen(['hesinfo', match_user.group(1), 'passwd'], stdout=subprocess.PIPE)
            out, _err_unused = proc.communicate()
            if proc.returncode == 0:
                return out.split(':')[4].split(',')[0]
    except:
        print >>sys.stderr, 'Error getting fullname for', username
        traceback.print_exc()

    return username.title().replace('@', ' at ').replace('.', ' dot ')

fullnames = {}
def username_to_fullname(username):
    if username not in fullnames:
        fullnames[username] = fetch_fullname(username)
    return fullnames[username]

def process_loop(log):
    while True:
        try:
            notice = zephyr.receive(block=True)
            zsig, body = notice.message.split("\x00", 1)
            is_personal = False
            is_huddle = False

            if zsig.endswith("   "):
                print "Skipping message from Humbug!"
                continue

            sender = notice.sender.lower().replace("athena.mit.edu", "mit.edu")
            recipient = notice.recipient.lower().replace("athena.mit.edu", "mit.edu")

            if (notice.cls == "message" and
                notice.instance == "personal"):
                is_personal = True
                if body.startswith("CC:"):
                    is_huddle = True
                    # Map "CC: sipbtest espuser" => "starnine@mit.edu,espuser@mit.edu"
                    huddle_recipients_list = [x + "@mit.edu" for x in
                                              body.split("\n")[0][4:].split()]
                    if sender not in huddle_recipients_list:
                        huddle_recipients_list.append(sender)
                    huddle_recipients = ",".join(huddle_recipients_list)

            if notice.opcode != "":
                # skip PING messages
                continue

            # Drop messages not to the listed subscriptions
            if (notice.cls not in mit_subs_list.all_subs) and not (is_personal and
                                                                   options.forward_personals):
                print "Skipping ...", notice.cls, notice.instance, is_personal
                continue

            if is_huddle:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : huddle_recipients,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body.split("\n", 1)[1] }
            elif is_personal:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : recipient,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }
            else:
                zeph = { 'type'      : 'class',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'class'     : notice.cls,
                         'instance'  : notice.instance,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }

            log.write(simplejson.dumps(zeph) + '\n')
            log.flush()

            print "received a message on %s/%s from %s..." % \
                (notice.cls, notice.instance, notice.sender)
            send_zephyr(zeph)
        except:
            print >>sys.stderr, 'Error relaying zephyr'
            traceback.print_exc()
            time.sleep(2)

def zephyr_to_humbug(options):
    browser_login()

    import mit_subs_list
    subs = zephyr.Subscriptions()
    if options.forward_class_messages:
        for sub in mit_subs_list.all_subs:
            subs.add((sub, '*', '*'))
    if options.forward_personals:
        subs.add(("message", "personal", "*"))

    if options.resend_log:
        with open('zephyrs', 'r') as log:
            try:
                for ln in log:
                    zeph = simplejson.loads(ln)
                    print "sending saved message to %s from %s..." % \
                        (zeph.get('class', zeph.get('recipient')), zeph['sender'])
                    send_zephyr(zeph)
            except:
                print >>sys.stderr, 'Could not send saved zephyr'
                traceback.print_exc()
                time.sleep(2)

    print "Starting receive loop"

    with open('zephyrs', 'a') as log:
        process_loop(log)

def get_zephyrs(last_received):
        browser.addheaders.append(('X-CSRFToken', csrf_token))
        submit_hash = {'last_received': last_received,
                       "mit_sync_bot": 'yes'}
        submit_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in submit_hash.items()])
        res = browser.open("https://app.humbughq.com/get_updates_longpoll", submit_data)
        return simplejson.loads(res.read())['zephyrs']

def humbug_to_zephyr(options):
    # Sync messages from zephyr to humbug
    browser_login()
    print "Starting get_updates_longpoll."
    zephyrs = get_zephyrs('0')
    while True:
        last_received = str(max([z["id"] for z in zephyrs]))
        new_zephyrs = get_zephyrs(last_received)
        for zephyr in new_zephyrs:
            print zephyr
        zephyrs.extend(new_zephyrs)

if options.forward_to_humbug:
    zephyr_to_humbug(options)
else:
    humbug_to_zephyr(options)
<commit_msg>zephyr_mirror: Rename send_zephyr to send_humbug.<commit_after>#!/usr/bin/python
import mechanize
import urllib
import sys
import logging
import zephyr
import traceback
import simplejson
import re
import time
import subprocess
import optparse

parser = optparse.OptionParser()
parser.add_option('--forward-class-messages',
                  dest='forward_class_messages',
                  default=False,
                  action='store_true')
parser.add_option('--resend-log',
                  dest='resend_log',
                  default=False,
                  action='store_true')
parser.add_option('--no-forward-personals',
                  dest='forward_personals',
                  default=True,
                  action='store_false')
parser.add_option('--forward-from-humbug',
                  dest='forward_to_humbug',
                  default=True,
                  action='store_false')
(options, args) = parser.parse_args()

browser = None
csrf_token = None

def browser_login():
    logger = logging.getLogger("mechanize")
    logger.addHandler(logging.StreamHandler(sys.stdout))
    logger.setLevel(logging.INFO)

    global browser
    browser = mechanize.Browser()
    browser.set_handle_robots(False)
    ## debugging code to consider
    # browser.set_debug_http(True)
    # browser.set_debug_responses(True)
    # browser.set_debug_redirects(True)
    # browser.set_handle_refresh(False)

    browser.add_password("https://app.humbughq.com/", "tabbott", "xxxxxxxxxxxxxxxxx", "wiki")
    browser.open("https://app.humbughq.com/")
    browser.follow_link(text_regex="\s*Log in\s*")
    browser.select_form(nr=0)
    browser["username"] = "starnine@mit.edu"
    browser["password"] = "xxxxxxxx"

    global csrf_token
    csrf_token = browser["csrfmiddlewaretoken"]

    browser.submit()

def send_humbug(zeph):
    zeph['fullname']  = username_to_fullname(zeph['sender'])
    zeph['shortname'] = zeph['sender'].split('@')[0]

    browser.addheaders.append(('X-CSRFToken', csrf_token))
    zephyr_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in zeph.items()])
    browser.open("https://app.humbughq.com/forge_zephyr/", zephyr_data)

def fetch_fullname(username):
    try:
        match_user = re.match(r'([a-zA-Z0-9_]+)@mit\.edu', username)
        if match_user:
            proc = subprocess.Popen(['hesinfo', match_user.group(1), 'passwd'], stdout=subprocess.PIPE)
            out, _err_unused = proc.communicate()
            if proc.returncode == 0:
                return out.split(':')[4].split(',')[0]
    except:
        print >>sys.stderr, 'Error getting fullname for', username
        traceback.print_exc()

    return username.title().replace('@', ' at ').replace('.', ' dot ')

fullnames = {}
def username_to_fullname(username):
    if username not in fullnames:
        fullnames[username] = fetch_fullname(username)
    return fullnames[username]

def process_loop(log):
    while True:
        try:
            notice = zephyr.receive(block=True)
            zsig, body = notice.message.split("\x00", 1)
            is_personal = False
            is_huddle = False

            if zsig.endswith("   "):
                print "Skipping message from Humbug!"
                continue

            sender = notice.sender.lower().replace("athena.mit.edu", "mit.edu")
            recipient = notice.recipient.lower().replace("athena.mit.edu", "mit.edu")

            if (notice.cls == "message" and
                notice.instance == "personal"):
                is_personal = True
                if body.startswith("CC:"):
                    is_huddle = True
                    # Map "CC: sipbtest espuser" => "starnine@mit.edu,espuser@mit.edu"
                    huddle_recipients_list = [x + "@mit.edu" for x in
                                              body.split("\n")[0][4:].split()]
                    if sender not in huddle_recipients_list:
                        huddle_recipients_list.append(sender)
                    huddle_recipients = ",".join(huddle_recipients_list)

            if notice.opcode != "":
                # skip PING messages
                continue

            # Drop messages not to the listed subscriptions
            if (notice.cls not in mit_subs_list.all_subs) and not (is_personal and
                                                                   options.forward_personals):
                print "Skipping ...", notice.cls, notice.instance, is_personal
                continue

            if is_huddle:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : huddle_recipients,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body.split("\n", 1)[1] }
            elif is_personal:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : recipient,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }
            else:
                zeph = { 'type'      : 'class',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'class'     : notice.cls,
                         'instance'  : notice.instance,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }

            log.write(simplejson.dumps(zeph) + '\n')
            log.flush()

            print "received a message on %s/%s from %s..." % \
                (notice.cls, notice.instance, notice.sender)
            send_humbug(zeph)
        except:
            print >>sys.stderr, 'Error relaying zephyr'
            traceback.print_exc()
            time.sleep(2)

def zephyr_to_humbug(options):
    browser_login()

    import mit_subs_list
    subs = zephyr.Subscriptions()
    if options.forward_class_messages:
        for sub in mit_subs_list.all_subs:
            subs.add((sub, '*', '*'))
    if options.forward_personals:
        subs.add(("message", "personal", "*"))

    if options.resend_log:
        with open('zephyrs', 'r') as log:
            try:
                for ln in log:
                    zeph = simplejson.loads(ln)
                    print "sending saved message to %s from %s..." % \
                        (zeph.get('class', zeph.get('recipient')), zeph['sender'])
                    send_humbug(zeph)
            except:
                print >>sys.stderr, 'Could not send saved zephyr'
                traceback.print_exc()
                time.sleep(2)

    print "Starting receive loop"

    with open('zephyrs', 'a') as log:
        process_loop(log)

def get_zephyrs(last_received):
        browser.addheaders.append(('X-CSRFToken', csrf_token))
        submit_hash = {'last_received': last_received,
                       "mit_sync_bot": 'yes'}
        submit_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in submit_hash.items()])
        res = browser.open("https://app.humbughq.com/get_updates_longpoll", submit_data)
        return simplejson.loads(res.read())['zephyrs']

def humbug_to_zephyr(options):
    # Sync messages from zephyr to humbug
    browser_login()
    print "Starting get_updates_longpoll."
    zephyrs = get_zephyrs('0')
    while True:
        last_received = str(max([z["id"] for z in zephyrs]))
        new_zephyrs = get_zephyrs(last_received)
        for zephyr in new_zephyrs:
            print zephyr
        zephyrs.extend(new_zephyrs)

if options.forward_to_humbug:
    zephyr_to_humbug(options)
else:
    humbug_to_zephyr(options)
<|endoftext|>


<commit_before>"""

This module contains all

"""

from flask import request
from app import db
from models.test_result import TestResult
from util.json_helpers import JSON_SUCCESS, JSON_FAILURE
from util.rest.rest_auth import requires_router_token
from . import rest_blueprint
from models.config import TestConfiguration
import json

@rest_blueprint.route("/router/get_config", methods=['POST'])
@requires_router_token()
def get_config():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if 'data' in request.form:
        data = json.loads(request.form['data'])
        rec.interface_stats = data

    if not rec:
        return JSON_FAILURE()

    rec.state = 'running'
    db.session.commit()
    rec.save()

    config = TestConfiguration()

    return JSON_SUCCESS(
            config=config.get_config()
        )

@rest_blueprint.route("/router/edit", methods=['POST'])
@requires_router_token()
def edit():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if not rec:
        return JSON_FAILURE(
                reason="Invalid token or IP."
                )
    if 'data' not in request.form:
        return JSON_FAILURE(
                reason="Missing 'data' parameter"
                )
    # Columns allowed to be updated and their types
    columns = TestResult.get_public_columns()

    data = json.loads(request.form['data'])

    for col in columns:
        if col in data:
            col_type = columns[col]
            datum = col_type(data[col])
            setattr(rec, col, datum)

    rec.save()
    return JSON_SUCCESS()
<commit_msg>Added packet loss ratio to router's edit route.<commit_after>"""

This module contains all

"""

from flask import request
from app import db
from models.test_result import TestResult
from util.json_helpers import JSON_SUCCESS, JSON_FAILURE
from util.rest.rest_auth import requires_router_token
from . import rest_blueprint
from models.config import TestConfiguration
import json

@rest_blueprint.route("/router/get_config", methods=['POST'])
@requires_router_token()
def get_config():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if 'data' in request.form:
        data = json.loads(request.form['data'])
        rec.interface_stats = data

    if not rec:
        return JSON_FAILURE()

    rec.state = 'running'
    db.session.commit()
    rec.save()

    config = TestConfiguration()

    return JSON_SUCCESS(
            config=config.get_config()
        )

@rest_blueprint.route("/router/edit", methods=['POST'])
@requires_router_token()
def edit():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if not rec:
        return JSON_FAILURE(
                reason="Invalid token or IP."
                )
    if 'data' not in request.form:
        return JSON_FAILURE(
                reason="Missing 'data' parameter"
                )
    # Columns allowed to be updated and their types
    columns = TestResult.get_public_columns()

    data = json.loads(request.form['data'])

    for col in columns:
        if col in data:
            col_type = columns[col]
            datum = col_type(data[col])
            setattr(rec, col, datum)

    # Calculate packet loss from the down/up latencies. (-1 = packet lost)
    ploss_cnt = rec.download_latencies.count(-1)
    ploss_cnt += rec.upload_latencies.count(-1)
    ploss_len = len(rec.download_latencies)
    ploss_len += len(rec.upload_latencies)
    rec.packet_loss_under_load = float(ploss_cnt) / float(ploss_len)
    rec.save()

    return JSON_SUCCESS()
<|endoftext|>