<commit_before># The MIT License (MIT)
#
# Copyright (c) 2015 University of East Anglia, Norwich, UK
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# Developed by Geoffrey French in collaboration with Dr. M. Fisher and
# Dr. M. Mackiewicz.
import argparse
import json
from matplotlib import pyplot as plt
import numpy as np
import os
import yaml

from flask import Flask, render_template, request, make_response, send_from_directory

from image_labelling_tool import labelling_tool

FILE_EXT = '.jpg'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Image labelling tool - Flask app')
    parser.add_argument('--slic', action='store_true', help='Use SLIC segmentation to generate initial labels')
    parser.add_argument('--readonly', action='store_true', help='Don\'t persist changes to disk')
    parser.add_argument('--image_dir')
    parser.add_argument('--label_names')
    args = parser.parse_args()

    # `LabelClass` parameters are: symbolic name, human readable name for UI, and RGB colour as list
    with open(args.label_names, 'r') as f:
        label_names = yaml.load(f)

    cmap = plt.get_cmap('spectral')
    colors = [(np.array(cmap(i)[:3]) * 255).astype(np.int32).tolist()
              for i in range(1, len(label_names) + 1)]
    label_classes = [labelling_tool.LabelClass(name, name, color)
                     for color, name in zip(colors, label_names)]

    img_dir = args.image_dir
    if args.slic:
        import glob
        from skimage.segmentation import slic

        for path in glob.glob(os.path.join(img_dir, '*{}'.format(FILE_EXT))):
            name = os.path.splitext(path)[0]
            out_name = name + '__labels.json'
            if os.path.exists(out_name):
                print('Label already exits at {}'.format(out_name))
                # raise ValueError
                continue

            print 'Segmenting {0}'.format(path)
            img = plt.imread(path)
            # slic_labels = slic(img, 1000, compactness=20.0)
            # slic_labels = slic(img, 1000, slic_zero=True) + 1
            slic_labels = slic(img, 1500, slic_zero=True) + 1

            print 'Converting SLIC labels to vector labels...'
            labels = labelling_tool.ImageLabels.from_label_image(slic_labels)

            with open(out_name, 'w') as f:
                json.dump(labels.labels_json, f)

    readonly = args.readonly
    # Load in .JPG images from the 'images' directory.
    labelled_images = labelling_tool.PersistentLabelledImage.for_directory(
        img_dir, image_filename_pattern='*{}'.format(FILE_EXT),
        readonly=readonly)
    print 'Loaded {0} images'.format(len(labelled_images))

    # Generate image IDs list
    image_ids = [str(i) for i in xrange(len(labelled_images))]
    # Generate images table mapping image ID to image so we can get an image by ID
    images_table = {image_id: img for image_id, img in zip(image_ids, labelled_images)}
    # Generate image descriptors list to hand over to the labelling tool
    # Each descriptor provides the image ID, the URL and the size
    image_descriptors = []
    for image_id, img in zip(image_ids, labelled_images):
        data, mimetype, width, height = img.data_and_mime_type_and_size()
        image_descriptors.append(labelling_tool.image_descriptor(
            image_id=image_id, url='/image/{}'.format(image_id),
            width=width, height=height
        ))

    app = Flask(__name__, static_folder='image_labelling_tool/static')
    config = {
        'tools': {
            'imageSelector': True,
            'labelClassSelector': True,
            'drawPolyLabel': True,
            'compositeLabel': True,
            'deleteLabel': True,
        }
    }

    @app.route('/')
    def index():
        label_classes_json = [{'name': cls.name, 'human_name': cls.human_name, 'colour': cls.colour}   for cls in label_classes]
        return render_template('labeller_page.jinja2',
                               tool_js_urls=labelling_tool.js_file_urls('/static/labelling_tool/'),
                               label_classes=json.dumps(label_classes_json),
                               image_descriptors=json.dumps(image_descriptors),
                               initial_image_index=0,
                               config=json.dumps(config))

    @app.route('/labelling/get_labels/<image_id>')
    def get_labels(image_id):
        image = images_table[image_id]

        labels = image.labels_json
        complete = False

        label_header = {
            'labels': labels,
            'image_id': image_id,
            'complete': complete
        }

        r = make_response(json.dumps(label_header))
        r.mimetype = 'application/json'
        return r

    @app.route('/labelling/set_labels', methods=['POST'])
    def set_labels():
        label_header = json.loads(request.form['labels'])
        image_id = label_header['image_id']
        complete = label_header['complete']
        labels = label_header['labels']

        image = images_table[image_id]
        image.labels_json = labels

        return make_response('')

    @app.route('/image/<image_id>')
    def get_image(image_id):
        image = images_table[image_id]
        data, mimetype, width, height = image.data_and_mime_type_and_size()
        r = make_response(data)
        r.mimetype = mimetype
        return r

    @app.route('/ext_static/<path:filename>')
    def base_static(filename):
        return send_from_directory(app.root_path + '/ext_static/', filename)

    # app.run(debug=True)
    app.run(debug=False)
<commit_msg>change flask to use png as default<commit_after># The MIT License (MIT)
#
# Copyright (c) 2015 University of East Anglia, Norwich, UK
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# Developed by Geoffrey French in collaboration with Dr. M. Fisher and
# Dr. M. Mackiewicz.
import argparse
import json
from matplotlib import pyplot as plt
import numpy as np
import os
import yaml

from flask import Flask, render_template, request, make_response, send_from_directory

from image_labelling_tool import labelling_tool

FILE_EXT = '.png'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Image labelling tool - Flask app')
    parser.add_argument('--slic', action='store_true', help='Use SLIC segmentation to generate initial labels')
    parser.add_argument('--readonly', action='store_true', help='Don\'t persist changes to disk')
    parser.add_argument('--image_dir')
    parser.add_argument('--label_names')
    args = parser.parse_args()

    # `LabelClass` parameters are: symbolic name, human readable name for UI, and RGB colour as list
    with open(args.label_names, 'r') as f:
        label_names = yaml.load(f)

    cmap = plt.get_cmap('spectral')
    colors = [(np.array(cmap(i)[:3]) * 255).astype(np.int32).tolist()
              for i in range(1, len(label_names) + 1)]
    label_classes = [labelling_tool.LabelClass(name, name, color)
                     for color, name in zip(colors, label_names)]

    img_dir = args.image_dir
    if args.slic:
        import glob
        from skimage.segmentation import slic

        for path in glob.glob(os.path.join(img_dir, '*{}'.format(FILE_EXT))):
            name = os.path.splitext(path)[0]
            out_name = name + '__labels.json'
            if os.path.exists(out_name):
                print('Label already exits at {}'.format(out_name))
                # raise ValueError
                continue

            print 'Segmenting {0}'.format(path)
            img = plt.imread(path)
            # slic_labels = slic(img, 1000, compactness=20.0)
            # slic_labels = slic(img, 1000, slic_zero=True) + 1
            slic_labels = slic(img, 1500, slic_zero=True) + 1

            print 'Converting SLIC labels to vector labels...'
            labels = labelling_tool.ImageLabels.from_label_image(slic_labels)

            with open(out_name, 'w') as f:
                json.dump(labels.labels_json, f)

    readonly = args.readonly
    # Load in .JPG images from the 'images' directory.
    labelled_images = labelling_tool.PersistentLabelledImage.for_directory(
        img_dir, image_filename_pattern='*{}'.format(FILE_EXT),
        readonly=readonly)
    print 'Loaded {0} images'.format(len(labelled_images))

    # Generate image IDs list
    image_ids = [str(i) for i in xrange(len(labelled_images))]
    # Generate images table mapping image ID to image so we can get an image by ID
    images_table = {image_id: img for image_id, img in zip(image_ids, labelled_images)}
    # Generate image descriptors list to hand over to the labelling tool
    # Each descriptor provides the image ID, the URL and the size
    image_descriptors = []
    for image_id, img in zip(image_ids, labelled_images):
        data, mimetype, width, height = img.data_and_mime_type_and_size()
        image_descriptors.append(labelling_tool.image_descriptor(
            image_id=image_id, url='/image/{}'.format(image_id),
            width=width, height=height
        ))

    app = Flask(__name__, static_folder='image_labelling_tool/static')
    config = {
        'tools': {
            'imageSelector': True,
            'labelClassSelector': True,
            'drawPolyLabel': True,
            'compositeLabel': True,
            'deleteLabel': True,
        }
    }

    @app.route('/')
    def index():
        label_classes_json = [{'name': cls.name, 'human_name': cls.human_name, 'colour': cls.colour}   for cls in label_classes]
        return render_template('labeller_page.jinja2',
                               tool_js_urls=labelling_tool.js_file_urls('/static/labelling_tool/'),
                               label_classes=json.dumps(label_classes_json),
                               image_descriptors=json.dumps(image_descriptors),
                               initial_image_index=0,
                               config=json.dumps(config))

    @app.route('/labelling/get_labels/<image_id>')
    def get_labels(image_id):
        image = images_table[image_id]

        labels = image.labels_json
        complete = False

        label_header = {
            'labels': labels,
            'image_id': image_id,
            'complete': complete
        }

        r = make_response(json.dumps(label_header))
        r.mimetype = 'application/json'
        return r

    @app.route('/labelling/set_labels', methods=['POST'])
    def set_labels():
        label_header = json.loads(request.form['labels'])
        image_id = label_header['image_id']
        complete = label_header['complete']
        labels = label_header['labels']

        image = images_table[image_id]
        image.labels_json = labels

        return make_response('')

    @app.route('/image/<image_id>')
    def get_image(image_id):
        image = images_table[image_id]
        data, mimetype, width, height = image.data_and_mime_type_and_size()
        r = make_response(data)
        r.mimetype = mimetype
        return r

    @app.route('/ext_static/<path:filename>')
    def base_static(filename):
        return send_from_directory(app.root_path + '/ext_static/', filename)

    # app.run(debug=True)
    app.run(debug=False)
<|endoftext|>


<commit_before># Copyright 2017 AT&T Intellectual Property.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from deckhand.client import base

class Revision(base.Resource):
    def __repr__(self):
        if hasattr(self, 'results'):
            return ', '.join(
                ["<Revision ID: %s>" % r['id'] for r in self.results])
        else:
            try:
                return ("<Revision ID: %s>" % base.getid(self))
            except Exception:
                return ("<Revision Diff>")

class RevisionManager(base.Manager):
    """Manage :class:`Revision` resources."""
    resource_class = Revision

    def list(self, **filters):
        """Get a list of revisions."""
        url = '/revisions'
        # Call `_get` instead of `_list` because the response from the server
        # is a dict of form `{"count": n, "results": []}`.
        return self._get(url, filters=filters)

    def get(self, revision_id):
        """Get details for a revision."""
        url = '/revisions/%s' % revision_id
        return self._get(url)

    def diff(self, revision_id, comparison_revision_id):
        """Get revision diff between two revisions."""
        url = '/revisions/%s/diff/%s' % (
            revision_id, comparison_revision_id)
        return self._get(url)

    def rollback(self, revision_id):
        """Rollback to a previous revision, effectively creating a new one."""
        url = '/rollback/%s' % revision_id
        return self._post(url)

    def documents(self, revision_id, rendered=True, **filters):
        """Get a list of revision documents or rendered documents.

        :param int revision_id: Revision ID.
        :param bool rendered: If True, returns list of rendered documents.
            Else returns list of unmodified, raw documents.
        :param filters: Filters to apply to response body.
        :returns: List of documents or rendered documents.
        :rtype: list[:class:`Revision`]
        """
        if rendered:
            url = '/revisions/%s/rendered-documents' % revision_id
        else:
            url = '/revisions/%s/documents' % revision_id
        return self._list(url, filters=filters)

    def delete_all(self):
        """Delete all revisions.

        .. warning::

            Effectively the same as purging the entire database.
        """
        url = '/revisions'
        return self._delete(url)
<commit_msg>Fix Revision Resource print out in Deckhand client<commit_after># Copyright 2017 AT&T Intellectual Property.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from deckhand.client import base

class Revision(base.Resource):
    def __repr__(self):
        # List revision documents.
        if hasattr(self, 'metadata') and hasattr(self, 'schema'):
            return "<Document [%s] %s>" % (self.schema, self.metadata['name'])
        # List revisions.
        elif hasattr(self, 'results'):
            return ', '.join(
                ["<Revision ID: %s>" % x['id'] for x in self.results])
        # Show revision.
        elif hasattr(self, 'id'):
            return "<Revision ID: %s>" % base.getid(self)
        # All options expired. Revision diff.
        else:
            return "<Revision Diff: %s>" % ', '.join(
                x for x in self.to_dict())

class RevisionManager(base.Manager):
    """Manage :class:`Revision` resources."""
    resource_class = Revision

    def list(self, **filters):
        """Get a list of revisions."""
        url = '/revisions'
        # Call `_get` instead of `_list` because the response from the server
        # is a dict of form `{"count": n, "results": []}`.
        return self._get(url, filters=filters)

    def get(self, revision_id):
        """Get details for a revision."""
        url = '/revisions/%s' % revision_id
        return self._get(url)

    def diff(self, revision_id, comparison_revision_id):
        """Get revision diff between two revisions."""
        url = '/revisions/%s/diff/%s' % (
            revision_id, comparison_revision_id)
        return self._get(url)

    def rollback(self, revision_id):
        """Rollback to a previous revision, effectively creating a new one."""
        url = '/rollback/%s' % revision_id
        return self._post(url)

    def documents(self, revision_id, rendered=True, **filters):
        """Get a list of revision documents or rendered documents.

        :param int revision_id: Revision ID.
        :param bool rendered: If True, returns list of rendered documents.
            Else returns list of unmodified, raw documents.
        :param filters: Filters to apply to response body.
        :returns: List of documents or rendered documents.
        :rtype: list[:class:`Revision`]
        """
        if rendered:
            url = '/revisions/%s/rendered-documents' % revision_id
        else:
            url = '/revisions/%s/documents' % revision_id
        return self._list(url, filters=filters)

    def delete_all(self):
        """Delete all revisions.

        .. warning::

            Effectively the same as purging the entire database.
        """
        url = '/revisions'
        return self._delete(url)
<|endoftext|>


<commit_before>"""
Callback functions that can be invoked while fitting a KerasModel.
"""
import sys

class ValidationCallback(object):
  """Performs validation while training a KerasModel.

  This is a callback that can be passed to fit().  It periodically computes a
  set of metrics over a validation set and writes them to a file.  In addition,
  it can save the best model parameters found so far to a directory on disk,
  updating them every time it finds a new best validation score.

  If Tensorboard logging is enabled on the KerasModel, the metrics are also
  logged to Tensorboard.  This only happens when validation coincides with a
  step on which the model writes to the log.  You should therefore make sure
  that this callback's reporting interval is an even fraction or multiple of
  the model's logging interval.
  """

  def __init__(self,
               dataset,
               interval,
               metrics,
               output_file=sys.stdout,
               save_dir=None,
               save_metric=0,
               save_on_minimum=True,
               transformers=[]):
    """Create a ValidationCallback.

    Parameters
    ----------
    dataset: dc.data.Dataset
      the validation set on which to compute the metrics
    interval: int
      the interval (in training steps) at which to perform validation
    metrics: list of dc.metrics.Metric
      metrics to compute on the validation set
    output_file: file
      to file to which results should be written
    save_dir: str
      if not None, the model parameters that produce the best validation score
      will be written to this directory
    save_metric: int
      the index of the metric to use when deciding whether to write a new set
      of parameters to disk
    save_on_minimum: bool
      if True, the best model is considered to be the one that minimizes the
      validation metric.  If False, the best model is considered to be the one
      that maximizes it.
    transformers: List[Transformer]
      List of `dc.trans.Transformer` objects. These transformations
      must have been applied to `dataset` previously. The dataset will
      be untransformed for metric evaluation.
    """
    self.dataset = dataset
    self.interval = interval
    self.metrics = metrics
    self.output_file = output_file
    self.save_dir = save_dir
    self.save_metric = save_metric
    self.save_on_minimum = save_on_minimum
    self._best_score = None
    self.transformers = transformers

  def __call__(self, model, step):
    """This is invoked by the KerasModel after every step of fitting.

    Parameters
    ----------
    model: KerasModel
      the model that is being trained
    step: int
      the index of the training step that has just completed
    """
    if step % self.interval != 0:
      return
    scores = model.evaluate(self.dataset, self.metrics, self.transformers)
    message = 'Step %d validation:' % step
    for key in scores:
      message += ' %s=%g' % (key, scores[key])
    print(message, file=self.output_file)
    if model.tensorboard:
      for key in scores:
        model._log_scalar_to_tensorboard(key, scores[key],
                                         model.get_global_step())
    if self.save_dir is not None:
      score = scores[self.metrics[self.save_metric].name]
      if not self.save_on_minimum:
        score = -score
      if self._best_score is None or score < self._best_score:
        model.save_checkpoint(model_dir=self.save_dir)
        self._best_score = score
    if model.wandb_logger is not None:
      # Log data to Wandb
      data = {'eval/' + k: v for k, v in scores.items()}
      model.wandb_logger.log_data(data, step, dataset_id=id(self.dataset))
<commit_msg>Update callbacks.py<commit_after>"""
Callback functions that can be invoked while fitting a KerasModel.
"""
import sys

class ValidationCallback(object):
  """Performs validation while training a KerasModel.

  This is a callback that can be passed to fit().  It periodically computes a
  set of metrics over a validation set, writes them to a file, and keeps track
  of the best score. In addition, it can save the best model parameters found
  so far to a directory on disk, updating them every time it finds a new best
  validation score.

  If Tensorboard logging is enabled on the KerasModel, the metrics are also
  logged to Tensorboard.  This only happens when validation coincides with a
  step on which the model writes to the log.  You should therefore make sure
  that this callback's reporting interval is an even fraction or multiple of
  the model's logging interval.
  """

  def __init__(self,
               dataset,
               interval,
               metrics,
               output_file=sys.stdout,
               save_dir=None,
               save_metric=0,
               save_on_minimum=True,
               transformers=[]):
    """Create a ValidationCallback.

    Parameters
    ----------
    dataset: dc.data.Dataset
      the validation set on which to compute the metrics
    interval: int
      the interval (in training steps) at which to perform validation
    metrics: list of dc.metrics.Metric
      metrics to compute on the validation set
    output_file: file
      to file to which results should be written
    save_dir: str
      if not None, the model parameters that produce the best validation score
      will be written to this directory
    save_metric: int
      the index of the metric to use when deciding whether to write a new set
      of parameters to disk
    save_on_minimum: bool
      if True, the best model is considered to be the one that minimizes the
      validation metric.  If False, the best model is considered to be the one
      that maximizes it.
    transformers: List[Transformer]
      List of `dc.trans.Transformer` objects. These transformations
      must have been applied to `dataset` previously. The dataset will
      be untransformed for metric evaluation.
    """
    self.dataset = dataset
    self.interval = interval
    self.metrics = metrics
    self.output_file = output_file
    self.save_dir = save_dir
    self.save_metric = save_metric
    self.save_on_minimum = save_on_minimum
    self._best_score = None
    self.transformers = transformers

  def __call__(self, model, step):
    """This is invoked by the KerasModel after every step of fitting.

    Parameters
    ----------
    model: KerasModel
      the model that is being trained
    step: int
      the index of the training step that has just completed
    """
    if step % self.interval != 0:
      return
    scores = model.evaluate(self.dataset, self.metrics, self.transformers)
    message = 'Step %d validation:' % step
    for key in scores:
      message += ' %s=%g' % (key, scores[key])
    print(message, file=self.output_file)
    if model.tensorboard:
      for key in scores:
        model._log_scalar_to_tensorboard(key, scores[key],
                                         model.get_global_step())
    score = scores[self.metrics[self.save_metric].name]
    if not self.save_on_minimum:
      score = -score
    if self._best_score is None or score < self._best_score:
      self._best_score = score
      if self.save_dir is not None:
        model.save_checkpoint(model_dir=self.save_dir)
    if model.wandb_logger is not None:
      # Log data to Wandb
      data = {'eval/' + k: v for k, v in scores.items()}
      model.wandb_logger.log_data(data, step, dataset_id=id(self.dataset))

  def get_best_score(self):
    """This getter returns the best score evaluated on the validation set.

    Returns
    -------
    float
      The best score.
    """
    if self.save_on_minimum:
      return self._best_score
    else:
      return -self._best_score
<|endoftext|>


<commit_before>import datetime
from pprint import pprint

from django.db import connections

from estoque import queries
from estoque.classes import TransacoesDeAjuste
from estoque.functions.base import transfo2_num_doc

def ajuste_por_inventario(
    dep,
    ref,
    tam,
    cor,
    qtd,
    data,
    hora,
    force=False,
    check=False,
):
    '''
        Recebe qtd, data e hora do inventário
        Calcula ajuste necessário.
        Devolve sucesso, mensagem e mais informações.
    '''
    infos = {}

    infos['dep'] = dep
    if dep not in ['101', '102', '231']:
        return False, 'Depósito inválido', infos

    cursor = connections['so'].cursor()
    infos['ref'] = ref
    infos['tam'] = tam
    infos['cor'] = cor
    produto = queries.get_preco_medio_ref_cor_tam(cursor, ref, cor, tam)
    try:
        preco_medio = produto[0]['preco_medio']
    except Exception:
        return False, 'Referência/Cor/Tamanho não encontrada', infos

    dt = datetime.datetime.strptime(data, '%Y-%m-%d').date()
    infos['data'] = dt

    tm = datetime.datetime.strptime(hora, '%Hh%M').time()
    infos['hora'] = tm

    num_doc = transfo2_num_doc(dt, tm)
    infos['num_doc'] = num_doc

    estoque_list = queries.get_estoque_dep_ref_cor_tam(
        cursor, dep, ref, cor, tam)
    if len(estoque_list) == 0:
        estoque = 0
    else:
        estoque = estoque_list[0]['estoque']
    infos['estoque'] = estoque

    movimento = 0
    movimento_list = queries.get_transfo2_deposito_ref(
        cursor, dep, ref, cor, tam, tipo='s', data=dt, hora=tm)
    if len(movimento_list) != 0:
        for row in movimento_list:
            if row['es'] == 'E':
                movimento += row['qtd']
            elif row['es'] == 'S':
                movimento -= row['qtd']
    infos['movimento'] = movimento

    infos['qtd'] = qtd
    ajuste = round(qtd - estoque + movimento)
    sinal = 1 if ajuste > 0 else -1
    ajuste *= sinal
    infos['ajuste'] = ajuste

    infos['check'] = check

    if check:
        if ajuste == 0:
            return True, 'Estoque correto', infos
        else:
            return False, 'Estoque errado', infos

    if ajuste == 0:
        return True, 'Nada a fazer', infos

    transacoes = TransacoesDeAjuste()
    trans, es, descr = transacoes.get(sinal)
    infos['trans'] = trans
    infos['es'] = es
    infos['descr'] = descr

    infos['force'] = force

    if not force:
        data = queries.get_transfo2(
            cursor,
            dep,
            num_doc,
            ref,
            cor,
            tam,
        )
        if len(data) != 0:
            return (
                True,
                'Já existe ajuste desse item nesse depósito com esse numdoc',
                infos)

    if queries.insert_transacao_ajuste(
            cursor,
            dep,
            ref,
            tam,
            cor,
            num_doc,
            trans,
            es,
            ajuste,
            preco_medio
            ):
        return True, 'Executado o ajuste por inventário no Systêxtil', infos
    return False, 'Erro durante inserção da transação no Systêxtil', infos
<commit_msg>Iniciado o tratamento do parâmetro delete. não funcional ainda<commit_after>import datetime
from pprint import pprint

from django.db import connections

from estoque import queries
from estoque.classes import TransacoesDeAjuste
from estoque.functions.base import transfo2_num_doc

def ajuste_por_inventario(
    dep,
    ref,
    tam,
    cor,
    qtd,
    data,
    hora,
    force=False,
    check=False,
    delete=False,
):
    '''
        Recebe qtd, data e hora do inventário
        Calcula ajuste necessário.
        Devolve sucesso, mensagem e mais informações.
    '''
    infos = {}

    infos['dep'] = dep
    if dep not in ['101', '102', '231']:
        return False, 'Depósito inválido', infos

    cursor = connections['so'].cursor()
    infos['ref'] = ref
    infos['tam'] = tam
    infos['cor'] = cor
    produto = queries.get_preco_medio_ref_cor_tam(cursor, ref, cor, tam)
    try:
        preco_medio = produto[0]['preco_medio']
    except Exception:
        return False, 'Referência/Cor/Tamanho não encontrada', infos

    dt = datetime.datetime.strptime(data, '%Y-%m-%d').date()
    infos['data'] = dt

    tm = datetime.datetime.strptime(hora, '%Hh%M').time()
    infos['hora'] = tm

    num_doc = transfo2_num_doc(dt, tm)
    infos['num_doc'] = num_doc

    infos['delete'] = delete
    if delete:
        transsacoes = queries.select_transacao_ajuste(
            cursor, dep, ref, tam, cor, num_doc)
        if len(transsacoes) == 0:
            return True, 'Sem transações', infos
        else:
            return (
                True,
                'Encontradas {} transações'.format(len(transsacoes)),
                infos
                )

    estoque_list = queries.get_estoque_dep_ref_cor_tam(
        cursor, dep, ref, cor, tam)
    if len(estoque_list) == 0:
        estoque = 0
    else:
        estoque = estoque_list[0]['estoque']
    infos['estoque'] = estoque

    movimento = 0
    movimento_list = queries.get_transfo2_deposito_ref(
        cursor, dep, ref, cor, tam, tipo='s', data=dt, hora=tm)
    if len(movimento_list) != 0:
        for row in movimento_list:
            if row['es'] == 'E':
                movimento += row['qtd']
            elif row['es'] == 'S':
                movimento -= row['qtd']
    infos['movimento'] = movimento

    infos['qtd'] = qtd
    ajuste = round(qtd - estoque + movimento)
    sinal = 1 if ajuste > 0 else -1
    ajuste *= sinal
    infos['ajuste'] = ajuste

    infos['check'] = check

    if check:
        if ajuste == 0:
            return True, 'Estoque correto', infos
        else:
            return False, 'Estoque errado', infos

    if ajuste == 0:
        return True, 'Nada a fazer', infos

    transacoes = TransacoesDeAjuste()
    trans, es, descr = transacoes.get(sinal)
    infos['trans'] = trans
    infos['es'] = es
    infos['descr'] = descr

    infos['force'] = force

    if not force:
        data = queries.get_transfo2(
            cursor,
            dep,
            num_doc,
            ref,
            cor,
            tam,
        )
        if len(data) != 0:
            return (
                True,
                'Já existe ajuste desse item nesse depósito com esse numdoc',
                infos)

    if queries.insert_transacao_ajuste(
            cursor,
            dep,
            ref,
            tam,
            cor,
            num_doc,
            trans,
            es,
            ajuste,
            preco_medio
            ):
        return True, 'Executado o ajuste por inventário no Systêxtil', infos
    return False, 'Erro durante inserção da transação no Systêxtil', infos
<|endoftext|>


<commit_before>from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from xml.dom.minidom import parseString
import libvirt
# Create your views here.

def index(request):
    return render_to_response('vmmanager/index.html')

def status(request, vmname):
    #con = libvirt.open('qemu+tls://g4hv.exp.ci.i.u-tokyo.ac.jp/system')
    con = libvirt.open('qemu:///system')
    dom = con.lookupByName(vmname)
    parsed = parseString(dom.XMLDesc(libvirt.VIR_DOMAIN_XML_SECURE))

    try:
        if request.POST["shutdown"] == "true":
            dom.destroy()
            return HttpResponseRedirect(reverse('vmmanager.views.index_top'))

    except KeyError:

        return render_to_response('vmmanager/status.html',
                                  context_instance=RequestContext(request, {
                                  'dom': dom,
                                  'info': dom.info,
                                  'mem': dom.memoryStats,
                                  'max_mem': dom.maxMemory,
				  'max_cpu': dom.maxVcpus,
                                  'getinfo': con.getInfo,
                                  'graphics_port': parsed.getElementsByTagName('graphics')[0].getAttribute('port')
                              }))

def create(request):
    return render_to_response('vmmanager/create.html')

def index_menu(request):
    #con = libvirt.open('qemu+tls://g4hv.exp.ci.i.u-tokyo.ac.jp/system')
    con = libvirt.open('qemu:///system')
    vmdoms = []
    for id in con.listDomainsID():
        dom = con.lookupByID(id)
        vmdoms.append(dom)
    return render_to_response('vmmanager/index_menu.html',
                              {'vmdoms': vmdoms},
                              context_instance=RequestContext(request))

def index_top(request):
    return render_to_response('vmmanager/index_top.html')

<commit_msg>changed the url to release version in view.py<commit_after>from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from xml.dom.minidom import parseString
import libvirt
# Create your views here.

def index(request):
    return render_to_response('vmmanager/index.html')

def status(request, vmname):
    con = libvirt.open('qemu+tls://g4hv.exp.ci.i.u-tokyo.ac.jp/system')
    dom = con.lookupByName(vmname)
    parsed = parseString(dom.XMLDesc(libvirt.VIR_DOMAIN_XML_SECURE))

    try:
        if request.POST["shutdown"] == "true":
            dom.destroy()
            return HttpResponseRedirect(reverse('vmmanager.views.index_top'))

    except KeyError:

        return render_to_response('vmmanager/status.html',
                                  context_instance=RequestContext(request, {
                                  'dom': dom,
                                  'info': dom.info,
                                  'mem': dom.memoryStats,
                                  'max_mem': dom.maxMemory,
				  'max_cpu': dom.maxVcpus,
                                  'getinfo': con.getInfo,
                                  'graphics_port': parsed.getElementsByTagName('graphics')[0].getAttribute('port')
                              }))

def create(request):
    return render_to_response('vmmanager/create.html')

def index_menu(request):
    con = libvirt.open('qemu+tls://g4hv.exp.ci.i.u-tokyo.ac.jp/system')
    vmdoms = []
    for id in con.listDomainsID():
        dom = con.lookupByID(id)
        vmdoms.append(dom)
    return render_to_response('vmmanager/index_menu.html',
                              {'vmdoms': vmdoms},
                              context_instance=RequestContext(request))

def index_top(request):
    return render_to_response('vmmanager/index_top.html')

<|endoftext|>


<commit_before>
from brick.iscsi.iscsi import TgtAdm

from vmthunder.singleton import SingleTon

@SingleTon
class TgtExecutor(TgtAdm):
    def __index__(self):
        TgtAdm.__init__(self, '', '/etc/tgt/conf.d')

tgt = TgtExecutor()

def create_iscsi_target(iqn, path):
    return tgt.create_iscsi_target(iqn, '', '', path)

def remove_iscsi_target(tid, lun, vol_id, vol_name, **kwargs):
    return tgt.remove_iscsi_target(tid, lun, vol_id, vol_name, **kwargs)<commit_msg>fix bug<commit_after>
from brick.iscsi.iscsi import TgtAdm

from vmthunder.singleton import SingleTon

@SingleTon
class TgtExecutor(TgtAdm):
    def __index__(self, root_helper='', volumes_dir= '/etc/tgt/conf.d'):
        TgtAdm.__init__(self, root_helper, volumes_dir)

tgt = TgtExecutor()

def create_iscsi_target(iqn, path):
    return tgt.create_iscsi_target(iqn, '', '', path)

def remove_iscsi_target(tid, lun, vol_id, vol_name, **kwargs):
    return tgt.remove_iscsi_target(tid, lun, vol_id, vol_name, **kwargs)<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

from os.path import abspath, join, dirname
import time

from pymongo import Connection
from pyvows import Vows, expect

from thumbor.storages.mongo_storage import Storage as MongoStorage
from thumbor.context import Context
from thumbor.config import Config
from fixtures.mongo_storage_fixture import IMAGE_URL, IMAGE_BYTES

class MongoDBContext(Vows.Context):
    def setup(self):
        self.fixtures_folder = join(abspath(dirname(__file__)), 'fixtures')

        self.connection = Connection('localhost', 27017)
        self.collection = self.connection['thumbor']['images']

    def teardown(self):
        self.connection.drop_database('thumbor')

@Vows.batch
class MongoStorageVows(MongoDBContext):
    class CanStoreImage(Vows.Context):
        def topic(self):
            storage = MongoStorage(Context(config=Config()))
            storage.put(IMAGE_URL % 1, IMAGE_BYTES)
            return self.parent.collection.find_one({'path': IMAGE_URL % 1})

        def should_be_in_catalog(self, topic):
            expect(topic).not_to_be_null()
            expect(topic).not_to_be_an_error()

    class CanGetImage(Vows.Context):
        def topic(self):
            storage = MongoStorage(Context(config=Config()))
            storage.put(IMAGE_URL % 2, IMAGE_BYTES)
            return storage.get(IMAGE_URL % 2)

        def should_not_be_null(self, topic):
            expect(topic).not_to_be_null()
            expect(topic).not_to_be_an_error()

        def should_have_proper_bytes(self, topic):
            expect(topic).to_equal(IMAGE_BYTES)

    class StoresCrypto(Vows.Context):
        class DoesNotStoreWhenConfigIsFalseInPutMethod(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=False)))
                storage.put(IMAGE_URL % 3, IMAGE_BYTES)

                return self.parent.parent.collection.find_one({'path': IMAGE_URL % 3})

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_not_have_crypto_key(self, topic):
                expect(topic.has_key('crypto')).to_be_false()

        class StoringEmptyKeyRaises(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='')))
                storage.put(IMAGE_URL % 4, IMAGE_BYTES)

            def should_be_an_error(self, topic):
                expect(topic).to_be_an_error_like(RuntimeError)
                expect(topic).to_have_an_error_message_of("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        class StoringProperKey(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC')))
                storage.put(IMAGE_URL % 5, IMAGE_BYTES)

                return self.parent.parent.collection.find_one({'path': IMAGE_URL % 5})

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_have_crypto_key(self, topic):
                expect(topic.has_key('crypto')).to_be_true()
                expect(topic['crypto']).to_equal('ACME-SEC')

        class GetProperKey(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC')))
                storage.put(IMAGE_URL % 6, IMAGE_BYTES)

                return storage.get_crypto(IMAGE_URL % 6)

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_have_crypto_key(self, topic):
                expect(topic).to_equal('ACME-SEC')

        class GetNoKey(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC')))
                return storage.get_crypto(IMAGE_URL % 7)

            def should_not_be_in_catalog(self, topic):
                expect(topic).to_be_null()

        class GetProperKeyBeforeExpiration(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC', STORAGE_EXPIRATION_SECONDS=5000)))
                storage.put(IMAGE_URL % 8, IMAGE_BYTES)
                return storage.get(IMAGE_URL % 8)

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_have_crypto_key(self, topic):
                expect(topic).to_equal(IMAGE_BYTES)

        class GetNothingAfterExpiration(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC', STORAGE_EXPIRATION_SECONDS=0)))
                storage.put(IMAGE_URL % 10, IMAGE_BYTES)

                item = storage.get(IMAGE_URL % 10)
                return item is None

            def should_be_expired(self, topic):
                expect(topic).to_be_true()

        class StoresCryptoAfterStoringImage(Vows.Context):
            def topic(self):
                conf = Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=False, SECURITY_KEY='ACME-SEC')
                storage = MongoStorage(Context(config=conf))
                storage.put(IMAGE_URL % 11, IMAGE_BYTES)

                conf.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True
                storage.put_crypto(IMAGE_URL % 11)

                item = storage.get_crypto(IMAGE_URL % 11)
                return item

            def should_be_acme_sec(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).to_equal('ACME-SEC')

<commit_msg>Storage vows<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

from os.path import abspath, join, dirname
import time

from pymongo import Connection
from pyvows import Vows, expect

from thumbor.storages.mongo_storage import Storage as MongoStorage
from thumbor.context import Context
from thumbor.config import Config
from fixtures.mongo_storage_fixture import IMAGE_URL, IMAGE_BYTES

class MongoDBContext(Vows.Context):
    def setup(self):
        self.fixtures_folder = join(abspath(dirname(__file__)), 'fixtures')

        self.connection = Connection('localhost', 27017)
        self.collection = self.connection['thumbor']['images']

    def teardown(self):
        self.connection.drop_database('thumbor')

@Vows.batch
class MongoStorageVows(MongoDBContext):
    class CanStoreImage(Vows.Context):
        def topic(self):
            storage = MongoStorage(Context(config=Config()))
            storage.put(IMAGE_URL % 1, IMAGE_BYTES)
            return self.parent.collection.find_one({'path': IMAGE_URL % 1})

        def should_be_in_catalog(self, topic):
            expect(topic).not_to_be_null()
            expect(topic).not_to_be_an_error()

    class CanGetImage(Vows.Context):
        def topic(self):
            storage = MongoStorage(Context(config=Config()))
            storage.put(IMAGE_URL % 2, IMAGE_BYTES)
            return storage.get(IMAGE_URL % 2)

        def should_not_be_null(self, topic):
            expect(topic).not_to_be_null()
            expect(topic).not_to_be_an_error()

        def should_have_proper_bytes(self, topic):
            expect(topic).to_equal(IMAGE_BYTES)

    class StoresCrypto(Vows.Context):
        class DoesNotStoreWhenConfigIsFalseInPutMethod(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=False)))
                storage.put(IMAGE_URL % 3, IMAGE_BYTES)

                return self.parent.parent.collection.find_one({'path': IMAGE_URL % 3})

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_not_have_crypto_key(self, topic):
                expect(topic.has_key('crypto')).to_be_false()

        class StoringEmptyKeyRaises(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='')))
                storage.put(IMAGE_URL % 4, IMAGE_BYTES)

            def should_be_an_error(self, topic):
                expect(topic).to_be_an_error_like(RuntimeError)
                expect(topic).to_have_an_error_message_of("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        class StoringProperKey(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC')))
                storage.put(IMAGE_URL % 5, IMAGE_BYTES)

                return self.parent.parent.collection.find_one({'path': IMAGE_URL % 5})

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_have_crypto_key(self, topic):
                expect(topic.has_key('crypto')).to_be_true()
                expect(topic['crypto']).to_equal('ACME-SEC')

        class GetProperKey(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC')))
                storage.put(IMAGE_URL % 6, IMAGE_BYTES)

                return storage.get_crypto(IMAGE_URL % 6)

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_have_crypto_key(self, topic):
                expect(topic).to_equal('ACME-SEC')

        class GetNoKey(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC')))
                return storage.get_crypto(IMAGE_URL % 7)

            def should_not_be_in_catalog(self, topic):
                expect(topic).to_be_null()

        class GetProperKeyBeforeExpiration(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC', STORAGE_EXPIRATION_SECONDS=5000)))
                storage.put(IMAGE_URL % 8, IMAGE_BYTES)
                return storage.get(IMAGE_URL % 8)

            def should_be_in_catalog(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).not_to_be_an_error()

            def should_have_crypto_key(self, topic):
                expect(topic).to_equal(IMAGE_BYTES)

        class GetNothingAfterExpiration(Vows.Context):
            def topic(self):
                storage = MongoStorage(Context(config=Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, SECURITY_KEY='ACME-SEC', STORAGE_EXPIRATION_SECONDS=0)))
                storage.put(IMAGE_URL % 10, IMAGE_BYTES)

                item = storage.get(IMAGE_URL % 10)
                return item is None

            def should_be_expired(self, topic):
                expect(topic).to_be_true()

        class StoresCryptoAfterStoringImage(Vows.Context):
            def topic(self):
                conf = Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=False, SECURITY_KEY='ACME-SEC')
                storage = MongoStorage(Context(config=conf))
                storage.put(IMAGE_URL % 11, IMAGE_BYTES)

                conf.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True
                storage.put_crypto(IMAGE_URL % 11)

                item = storage.get_crypto(IMAGE_URL % 11)
                return item

            def should_be_acme_sec(self, topic):
                expect(topic).not_to_be_null()
                expect(topic).to_equal('ACME-SEC')

        class DoesNotStoreCryptoIfNoNeed(Vows.Context):
            def topic(self):
                conf = Config(STORES_CRYPTO_KEY_FOR_EACH_IMAGE=False, SECURITY_KEY='ACME-SEC')
                storage = MongoStorage(Context(config=conf))
                storage.put(IMAGE_URL % 12, IMAGE_BYTES)
                storage.put_crypto(IMAGE_URL % 12)

                item = storage.get_crypto(IMAGE_URL % 12)
                return item

            def should_be_null(self, topic):
                expect(topic).to_be_null()

<|endoftext|>


<commit_before>from pyrr import matrix44
from demosys.opengl import FBO
from demosys.opengl import Texture2D, DepthTexture
from OpenGL import GL
from demosys import resources
from demosys import geometry

class PointLight:
    """A point light and its properties"""
    def __init__(self, position, radius):
        self._position = position
        self.radius = radius
        self.matrix = None

    @property
    def position(self):
        return self._position

    @position.setter
    def position(self, pos):
        self._position = pos
        self.matrix = matrix44.create_from_translation(pos)

class DeferredRenderer:

    def __init__(self, width, height, gbuffer=None, lightbuffer=None):
        self.width = width
        self.height = height
        self.size = (width, height)

        # FBOs
        self.gbuffer = gbuffer
        self.lightbuffer = lightbuffer

        # Light Info
        self.point_lights = []

        # Create geometry buffer if not supplied
        depth_buffer = DepthTexture(self.size)

        if not self.gbuffer:
            self.gbuffer = FBO.create_from_textures(
                [
                    Texture2D.create(self.size, 4, dtype='f1'),
                    Texture2D.create(self.size, 3, dtype='f2'),
                ],
                depth_buffer=depth_buffer,
            )

        if not self.lightbuffer:
            self.lightbuffer = FBO.create_from_textures(
                [Texture2D.create(self.size, 1)],
                depth_buffer=depth_buffer,
            )

        # Unit cube for point lights (cube with radius 1.0)
        self.unit_cube = geometry.cube(width=2, height=2, depth=2)
        self.point_light_shader = resources.shaders.get("deferred/light_point.glsl", create=True)

        # Debug draw lights
        self.debug_shader = resources.shaders.get("deferred/debug.glsl", create=True)

        # Combine shader
        self.combine_shader = resources.shaders.get("deferred/combine.glsl", create=True)
        self.quad = geometry.quad_fs()

    def draw_buffers(self, near, far):
        """
        Draw framebuffers for debug purposes.
        We need to supply near and far plane so the depth buffer can be linearized when visualizing.

        :param near: Projection near value
        :param far: Projection far value
        """
        self.gbuffer.draw_color_layer(layer=0, pos=(0.0, 0.0), scale=(0.25, 0.25))
        self.gbuffer.draw_color_layer(layer=1, pos=(0.5, 0.0), scale=(0.25, 0.25))
        self.gbuffer.draw_depth(near, far, pos=(1.0, 0.0), scale=(0.25, 0.25))
        self.lightbuffer.draw_color_layer(layer=0, pos=(1.5, 0.0), scale=(0.25, 0.25))

    def add_point_light(self, position, radius):
        """Add point light"""
        self.point_lights.append(PointLight(position, radius))

    def render_lights(self, camera_matrix, projection):
        """Render light volumes"""
        # Disable culling so lights can be rendered when inside volumes
        GL.glEnable(GL.GL_CULL_FACE)
        GL.glFrontFace(GL.GL_CW)
        # No depth testing
        GL.glDisable(GL.GL_DEPTH_TEST)
        # Enable additive blending
        GL.glEnable(GL.GL_BLEND)
        GL.glBlendFunc(GL.GL_ONE, GL.GL_ONE)

        with self.lightbuffer:
            for light in self.point_lights:
                # Calc light properties
                light_size = light.radius
                m_light = matrix44.multiply(light.matrix, camera_matrix)
                # Draw the light volume
                self.point_light_shader.uniform("m_proj", projection.tobytes())
                self.point_light_shader.uniform("m_light", m_light.astype('f4').tobytes())
                self.gbuffer.color_buffers[1].use(location=0)
                self.point_light_shader.uniform("g_normal", 0)
                self.gbuffer.depth_buffer.use(location=1)
                self.point_light_shader.uniform("g_depth", 1)
                self.point_light_shader.uniform("screensize", (self.width, self.height))
                self.point_light_shader.uniform("proj_const", projection.projection_constants)
                self.point_light_shader.uniform("radius", light_size)
                self.unit_cube.draw(self.point_light_shader)

        GL.glDisable(GL.GL_BLEND)
        GL.glDisable(GL.GL_CULL_FACE)

    def render_lights_debug(self, camera_matrix, projection):
        """Render outlines of light volumes"""
        GL.glEnable(GL.GL_BLEND)
        GL.glBlendFunc(GL.GL_SRC_ALPHA, GL.GL_ONE_MINUS_SRC_ALPHA)

        for light in self.point_lights:
            m_mv = matrix44.multiply(light.matrix, camera_matrix)
            light_size = light.radius
            self.debug_shader.uniform("m_proj", projection.tobytes())
            self.debug_shader.uniform("m_mv", m_mv.astype('f4'))
            self.debug_shader.uniform("size", light_size)
            self.unit_cube.draw(self.debug_shader, mode=GL.GL_LINE_STRIP)

        GL.glDisable(GL.GL_BLEND)

    def render_geometry(self, cam_matrix, projection):
        raise NotImplementedError("render_geometry() not implemented")

    def combine(self):
        """Combine diffuse and light buffer"""
        self.gbuffer.color_buffers[0].use(location=0)
        self.combine_shader.uniform("diffuse_buffer", 0)
        self.lightbuffer.color_buffers[0].use(location=1)
        self.combine_shader.uniform("light_buffer", 1)
        self.quad.draw(self.combine_shader)

    def clear(self):
        """clear all buffers"""
        self.gbuffer.clear()
        self.lightbuffer.clear()
<commit_msg>Deferred: Light buffer only has 1 component<commit_after>from pyrr import matrix44
from demosys.opengl import FBO
from demosys.opengl import Texture2D, DepthTexture
from OpenGL import GL
from demosys import resources
from demosys import geometry

class PointLight:
    """A point light and its properties"""
    def __init__(self, position, radius):
        self._position = position
        self.radius = radius
        self.matrix = None

    @property
    def position(self):
        return self._position

    @position.setter
    def position(self, pos):
        self._position = pos
        self.matrix = matrix44.create_from_translation(pos)

class DeferredRenderer:

    def __init__(self, width, height, gbuffer=None, lightbuffer=None):
        self.width = width
        self.height = height
        self.size = (width, height)

        # FBOs
        self.gbuffer = gbuffer
        self.lightbuffer = lightbuffer

        # Light Info
        self.point_lights = []

        # Create geometry buffer if not supplied
        depth_buffer = DepthTexture(self.size)

        if not self.gbuffer:
            self.gbuffer = FBO.create_from_textures(
                [
                    Texture2D.create(self.size, 4, dtype='f1'),
                    Texture2D.create(self.size, 3, dtype='f2'),
                ],
                depth_buffer=depth_buffer,
            )

        if not self.lightbuffer:
            self.lightbuffer = FBO.create_from_textures(
                [Texture2D.create(self.size, 4)],
                depth_buffer=depth_buffer,
            )

        # Unit cube for point lights (cube with radius 1.0)
        self.unit_cube = geometry.cube(width=2, height=2, depth=2)
        self.point_light_shader = resources.shaders.get("deferred/light_point.glsl", create=True)

        # Debug draw lights
        self.debug_shader = resources.shaders.get("deferred/debug.glsl", create=True)

        # Combine shader
        self.combine_shader = resources.shaders.get("deferred/combine.glsl", create=True)
        self.quad = geometry.quad_fs()

    def draw_buffers(self, near, far):
        """
        Draw framebuffers for debug purposes.
        We need to supply near and far plane so the depth buffer can be linearized when visualizing.

        :param near: Projection near value
        :param far: Projection far value
        """
        self.gbuffer.draw_color_layer(layer=0, pos=(0.0, 0.0), scale=(0.25, 0.25))
        self.gbuffer.draw_color_layer(layer=1, pos=(0.5, 0.0), scale=(0.25, 0.25))
        self.gbuffer.draw_depth(near, far, pos=(1.0, 0.0), scale=(0.25, 0.25))
        self.lightbuffer.draw_color_layer(layer=0, pos=(1.5, 0.0), scale=(0.25, 0.25))

    def add_point_light(self, position, radius):
        """Add point light"""
        self.point_lights.append(PointLight(position, radius))

    def render_lights(self, camera_matrix, projection):
        """Render light volumes"""
        # Disable culling so lights can be rendered when inside volumes
        GL.glEnable(GL.GL_CULL_FACE)
        GL.glFrontFace(GL.GL_CW)
        # No depth testing
        GL.glDisable(GL.GL_DEPTH_TEST)
        # Enable additive blending
        GL.glEnable(GL.GL_BLEND)
        GL.glBlendFunc(GL.GL_ONE, GL.GL_ONE)

        with self.lightbuffer:
            for light in self.point_lights:
                # Calc light properties
                light_size = light.radius
                m_light = matrix44.multiply(light.matrix, camera_matrix)
                # Draw the light volume
                self.point_light_shader.uniform("m_proj", projection.tobytes())
                self.point_light_shader.uniform("m_light", m_light.astype('f4').tobytes())
                self.gbuffer.color_buffers[1].use(location=0)
                self.point_light_shader.uniform("g_normal", 0)
                self.gbuffer.depth_buffer.use(location=1)
                self.point_light_shader.uniform("g_depth", 1)
                self.point_light_shader.uniform("screensize", (self.width, self.height))
                self.point_light_shader.uniform("proj_const", projection.projection_constants)
                self.point_light_shader.uniform("radius", light_size)
                self.unit_cube.draw(self.point_light_shader)

        GL.glDisable(GL.GL_BLEND)
        GL.glDisable(GL.GL_CULL_FACE)

    def render_lights_debug(self, camera_matrix, projection):
        """Render outlines of light volumes"""
        GL.glEnable(GL.GL_BLEND)
        GL.glBlendFunc(GL.GL_SRC_ALPHA, GL.GL_ONE_MINUS_SRC_ALPHA)

        for light in self.point_lights:
            m_mv = matrix44.multiply(light.matrix, camera_matrix)
            light_size = light.radius
            self.debug_shader.uniform("m_proj", projection.tobytes())
            self.debug_shader.uniform("m_mv", m_mv.astype('f4'))
            self.debug_shader.uniform("size", light_size)
            self.unit_cube.draw(self.debug_shader, mode=GL.GL_LINE_STRIP)

        GL.glDisable(GL.GL_BLEND)

    def render_geometry(self, cam_matrix, projection):
        raise NotImplementedError("render_geometry() not implemented")

    def combine(self):
        """Combine diffuse and light buffer"""
        self.gbuffer.color_buffers[0].use(location=0)
        self.combine_shader.uniform("diffuse_buffer", 0)
        self.lightbuffer.color_buffers[0].use(location=1)
        self.combine_shader.uniform("light_buffer", 1)
        self.quad.draw(self.combine_shader)

    def clear(self):
        """clear all buffers"""
        self.gbuffer.clear()
        self.lightbuffer.clear()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -*- For Facebook Hackathon -*-

import socket
import urllib2
import os
import sys
import platform

class SystemUtils(object):

    VERSION_CENTOS_6_7 = 'Centos 6.7'
    VERSION_CENTOS_6_6 = 'Centos 6.6'
    VERSION_CENTOS_6_5 = 'Centos 6.5'
    VERSION_CENTOS_7 = 'Centos 7'

    # Check if Internet connection is working
    @staticmethod
    def check_internet_connection():
        try:
            url = 'http://www.google.com'
            data = urllib2.urlopen(url, timeout=1)
            print data
            print 'Connection to Internet is Ok \n'
        except urllib2.URLError as err:
            pass
            print 'You are not connected to Internet \n'

    # Check local hostname
    @staticmethod
    def check_host_name():
        host_name = socket.getfqdn()
        print "Hostname is: ", host_name

    # Check local IP address
    @staticmethod
    def check_local_ip():
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 0))
        local_ip_address = s.getsockname()[0]
        return local_ip_address

    @staticmethod
    def disable_selinux():
        print "Disable SELinux"
        if os.path.exists("/etc/sysconfig/selinux"):
            with open("/etc/sysconfig/selinux", 'r') as file:
                selinux = ''
                for selinux in selinux:
                    selinux += line
                    if 'SELINUX=enforcing' in selinux:
                        print selinux

    # Check if user run CentOS 6.5 or 6.6
    # This part is actually very-very bad :(
    @staticmethod
    def check_centos_version():
        if os.path.exists("/etc/redhat-release"):
            with open("/etc/redhat-release", 'r') as file:
                version = ''
                for line in file:
                    version += line
                if 'release 6.6' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_6_6
                elif 'release 6.5' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_6_5
                elif 'release 6.7' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_6_7

                elif 'CentOS Linux release 7' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_7
                    file.close()
        else:
            print '\n<------------SYSTEM INFO------------>'
            print 'Architecture: ' + platform.machine()
            print 'Platform: ' + platform.platform()
            print 'Platform details: ' + str(platform.uname())
            print 'Your version of Linux incompatible with this script, exit...'
            print '<------------SYSTEM INFO------------>\n'
            sys.exit()<commit_msg>selinux continue <commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -*- For Facebook Hackathon -*-

import socket
import urllib2
import os
import sys
import platform

class SystemUtils(object):

    VERSION_CENTOS_6_7 = 'Centos 6.7'
    VERSION_CENTOS_6_6 = 'Centos 6.6'
    VERSION_CENTOS_6_5 = 'Centos 6.5'
    VERSION_CENTOS_7 = 'Centos 7'

    # Check if Internet connection is working
    @staticmethod
    def check_internet_connection():
        try:
            url = 'http://www.google.com'
            data = urllib2.urlopen(url, timeout=1)
            print data
            print 'Connection to Internet is Ok \n'
        except urllib2.URLError as err:
            pass
            print 'You are not connected to Internet \n'

    # Check local hostname
    @staticmethod
    def check_host_name():
        host_name = socket.getfqdn()
        print "Hostname is: ", host_name

    # Check local IP address
    @staticmethod
    def check_local_ip():
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 0))
        local_ip_address = s.getsockname()[0]
        return local_ip_address

    @staticmethod
    def disable_selinux():
        if os.path.exists("/etc/sysconfig/selinux"):
            with open("/etc/sysconfig/selinux", 'r') as file:
                selinux = ''
                for line in file:
                    selinux += line
                    if 'SELINUX=enforcing' in file:
                        print selinux

    # Check if user run CentOS 6.5 or 6.6
    # This part is actually very-very bad :(
    @staticmethod
    def check_centos_version():
        if os.path.exists("/etc/redhat-release"):
            with open("/etc/redhat-release", 'r') as file:
                version = ''
                for line in file:
                    version += line
                if 'release 6.6' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_6_6
                elif 'release 6.5' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_6_5
                elif 'release 6.7' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_6_7

                elif 'CentOS Linux release 7' in version:
                    print 'You are using: ' + SystemUtils.VERSION_CENTOS_7
                    file.close()
        else:
            print '\n<------------SYSTEM INFO------------>'
            print 'Architecture: ' + platform.machine()
            print 'Platform: ' + platform.platform()
            print 'Platform details: ' + str(platform.uname())
            print 'Your version of Linux incompatible with this script, exit...'
            print '<------------SYSTEM INFO------------>\n'
            sys.exit()<|endoftext|>


<commit_before>#!/usr/bin/python

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.web.http_headers import Headers
from twisted.python.util import println
from twisted.python.failure import Failure

import sys, re, string, htmlentitydefs

entityPattern = re.compile("&(\w+?);")
decPattern = re.compile("(&#(\d+?);)")
whitespacePattern = re.compile("\s+")
titlePattern = re.compile(r'<title>(.*?)</title>', re.S | re.I )
cookiePattern = re.compile("^(.+?)=(.*?);")
charsetPattern = re.compile(r'charset=([^\s]+)', re.I)
domainPattern = re.compile(r'^(https?://[^/]+)/', re.I)
metaCharsetPattern = re.compile(
	"meta http-equiv=\"content-type\".+?charset=([^\s]+)", re.I)

entityDefs = htmlentitydefs.entitydefs

class Getter(Agent):
	""" A title fetcher """

	def __init__(self, reactor):
		Agent.__init__(self, reactor)

	def Get(self, url, context, maxRetries=10):
		""" Start the lookup process for a title.

		context dict stores any information that has to
		come back with the response object

		context['cookies']:
			type dict
			cookie jar

		context['retry_count']:
			type int
			number of retries used so far

		context['original_url']:
			type str
			original url, in case we have to retry after
			following redirects

		context['charset']:
			type str
			charset received from server, so we can convert
			to utf-8

		context ['domain']:
			type str
			stores the scheme and domain portion of the url
			in case we get a relative redirect

		maxRetries defines the maximum number of times
		we retry the fetch after a 4xx or 5xx response code

		Returns deferred."""

		url = stripNoPrint(url)
		println('Get: ' + url)

		m = domainPattern.search(url)
		context['domain'] = m.group(1)

		try:
			context['cookies']
		except KeyError:
			context['cookies'] = dict()

		if not 'retry_count' in context:
			context['retry_count'] = 0
		elif context['retry_count'] >= maxRetries:
			println('exceeded max retry count for ' + str(url))
			return
		else:
			context['retry_count'] += 1
		context['original_url'] = url

		cookies = ''
		for k in context['cookies']:
			cookies += k + '=' + context['cookies'][k] + '; '

		d = self.request('GET', url, Headers({
				'User-Agent': ['Mozilla/5.0 (compatible; Subtitle/0.1)'],
				'Cookie': [cookies]
				}),	None)

		def addContext(response):
			response.context = context
			return response

		d.addCallback(addContext)
		d.addCallback(self.__checkResponse)
		d.addCallback(self.__extractTitle)
		d.addErrback(self.__err)
		d.addCallback(self.Output)

	def __checkResponse(self, response):
		""" Checks the response from an http get and
		takes appropriate actions based on that response.

		should only be used as a callback attached to deferred
		returned by Get()

		Returns response object."""

		rcode = str(response.code)
		headers = response.headers
		if headers.hasHeader('content-type'):
			ctype = [headers.getRawHeaders('content-type')][0][0]
			m = charsetPattern.search(ctype)
			if not m is None:
				response.context['charset'] = m.group(1)
		if headers.hasHeader('set-cookie'):
			cookies = headers.getRawHeaders('set-cookie')
			for c in cookies:
				m = cookiePattern.search(c)
				(k, v) = m.group(1, 2)
				response.context['cookies'][k] = v
		if rcode.startswith('2'):
			return response
		elif headers.hasHeader('location'):       # we got a redirect
			location = headers.getRawHeaders('location')[0]
			domain = response.context['domain']
			if not location.startswith('http'):
				if not domain.startswith('/'):
					location = domain + '/' + location
				else:
					location = domain + location
			self.Get(location, response.context)
			return None
		elif rcode.startswith('4') or rcode.startswith('5'):
			self.Get(response.context['original_url'], response.context)
			return None
		else:
			print 'got something weird'
			print str(headers)
			return response

	def __extractTitle(self, response):
		""" We need to fire another deferred to get the body

		this method handles that.
		returns deferred. """

		if response is None:
			return None
		finished = Deferred()
		response.deliverBody(TitleGetter(finished, response.context))
		return finished

	def __err(self, failure):
		println("an error occurred in the title getter:", failure)

	def Output(self, data):
		""" default Output method.

		Should be overridden, ancestor can be called for debugging
		Should be at the end of the callback chain """

		if data is None:
			return None
		title, context = data[:2]
		print title + ' ' + str(context)

class TitleGetter(Protocol):
	""" Body handler class

	gets the title from the body"""

	def __init__(self, finished, context):
		self.context = context
		self.finished = finished
		self.remaining = 1024 * 1024
		self.bodyStr = ''
		self.titleSent = 0

	def dataReceived(self, bytes):
		""" Gets a chunk of data and checks it for a title

		if no title, adds the chunk to the bodystring

		does not return, if title is found, returns control
		to the callback chain """

		if self.remaining:
			chunk = bytes[:self.remaining]
			title = self.extractTitle(chunk)
			charset = getCharsetFromMetaTag(chunk)
			if not charset is None:
				self.context['charset'] = charset
			if title and self.titleSent == 0:
				self.titleSent = 1
				self.finished.callback([title, self.context])
			self.bodyStr = self.bodyStr + chunk
			self.remaining -= len(chunk)

	def connectionLost(self, reason):
		""" We've got every byte we're going to get,
		check for title now in case it was split between
		two chunks sent to dataReceived

		does not return, if title is found, returns control
		to the callback chain """

		title = self.extractTitle(self.bodyStr)
		charset = getCharsetFromMetaTag(self.bodyStr)
		if not charset is None:
			self.context['charset'] = charset
		if title and self.titleSent == 0:
			self.titleSent = 1
			self.finished.callback([title, self.context])

	def extractTitle(self, body):
		""" Extracts everything between title tags

		returns string """

		m = titlePattern.search(body)
		if m:
			title = m.group(1)
			try:
				self.context['charset']
			except KeyError:
				pass
			else:
				c = self.context['charset']
				c = c.lower()
				if not c.startswith('utf8') and not c.startswith('utf-8'):
					title = unicode(title, self.context['charset'], errors='replace')
				else:
					title = unicode(title, 'utf8', errors='replace')
			if not isinstance(title, unicode):
				title = unicode(title, 'utf8', errors='replace')
			title = processTitle(title)
			title = title.encode("utf-8", "ignore")
			return title

def getCharsetFromMetaTag(str):
	""" looks for a meta http-equiv content-type tag
	and gets the charset from it

	returns string"""
	m = metaCharsetPattern.search(str)
	if m:
		charset = m.group(1)
		return charset

def processTitle(title):
	""" performs miscellaneous processing on title

	returns string """

	title = string.strip(title)
	title = descape_ents(title)
	title = descape_decs(title)
	title = normalizeWhitespace(title)
	return title

def descape_dec(m):
	""" de-escape one html decimal entity

	ex: "

	returns string """

	return unichr(int(m.group(2)))

def descape_ent(m, defs=entityDefs):
	""" de-escape one html named entity

	ex: "

	returns string """

	try:
		return defs[m.group(1)]
	except KeyError:
		return m.group(0) # use as is

def descape_decs(string):
	""" de-escape all decimal entities in a string

	returns string """

	return decPattern.sub(descape_dec, string)

def descape_ents(string):
	""" de-escape all named entities in a string

	returns string """

	return entityPattern.sub(descape_ent, string)

def stripNoPrint(str):
	""" strips non-printable characters from a string

	***this function is incomplete, it works for our
	current needs - obviously there are other non-printable
	characters besides those below ascii 32***

	returns string """

	results = ""
	for char in str:
		if not int(ord(char)) <= 31:
			results += char
	return results

def normalizeWhitespace(str):
	""" replaces sequential whitespaces with a single space

	returns string """

	str = re.sub(whitespacePattern, ' ', str)
	return str

if __name__ == '__main__':
	g = Getter(reactor)
	for n in sys.argv[1:]:
		g.Get(n, dict())
	reactor.run()
<commit_msg>lil fix<commit_after>#!/usr/bin/python

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.web.http_headers import Headers
from twisted.python.util import println
from twisted.python.failure import Failure

import sys, re, string, htmlentitydefs

entityPattern = re.compile("&(\w+?);")
decPattern = re.compile("&#(\d+?);")
whitespacePattern = re.compile("\s+")
titlePattern = re.compile(r'<title>(.*?)</title>', re.S | re.I )
cookiePattern = re.compile("^(.+?)=(.*?);")
charsetPattern = re.compile(r'charset=([^\s]+)', re.I)
domainPattern = re.compile(r'^(https?://[^/]+)/', re.I)
metaCharsetPattern = re.compile(
	"meta http-equiv=\"content-type\".+?charset=([^\s]+)", re.I)

entityDefs = htmlentitydefs.entitydefs

class Getter(Agent):
	""" A title fetcher """

	def __init__(self, reactor):
		Agent.__init__(self, reactor)

	def Get(self, url, context, maxRetries=10):
		""" Start the lookup process for a title.

		context dict stores any information that has to
		come back with the response object

		context['cookies']:
			type dict
			cookie jar

		context['retry_count']:
			type int
			number of retries used so far

		context['original_url']:
			type str
			original url, in case we have to retry after
			following redirects

		context['charset']:
			type str
			charset received from server, so we can convert
			to utf-8

		context ['domain']:
			type str
			stores the scheme and domain portion of the url
			in case we get a relative redirect

		maxRetries defines the maximum number of times
		we retry the fetch after a 4xx or 5xx response code

		Returns deferred."""

		url = stripNoPrint(url)
		println('Get: ' + url)

		m = domainPattern.search(url)
		context['domain'] = m.group(1)

		try:
			context['cookies']
		except KeyError:
			context['cookies'] = dict()

		if not 'retry_count' in context:
			context['retry_count'] = 0
		elif context['retry_count'] >= maxRetries:
			println('exceeded max retry count for ' + str(url))
			return
		else:
			context['retry_count'] += 1
		context['original_url'] = url

		cookies = ''
		for k in context['cookies']:
			cookies += k + '=' + context['cookies'][k] + '; '

		d = self.request('GET', url, Headers({
				'User-Agent': ['Mozilla/5.0 (compatible; Subtitle/0.1)'],
				'Cookie': [cookies]
				}),	None)

		def addContext(response):
			response.context = context
			return response

		d.addCallback(addContext)
		d.addCallback(self.__checkResponse)
		d.addCallback(self.__extractTitle)
		d.addErrback(self.__err)
		d.addCallback(self.Output)

	def __checkResponse(self, response):
		""" Checks the response from an http get and
		takes appropriate actions based on that response.

		should only be used as a callback attached to deferred
		returned by Get()

		Returns response object."""

		rcode = str(response.code)
		headers = response.headers
		if headers.hasHeader('content-type'):
			ctype = [headers.getRawHeaders('content-type')][0][0]
			m = charsetPattern.search(ctype)
			if not m is None:
				response.context['charset'] = m.group(1)
		if headers.hasHeader('set-cookie'):
			cookies = headers.getRawHeaders('set-cookie')
			for c in cookies:
				m = cookiePattern.search(c)
				(k, v) = m.group(1, 2)
				response.context['cookies'][k] = v
		if rcode.startswith('2'):
			return response
		elif headers.hasHeader('location'):       # we got a redirect
			location = headers.getRawHeaders('location')[0]
			domain = response.context['domain']
			if not location.startswith('http'):
				if not domain.startswith('/'):
					location = domain + '/' + location
				else:
					location = domain + location
			self.Get(location, response.context)
			return None
		elif rcode.startswith('4') or rcode.startswith('5'):
			self.Get(response.context['original_url'], response.context)
			return None
		else:
			print 'got something weird'
			print str(headers)
			return response

	def __extractTitle(self, response):
		""" We need to fire another deferred to get the body

		this method handles that.
		returns deferred. """

		if response is None:
			return None
		finished = Deferred()
		response.deliverBody(TitleGetter(finished, response.context))
		return finished

	def __err(self, failure):
		println("an error occurred in the title getter:", failure)

	def Output(self, data):
		""" default Output method.

		Should be overridden, ancestor can be called for debugging
		Should be at the end of the callback chain """

		if data is None:
			return None
		title, context = data[:2]
		print title + ' ' + str(context)

class TitleGetter(Protocol):
	""" Body handler class

	gets the title from the body"""

	def __init__(self, finished, context):
		self.context = context
		self.finished = finished
		self.remaining = 1024 * 1024
		self.bodyStr = ''
		self.titleSent = 0

	def dataReceived(self, bytes):
		""" Gets a chunk of data and checks it for a title

		if no title, adds the chunk to the bodystring

		does not return, if title is found, returns control
		to the callback chain """

		if self.remaining:
			chunk = bytes[:self.remaining]
			title = self.extractTitle(chunk)
			charset = getCharsetFromMetaTag(chunk)
			if not charset is None:
				self.context['charset'] = charset
			if title and self.titleSent == 0:
				self.titleSent = 1
				self.finished.callback([title, self.context])
			self.bodyStr = self.bodyStr + chunk
			self.remaining -= len(chunk)

	def connectionLost(self, reason):
		""" We've got every byte we're going to get,
		check for title now in case it was split between
		two chunks sent to dataReceived

		does not return, if title is found, returns control
		to the callback chain """

		title = self.extractTitle(self.bodyStr)
		charset = getCharsetFromMetaTag(self.bodyStr)
		if not charset is None:
			self.context['charset'] = charset
		if title and self.titleSent == 0:
			self.titleSent = 1
			self.finished.callback([title, self.context])

	def extractTitle(self, body):
		""" Extracts everything between title tags

		returns string """

		m = titlePattern.search(body)
		if m:
			title = m.group(1)
			try:
				self.context['charset']
			except KeyError:
				pass
			else:
				c = self.context['charset']
				c = c.lower()
				if not c.startswith('utf8') and not c.startswith('utf-8'):
					title = unicode(title, self.context['charset'], errors='replace')
				else:
					title = unicode(title, 'utf8', errors='replace')
			if not isinstance(title, unicode):
				title = unicode(title, 'utf8', errors='replace')
			title = processTitle(title)
			title = title.encode("utf-8", "ignore")
			return title

def getCharsetFromMetaTag(str):
	""" looks for a meta http-equiv content-type tag
	and gets the charset from it

	returns string"""
	m = metaCharsetPattern.search(str)
	if m:
		charset = m.group(1)
		return charset

def processTitle(title):
	""" performs miscellaneous processing on title

	returns string """

	title = string.strip(title)
	title = descape_ents(title)
	title = descape_decs(title)
	title = normalizeWhitespace(title)
	return title

def descape_dec(m):
	""" de-escape one html decimal entity

	ex: "

	returns string """

	return unichr(int(m.group(1)))

def descape_ent(m, defs=entityDefs):
	""" de-escape one html named entity

	ex: "

	returns string """

	try:
		return defs[m.group(1)]
	except KeyError:
		return m.group(0) # use as is

def descape_decs(string):
	""" de-escape all decimal entities in a string

	returns string """

	return decPattern.sub(descape_dec, string)

def descape_ents(string):
	""" de-escape all named entities in a string

	returns string """

	return entityPattern.sub(descape_ent, string)

def stripNoPrint(str):
	""" strips non-printable characters from a string

	***this function is incomplete, it works for our
	current needs - obviously there are other non-printable
	characters besides those below ascii 32***

	returns string """

	results = ""
	for char in str:
		if not int(ord(char)) <= 31:
			results += char
	return results

def normalizeWhitespace(str):
	""" replaces sequential whitespaces with a single space

	returns string """

	str = re.sub(whitespacePattern, ' ', str)
	return str

if __name__ == '__main__':
	g = Getter(reactor)
	for n in sys.argv[1:]:
		g.Get(n, dict())
	reactor.run()
<|endoftext|>